Update README.md

[web] Add option to disable subnet rate limiting (+CE/SP Hotfix `5.2.1`)

GitOrigin-RevId: 78d60c9638cede729dd93c3c2421f55b34c0dbfe

.editorconfig

@@ -1,25 +0,0 @@
-root = true
-
-[*]
-charset = utf-8
-indent_style = space
-indent_size = 2
-end_of_line = lf
-insert_final_newline = true
-trim_trailing_whitespace = true
-
-[Makefile]
-indent_style = tab
-
-[*.go]
-indent_style = tab
-
-[*.{pug,coffee}]
-indent_style = tab
-
-[*.{pug,patch}]
-trim_trailing_whitespace = false
-
-[Jenkinsfile]
-insert_final_newline = false
-max_line_length = off

.github/ISSUE_TEMPLATE.md

@@ -0,0 +1,44 @@
+<!--
+
+Note: If you are using www.overleaf.com and have a problem, 
+      or if you would like to request a new feature please contact
+      the support team at support@overleaf.com
+      
+      This form should only be used to report bugs in the 
+      Community Edition release of Overleaf.
+
+-->
+
+
+
+<!-- BUG REPORT TEMPLATE -->
+
+## Steps to Reproduce
+<!-- Describe the steps leading up to when / where you found the bug. -->
+<!-- Screenshots may be helpful here. -->
+
+1.
+2.
+3.
+
+## Expected Behaviour
+<!-- What should have happened when you completed the steps above? -->
+
+## Observed Behaviour
+<!-- What actually happened when you completed the steps above? -->
+<!-- Screenshots may be helpful here. -->
+
+## Context
+<!-- How has this issue affected you? What were you trying to accomplish? -->
+
+## Technical Info
+<!-- Provide any technical details that may be applicable (or N/A if not applicable). -->
+
+* URL:
+* Browser Name and version:
+* Operating System and version (desktop or mobile):
+* Signed in as:
+* Project and/or file:
+
+## Analysis
+<!--- Optionally, document investigation of / suggest a fix for the bug, e.g. 'comes from this line / commit' -->

.github/ISSUE_TEMPLATE/bug_report.md

@@ -1,53 +0,0 @@
----
-name: Bug report
-about: Report a bug
-title: ''
-labels: type:bug
-assignees: ''
-
----
-
-<!--
-
-Note: If you are using www.overleaf.com and have a problem,
-      or if you would like to request a new feature please contact
-      the support team at support@overleaf.com
-
-      This form should only be used to report bugs in the
-      Community Edition release of Overleaf.
-
--->
-
-
-
-<!-- BUG REPORT TEMPLATE -->
-
-## Steps to Reproduce
-<!-- Describe the steps leading up to when / where you found the bug. -->
-<!-- Screenshots may be helpful here. -->
-
-1.
-2.
-3.
-
-## Expected Behaviour
-<!-- What should have happened when you completed the steps above? -->
-
-## Observed Behaviour
-<!-- What actually happened when you completed the steps above? -->
-<!-- Screenshots may be helpful here. -->
-
-## Context
-<!-- How has this issue affected you? What were you trying to accomplish? -->
-
-## Technical Info
-<!-- Provide any technical details that may be applicable (or N/A if not applicable). -->
-
-* URL:
-* Browser Name and version:
-* Operating System and version (desktop or mobile):
-* Signed in as:
-* Project and/or file:
-
-## Analysis
-<!--- Optionally, document investigation of / suggest a fix for the bug, e.g. 'comes from this line / commit' -->

.github/PULL_REQUEST_TEMPLATE.md

@@ -8,4 +8,4 @@
 
 ## Contributor Agreement
 
-- [ ] I confirm I have signed the [Contributor License Agreement](https://github.com/overleaf/overleaf/blob/main/CONTRIBUTING.md#contributor-license-agreement)
+- [ ] I confirm I have signed the [Contributor License Agreement](https://github.com/overleaf/overleaf/blob/master/CONTRIBUTING.md#contributor-license-agreement)

CONTRIBUTING.md

@@ -14,8 +14,6 @@ When submitting an issue please describe the issue as clearly as possible, inclu
 reproduce the bug, which situations it appears in, what you expected to happen, and what actually happens.
 If you can include a screenshot for front end issues that is very helpful.
 
-**Note**: If you are using [www.overleaf.com](www.overleaf.com) and have a problem, or if you would like to request a new feature, please contact the Support team at support@overleaf.com. Raise an issue here only to report bugs in the Community Edition release of Overleaf.
-
 Pull Requests
 -------------
 
@@ -36,7 +34,7 @@ Please see [our security policy](https://github.com/overleaf/overleaf/security/p
 Contributor License Agreement
 -----------------------------
 
-Before we can accept any contributions of code, we need you to agree to our
+Before we can accept any contributions of code, we need you to agree to our 
 [Contributor License Agreement](https://docs.google.com/forms/d/e/1FAIpQLSef79XH3mb7yIiMzZw-yALEegS-wyFetvjTiNBfZvf_IHD2KA/viewform?usp=sf_link).
 This is to ensure that you own the copyright of your contribution, and that you
 agree to give us a license to use it in both the open source version, and the version

README.md

@@ -6,8 +6,10 @@
 <h4 align="center">An open-source online real-time collaborative LaTeX editor.</h4>
 
 <p align="center">
-  <a href="https://github.com/yu-i-i/overleaf-cep/wiki">Wiki</a> •
+  <a href="https://github.com/overleaf/overleaf/wiki">Wiki</a> •
   <a href="https://www.overleaf.com/for/enterprises">Server Pro</a> •
+  <a href="#contributing">Contributing</a> •
+  <a href="https://mailchi.mp/overleaf.com/community-edition-and-server-pro">Mailing List</a> •
   <a href="#authors">Authors</a> •
   <a href="#license">License</a>
 </p>
@@ -21,26 +23,16 @@
 
 [Overleaf](https://www.overleaf.com) is an open-source online real-time collaborative LaTeX editor. Overleaf runs a hosted version at [www.overleaf.com](https://www.overleaf.com), but you can also run your own local version, and contribute to the development of Overleaf.
 
-## Extended Community Edition (CE+)
+## Extended Community Edition
 
 The present "extended" version of Overleaf CE includes:
 
-- Template Gallery
 - Sandboxed Compiles with TeX Live image selection
 - LDAP authentication
 - SAML authentication
-- OpenID Connect authentication
 - Real-time track changes and comments
 - Autocomplete of reference keys
 - Symbol Palette
-- "From External URL" feature
-- Advanced administrator tools for managing user accounts and projects
-
-> [!CAUTION]
-> Overleaf Community Edition is intended for use in environments where **all** users are trusted. Community Edition is **not** appropriate for scenarios where isolation of users is required due to Sandbox Compiles not being available. When not using Sandboxed Compiles, users have full read and write access to the `sharelatex` container resources (filesystem, network, environment variables) when running LaTeX compiles. 
-Therefore, in any environment where not all users can be fully trusted, it is strongly recommended to enable the Sandboxed Compiles feature available in the Extended Community Edition.
-
-For more information on Sandbox Compiles check out Overleaf [documentation](https://docs.overleaf.com/on-premises/configuration/overleaf-toolkit/server-pro-only-configuration/sandboxed-compiles).
 
 ## Enterprise
 
@@ -49,16 +41,658 @@ If you want help installing and maintaining Overleaf in your lab or workplace, O
 ## Installation
 
 Detailed installation instructions can be found in the [Overleaf Toolkit](https://github.com/overleaf/toolkit/).
-Configuration details and release history for the Extended Community Edition can be found on the [Extended CE Wiki Page](https://github.com/yu-i-i/overleaf-cep/wiki).
+To run a custom image, add a file named docker-compose.override.yml with the following or similar content into the `overleaf-toolkit/config directory`:
+
+```yml
+---
+version: '2.2'
+services:
+    sharelatex:
+        image: sharelatex/sharelatex:ext-ce
+        volumes:
+          - ../config/certs:/overleaf/certs
+```
+Here, the attached volume provides convenient access for the container to the certificates needed for SAML or LDAP authentication.
+
+If you want to build a Docker image of the extended CE based on the upstream v5.2.1 codebase, you can check out the corresponding tag by running:
+```
+git checkout v5.2.1-ext
+```
+Alternatively, you can download a prebuilt image from Docker Hub:
+```
+docker pull overleafcep/sharelatex:5.2.1-ext
+```
+Make sure to update the image name in overleaf-toolkit/config/docker-compose.override.yml accordingly.
+
+## Sandboxed Compiles
+
+To enable sandboxed compiles (also known as "Sibling containers"), set the following configuration options in `overleaf-toolkit/config/overleaf.rc`:
+
+```
+SERVER_PRO=true
+SIBLING_CONTAINERS_ENABLED=true
+```
+
+The following environment variables are used to specify which TeX Live images to use for sandboxed compiles:
+
+- `ALL_TEX_LIVE_DOCKER_IMAGES` **(required)**
+    * A comma-separated list of TeX Live images to use. These images will be downloaded or updated.
+      To skip downloading the images, set `SIBLING_CONTAINERS_PULL=false` in `config/overleaf.rc`.
+- `ALL_TEX_LIVE_DOCKER_IMAGE_NAMES`
+    * A comma-separated list of friendly names for the images. If omitted, the version name will be used (e.g., `latest-full`).
+- `TEX_LIVE_DOCKER_IMAGE` **(required)**
+    * The default TeX Live image that will be used for compiling new projects. The environment variable `ALL_TEX_LIVE_DOCKER_IMAGES` must include this image.
+
+Users can select the image for their project in the project menu.
+
+Here is an example where the default TeX Live image is `latest-full` from Docker Hub, but the `TL2023-historic` image can be used for older projects:
+```
+ALL_TEX_LIVE_DOCKER_IMAGES=texlive/texlive:latest-full, texlive/texlive:TL2023-historic
+ALL_TEX_LIVE_DOCKER_IMAGE_NAMES=TeXLive 2024, TeXLive 2023
+TEX_LIVE_DOCKER_IMAGE=texlive/texlive:latest-full
+```
+For additional details refer to
+[Server Pro: Sandboxed Compiles](https://github.com/overleaf/overleaf/wiki/Server-Pro:-Sandboxed-Compiles) and
+[Toolkit: Sandboxed Compiles](https://github.com/overleaf/toolkit/blob/master/doc/sandboxed-compiles.md).
+
+<details>
+<summary><h4>Sample variables.env file</h4></summary>
+
+```
+OVERLEAF_APP_NAME="Our Overleaf Instance"
+
+ENABLED_LINKED_FILE_TYPES=project_file,project_output_file
+
+# Enables Thumbnail generation using ImageMagick
+ENABLE_CONVERSIONS=true
+
+# Disables email confirmation requirement
+EMAIL_CONFIRMATION_DISABLED=true
+
+## Nginx
+# NGINX_WORKER_PROCESSES=4
+# NGINX_WORKER_CONNECTIONS=768
+
+## Set for TLS via nginx-proxy
+# OVERLEAF_BEHIND_PROXY=true
+# OVERLEAF_SECURE_COOKIE=true
+
+OVERLEAF_SITE_URL=http://my-overleaf-instance.com
+OVERLEAF_NAV_TITLE=Our Overleaf Instance
+# OVERLEAF_HEADER_IMAGE_URL=http://somewhere.com/mylogo.png
+OVERLEAF_ADMIN_EMAIL=support@example.com
+
+OVERLEAF_LEFT_FOOTER=[{"text": "Contact your support team", "url": "mailto:support@example.com"}]
+OVERLEAF_RIGHT_FOOTER=[{"text":"Hello, I am on the Right", "url":"https://github.com/yu-i-i/overleaf-cep"}]
+
+OVERLEAF_EMAIL_FROM_ADDRESS=team@example.com
+OVERLEAF_EMAIL_SMTP_HOST=smtp.example.com
+OVERLEAF_EMAIL_SMTP_PORT=587
+OVERLEAF_EMAIL_SMTP_SECURE=false
+# OVERLEAF_EMAIL_SMTP_USER=
+# OVERLEAF_EMAIL_SMTP_PASS=
+# OVERLEAF_EMAIL_SMTP_NAME=
+OVERLEAF_EMAIL_SMTP_LOGGER=false
+OVERLEAF_EMAIL_SMTP_TLS_REJECT_UNAUTH=true
+OVERLEAF_EMAIL_SMTP_IGNORE_TLS=false
+OVERLEAF_CUSTOM_EMAIL_FOOTER=This system is run by department x
+
+OVERLEAF_PROXY_LEARN=true
+NAV_HIDE_POWERED_BY=true
+
+########################
+## Sandboxed Compiles ##
+########################
+
+ALL_TEX_LIVE_DOCKER_IMAGES=texlive/texlive:latest-full, texlive/texlive:TL2023-historic
+ALL_TEX_LIVE_DOCKER_IMAGE_NAMES=TeXLive 2024, TeXLive 2023
+TEX_LIVE_DOCKER_IMAGE=texlive/texlive:latest-full
+```
+</details>
+
+## Authentication Methods
+
+The following authentication methods are supported: Local authentication, LDAP authentication, and SAML authentication. Local authentication is always active.
+To enable LDAP or SAML authentication, the environment variable `EXTERNAL_AUTH` must be set to `ldap` or `saml`, respectively.
+
+<details>
+<summary><h3>Local Authentication</h3></summary>
+
+Password of local users stored in the MongoDB database. An admin user can create a new local user. For details, visit the
+[wiki of Overleaf project](https://github.com/overleaf/overleaf/wiki/Creating-and-managing-users).
+
+It is possible to enforce password restrictions on local users:
+
+
+* `OVERLEAF_PASSWORD_VALIDATION_MIN_LENGTH`: The minimum length required
+
+* `OVERLEAF_PASSWORD_VALIDATION_MAX_LENGTH`: The maximum length allowed
+
+* `OVERLEAF_PASSWORD_VALIDATION_PATTERN`: is used to validate password strength
+
+  - `abc123` – password requires 3 letters and 3 numbers and be at least 6 characters long
+  - `aA` – password requires lower and uppercase letters and be at least 2 characters long
+  - `ab$3` – it must contain letters, digits and symbols and be at least 4 characters long
+  - There are 4 groups of characters: letters, UPPERcase letters, digits, symbols. Anything that is neither a letter nor a digit is considered to be a symbol.
+
+</details>
+
+<details>
+<summary><h3>LDAP Authentication</h3></summary>
+
+Internally, Overleaf LDAP uses the [passport-ldapauth](https://github.com/vesse/passport-ldapauth) library. Most of these configuration options are passed through to the `server` config object which is used to configure `passport-ldapauth`. If you are having issues configuring LDAP, it is worth reading the README for `passport-ldapauth` to understand the configuration it expects.
+
+#### Environment Variables
+
+- `OVERLEAF_LDAP_URL` **(required)**
+    * URL of the LDAP server.
+
+      - Example: `ldaps://ldap.example.com:636` (LDAP over SSL)
+      - Example: `ldap://ldap.example.com:389` (unencrypted or STARTTLS, if configured).
+
+- `OVERLEAF_LDAP_EMAIL_ATT`
+    * The email attribute returned by the LDAP server, default `mail`. Each LDAP user must have at least one email address.
+      If multiple addresses are provided, only the first one will be used.
+
+- `OVERLEAF_LDAP_FIRST_NAME_ATT`
+    * The property name holding the first name of the user which is used in the application, usually `givenName`.
+
+- `OVERLEAF_LDAP_LAST_NAME_ATT`
+    * The property name holding the family name of the user which is used in the application, usually `sn`.
+
+- `OVERLEAF_LDAP_NAME_ATT`
+    * The property name holding the full name of the user, usually `cn`. If either of the two previous variables is not defined,
+      the first and/or last name of the user is extracted from this variable. Otherwise, it is not used.
+
+- `OVERLEAF_LDAP_PLACEHOLDER`
+    * The placeholder for the login form, defaults to `Username`.
+
+- `OVERLEAF_LDAP_UPDATE_USER_DETAILS_ON_LOGIN`
+    * If set to `true`, updates the LDAP user `first_name` and `last_name` field on login, and turn off the user details form on the `/user/settings`
+      page for LDAP users. Otherwise, details will be fetched only on first login.
+
+- `OVERLEAF_LDAP_BIND_DN`
+    * The distinguished name of the LDAP user that should be used for the LDAP connection
+      (this user should be able to search/list accounts on the LDAP server),
+      e.g., `cn=ldap_reader,dc=example,dc=com`. If not defined, anonymous binding is used.
+
+- `OVERLEAF_LDAP_BIND_CREDENTIALS`
+    * Password for `OVERLEAF_LDAP_BIND_DN`.
+
+- `OVERLEAF_LDAP_BIND_PROPERTY`
+    * Property of the user to bind against the client, defaults to `dn`.
+
+- `OVERLEAF_LDAP_SEARCH_BASE` **(required)**
+    * The base DN from which to search for users. E.g., `ou=people,dc=example,dc=com`.
+
+- `OVERLEAF_LDAP_SEARCH_FILTER`
+    * LDAP search filter with which to find a user. Use the literal '{{username}}' to have the given username be interpolated in for the LDAP search.
+
+      - Example: `(|(uid={{username}})(mail={{username}}))` (user can login with email or with login name).
+      - Example: `(sAMAccountName={{username}})` (Active Directory).
+
+- `OVERLEAF_LDAP_SEARCH_SCOPE`
+    * The scope of the search can be `base`, `one`, or `sub` (default).
+
+- `OVERLEAF_LDAP_SEARCH_ATTRIBUTES`
+    * JSON array of attributes to fetch from the LDAP server, e.g., `["uid", "mail", "givenName", "sn"]`.
+      By default, all attributes are fetched.
+
+- `OVERLEAF_LDAP_STARTTLS`
+    * If `true`, LDAP over TLS is used.
+
+- `OVERLEAF_LDAP_TLS_OPTS_CA_PATH`
+    * Path to the file containing the CA certificate used to verify the LDAP server's SSL/TLS certificate. If there are multiple certificates, then
+      it can be a JSON array of paths to the certificates. The files must be accessible to the docker container.
+
+      - Example (one certificate): `/overleaf/certs/ldap_ca_cert.pem`
+      - Example (multiple certificates): `["/overleaf/certs/ldap_ca_cert1.pem", "/overleaf/certs/ldap_ca_cert2.pem"]`
+
+- `OVERLEAF_LDAP_TLS_OPTS_REJECT_UNAUTH`
+    * If `true`, the server certificate is verified against the list of supplied CAs.
+
+- `OVERLEAF_LDAP_CACHE`
+    * If `true`, then up to 100 credentials at a time will be cached for 5 minutes.
+
+- `OVERLEAF_LDAP_TIMEOUT`
+    * How long the client should let operations live for before timing out, ms (Default: Infinity).
+
+- `OVERLEAF_LDAP_CONNECT_TIMEOUT`
+    * How long the client should wait before timing out on TCP connections, ms (Default: OS default).
+
+- `OVERLEAF_LDAP_IS_ADMIN_ATT` and `OVERLEAF_LDAP_IS_ADMIN_ATT_VALUE`
+    * When both environment variables are set, the login process updates `user.isAdmin = true` if the LDAP profile contains the attribute specified by
+     `OVERLEAF_LDAP_IS_ADMIN_ATT` and its value either matches `OVERLEAF_LDAP_IS_ADMIN_ATT_VALUE` or is an array containing `OVERLEAF_LDAP_IS_ADMIN_ATT_VALUE`,
+      otherwise `user.isAdmin` is set to `false`. If either of these variables is not set, then the admin status is only set to `true` during admin user
+      creation in Launchpad.
+
+The following five variables are used to configure how user contacts are retrieved from the LDAP server.
+
+- `OVERLEAF_LDAP_CONTACTS_FILTER`
+    * The filter used to search for users in the LDAP server to be loaded into contacts. The placeholder '{{userProperty}}' within the filter is replaced with the value of
+    the property specified by `OVERLEAF_LDAP_CONTACTS_PROPERTY` from the LDAP user initiating the search. If not defined, no users are retrieved from the LDAP server into contacts.
+
+- `OVERLEAF_LDAP_CONTACTS_SEARCH_BASE`
+    * Specifies the base DN from which to start searching for the contacts. Defaults to `OVERLEAF_LDAP_SEARCH_BASE`.
+
+- `OVERLEAF_LDAP_CONTACTS_SEARCH_SCOPE`
+    * The scope of the search can be `base`, `one`, or `sub` (default).
+
+- `OVERLEAF_LDAP_CONTACTS_PROPERTY`
+    * Specifies the property of the user object that will replace the '{{userProperty}}' placeholder in the `OVERLEAF_LDAP_CONTACTS_FILTER`.
+
+- `OVERLEAF_LDAP_CONTACTS_NON_LDAP_VALUE`
+    * Specifies the value of the `OVERLEAF_LDAP_CONTACTS_PROPERTY` if the search is initiated by a non-LDAP user. If this variable is not defined, the resulting filter
+      will match nothing. The value `*` can be used as a wildcard.
+
+<details>
+<summary><h5>Example</h5></summary>
+
+    OVERLEAF_LDAP_CONTACTS_FILTER=(gidNumber={{userProperty}})
+    OVERLEAF_LDAP_CONTACTS_PROPERTY=gidNumber
+    OVERLEAF_LDAP_CONTACTS_NON_LDAP_VALUE=1000
+
+The above example results in loading into the contacts of the current LDAP user all LDAP users who have the same UNIX `gid`. Non-LDAP users will have all LDAP users with UNIX `gid=1000` in their contacts.
+
+</details>
+
+
+<details>
+<summary><h4>Sample variables.env file</h4></summary>
+
+```
+OVERLEAF_APP_NAME="Our Overleaf Instance"
+
+ENABLED_LINKED_FILE_TYPES=project_file,project_output_file
+
+# Enables Thumbnail generation using ImageMagick
+ENABLE_CONVERSIONS=true
+
+# Disables email confirmation requirement
+EMAIL_CONFIRMATION_DISABLED=true
+
+## Nginx
+# NGINX_WORKER_PROCESSES=4
+# NGINX_WORKER_CONNECTIONS=768
+
+## Set for TLS via nginx-proxy
+# OVERLEAF_BEHIND_PROXY=true
+# OVERLEAF_SECURE_COOKIE=true
+
+OVERLEAF_SITE_URL=http://my-overleaf-instance.com
+OVERLEAF_NAV_TITLE=Our Overleaf Instance
+# OVERLEAF_HEADER_IMAGE_URL=http://somewhere.com/mylogo.png
+OVERLEAF_ADMIN_EMAIL=support@example.com
+
+OVERLEAF_LEFT_FOOTER=[{"text": "Contact your support team", "url": "mailto:support@example.com"}]
+OVERLEAF_RIGHT_FOOTER=[{"text":"Hello, I am on the Right", "url":"https://github.com/yu-i-i/overleaf-cep"}]
+
+OVERLEAF_EMAIL_FROM_ADDRESS=team@example.com
+OVERLEAF_EMAIL_SMTP_HOST=smtp.example.com
+OVERLEAF_EMAIL_SMTP_PORT=587
+OVERLEAF_EMAIL_SMTP_SECURE=false
+# OVERLEAF_EMAIL_SMTP_USER=
+# OVERLEAF_EMAIL_SMTP_PASS=
+# OVERLEAF_EMAIL_SMTP_NAME=
+OVERLEAF_EMAIL_SMTP_LOGGER=false
+OVERLEAF_EMAIL_SMTP_TLS_REJECT_UNAUTH=true
+OVERLEAF_EMAIL_SMTP_IGNORE_TLS=false
+OVERLEAF_CUSTOM_EMAIL_FOOTER=This system is run by department x
+
+OVERLEAF_PROXY_LEARN=true
+NAV_HIDE_POWERED_BY=true
+
+#################
+## LDAP for CE ##
+#################
+
+EXTERNAL_AUTH=ldap
+OVERLEAF_LDAP_URL=ldap://ldap.example.com:389
+OVERLEAF_LDAP_STARTTLS=true
+OVERLEAF_LDAP_TLS_OPTS_CA_PATH=/overleaf/certs/ldap_ca_cert.pem
+OVERLEAF_LDAP_SEARCH_BASE=ou=people,dc=example,dc=com
+OVERLEAF_LDAP_SEARCH_FILTER=(|(uid={{username}})(mail={{username}}))
+OVERLEAF_LDAP_BIND_DN=cn=ldap_reader,dc=example,dc=com
+OVERLEAF_LDAP_BIND_CREDENTIALS=GoodNewsEveryone
+OVERLEAF_LDAP_EMAIL_ATT=mail
+OVERLEAF_LDAP_FIRST_NAME_ATT=givenName
+OVERLEAF_LDAP_LAST_NAME_ATT=sn
+# OVERLEAF_LDAP_NAME_ATT=cn
+OVERLEAF_LDAP_SEARCH_ATTRIBUTES=["uid", "sn", "givenName", "mail"]
+
+OVERLEAF_LDAP_UPDATE_USER_DETAILS_ON_LOGIN=true
+
+OVERLEAF_LDAP_PLACEHOLDER='Username or email address'
+
+OVERLEAF_LDAP_IS_ADMIN_ATT=mail
+OVERLEAF_LDAP_IS_ADMIN_ATT_VALUE=admin@example.com
+
+OVERLEAF_LDAP_CONTACTS_FILTER=(gidNumber={{userProperty}})
+OVERLEAF_LDAP_CONTACTS_PROPERTY=gidNumber
+OVERLEAF_LDAP_CONTACTS_NON_LDAP_VALUE='*'
+```
+</details>
+
+<details>
+<summary><i>Deprecated variables</i></summary>
+
+**These variables will be removed soon**, use `OVERLEAF_LDAP_IS_ADMIN_ATT` and `OVERLEAF_LDAP_IS_ADMIN_ATT_VALUE` instead.
+
+The following variables are used to determine if the user has admin rights.
+Please note: the user gains admin status if the search result is not empty, not when the user is explicitly included in the search results.
+
+- `OVERLEAF_LDAP_ADMIN_SEARCH_BASE`
+    * Specifies the base DN from which to start searching for the admin group. If this variable is defined,
+    `OVERLEAF_LDAP_ADMIN_SEARCH_FILTER` must also be defined for the search to function properly.
+
+- `OVERLEAF_LDAP_ADMIN_SEARCH_FILTER`
+    * Defines the LDAP search filter used to identify the admin group. The placeholder `{{dn}}` within the filter
+      is replaced with the value of the property specified by `OVERLEAF_LDAP_ADMIN_DN_PROPERTY`. The placeholder `{{username}}` is also supported.
+
+- `OVERLEAF_LDAP_ADMIN_DN_PROPERTY`
+    * Specifies the property of the user object that will replace the '{{dn}}' placeholder
+      in the `OVERLEAF_LDAP_ADMIN_SEARCH_FILTER`, defaults to `dn`.
+
+- `OVERLEAF_LDAP_ADMIN_SEARCH_SCOPE`
+    * The scope of the LDAP search can be `base`, `one`, or `sub` (default)
+
+<details>
+<summary><h5>Example</h5></summary>
+
+In the following example admins are members of a group `admins`, the objectClass of the entry `admins` is `groupOfNames`:
+
+    OVERLEAF_LDAP_ADMIN_SEARCH_BASE='cn=admins,ou=group,dc=example,dc=com'
+    OVERLEAF_LDAP_ADMIN_SEARCH_FILTER='(member={{dn}})'
+
+In the following example admins are members of a group 'admins', the objectClass of the entry `admins` is `posixGroup`:
+
+    OVERLEAF_LDAP_ADMIN_SEARCH_BASE='cn=admins,ou=group,dc=example,dc=com'
+    OVERLEAF_LDAP_ADMIN_SEARCH_FILTER='(memberUid={{username}})'
+
+In the following example admins are users with UNIX gid=1234:
+
+    OVERLEAF_LDAP_ADMIN_SEARCH_BASE='ou=people,dc=example,dc=com'
+    OVERLEAF_LDAP_ADMIN_SEARCH_FILTER='(&(gidNumber=1234)(uid={{username}}))'
+
+In the following example admin is the user with `uid=someuser`:
+
+    OVERLEAF_LDAP_ADMIN_SEARCH_BASE='ou=people,dc=example,dc=com'
+    OVERLEAF_LDAP_ADMIN_SEARCH_FILTER='(&(uid=someuser)(uid={{username}}))'
+
+The filter
+
+    OVERLEAF_LDAP_ADMIN_SEARCH_FILTER='(uid=someuser)'
+
+where `someuser` is the uid of an existing user, will always produce a non-empty search result.
+As a result, **every user will be granted admin rights**, not just `someuser`, as one might expect.
+
+</details>
+</details>
+</details>
+
+<details>
+<summary><h3>SAML Authentication</h3></summary>
+
+Internally, Overleaf SAML module uses the [passport-saml](https://github.com/node-saml/passport-saml) library, most of the following
+configuration options are passed through to `passport-saml`. If you are having issues configuring SAML, it is worth reading the README
+for `passport-saml` to get a feel for the configuration it expects.
+
+#### Environment Variables
+
+- `OVERLEAF_SAML_IDENTITY_SERVICE_NAME`
+    * Display name for the identity service, used on the login page (default: `Login with SAML IdP`).
+
+- `OVERLEAF_SAML_EMAIL_FIELD`
+    * Name of the Email field in user profile, default to 'nameID'.
+
+- `OVERLEAF_SAML_FIRST_NAME_FIELD`
+    * Name of the firstName field in user profile, default to 'givenName'.
+
+- `OVERLEAF_SAML_LAST_NAME_FIELD`
+    * Name of the lastName field in user profile, default to 'lastName'
+
+- `OVERLEAF_SAML_UPDATE_USER_DETAILS_ON_LOGIN`
+    * If set to `true`, updates the user `first_name` and `last_name` field on login,
+      and turn off the user details form on `/user/settings` page.
+
+- `OVERLEAF_SAML_ENTRYPOINT` **(required)**
+    * Entrypoint URL for the SAML identity service.
+
+      - Example: `https://idp.example.com/simplesaml/saml2/idp/SSOService.php`
+      - Azure Example: `https://login.microsoftonline.com/8b26b46a-6dd3-45c7-a104-f883f4db1f6b/saml2`
+
+- `OVERLEAF_SAML_CALLBACK_URL` **(required)**
+    * Callback URL for Overleaf service. Should be the full URL of the `/saml/login/callback` path.
+
+      - Example: `https://my-overleaf-instance.com/saml/login/callback`
+
+- `OVERLEAF_SAML_ISSUER` **(required)**
+    * The Issuer name.
+
+- `OVERLEAF_SAML_AUDIENCE`
+    * Expected saml response Audience, defaults to value of `OVERLEAF_SAML_ISSUER`.
+
+- `OVERLEAF_SAML_IDP_CERT` **(required)**
+    * Path to a file containing the Identity Provider's public certificate, used to validate the signatures of incoming SAML responses. If the Identity Provider has multiple valid signing certificates, then
+      it can be a JSON array of paths to the certificates.
+
+        - Example (one certificate): `/overleaf/certs/idp_cert.pem`
+        - Example (multiple certificates): `["/overleaf/certs/idp_cert.pem", "/overleaf/certs/idp_cert_old.pem"]`
+
+- `OVERLEAF_SAML_PUBLIC_CERT`
+    * Path to a file containing public signing certificate used to embed in auth requests in order for the IdP to validate the signatures of the incoming SAML Request. It's required when setting up the [metadata endpoint](#metadata-for-the-identity-provider)
+      when the strategy is configured with a `OVERLEAF_SAML_PRIVATE_KEY`. A JSON array of paths to certificates can be provided to support certificate rotation. When supplying an array of certificates, the first entry in the array should match the
+      current `OVERLEAF_SAML_PRIVATE_KEY`. Additional entries in the array can be used to publish upcoming certificates to IdPs before changing the `OVERLEAF_SAML_PRIVATE_KEY`.
+
+- `OVERLEAF_SAML_PRIVATE_KEY`
+    * Path to a file containing a PEM-formatted private key matching the `OVERLEAF_SAML_PUBLIC_CERT` used to sign auth requests sent by passport-saml.
+
+- `OVERLEAF_SAML_DECRYPTION_CERT`
+    *  Path to a file containing public certificate, used for the [metadata endpoint](#metadata-for-the-identity-provider).
+
+- `OVERLEAF_SAML_DECRYPTION_PVK`
+    * Path to a file containing private key  matching the `OVERLEAF_SAML_DECRYPTION_CERT` that will be used to attempt to decrypt any encrypted assertions that are received.
+
+- `OVERLEAF_SAML_SIGNATURE_ALGORITHM`
+    * Optionally set the signature algorithm for signing requests,
+          valid values are 'sha1' (default), 'sha256' (prefered), 'sha512' (most secure, check if your IdP supports it).
+
+- `OVERLEAF_SAML_ADDITIONAL_PARAMS`
+    * JSON dictionary of additional query params to add to all requests.
+
+- `OVERLEAF_SAML_ADDITIONAL_AUTHORIZE_PARAMS`
+    * JSON dictionary of additional query params to add to 'authorize' requests.
+
+          - Example: `{"some_key": "some_value"}`
+
+- `OVERLEAF_SAML_IDENTIFIER_FORMAT`
+    * Name identifier format to request from identity provider (default: `urn:oasis:names:tc:SAML:1.1:nameid-format:emailAddress`).
+
+- `OVERLEAF_SAML_ACCEPTED_CLOCK_SKEW_MS`
+    * Time in milliseconds of skew that is acceptable between client and server when checking OnBefore and NotOnOrAfter assertion
+          condition validity timestamps. Setting to -1 will disable checking these conditions entirely. Default is 0.
+
+- `OVERLEAF_SAML_ATTRIBUTE_CONSUMING_SERVICE_INDEX`
+    * `AttributeConsumingServiceIndex` attribute to add to AuthnRequest to instruct the IdP which attribute set to attach
+          to the response ([link](http://blog.aniljohn.com/2014/01/data-minimization-front-channel-saml-attribute-requests.html)).
+
+- `OVERLEAF_SAML_AUTHN_CONTEXT`
+    * JSON array of name identifier format values to request auth context. Default: `["urn:oasis:names:tc:SAML:2.0:ac:classes:PasswordProtectedTransport"]`.
+
+- `OVERLEAF_SAML_FORCE_AUTHN`
+    * If `true`, the initial SAML request from the service provider specifies that the IdP should force re-authentication of the user,
+          even if they possess a valid session.
+
+- `OVERLEAF_SAML_DISABLE_REQUESTED_AUTHN_CONTEXT`
+    * If `true`, do not request a specific auth context. For example, you can this this to `true` to allow additional contexts such as password-less logins (`urn:oasis:names:tc:SAML:2.0:ac:classes:X509`). Support for additional contexts is dependant on your IdP.
+
+- `OVERLEAF_SAML_AUTHN_REQUEST_BINDING`
+    * If set to `HTTP-POST`, will request authentication from IdP via HTTP POST binding, otherwise defaults to HTTP-Redirect.
+
+- `OVERLEAF_SAML_VALIDATE_IN_RESPONSE_TO`
+    * If `always`, then InResponseTo will be validated from incoming SAML responses.
+    * If `never`, then InResponseTo  won't be validated (default).
+    * If `ifPresent`, then InResponseTo will only be validated if present in the incoming SAML response.
+
+- `OVERLEAF_SAML_REQUEST_ID_EXPIRATION_PERIOD_MS`
+    * Defines the expiration time when a Request ID generated for a SAML request will not be valid if seen
+          in a SAML response in the `InResponseTo` field. Default: 28800000 (8 hours).
+
+- `OVERLEAF_SAML_LOGOUT_URL`
+    * base address to call with logout requests (default: `entryPoint`).
+
+      - Example: `https://idp.example.com/simplesaml/saml2/idp/SingleLogoutService.php`
+
+- `OVERLEAF_SAML_LOGOUT_CALLBACK_URL`
+    * Callback URL for IdP initiated logout. Should be the full URL of the `/saml/logot/callback` path.
+       With this value the `Location` attribute in the `SingleLogoutService` elements in the generated service provider metadata is populated with this value.
+
+      - Example: `https://my-overleaf-instance.com/saml/logout/callback`
+
+- `OVERLEAF_SAML_ADDITIONAL_LOGOUT_PARAMS`
+    * JSON dictionary of additional query params to add to 'logout' requests.
+
+
+- `OVERLEAF_SAML_IS_ADMIN_FIELD` and `OVERLEAF_SAML_IS_ADMIN_FIELD_VALUE`
+    * When both environment variables are set, the login process updates `user.isAdmin = true` if the profile returned by the SAML IdP contains the attribute specified by
+     `OVERLEAF_SAML_IS_ADMIN_FIELD` and its value either matches `OVERLEAF_SAML_IS_ADMIN_FIELD_VALUE` or is an array containing `OVERLEAF_SAML_IS_ADMIN_FIELD_VALUE`,
+      otherwise `user.isAdmin` is set to `false`. If either of these variables is not set, then the admin status is only set to `true` during admin user.
+      creation in Launchpad.
+
+#### Metadata for the Identity Provider
+
+The current version of Overleaf CE includes and endpoint to retrieve Service Provider Metadata: `http://my-overleaf-instance.com/saml/meta`
+
+The Identity Provider will need to be configured to recognize the Overleaf server as a "Service Provider". Consult the documentation for your SAML server for instructions on how to do this.
+
+Below is an example of appropriate Service Provider metadata:
+
+<details>
+<summary><h5>ol-meta.xml</h5></summary>
+
+```
+<?xml version="1.0"?>
+<EntityDescriptor xmlns="urn:oasis:names:tc:SAML:2.0:metadata"
+                  xmlns:ds="http://www.w3.org/2000/09/xmldsig#"
+                  entityID="MyOverleaf"
+                  ID="_b508c83b7dda452f5b269383fb391107116f8f57">
+  <SPSSODescriptor protocolSupportEnumeration="urn:oasis:names:tc:SAML:2.0:protocol" AuthnRequestsSigned="true" WantAssertionsSigned="true">
+    <KeyDescriptor use="signing">
+      <ds:KeyInfo>
+        <ds:X509Data>
+          <ds:X509Certificate>MII...
+[skipped]
+</ds:X509Certificate>
+        </ds:X509Data>
+      </ds:KeyInfo>
+    </KeyDescriptor>
+    <KeyDescriptor use="encryption">
+      <ds:KeyInfo>
+        <ds:X509Data>
+          <ds:X509Certificate>MII...
+[skipped]
+</ds:X509Certificate>
+        </ds:X509Data>
+      </ds:KeyInfo>
+      <EncryptionMethod Algorithm="http://www.w3.org/2009/xmlenc11#aes256-gcm"/>
+      <EncryptionMethod Algorithm="http://www.w3.org/2009/xmlenc11#aes128-gcm"/>
+      <EncryptionMethod Algorithm="http://www.w3.org/2001/04/xmlenc#aes256-cbc"/>
+      <EncryptionMethod Algorithm="http://www.w3.org/2001/04/xmlenc#aes128-cbc"/>
+    </KeyDescriptor>
+    <SingleLogoutService Binding="urn:oasis:names:tc:SAML:2.0:bindings:HTTP-POST"
+                         Location="https://my-overleaf-instance.com/saml/logout/callback"/>
+    <NameIDFormat>urn:oasis:names:tc:SAML:1.1:nameid-format:emailAddress</NameIDFormat>
+    <AssertionConsumerService index="1"
+                              isDefault="true"
+                              Binding="urn:oasis:names:tc:SAML:2.0:bindings:HTTP-POST"
+                              Location="https://my-overleaf-instance.com/saml/login/callback"/>
+  </SPSSODescriptor>
+</EntityDescriptor>
+
+```
+</details>
+
+Note the certificates, `AssertionConsumerService.Location`, `SingleLogoutService.Location` and `EntityDescriptor.entityID`
+and set as appropriate in your IdP configuration, or send the metadata file to the IdP admin.
+
+<details>
+<summary><h4>Sample variables.env file</h4></summary>
+
+```
+OVERLEAF_APP_NAME="Our Overleaf Instance"
+
+ENABLED_LINKED_FILE_TYPES=project_file,project_output_file
+
+# Enables Thumbnail generation using ImageMagick
+ENABLE_CONVERSIONS=true
+
+# Disables email confirmation requirement
+EMAIL_CONFIRMATION_DISABLED=true
+
+## Nginx
+# NGINX_WORKER_PROCESSES=4
+# NGINX_WORKER_CONNECTIONS=768
+
+## Set for TLS via nginx-proxy
+# OVERLEAF_BEHIND_PROXY=true
+# OVERLEAF_SECURE_COOKIE=true
+
+OVERLEAF_SITE_URL=http://my-overleaf-instance.com
+OVERLEAF_NAV_TITLE=Our Overleaf Instance
+# OVERLEAF_HEADER_IMAGE_URL=http://somewhere.com/mylogo.png
+OVERLEAF_ADMIN_EMAIL=support@example.com
+
+OVERLEAF_LEFT_FOOTER=[{"text": "Contact your support team", "url": "mailto:support@example.com"}]
+OVERLEAF_RIGHT_FOOTER=[{"text":"Hello, I am on the Right", "url":"https://github.com/yu-i-i/overleaf-cep"}]
+
+OVERLEAF_EMAIL_FROM_ADDRESS=team@example.com
+OVERLEAF_EMAIL_SMTP_HOST=smtp.example.com
+OVERLEAF_EMAIL_SMTP_PORT=587
+OVERLEAF_EMAIL_SMTP_SECURE=false
+# OVERLEAF_EMAIL_SMTP_USER=
+# OVERLEAF_EMAIL_SMTP_PASS=
+# OVERLEAF_EMAIL_SMTP_NAME=
+OVERLEAF_EMAIL_SMTP_LOGGER=false
+OVERLEAF_EMAIL_SMTP_TLS_REJECT_UNAUTH=true
+OVERLEAF_EMAIL_SMTP_IGNORE_TLS=false
+OVERLEAF_CUSTOM_EMAIL_FOOTER=This system is run by department x
+
+OVERLEAF_PROXY_LEARN=true
+NAV_HIDE_POWERED_BY=true
+
+#################
+## SAML for CE ##
+#################
+
+EXTERNAL_AUTH=saml
+OVERLEAF_SAML_IDENTITY_SERVICE_NAME='Login with My IdP'
+OVERLEAF_SAML_EMAIL_FIELD=mail
+OVERLEAF_SAML_FIRST_NAME_FIELD=givenName
+OVERLEAF_SAML_LAST_NAME_FIELD=sn
+OVERLEAF_SAML_ENTRYPOINT=https://idp.example.com/simplesamlphp/saml2/idp/SSOService.php
+OVERLEAF_SAML_CALLBACK_URL=https://my-overleaf-instance.com/saml/login/callback
+OVERLEAF_SAML_LOGOUT_URL=https://idp.example.com/simplesamlphp/saml2/idp/SingleLogoutService.php
+OVERLEAF_SAML_LOGOUT_CALLBACK_URL=https://my-overleaf-instance.com/saml/logout/callback
+OVERLEAF_SAML_ISSUER=MyOverleaf
+OVERLEAF_SAML_IDP_CERT=/overleaf/certs/idp_cert.pem
+OVERLEAF_SAML_PUBLIC_CERT=/overleaf/certs/myol_cert.pem
+OVERLEAF_SAML_PRIVATE_KEY=/overleaf/certs/myol_key.pem
+OVERLEAF_SAML_DECRYPTION_CERT=/overleaf/certs/myol_decr_cert.pem
+OVERLEAF_SAML_DECRYPTION_PVK=/overleaf/certs/myol_decr_key.pem
+OVERLEAF_SAML_IS_ADMIN_FIELD=mail
+OVERLEAF_SAML_IS_ADMIN_FIELD_VALUE=overleaf.admin@example.com
+```
+</details>
+</details>
 
 ## Overleaf Docker Image
 
 This repo contains two dockerfiles, [`Dockerfile-base`](server-ce/Dockerfile-base), which builds the
-`sharelatex/sharelatex-base:ext-ce` image, and [`Dockerfile`](server-ce/Dockerfile) which builds the
-`sharelatex/sharelatex:ext-ce` image.
+`sharelatex/sharelatex-base` image, and [`Dockerfile`](server-ce/Dockerfile) which builds the
+`sharelatex/sharelatex` (or "community") image.
 
-The Base image generally contains the basic dependencies like `wget`, plus `texlive`.
-This is split out because it's a pretty heavy set of
+The Base image generally contains the basic dependencies like `wget` and
+`aspell`, plus `texlive`. We split this out because it's a pretty heavy set of
 dependencies, and it's nice to not have to rebuild all of that every time.
 
 The `sharelatex/sharelatex` image extends the base image and adds the actual Overleaf code
@@ -66,19 +700,20 @@ and services.
 
 Use `make build-base` and `make build-community` from `server-ce/` to build these images.
 
-The [Phusion base-image](https://github.com/phusion/baseimage-docker)
-(which is extended by the `base` image) provides a VM-like container
+We use the [Phusion base-image](https://github.com/phusion/baseimage-docker)
+(which is extended by our `base` image) to provide us with a VM-like container
 in which to run the Overleaf services. Baseimage uses the `runit` service
-manager to manage services, and init scripts from the `server-ce/runit`
-folder are added.
+manager to manage services, and we add our init-scripts from the `server-ce/runit`
+folder.
 
 ## Authors
 
-[The Overleaf Team](https://www.overleaf.com/about)  
-[yu-i-i](https://github.com/yu-i-i/overleaf-cep) — Extensions for CE unless otherwise noted
+[The Overleaf Team](https://www.overleaf.com/about)
+<br>
+Extensions for CE by: [yu-i-i](https://github.com/yu-i-i/overleaf-cep)
 
 ## License
 
 The code in this repository is released under the GNU AFFERO GENERAL PUBLIC LICENSE, version 3. A copy can be found in the [`LICENSE`](LICENSE) file.
 
-Copyright (c) Overleaf, 2014-2025.
+Copyright (c) Overleaf, 2014-2024.

bin/shared/mongodb-init-replica-set.js

@@ -1,3 +0,0 @@
-/* eslint-disable no-undef */
-
-rs.initiate({ _id: 'overleaf', members: [{ _id: 0, host: 'mongo:27017' }] })

develop/README.md

@@ -11,6 +11,12 @@ bin/build
 > [!NOTE]
 > If Docker is running out of RAM while building the services in parallel, create a `.env` file in this directory containing `COMPOSE_PARALLEL_LIMIT=1`.
 
+Next, initialize the database:
+
+```shell
+bin/init
+```
+
 Then start the services:
 
 ```shell
@@ -42,7 +48,7 @@ To do this, use the included `bin/dev` script:
 bin/dev
 ```
 
-This will start all services using `node --watch`, which will automatically monitor the code and restart the services as necessary.
+This will start all services using `nodemon`, which will automatically monitor the code and restart the services as necessary.
 
 To improve performance, you can start only a subset of the services in development mode by providing a space-separated list to the `bin/dev` script:
 
@@ -77,10 +83,8 @@ each service:
 | `filestore`        | 9235 |
 | `notifications`    | 9236 |
 | `real-time`        | 9237 |
-| `references`       | 9238 |
 | `history-v1`       | 9239 |
 | `project-history`  | 9240 |
-| `linked-url-proxy` | 9241 |
 
 To attach to a service using Chrome's _remote debugging_, go to
 <chrome://inspect/> and make sure _Discover network targets_ is checked. Next

develop/bin/init

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+docker compose up --detach mongo
+curl --max-time 10 --retry 5 --retry-delay 5 --retry-all-errors --silent --output /dev/null localhost:27017
+docker compose exec mongo mongosh --eval "rs.initiate({ _id: 'overleaf', members: [{ _id: 0, host: 'mongo:27017' }] })"
+docker compose down mongo

develop/dev.env

@@ -6,20 +6,15 @@ DOCUMENT_UPDATER_HOST=document-updater
 FILESTORE_HOST=filestore
 GRACEFUL_SHUTDOWN_DELAY_SECONDS=0
 HISTORY_V1_HOST=history-v1
-HISTORY_REDIS_HOST=redis
-LINKED_URL_PROXY_HOST=linked-url-proxy
 LISTEN_ADDRESS=0.0.0.0
 MONGO_HOST=mongo
 MONGO_URL=mongodb://mongo/sharelatex?directConnection=true
 NOTIFICATIONS_HOST=notifications
 PROJECT_HISTORY_HOST=project-history
-QUEUES_REDIS_HOST=redis
-DSMP_REDIS_HOST=redis
 REALTIME_HOST=real-time
 REDIS_HOST=redis
 REFERENCES_HOST=references
-SESSION_SECRET=foo
-V1_HISTORY_HOST=history-v1
+SPELLING_HOST=spelling
 WEBPACK_HOST=webpack
 WEB_API_PASSWORD=overleaf
 WEB_API_USER=overleaf

develop/docker-compose.dev.yml

@@ -79,26 +79,15 @@ services:
       - ../services/history-v1/knexfile.js:/overleaf/services/history-v1/knexfile.js
       - ../services/history-v1/migrations:/overleaf/services/history-v1/migrations
 
-  linked-url-proxy:
-    command: ["node", "--watch", "app.mjs"]
-    environment:
-      - NODE_OPTIONS=--inspect=0.0.0.0:9229
-    ports:
-      - "127.0.0.1:9241:9229"
-    volumes:
-      - ../services/linked-url-proxy/app:/overleaf/services/linked-url-proxy/app
-      - ../services/linked-url-proxy/config:/overleaf/services/linked-url-proxy/config
-      - ../services/linked-url-proxy/app.js:/overleaf/services/linked-url-proxy/app.js
-
   notifications:
-    command: ["node", "--watch", "app.ts"]
+    command: ["node", "--watch", "app.js"]
     environment:
       - NODE_OPTIONS=--inspect=0.0.0.0:9229
     ports:
       - "127.0.0.1:9236:9229"
     volumes:
       - ../services/notifications/app:/overleaf/services/notifications/app
-      - ../services/notifications/app.ts:/overleaf/services/notifications/app.ts
+      - ../services/notifications/app.js:/overleaf/services/notifications/app.js
       - ../services/notifications/config:/overleaf/services/notifications/config
 
   project-history:
@@ -128,14 +117,14 @@ services:
     environment:
       - NODE_OPTIONS=--inspect=0.0.0.0:9229
     ports:
-      - "127.0.0.1:9238:9229"
+      - "127.0.0.1:9236:9229"
     volumes:
       - ../services/references/app:/overleaf/services/references/app
       - ../services/references/config:/overleaf/services/references/config
       - ../services/references/app.js:/overleaf/services/references/app.js
 
   web:
-    command: ["node", "--watch", "app.mjs", "--watch-locales"]
+    command: ["node", "--watch", "app.js", "--watch-locales"]
     environment:
       - NODE_OPTIONS=--inspect=0.0.0.0:9229
     ports:

develop/docker-compose.yml

@@ -1,5 +1,6 @@
 volumes:
   clsi-cache:
+  clsi-output:
   filestore-public-files:
   filestore-template-files:
   filestore-uploads:
@@ -7,6 +8,7 @@ volumes:
   mongo-data:
   redis-data:
   sharelatex-data:
+  spelling-cache:
   web-data:
   history-v1-buckets:
 
@@ -25,16 +27,15 @@ services:
     env_file:
       - dev.env
     environment:
+      - DOCKER_RUNNER=true
       - TEXLIVE_IMAGE=texlive-full # docker build texlive -t texlive-full
-      - SANDBOXED_COMPILES=true
-      - SANDBOXED_COMPILES_HOST_DIR_COMPILES=${PWD}/compiles
-      - SANDBOXED_COMPILES_HOST_DIR_OUTPUT=${PWD}/output
+      - COMPILES_HOST_DIR=${PWD}/compiles
     user: root
     volumes:
       - ${PWD}/compiles:/overleaf/services/clsi/compiles
-      - ${PWD}/output:/overleaf/services/clsi/output
       - ${DOCKER_SOCKET_PATH:-/var/run/docker.sock}:/var/run/docker.sock
       - clsi-cache:/overleaf/services/clsi/cache
+      - clsi-output:/overleaf/services/clsi/output
 
   contacts:
     build:
@@ -69,6 +70,7 @@ services:
       - filestore-public-files:/overleaf/services/filestore/public_files
       - filestore-template-files:/overleaf/services/filestore/template_files
       - filestore-uploads:/overleaf/services/filestore/uploads
+      - filestore-user-files:/overleaf/services/filestore/user_files
 
   history-v1:
     build:
@@ -86,28 +88,13 @@ services:
     volumes:
         - history-v1-buckets:/buckets
 
-  linked-url-proxy:
-    build:
-      context: ..
-      dockerfile: services/linked-url-proxy/Dockerfile
-    env_file:
-      - dev.env
-
   mongo:
-    image: mongo:6.0
+    image: mongo:5
     command: --replSet overleaf
     ports:
       - "127.0.0.1:27017:27017" # for debugging
     volumes:
       - mongo-data:/data/db
-      - ../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js
-    environment:
-      MONGO_INITDB_DATABASE: sharelatex
-    extra_hosts:
-      # Required when using the automatic database setup for initializing the
-      # replica set. This override is not needed when running the setup after
-      # starting up mongo.
-      - mongo:127.0.0.1
 
   notifications:
     build:
@@ -144,6 +131,15 @@ services:
     env_file:
       - dev.env
 
+  spelling:
+    build:
+      context: ..
+      dockerfile: services/spelling/Dockerfile
+    env_file:
+      - dev.env
+    volumes:
+      - spelling-cache:/overleaf/services/spelling/cache
+
   web:
     build:
       context: ..
@@ -157,7 +153,7 @@ services:
       - EMAIL_CONFIRMATION_DISABLED=true
       - NODE_ENV=development
       - OVERLEAF_ALLOW_PUBLIC_ACCESS=true
-    command: ["node", "app.mjs"]
+    command: ["node", "app.js"]
     volumes:
       - sharelatex-data:/var/lib/overleaf
       - web-data:/overleaf/services/web/data
@@ -171,17 +167,17 @@ services:
       - document-updater
       - filestore
       - history-v1
-      - linked-url-proxy
       - notifications
       - project-history
       - real-time
       - references
+      - spelling
 
   webpack:
     build:
       context: ..
       dockerfile: services/web/Dockerfile
-      target: webpack
+      target: dev
     command: ["npx", "webpack", "serve", "--config", "webpack.config.dev-env.js"]
     ports:
       - "127.0.0.1:80:3808"

docker-compose.yml

@@ -1,3 +1,4 @@
+version: '2.2'
 services:
     sharelatex:
         restart: always
@@ -39,6 +40,10 @@ services:
             # Disables email confirmation requirement
             EMAIL_CONFIRMATION_DISABLED: 'true'
 
+            # temporary fix for LuaLaTex compiles
+            # see https://github.com/overleaf/overleaf/issues/695
+            TEXMFVAR: /var/lib/overleaf/tmp/texmf-var
+
             ## Set for SSL via nginx-proxy
             #VIRTUAL_HOST: 103.112.212.22
 
@@ -72,19 +77,11 @@ services:
             ## Server Pro ##
             ################
 
-            ## The Community Edition is intended for use in environments where all users are trusted and is not appropriate for
-            ## scenarios where isolation of users is required. Sandboxed Compiles are not available in the Community Edition,
-            ## so the following environment variables must be commented out to avoid compile issues.
-            ##
-            ## Sandboxed Compiles: https://docs.overleaf.com/on-premises/configuration/overleaf-toolkit/server-pro-only-configuration/sandboxed-compiles
+            ## Sandboxed Compiles: https://github.com/overleaf/overleaf/wiki/Server-Pro:-Sandboxed-Compiles
             SANDBOXED_COMPILES: 'true'
-            ### Bind-mount source for /var/lib/overleaf/data/compiles inside the container.
-            SANDBOXED_COMPILES_HOST_DIR_COMPILES: '/home/user/sharelatex_data/data/compiles'
-            ### Bind-mount source for /var/lib/overleaf/data/output inside the container.
-            SANDBOXED_COMPILES_HOST_DIR_OUTPUT: '/home/user/sharelatex_data/data/output'
-            ### Backwards compatibility (before Server Pro 5.5)
-            DOCKER_RUNNER: 'true'
             SANDBOXED_COMPILES_SIBLING_CONTAINERS: 'true'
+            ### Bind-mount source for /var/lib/overleaf/data/compiles inside the container.
+            SANDBOXED_COMPILES_HOST_DIR: '/home/user/sharelatex_data/data/compiles'
 
             ## Works with test LDAP server shown at bottom of docker compose
             # OVERLEAF_LDAP_URL: 'ldap://ldap:389'
@@ -105,12 +102,12 @@ services:
 
     mongo:
         restart: always
-        image: mongo:6.0
+        image: mongo:5.0
         container_name: mongo
         command: '--replSet overleaf'
         volumes:
             - ~/mongo_data:/data/db
-            - ./bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js
+            - ./mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js
         environment:
           MONGO_INITDB_DATABASE: sharelatex
         extra_hosts:
@@ -118,7 +115,7 @@ services:
           # This override is not needed when running the setup after starting up mongo.
           - mongo:127.0.0.1
         healthcheck:
-            test: echo 'db.stats().ok' | mongosh localhost:27017/test --quiet
+            test: echo 'db.stats().ok' | mongo localhost:27017/test --quiet
             interval: 10s
             timeout: 10s
             retries: 5

libraries/access-token-encryptor/.dockerignore

@@ -0,0 +1 @@
+node_modules/

libraries/access-token-encryptor/.gitignore

@@ -0,0 +1,46 @@
+compileFolder
+
+Compiled source #
+###################
+*.com
+*.class
+*.dll
+*.exe
+*.o
+*.so
+
+# Packages #
+############
+# it's better to unpack these files and commit the raw source
+# git has its own built in compression methods
+*.7z
+*.dmg
+*.gz
+*.iso
+*.jar
+*.rar
+*.tar
+*.zip
+
+# Logs and databases #
+######################
+*.log
+*.sql
+*.sqlite
+
+# OS generated files #
+######################
+.DS_Store?
+ehthumbs.db
+Icon?
+Thumbs.db
+
+/node_modules/*
+data/*/*
+
+**.swp
+
+/log.json
+hash_folder
+
+.npmrc

libraries/access-token-encryptor/.mocharc.cjs

@@ -1,13 +0,0 @@
-let reporterOptions = {}
-if (process.env.CI) {
-  reporterOptions = {
-    reporter: '/overleaf/node_modules/mocha-multi-reporters',
-    'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
-  }
-}
-const all = {
-  require: 'test/setup.js',
-  ...reporterOptions,
-}
-
-module.exports = all

libraries/access-token-encryptor/.mocharc.json

@@ -0,0 +1,6 @@
+{
+  "ui": "bdd",
+  "recursive": "true",
+  "reporter": "spec",
+  "require": "test/setup.js"
+}

libraries/access-token-encryptor/.nvmrc

@@ -1 +1 @@
-22.18.0
+18.20.2

libraries/access-token-encryptor/buildscript.txt

@@ -1,9 +1,10 @@
 access-token-encryptor
 --dependencies=None
+--docker-repos=gcr.io/overleaf-ops
 --env-add=
 --env-pass-through=
 --esmock-loader=False
 --is-library=True
---node-version=22.18.0
---pipeline-owner=🚉 Platform
+--node-version=18.20.2
 --public-repo=False
+--script-version=4.5.0

libraries/access-token-encryptor/lib/js/AccessTokenEncryptor.js

@@ -1,5 +1,5 @@
-const { promisify } = require('node:util')
-const crypto = require('node:crypto')
+const { promisify } = require('util')
+const crypto = require('crypto')
 
 const ALGORITHM = 'aes-256-ctr'
 

libraries/access-token-encryptor/package.json

@@ -5,8 +5,8 @@
   "main": "index.js",
   "scripts": {
     "test": "npm run lint && npm run format && npm run types:check && npm run test:unit",
-    "lint": "eslint --ext .cjs,.js,.jsx,.mjs,.ts --max-warnings 0 --format unix .",
-    "lint:fix": "eslint --fix --ext .cjs,.js,.jsx,.mjs,.ts .",
+    "lint": "eslint --ext .js --ext .cjs --ext .ts --max-warnings 0 --format unix .",
+    "lint:fix": "eslint --fix --ext .js --ext .cjs --ext .ts .",
     "format": "prettier --list-different $PWD/'**/*.{js,cjs,ts}'",
     "format:fix": "prettier --write $PWD/'**/*.{js,cjs,ts}'",
     "test:ci": "npm run test:unit",
@@ -21,9 +21,7 @@
   "devDependencies": {
     "chai": "^4.3.6",
     "chai-as-promised": "^7.1.1",
-    "mocha": "^11.1.0",
-    "mocha-junit-reporter": "^2.2.1",
-    "mocha-multi-reporters": "^1.5.1",
+    "mocha": "^10.2.0",
     "sandboxed-module": "^2.0.4",
     "typescript": "^5.0.4"
   }

libraries/access-token-encryptor/test/mocha-multi-reporters.cjs

@@ -1,9 +0,0 @@
-module.exports = {
-  reporterEnabled: 'spec, mocha-junit-reporter',
-  mochaJunitReporterReporterOptions: {
-    mochaFile: `reports/junit-mocha-${process.env.MOCHA_GREP}.xml`,
-    includePending: true,
-    jenkinsMode: true,
-    output: true,
-  },
-}

libraries/access-token-encryptor/test/setup.js

@@ -1,13 +1,4 @@
 const chai = require('chai')
 const chaiAsPromised = require('chai-as-promised')
-const SandboxedModule = require('sandboxed-module')
 
 chai.use(chaiAsPromised)
-
-SandboxedModule.configure({
-  sourceTransformers: {
-    removeNodePrefix: function (source) {
-      return source.replace(/require\(['"]node:/g, "require('")
-    },
-  },
-})

libraries/access-token-encryptor/tsconfig.json

@@ -2,7 +2,6 @@
   "extends": "../../tsconfig.backend.json",
   "include": [
     "**/*.js",
-    "**/*.cjs",
-    "**/*.ts"
+    "**/*.cjs"
   ]
 }

libraries/eslint-plugin/index.js

@@ -1,11 +0,0 @@
-module.exports = {
-  rules: {
-    'no-unnecessary-trans': require('./no-unnecessary-trans'),
-    'prefer-kebab-url': require('./prefer-kebab-url'),
-    'should-unescape-trans': require('./should-unescape-trans'),
-    'no-generated-editor-themes': require('./no-generated-editor-themes'),
-    'require-script-runner': require('./require-script-runner'),
-    'require-vi-doMock-valid-path': require('./require-vi-doMock-valid-path'),
-    'require-loading-label': require('./require-loading-label'),
-  },
-}

libraries/eslint-plugin/no-generated-editor-themes.js

@@ -1,21 +0,0 @@
-module.exports = {
-  meta: {
-    type: 'error',
-    docs: {
-      description:
-        'Prohibit CodeMirror themes that are generated in a function',
-    },
-  },
-  create(context) {
-    return {
-      ':matches(ArrowFunctionExpression, FunctionDeclaration, FunctionExpression) CallExpression > MemberExpression[object.name="EditorView"]:matches([property.name="theme"],[property.name="baseTheme"])'(
-        node
-      ) {
-        context.report({
-          node,
-          message: `EditorView.theme and EditorView.baseTheme each add CSS to the page for every instance of the theme. Store the theme in a variable and reuse it instead.`,
-        })
-      },
-    }
-  },
-}

libraries/eslint-plugin/no-unnecessary-trans.js

@@ -1,43 +0,0 @@
-module.exports = {
-  meta: {
-    type: 'problem',
-    fixable: 'code',
-    docs: {
-      description: 'Prohibit Trans with no components or values',
-    },
-  },
-  create(context) {
-    return {
-      'JSXOpeningElement[name.name="Trans"]'(node) {
-        const attributes = new Map(
-          node.attributes.map(attr => [attr.name.name, attr])
-        )
-
-        if (!attributes.has('components')) {
-          if (node.parent.children.length > 0) {
-            context.report({
-              node,
-              message: `Trans components must not have child elements`,
-            })
-          } else if (attributes.has('values')) {
-            context.report({
-              node,
-              message: `Use t('…') when there are no components`,
-            })
-          } else {
-            context.report({
-              node,
-              message: `Use t('…') when there are no components`,
-              fix(fixer) {
-                const i18nKey = attributes.get('i18nKey').value.value
-
-                // Note: Prettier can fix indentation
-                return fixer.replaceText(node.parent, `{t('${i18nKey}')}`)
-              },
-            })
-          }
-        }
-      },
-    }
-  },
-}

libraries/eslint-plugin/package.json

@@ -1,17 +0,0 @@
-{
-  "name": "@overleaf/eslint-plugin",
-  "version": "0.1.0",
-  "author": "Overleaf (https://www.overleaf.com)",
-  "license": "AGPL-3.0-only",
-  "main": "index.js",
-  "dependencies": {
-    "eslint": "^8.51.0",
-    "lodash": "^4.17.21"
-  },
-  "devDependencies": {
-    "@typescript-eslint/parser": "^8.50.0"
-  },
-  "scripts": {
-    "test": "node rules.test.js"
-  }
-}

libraries/eslint-plugin/prefer-kebab-url-ignore.js

@@ -1,83 +0,0 @@
-// URL parts should be kebab-case, but we didn't have this rule in the past.
-// The ESLint rule `prefer-kebab-url` will ignore these "legacy" URL parts.
-
-const ignoreWords = {
-  snake: new Set([
-    'clear_saml_data',
-    'confirm_link',
-    'confirm_university_domain',
-    'create_recurly_account',
-    'current_history_content',
-    'current_user',
-    'default_email',
-    'disable_managed_users',
-    'doc_snapshot',
-    'enable_history_ranges_support',
-    'features_override',
-    'generate_password_reset_url',
-    'get_assignment',
-    'get_clone',
-    'health_check',
-    'institutional_emails',
-    'latest_template',
-    'link_after_saml_response',
-    'linked_file',
-    'metrics_segmentation',
-    'new_users',
-    'no_autostart_post_gateway',
-    'personal_info',
-    'planned_maintenance',
-    'refresh_features',
-    'register_admin',
-    'register_ldap_admin',
-    'register_saml_admin',
-    'restore_file',
-    'revert_file',
-    'saved_vers',
-    'send_test_email',
-    'session_maintenance',
-    'set_in_session',
-    'sign_in_to_link',
-    'split_test',
-    'sso_configuration_test',
-    'sso_email',
-    'sso_enrollment',
-    'track_changes',
-    'update_admin',
-    'user_details',
-  ]),
-  camel: new Set([
-    'addWorkflowScope',
-    'aiErrorAssistant',
-    'beginAuth',
-    'brandVariationId',
-    'closeEditor',
-    'completeRegistration',
-    'deactivateOldProjects',
-    'deletedSubscription',
-    'disconnectAllUsers',
-    'editingSession',
-    'emailSubscription',
-    'enableManagedUsers',
-    'externalCollaboration',
-    'flushProjectToTpds',
-    'indexAll',
-    'offboardManagedUser',
-    'openEditor',
-    'perfTest',
-    'pollDropboxForUser',
-    'resendInvite',
-    'resendManagedUserInvite',
-    'salesContactForm',
-    'showSupport',
-  ]),
-  other: new Set([
-    'Project',
-    'disableSSO',
-    'enableSSO',
-    'resendSSOLinkInvite',
-    'usersCSV',
-  ]),
-}
-
-module.exports = { ignoreWords }

libraries/eslint-plugin/prefer-kebab-url.js

@@ -1,91 +0,0 @@
-const _ = require('lodash')
-const { ignoreWords } = require('./prefer-kebab-url-ignore')
-
-const removeTextBetweenBrackets = text => {
-  while (text.includes('[') || text.includes('(')) {
-    text = text.replaceAll(/\[[^[\]]*]/g, '')
-    text = text.replaceAll(/\([^()]*\)/g, '')
-  }
-  return text
-}
-
-const shouldIgnoreWord = str =>
-  str.includes(':') ||
-  str.includes('(') ||
-  str === '*' ||
-  str.match(/^[a-z0-9.]+$/) ||
-  ignoreWords.snake.has(str) ||
-  ignoreWords.camel.has(str) ||
-  ignoreWords.other.has(str)
-
-const getSuggestion = routePath => {
-  if (typeof routePath === 'string') {
-    const kebabed = routePath
-      .split('/')
-      .map(word => (shouldIgnoreWord(word) ? word : _.kebabCase(word)))
-      .join('/')
-    return kebabed === routePath ? null : `'${kebabed}'`
-  }
-
-  if (routePath instanceof RegExp) {
-    const words = removeTextBetweenBrackets(routePath.source).match(/[\w-]+/g)
-    if (!words) return routePath
-
-    let newSource = routePath.source
-    for (const word of words) {
-      if (!shouldIgnoreWord(word)) {
-        newSource = newSource.replaceAll(
-          new RegExp(`\\b${word}\\b`, 'g'),
-          _.kebabCase(word)
-        )
-      }
-    }
-
-    const kebabed = new RegExp(newSource, routePath.flags)
-    return kebabed.source.toString() === routePath.source.toString()
-      ? null
-      : kebabed
-  }
-}
-
-module.exports = {
-  meta: {
-    type: 'problem',
-    fixable: 'code',
-    hasSuggestions: true,
-    docs: {
-      description: 'Enforce using kebab-case for URL paths',
-    },
-  },
-  create: context => ({
-    CallExpression(node) {
-      if (
-        node.callee.type === 'MemberExpression' &&
-        node.arguments[0]?.type === 'Literal' &&
-        [/app/i, /router/i].some(callee =>
-          typeof callee === 'string'
-            ? node.callee.object.name === callee
-            : callee.test(node.callee.object.name)
-        ) &&
-        ['get', 'post', 'put', 'delete'].includes(node.callee.property.name)
-      ) {
-        const routePath = node.arguments[0].value
-
-        const suggestion = getSuggestion(routePath)
-
-        if (suggestion) {
-          context.report({
-            node: node.arguments[0],
-            message: 'Route path should be in kebab-case.',
-            suggest: [
-              {
-                desc: `Change to kebab-case: ${suggestion}`,
-                fix: fixer => fixer.replaceText(node.arguments[0], suggestion),
-              },
-            ],
-          })
-        }
-      }
-    },
-  }),
-}

libraries/eslint-plugin/require-loading-label.js

@@ -1,49 +0,0 @@
-module.exports = {
-  meta: {
-    type: 'problem',
-    fixable: null,
-    docs: {
-      description:
-        'Require loadingLabel prop when isLoading is specified on OLButton',
-    },
-    schema: [],
-  },
-  create(context) {
-    return {
-      'JSXOpeningElement[name.name="OLButton"]'(node) {
-        const attributes = new Map(
-          node.attributes.map(attr => [attr.name?.name, attr])
-        )
-
-        const isLoadingAttr = attributes.get('isLoading')
-        const loadingLabelAttr = attributes.get('loadingLabel')
-
-        if (isLoadingAttr && !loadingLabelAttr) {
-          const isLoadingValue = isLoadingAttr.value
-
-          if (
-            !isLoadingValue ||
-            (isLoadingValue.type === 'JSXExpressionContainer' &&
-              isLoadingValue.expression.type === 'Literal' &&
-              isLoadingValue.expression.value === true)
-          ) {
-            context.report({
-              node: isLoadingAttr,
-              message:
-                'Button with isLoading prop must also specify loadingLabel',
-            })
-          } else if (
-            isLoadingValue.type === 'JSXExpressionContainer' &&
-            isLoadingValue.expression.type !== 'Literal'
-          ) {
-            context.report({
-              node: isLoadingAttr,
-              message:
-                'Button with isLoading prop must also specify loadingLabel',
-            })
-          }
-        }
-      },
-    }
-  },
-}

libraries/eslint-plugin/require-script-runner.js

@@ -1,28 +0,0 @@
-module.exports = {
-  meta: {
-    type: 'suggestion',
-    docs: {
-      description: 'Require Script Runner for scripts',
-    },
-  },
-  create(context) {
-    let hasImport = false
-
-    return {
-      ImportDeclaration(node) {
-        if (node.source.value.endsWith('lib/ScriptRunner.mjs')) {
-          hasImport = true
-        }
-      },
-      'Program:exit'() {
-        if (!hasImport) {
-          context.report({
-            loc: { line: 1, column: 0 },
-            message:
-              'Please use Script Runner for scripts. Refer to the developer manual (https://manual.dev-overleaf.com/development/code/web_scripts/#monitor-script-execution-and-usage-with-script-runner) for more information.',
-          })
-        }
-      },
-    }
-  },
-}

libraries/eslint-plugin/require-vi-doMock-valid-path.js

@@ -1,138 +0,0 @@
-const path = require('node:path')
-const fs = require('node:fs')
-
-module.exports = {
-  meta: {
-    type: 'problem',
-    docs: {
-      description: 'Ensure vi.doMock first argument is a resolvable path.',
-      category: 'Best Practices',
-      recommended: false,
-      url: '',
-    },
-    fixable: 'code',
-    hasSuggestions: true,
-    schema: [],
-    messages: {
-      unresolvablePath:
-        'The path "{{pathValue}}" in vi.doMock() cannot be resolved relative to the current file.',
-      notAStringLiteral:
-        'The first argument of vi.doMock() must be (or resolve to) a string literal representing a path.',
-      noArguments: 'vi.doMock() called with no arguments.',
-    },
-  },
-  create(context) {
-    const currentFilePath = context.getFilename()
-    // ESLint can sometimes pass <text> or <input> for snippets not in a file
-    if (currentFilePath === '<text>' || currentFilePath === '<input>') {
-      return {}
-    }
-    const currentDirectory = path.dirname(currentFilePath)
-
-    function canResolve(modulePath) {
-      try {
-        require.resolve(path.resolve(currentDirectory, modulePath))
-        return true
-      } catch (e) {
-        const absolutePath = path.resolve(currentDirectory, modulePath)
-        const extensions = [
-          '',
-          '.js',
-          '.mjs',
-          '.ts',
-          '.jsx',
-          '.tsx',
-          '.json',
-          '.node',
-          '/index.js',
-          '/index.ts',
-        ] // Add common extensions
-        for (const ext of extensions) {
-          if (fs.existsSync(absolutePath + ext)) {
-            return true
-          }
-        }
-        return false
-      }
-    }
-
-    return {
-      CallExpression(node) {
-        if (
-          node.callee.type === 'MemberExpression' &&
-          node.callee.object.type === 'Identifier' &&
-          node.callee.object.name === 'vi' &&
-          node.callee.property.type === 'Identifier' &&
-          node.callee.property.name === 'doMock'
-        ) {
-          if (node.arguments.length === 0) {
-            context.report({
-              node,
-              messageId: 'noArguments',
-            })
-            return
-          }
-
-          const firstArg = node.arguments[0]
-          let pathValue = firstArg.value
-
-          if (
-            firstArg.type !== 'Literal' ||
-            typeof firstArg.value !== 'string'
-          ) {
-            if (firstArg.type === 'Identifier') {
-              const variable = context
-                .getScope()
-                .variables.find(v => v.name === firstArg.name)
-              if (
-                variable &&
-                variable.defs.length > 0 &&
-                variable.defs[0].node.init &&
-                variable.defs[0].node.init.type === 'Literal' &&
-                typeof variable.defs[0].node.init.value === 'string'
-              ) {
-                pathValue = variable.defs[0].node.init.value
-                if (canResolve(pathValue)) {
-                  return
-                }
-                // If the first argument was a variable that didn't resolve then we can't auto-fix it
-              }
-            }
-            context.report({
-              node: firstArg,
-              messageId: 'notAStringLiteral',
-            })
-            return
-          }
-
-          if (!pathValue.startsWith('.')) {
-            return
-          }
-
-          if (!canResolve(pathValue)) {
-            const mjsPath = pathValue.replace('.js', '.mjs')
-            const additionalReportOptions = {}
-            if (canResolve(mjsPath)) {
-              additionalReportOptions.fix = fixer =>
-                fixer.replaceText(firstArg, `'${mjsPath}'`)
-              additionalReportOptions.suggest = [
-                {
-                  desc: `Replace with "${pathValue.replace('.js', '.mjs')}"`,
-                  fix: fixer => fixer.replaceText(firstArg, `'${mjsPath}'`),
-                },
-              ]
-            }
-            context.report({
-              node: firstArg,
-              messageId: 'unresolvablePath',
-              data: {
-                pathValue,
-              },
-              ...additionalReportOptions,
-            })
-          }
-        }
-      },
-    }
-  },
-}

libraries/eslint-plugin/rules.test.js

@@ -1,168 +0,0 @@
-const { RuleTester } = require('eslint')
-const preferKebabUrl = require('./prefer-kebab-url')
-const noUnnecessaryTrans = require('./no-unnecessary-trans')
-const shouldUnescapeTrans = require('./should-unescape-trans')
-const noGeneratedEditorThemes = require('./no-generated-editor-themes')
-const viDoMockValidPath = require('./require-vi-doMock-valid-path')
-
-const ruleTester = new RuleTester({
-  parser: require.resolve('@typescript-eslint/parser'),
-  parserOptions: {
-    ecmaVersion: 'latest',
-    ecmaFeatures: { jsx: true },
-  },
-})
-
-ruleTester.run('prefer-kebab-url', preferKebabUrl, {
-  valid: [
-    { code: `app.get('/foo-bar')` },
-    { code: `app.get('/foo-bar/:id')` },
-    { code: `router.post('/foo-bar')` },
-    { code: `router.get('/foo-bar/:id/:name/:age')` },
-    { code: `webRouter.get('/foo-bar/:user_id/(ProjectName)/get-info')` },
-    { code: `webApp.post('/foo-bar/:user_id/(ProjectName)/get-info')` },
-    {
-      code: `router.get(/^\\/download\\/project\\/([^/]*)\\/output\\/output\\.pdf$/)`,
-    },
-    {
-      code: `webRouter.get(/^\\/project\\/([^/]*)\\/user\\/([0-9a-f]+)\\/build\\/([0-9a-f-]+)\\/output\\/(.*)$/)`,
-    },
-  ],
-  invalid: [
-    {
-      code: `app.get('/fooBar')`,
-      errors: [{ message: 'Route path should be in kebab-case.' }],
-    },
-    {
-      code: `app.get('/fooBar/:id')`,
-      errors: [{ message: 'Route path should be in kebab-case.' }],
-    },
-    {
-      code: `webRouter.get('/foo_bar/:id/FooBar/:name/fooBar')`,
-      errors: [{ message: 'Route path should be in kebab-case.' }],
-    },
-    {
-      code: `router.get(/^\\/downLoad\\/pro-ject\\/([^/]*)\\/OutPut\\/out-put\\.pdf$/)`,
-      errors: [{ message: 'Route path should be in kebab-case.' }],
-    },
-  ],
-})
-
-ruleTester.run('no-unnecessary-trans', noUnnecessaryTrans, {
-  valid: [
-    { code: `<Trans i18nKey="test" components={{ strong: <strong/> }}/>` },
-  ],
-  invalid: [
-    {
-      code: `<Trans i18nKey="test" values={{ test: 'foo '}}/>`,
-      errors: [{ message: `Use t('…') when there are no components` }],
-    },
-    {
-      code: `<Trans i18nKey="test" />`,
-      errors: [{ message: `Use t('…') when there are no components` }],
-      output: `{t('test')}`,
-    },
-  ],
-})
-
-ruleTester.run('should-unescape-trans', shouldUnescapeTrans, {
-  valid: [
-    {
-      code: `<Trans i18nKey="test" components={{ strong: <strong/> }}/>`,
-    },
-    {
-      code: `<Trans i18nKey="test" values={{ foo: 'bar' }} components={{ strong: <strong/> }} shouldUnescape tOptions={{ interpolation: { escapeValue: true } }}/>`,
-    },
-  ],
-  invalid: [
-    {
-      code: `<Trans i18nKey="test" values={{ foo: 'bar' }} components={{ strong: <strong/> }} />`,
-      errors: [{ message: 'Trans with values must have shouldUnescape' }],
-      output: `<Trans i18nKey="test" values={{ foo: 'bar' }}\nshouldUnescape components={{ strong: <strong/> }} />`,
-    },
-    {
-      code: `<Trans i18nKey="test" values={{ foo: 'bar' }} components={{ strong: <strong/> }} shouldUnescape />`,
-      errors: [
-        {
-          message:
-            'Trans with shouldUnescape must have tOptions.interpolation.escapeValue',
-        },
-      ],
-      output: `<Trans i18nKey="test" values={{ foo: 'bar' }} components={{ strong: <strong/> }} shouldUnescape\ntOptions={{ interpolation: { escapeValue: true } }} />`,
-    },
-  ],
-})
-
-const noGeneratedEditorThemesError =
-  'EditorView.theme and EditorView.baseTheme each add CSS to the page for every instance of the theme. Store the theme in a variable and reuse it instead.'
-ruleTester.run('no-generated-editor-themes', noGeneratedEditorThemes, {
-  valid: [
-    {
-      code: `EditorView.theme({ '.cm-editor': { color: 'black' } })`,
-    },
-    {
-      code: `const theme = EditorView.theme({ '.cm-editor': { color: 'black' } })`,
-    },
-  ],
-  invalid: [
-    {
-      code: `function createTheme() { return EditorView.theme({ '.cm-editor': { color: 'black' } }) }`,
-      errors: [
-        {
-          message: noGeneratedEditorThemesError,
-        },
-      ],
-    },
-    {
-      code: `() => EditorView.theme({ '.cm-editor': { color: 'black' } })`,
-      errors: [
-        {
-          message: noGeneratedEditorThemesError,
-        },
-      ],
-    },
-    {
-      code: `class Foo { createTheme() { return EditorView.theme({ '.cm-editor': { color: 'black' } }) } }`,
-      errors: [
-        {
-          message: noGeneratedEditorThemesError,
-        },
-      ],
-    },
-  ],
-})
-
-ruleTester.run('domock-require-valid-path', viDoMockValidPath, {
-  valid: [
-    {
-      code: 'vi.doMock("./require-vi-doMock-valid-path.js")',
-      filename: __filename,
-    },
-    {
-      code: 'const filename = "./require-vi-doMock-valid-path.js"; vi.doMock(filename);',
-      filename: __filename,
-    },
-  ],
-  invalid: [
-    {
-      code: "vi.doMock('./require-vi-doMock-valid-path2')",
-      filename: __filename,
-      errors: [
-        {
-          message:
-            'The path "./require-vi-doMock-valid-path2" in vi.doMock() cannot be resolved relative to the current file.',
-        },
-      ],
-    },
-    {
-      code: 'const filename = "./require-vi-doMock-valid-path2.js"; vi.doMock(filename);',
-      filename: __filename,
-      errors: [
-        {
-          message:
-            'The first argument of vi.doMock() must be (or resolve to) a string literal representing a path.',
-        },
-      ],
-    },
-  ],
-})

libraries/eslint-plugin/should-unescape-trans.js

@@ -1,60 +0,0 @@
-module.exports = {
-  meta: {
-    type: 'problem',
-    fixable: 'code',
-    docs: {
-      description: 'Ensure that Trans with values has shouldUnescape',
-    },
-  },
-  create(context) {
-    return {
-      'JSXOpeningElement[name.name="Trans"]'(node) {
-        const attributes = new Map(
-          node.attributes.map(attr => [attr.name.name, attr])
-        )
-
-        if (attributes.has('values') && !attributes.has('shouldUnescape')) {
-          context.report({
-            node,
-            message: 'Trans with values must have shouldUnescape',
-            fix(fixer) {
-              return fixer.insertTextAfter(
-                attributes.get('values'),
-                '\nshouldUnescape' // Note: Prettier can fix indentation
-              )
-            },
-          })
-        }
-
-        if (attributes.has('values') && attributes.has('shouldUnescape')) {
-          const tOptions = attributes.get('tOptions')
-          if (!tOptions) {
-            context.report({
-              node,
-              message:
-                'Trans with shouldUnescape must have tOptions.interpolation.escapeValue',
-              fix(fixer) {
-                return fixer.insertTextAfter(
-                  attributes.get('shouldUnescape'),
-                  '\ntOptions={{ interpolation: { escapeValue: true } }}' // Note: Prettier can fix indentation
-                )
-              },
-            })
-          } else {
-            const property = tOptions.value.expression.properties
-              .find(p => p.key.name === 'interpolation')
-              ?.value.properties.find(p => p.key.name === 'escapeValue')
-
-            if (property?.value.value !== true) {
-              context.report({
-                node,
-                message:
-                  'Trans with shouldUnescape must have tOptions.interpolation.escapeValue set to true',
-              })
-            }
-          }
-        }
-      },
-    }
-  },
-}

libraries/eslint-plugin/tsconfig.json

@@ -1,6 +0,0 @@
-{
-  "extends": "../../tsconfig.backend.json",
-  "include": [
-    "**/*.js",
-  ]
-}

libraries/fetch-utils/.dockerignore

@@ -0,0 +1 @@
+node_modules/

libraries/fetch-utils/.gitignore

@@ -0,0 +1,3 @@
+
+# managed by monorepo$ bin/update_build_scripts
+.npmrc

libraries/fetch-utils/.mocharc.cjs

@@ -1,13 +0,0 @@
-let reporterOptions = {}
-if (process.env.CI) {
-  reporterOptions = {
-    reporter: '/overleaf/node_modules/mocha-multi-reporters',
-    'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
-  }
-}
-const all = {
-  require: 'test/setup.js',
-  ...reporterOptions,
-}
-
-module.exports = all

libraries/fetch-utils/.mocharc.json

@@ -0,0 +1,6 @@
+{
+  "ui": "bdd",
+  "recursive": "true",
+  "reporter": "spec",
+  "require": "test/setup.js"
+}

libraries/fetch-utils/.nvmrc

@@ -1 +1 @@
-22.18.0
+18.20.2

libraries/fetch-utils/buildscript.txt

@@ -1,9 +1,10 @@
 fetch-utils
 --dependencies=None
+--docker-repos=gcr.io/overleaf-ops
 --env-add=
 --env-pass-through=
 --esmock-loader=False
 --is-library=True
---node-version=22.18.0
---pipeline-owner=🚉 Platform
+--node-version=18.20.2
 --public-repo=False
+--script-version=4.5.0

libraries/fetch-utils/index.js

@@ -1,9 +1,9 @@
 const _ = require('lodash')
-const { Readable } = require('node:stream')
+const { Readable } = require('stream')
 const OError = require('@overleaf/o-error')
 const fetch = require('node-fetch')
-const http = require('node:http')
-const https = require('node:https')
+const http = require('http')
+const https = require('https')
 
 /**
  * @import { Response } from 'node-fetch'
@@ -23,11 +23,11 @@ async function fetchJson(url, opts = {}) {
 }
 
 async function fetchJsonWithResponse(url, opts = {}) {
-  const { fetchOpts, detachSignal } = parseOpts(opts)
+  const { fetchOpts } = parseOpts(opts)
   fetchOpts.headers = fetchOpts.headers ?? {}
   fetchOpts.headers.Accept = fetchOpts.headers.Accept ?? 'application/json'
 
-  const response = await performRequest(url, fetchOpts, detachSignal)
+  const response = await performRequest(url, fetchOpts)
   if (!response.ok) {
     const body = await maybeGetResponseBody(response)
     throw new RequestFailedError(url, opts, response, body)
@@ -53,8 +53,8 @@ async function fetchStream(url, opts = {}) {
 }
 
 async function fetchStreamWithResponse(url, opts = {}) {
-  const { fetchOpts, abortController, detachSignal } = parseOpts(opts)
-  const response = await performRequest(url, fetchOpts, detachSignal)
+  const { fetchOpts, abortController } = parseOpts(opts)
+  const response = await performRequest(url, fetchOpts)
 
   if (!response.ok) {
     const body = await maybeGetResponseBody(response)
@@ -76,8 +76,8 @@ async function fetchStreamWithResponse(url, opts = {}) {
  * @throws {RequestFailedError} if the response has a failure status code
  */
 async function fetchNothing(url, opts = {}) {
-  const { fetchOpts, detachSignal } = parseOpts(opts)
-  const response = await performRequest(url, fetchOpts, detachSignal)
+  const { fetchOpts } = parseOpts(opts)
+  const response = await performRequest(url, fetchOpts)
   if (!response.ok) {
     const body = await maybeGetResponseBody(response)
     throw new RequestFailedError(url, opts, response, body)
@@ -95,22 +95,9 @@ async function fetchNothing(url, opts = {}) {
  * @throws {RequestFailedError} if the response has a non redirect status code or missing Location header
  */
 async function fetchRedirect(url, opts = {}) {
-  const { location } = await fetchRedirectWithResponse(url, opts)
-  return location
-}
-
-/**
- * Make a request and extract the redirect from the response.
- *
- * @param {string | URL} url - request URL
- * @param {object} opts - fetch options
- * @return {Promise<{location: string, response: Response}>}
- * @throws {RequestFailedError} if the response has a non redirect status code or missing Location header
- */
-async function fetchRedirectWithResponse(url, opts = {}) {
-  const { fetchOpts, detachSignal } = parseOpts(opts)
+  const { fetchOpts } = parseOpts(opts)
   fetchOpts.redirect = 'manual'
-  const response = await performRequest(url, fetchOpts, detachSignal)
+  const response = await performRequest(url, fetchOpts)
   if (response.status < 300 || response.status >= 400) {
     const body = await maybeGetResponseBody(response)
     throw new RequestFailedError(url, opts, response, body)
@@ -125,7 +112,7 @@ async function fetchRedirectWithResponse(url, opts = {}) {
     )
   }
   await discardResponseBody(response)
-  return { location, response }
+  return location
 }
 
 /**
@@ -142,8 +129,8 @@ async function fetchString(url, opts = {}) {
 }
 
 async function fetchStringWithResponse(url, opts = {}) {
-  const { fetchOpts, detachSignal } = parseOpts(opts)
-  const response = await performRequest(url, fetchOpts, detachSignal)
+  const { fetchOpts } = parseOpts(opts)
+  const response = await performRequest(url, fetchOpts)
   if (!response.ok) {
     const body = await maybeGetResponseBody(response)
     throw new RequestFailedError(url, opts, response, body)
@@ -178,14 +165,13 @@ function parseOpts(opts) {
 
   const abortController = new AbortController()
   fetchOpts.signal = abortController.signal
-  let detachSignal = () => {}
   if (opts.signal) {
-    detachSignal = abortOnSignal(abortController, opts.signal)
+    abortOnSignal(abortController, opts.signal)
   }
   if (opts.body instanceof Readable) {
     abortOnDestroyedRequest(abortController, fetchOpts.body)
   }
-  return { fetchOpts, abortController, detachSignal }
+  return { fetchOpts, abortController }
 }
 
 function setupJsonBody(fetchOpts, json) {
@@ -209,9 +195,6 @@ function abortOnSignal(abortController, signal) {
     abortController.abort(signal.reason)
   }
   signal.addEventListener('abort', listener)
-  return () => {
-    signal.removeEventListener('abort', listener)
-  }
 }
 
 function abortOnDestroyedRequest(abortController, stream) {
@@ -230,12 +213,11 @@ function abortOnDestroyedResponse(abortController, response) {
   })
 }
 
-async function performRequest(url, fetchOpts, detachSignal) {
+async function performRequest(url, fetchOpts) {
   let response
   try {
     response = await fetch(url, fetchOpts)
   } catch (err) {
-    detachSignal()
     if (fetchOpts.body instanceof Readable) {
       fetchOpts.body.destroy()
     }
@@ -244,7 +226,6 @@ async function performRequest(url, fetchOpts, detachSignal) {
       method: fetchOpts.method ?? 'GET',
     })
   }
-  response.body.on('close', detachSignal)
   if (fetchOpts.body instanceof Readable) {
     response.body.on('close', () => {
       if (!fetchOpts.body.readableEnded) {
@@ -316,7 +297,6 @@ module.exports = {
   fetchStreamWithResponse,
   fetchNothing,
   fetchRedirect,
-  fetchRedirectWithResponse,
   fetchString,
   fetchStringWithResponse,
   RequestFailedError,

libraries/fetch-utils/package.json

@@ -5,8 +5,8 @@
   "main": "index.js",
   "scripts": {
     "test": "npm run lint && npm run format && npm run types:check && npm run test:unit",
-    "lint": "eslint --ext .cjs,.js,.jsx,.mjs,.ts --max-warnings 0 --format unix .",
-    "lint:fix": "eslint --fix --ext .cjs,.js,.jsx,.mjs,.ts .",
+    "lint": "eslint --ext .js --ext .cjs --ext .ts --max-warnings 0 --format unix .",
+    "lint:fix": "eslint --fix --ext .js --ext .cjs --ext .ts .",
     "format": "prettier --list-different $PWD/'**/*.{js,cjs,ts}'",
     "format:fix": "prettier --write $PWD/'**/*.{js,cjs,ts}'",
     "test:ci": "npm run test:unit",
@@ -16,14 +16,12 @@
   "author": "Overleaf (https://www.overleaf.com)",
   "license": "AGPL-3.0-only",
   "devDependencies": {
-    "@types/node-fetch": "^2.6.13",
-    "body-parser": "1.20.4",
+    "@types/node-fetch": "^2.6.11",
+    "body-parser": "^1.20.3",
     "chai": "^4.3.6",
     "chai-as-promised": "^7.1.1",
-    "express": "4.22.1",
-    "mocha": "^11.1.0",
-    "mocha-junit-reporter": "^2.2.1",
-    "mocha-multi-reporters": "^1.5.1",
+    "express": "^4.21.0",
+    "mocha": "^10.2.0",
     "typescript": "^5.0.4"
   },
   "dependencies": {

libraries/fetch-utils/test/mocha-multi-reporters.cjs

@@ -1,9 +0,0 @@
-module.exports = {
-  reporterEnabled: 'spec, mocha-junit-reporter',
-  mochaJunitReporterReporterOptions: {
-    mochaFile: `reports/junit-mocha-${process.env.MOCHA_GREP}.xml`,
-    includePending: true,
-    jenkinsMode: true,
-    output: true,
-  },
-}

libraries/fetch-utils/test/unit/FetchUtilsTests.js

@@ -1,10 +1,7 @@
 const { expect } = require('chai')
-const fs = require('node:fs')
-const events = require('node:events')
 const { FetchError, AbortError } = require('node-fetch')
-const { Readable } = require('node:stream')
-const { pipeline } = require('node:stream/promises')
-const { once } = require('node:events')
+const { Readable } = require('stream')
+const { once } = require('events')
 const { TestServer } = require('./helpers/TestServer')
 const selfsigned = require('selfsigned')
 const {
@@ -27,17 +24,13 @@ const pems = selfsigned.generate(attrs, { days: 365 })
 const PRIVATE_KEY = pems.private
 const PUBLIC_CERT = pems.cert
 
-const dns = require('node:dns')
+const dns = require('dns')
 const _originalLookup = dns.lookup
 // Custom DNS resolver function
 dns.lookup = (hostname, options, callback) => {
   if (hostname === 'example.com') {
     // If the hostname is our test case, return the ip address for the test server
-    if (options?.all) {
-      callback(null, [{ address: '127.0.0.1', family: 4 }])
-    } else {
-      callback(null, '127.0.0.1', 4)
-    }
+    callback(null, '127.0.0.1', 4)
   } else {
     // Otherwise, use the default lookup
     _originalLookup(hostname, options, callback)
@@ -206,31 +199,6 @@ describe('fetch-utils', function () {
       ).to.be.rejectedWith(AbortError)
       expect(stream.destroyed).to.be.true
     })
-
-    it('detaches from signal on success', async function () {
-      const signal = AbortSignal.timeout(10_000)
-      for (let i = 0; i < 20; i++) {
-        const s = await fetchStream(this.url('/hello'), { signal })
-        expect(events.getEventListeners(signal, 'abort')).to.have.length(1)
-        await pipeline(s, fs.createWriteStream('/dev/null'))
-        expect(events.getEventListeners(signal, 'abort')).to.have.length(0)
-      }
-    })
-
-    it('detaches from signal on error', async function () {
-      const signal = AbortSignal.timeout(10_000)
-      for (let i = 0; i < 20; i++) {
-        try {
-          await fetchStream(this.url('/500'), { signal })
-        } catch (err) {
-          if (err instanceof RequestFailedError && err.response.status === 500)
-            continue
-          throw err
-        } finally {
-          expect(events.getEventListeners(signal, 'abort')).to.have.length(0)
-        }
-      }
-    })
   })
 
   describe('fetchNothing', function () {
@@ -419,16 +387,9 @@ async function* infiniteIterator() {
 async function abortOnceReceived(func, server) {
   const controller = new AbortController()
   const promise = func(controller.signal)
-  expect(events.getEventListeners(controller.signal, 'abort')).to.have.length(1)
   await once(server.events, 'request-received')
   controller.abort()
-  try {
-    return await promise
-  } finally {
-    expect(events.getEventListeners(controller.signal, 'abort')).to.have.length(
-      0
-    )
-  }
+  return await promise
 }
 
 async function expectRequestAborted(req) {

libraries/fetch-utils/test/unit/helpers/TestServer.js

@@ -1,9 +1,9 @@
 const express = require('express')
 const bodyParser = require('body-parser')
-const { EventEmitter } = require('node:events')
-const http = require('node:http')
-const https = require('node:https')
-const { promisify } = require('node:util')
+const { EventEmitter } = require('events')
+const http = require('http')
+const https = require('https')
+const { promisify } = require('util')
 
 class TestServer {
   constructor() {

libraries/fetch-utils/tsconfig.json

@@ -2,7 +2,6 @@
   "extends": "../../tsconfig.backend.json",
   "include": [
     "**/*.js",
-    "**/*.cjs",
-    "**/*.ts"
+    "**/*.cjs"
   ]
 }

libraries/logger/.dockerignore

@@ -0,0 +1 @@
+node_modules/

libraries/logger/.gitignore

@@ -0,0 +1,3 @@
+node_modules
+
+.npmrc

libraries/logger/.mocharc.cjs

@@ -1,13 +0,0 @@
-let reporterOptions = {}
-if (process.env.CI) {
-  reporterOptions = {
-    reporter: '/overleaf/node_modules/mocha-multi-reporters',
-    'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
-  }
-}
-const all = {
-  require: 'test/setup.js',
-  ...reporterOptions,
-}
-
-module.exports = all

libraries/logger/.mocharc.json

@@ -0,0 +1,3 @@
+{
+  "require": "test/setup.js"
+}

libraries/logger/.nvmrc

@@ -1 +1 @@
-22.18.0
+18.20.2

libraries/logger/buildscript.txt

@@ -1,9 +1,10 @@
 logger
 --dependencies=None
+--docker-repos=gcr.io/overleaf-ops
 --env-add=
 --env-pass-through=
 --esmock-loader=False
 --is-library=True
---node-version=22.18.0
---pipeline-owner=🚉 Platform
+--node-version=18.20.2
 --public-repo=False
+--script-version=4.5.0

libraries/logger/log-level-checker.js

@@ -1,5 +1,5 @@
 const { fetchString } = require('@overleaf/fetch-utils')
-const fs = require('node:fs')
+const fs = require('fs')
 
 class LogLevelChecker {
   constructor(logger, defaultLevel) {

libraries/logger/logging-manager.js

@@ -1,6 +1,7 @@
-const Stream = require('node:stream')
+const Stream = require('stream')
 const bunyan = require('bunyan')
 const GCPManager = require('./gcp-manager')
+const SentryManager = require('./sentry-manager')
 const Serializers = require('./serializers')
 const {
   FileLogLevelChecker,
@@ -11,13 +12,11 @@ const LoggingManager = {
   /**
    * @param {string} name - The name of the logger
    */
-  initialize(name, options = {}) {
+  initialize(name) {
     this.isProduction =
       (process.env.NODE_ENV || '').toLowerCase() === 'production'
-    const isTest = (process.env.NODE_ENV || '').toLowerCase() === 'test'
     this.defaultLevel =
-      process.env.LOG_LEVEL ||
-      (this.isProduction ? 'info' : isTest ? 'fatal' : 'debug')
+      process.env.LOG_LEVEL || (this.isProduction ? 'info' : 'debug')
     this.loggerName = name
     this.logger = bunyan.createLogger({
       name,
@@ -27,13 +26,17 @@ const LoggingManager = {
         req: Serializers.req,
         res: Serializers.res,
       },
-      streams: options.streams ?? [this._getOutputStreamConfig()],
+      streams: [this._getOutputStreamConfig()],
     })
     this._setupRingBuffer()
     this._setupLogLevelChecker()
     return this
   },
 
+  initializeErrorReporting(dsn, options) {
+    this.sentryManager = new SentryManager()
+  },
+
   /**
    * @param {Record<string, any>|string} attributes - Attributes to log (nice serialization for err, req, res)
    * @param {string} [message] - Optional message
@@ -65,6 +68,9 @@ const LoggingManager = {
       })
     }
     this.logger.error(attributes, message, ...Array.from(args))
+    if (this.sentryManager) {
+      this.sentryManager.captureExceptionRateLimited(attributes, message)
+    }
   },
 
   /**
@@ -92,6 +98,9 @@ const LoggingManager = {
    */
   fatal(attributes, message) {
     this.logger.fatal(attributes, message)
+    if (this.sentryManager) {
+      this.sentryManager.captureException(attributes, message, 'fatal')
+    }
   },
 
   _getOutputStreamConfig() {

libraries/logger/package.json

@@ -13,8 +13,8 @@
     "test": "npm run lint && npm run format && npm run types:check && npm run test:unit",
     "format": "prettier --list-different $PWD/'**/*.{js,cjs,ts}'",
     "format:fix": "prettier --write $PWD/'**/*.{js,cjs,ts}'",
-    "lint": "eslint --ext .cjs,.js,.jsx,.mjs,.ts --max-warnings 0 --format unix .",
-    "lint:fix": "eslint --fix --ext .cjs,.js,.jsx,.mjs,.ts .",
+    "lint": "eslint --ext .js --ext .cjs --ext .ts --max-warnings 0 --format unix .",
+    "lint:fix": "eslint --fix --ext .js --ext .cjs --ext .ts .",
     "test:ci": "npm run test:unit",
     "test:unit": "mocha --exit test/**/*.{js,cjs}",
     "types:check": "tsc --noEmit"
@@ -23,13 +23,12 @@
     "@google-cloud/logging-bunyan": "^5.1.0",
     "@overleaf/fetch-utils": "*",
     "@overleaf/o-error": "*",
+    "@sentry/node": "^6.13.2",
     "bunyan": "^1.8.14"
   },
   "devDependencies": {
     "chai": "^4.3.6",
-    "mocha": "^11.1.0",
-    "mocha-junit-reporter": "^2.2.1",
-    "mocha-multi-reporters": "^1.5.1",
+    "mocha": "^10.2.0",
     "sandboxed-module": "^2.0.4",
     "sinon": "^9.2.4",
     "sinon-chai": "^3.7.0",

libraries/logger/sentry-manager.js

@@ -0,0 +1,106 @@
+const Serializers = require('./serializers')
+const RATE_LIMIT_MAX_ERRORS = 5
+const RATE_LIMIT_INTERVAL_MS = 60000
+
+class SentryManager {
+  constructor(dsn, options) {
+    this.Sentry = require('@sentry/node')
+    this.Sentry.init({ dsn, ...options })
+    // for rate limiting on sentry reporting
+    this.lastErrorTimeStamp = 0
+    this.lastErrorCount = 0
+  }
+
+  captureExceptionRateLimited(attributes, message) {
+    const now = Date.now()
+    // have we recently reported an error?
+    const recentSentryReport =
+      now - this.lastErrorTimeStamp < RATE_LIMIT_INTERVAL_MS
+    // if so, increment the error count
+    if (recentSentryReport) {
+      this.lastErrorCount++
+    } else {
+      this.lastErrorCount = 0
+      this.lastErrorTimeStamp = now
+    }
+    // only report 5 errors every minute to avoid overload
+    if (this.lastErrorCount < RATE_LIMIT_MAX_ERRORS) {
+      // add a note if the rate limit has been hit
+      const note =
+        this.lastErrorCount + 1 === RATE_LIMIT_MAX_ERRORS
+          ? '(rate limited)'
+          : ''
+      // report the exception
+      this.captureException(attributes, message, `error${note}`)
+    }
+  }
+
+  captureException(attributes, message, level) {
+    // handle case of logger.error "message"
+    if (typeof attributes === 'string') {
+      attributes = { err: new Error(attributes) }
+    }
+
+    // extract any error object
+    let error = Serializers.err(attributes.err || attributes.error)
+
+    // avoid reporting errors twice
+    for (const key in attributes) {
+      const value = attributes[key]
+      if (value instanceof Error && value.reportedToSentry) {
+        return
+      }
+    }
+
+    // include our log message in the error report
+    if (error == null) {
+      if (typeof message === 'string') {
+        error = { message }
+      }
+    } else if (message != null) {
+      attributes.description = message
+    }
+
+    // report the error
+    if (error != null) {
+      // capture attributes and use *_id objects as tags
+      const tags = {}
+      const extra = {}
+      for (const key in attributes) {
+        let value = attributes[key]
+        if (Serializers[key]) {
+          value = Serializers[key](value)
+        }
+        if (key.match(/_id/) && typeof value === 'string') {
+          tags[key] = value
+        }
+        extra[key] = value
+      }
+
+      // OError integration
+      extra.info = error.info
+      delete error.info
+
+      // Sentry wants to receive an Error instance.
+      const errInstance = new Error(error.message)
+      Object.assign(errInstance, error)
+
+      try {
+        // send the error to sentry
+        this.Sentry.captureException(errInstance, { tags, extra, level })
+
+        // put a flag on the errors to avoid reporting them multiple times
+        for (const key in attributes) {
+          const value = attributes[key]
+          if (value instanceof Error) {
+            value.reportedToSentry = true
+          }
+        }
+      } catch (err) {
+        // ignore Sentry errors
+      }
+    }
+  }
+}
+
+module.exports = SentryManager

libraries/logger/test-log-recorder.js

@@ -1,20 +0,0 @@
-const logger = require('./')
-const bunyan = require('bunyan')
-const serializers = require('./serializers')
-
-function testLogRecorder() {
-  const currentTest = this.currentTest
-  for (const level of ['error', 'fatal']) {
-    logger[level] = (info, msg) => {
-      const entry = { level, ...info, msg }
-      for (const [name, fn] of Object.entries(serializers)) {
-        if (name in entry) entry[name] = fn(entry[name])
-      }
-      currentTest.consoleErrors = (currentTest.consoleErrors || []).concat(
-        JSON.stringify(info, bunyan.safeCycles())
-      )
-    }
-  }
-}
-
-module.exports = testLogRecorder

libraries/logger/test/mocha-multi-reporters.cjs

@@ -1,9 +0,0 @@
-module.exports = {
-  reporterEnabled: 'spec, mocha-junit-reporter',
-  mochaJunitReporterReporterOptions: {
-    mochaFile: `reports/junit-mocha-${process.env.MOCHA_GREP}.xml`,
-    includePending: true,
-    jenkinsMode: true,
-    output: true,
-  },
-}

libraries/logger/test/setup.js

@@ -8,9 +8,4 @@ chai.use(sinonChai)
 
 SandboxedModule.configure({
   globals: { Buffer, JSON, console, process },
-  sourceTransformers: {
-    removeNodePrefix: function (source) {
-      return source.replace(/require\(['"]node:/g, "require('")
-    },
-  },
 })

libraries/logger/test/unit/log-level-checker-tests.js

@@ -1,5 +1,5 @@
-const Path = require('node:path')
-const { promisify } = require('node:util')
+const Path = require('path')
+const { promisify } = require('util')
 const SandboxedModule = require('sandboxed-module')
 const sinon = require('sinon')
 const { expect } = require('chai')

libraries/logger/test/unit/logging-manager-tests.js

@@ -1,7 +1,7 @@
 const SandboxedModule = require('sandboxed-module')
 const bunyan = require('bunyan')
 const { expect } = require('chai')
-const path = require('node:path')
+const path = require('path')
 const sinon = require('sinon')
 
 const MODULE_PATH = path.join(__dirname, '../../logging-manager.js')
@@ -43,14 +43,20 @@ describe('LoggingManager', function () {
         .stub()
         .returns(this.GCEMetadataLogLevelChecker),
     }
+    this.SentryManager = {
+      captureException: sinon.stub(),
+      captureExceptionRateLimited: sinon.stub(),
+    }
     this.LoggingManager = SandboxedModule.require(MODULE_PATH, {
       requires: {
         bunyan: this.Bunyan,
         './log-level-checker': this.LogLevelChecker,
+        './sentry-manager': sinon.stub().returns(this.SentryManager),
       },
     })
     this.loggerName = 'test'
     this.logger = this.LoggingManager.initialize(this.loggerName)
+    this.logger.initializeErrorReporting('test_dsn')
   })
 
   afterEach(function () {
@@ -154,6 +160,13 @@ describe('LoggingManager', function () {
     })
   })
 
+  describe('logger.error', function () {
+    it('should report errors to Sentry', function () {
+      this.logger.error({ foo: 'bar' }, 'message')
+      expect(this.SentryManager.captureExceptionRateLimited).to.have.been.called
+    })
+  })
+
   describe('ringbuffer', function () {
     beforeEach(function () {
       this.logBufferMock = [

libraries/logger/test/unit/sentry-manager-tests.js

@@ -0,0 +1,247 @@
+const Path = require('path')
+const SandboxedModule = require('sandboxed-module')
+const { expect } = require('chai')
+const sinon = require('sinon')
+
+const MODULE_PATH = Path.join(__dirname, '../../sentry-manager.js')
+
+describe('SentryManager', function () {
+  beforeEach(function () {
+    this.clock = sinon.useFakeTimers(Date.now())
+    this.Sentry = {
+      init: sinon.stub(),
+      captureException: sinon.stub(),
+    }
+    this.SentryManager = SandboxedModule.require(MODULE_PATH, {
+      requires: {
+        '@sentry/node': this.Sentry,
+      },
+    })
+    this.sentryManager = new this.SentryManager('test_dsn')
+  })
+
+  afterEach(function () {
+    this.clock.restore()
+  })
+
+  describe('captureExceptionRateLimited', function () {
+    it('should report a single error to sentry', function () {
+      this.sentryManager.captureExceptionRateLimited({ foo: 'bar' }, 'message')
+      expect(this.Sentry.captureException).to.have.been.calledOnce
+    })
+
+    it('should report the same error to sentry only once', function () {
+      const error1 = new Error('this is the error')
+      this.sentryManager.captureExceptionRateLimited(
+        { foo: error1 },
+        'first message'
+      )
+      this.sentryManager.captureExceptionRateLimited(
+        { bar: error1 },
+        'second message'
+      )
+      expect(this.Sentry.captureException).to.have.been.calledOnce
+    })
+
+    it('should report two different errors to sentry individually', function () {
+      const error1 = new Error('this is the error')
+      const error2 = new Error('this is the error')
+      this.sentryManager.captureExceptionRateLimited(
+        { foo: error1 },
+        'first message'
+      )
+      this.sentryManager.captureExceptionRateLimited(
+        { bar: error2 },
+        'second message'
+      )
+      expect(this.Sentry.captureException).to.have.been.calledTwice
+    })
+
+    it('for multiple errors should only report a maximum of 5 errors to sentry', function () {
+      for (let i = 0; i < 10; i++) {
+        this.sentryManager.captureExceptionRateLimited(
+          { foo: 'bar' },
+          'message'
+        )
+      }
+      expect(this.Sentry.captureException).to.have.callCount(5)
+    })
+
+    it('for multiple errors with a minute delay should report 10 errors to sentry', function () {
+      for (let i = 0; i < 10; i++) {
+        this.sentryManager.captureExceptionRateLimited(
+          { foo: 'bar' },
+          'message'
+        )
+      }
+      expect(this.Sentry.captureException).to.have.callCount(5)
+
+      // allow a minute to pass
+      this.clock.tick(61 * 1000)
+
+      for (let i = 0; i < 10; i++) {
+        this.sentryManager.captureExceptionRateLimited(
+          { foo: 'bar' },
+          'message'
+        )
+      }
+      expect(this.Sentry.captureException).to.have.callCount(10)
+    })
+  })
+
+  describe('captureException', function () {
+    it('should remove the path from fs errors', function () {
+      const fsError = new Error(
+        "Error: ENOENT: no such file or directory, stat '/tmp/3279b8d0-da10-11e8-8255-efd98985942b'"
+      )
+      fsError.path = '/tmp/3279b8d0-da10-11e8-8255-efd98985942b'
+      this.sentryManager.captureException({ err: fsError }, 'message', 'error')
+      expect(this.Sentry.captureException).to.have.been.calledWith(
+        sinon.match.has(
+          'message',
+          'Error: ENOENT: no such file or directory, stat'
+        )
+      )
+    })
+
+    it('should sanitize error', function () {
+      const err = {
+        name: 'CustomError',
+        message: 'hello',
+        _oErrorTags: [{ stack: 'here:1', info: { one: 1 } }],
+        stack: 'here:0',
+        info: { key: 'value' },
+        code: 42,
+        signal: 9,
+        path: '/foo',
+      }
+      this.sentryManager.captureException({ err }, 'message', 'error')
+      const expectedErr = {
+        name: 'CustomError',
+        message: 'hello',
+        stack: 'here:0\nhere:1',
+        code: 42,
+        signal: 9,
+        path: '/foo',
+      }
+      expect(this.Sentry.captureException).to.have.been.calledWith(
+        sinon.match(expectedErr),
+        sinon.match({
+          tags: sinon.match.any,
+          level: sinon.match.any,
+          extra: {
+            description: 'message',
+            info: sinon.match({
+              one: 1,
+              key: 'value',
+            }),
+          },
+        })
+      )
+      // Chai is very picky with comparing Error instances. Go the long way of comparing all the fields manually.
+      const gotErr = this.Sentry.captureException.args[0][0]
+      for (const [key, wanted] of Object.entries(expectedErr)) {
+        expect(gotErr).to.have.property(key, wanted)
+      }
+    })
+    it('should sanitize request', function () {
+      const req = {
+        ip: '1.2.3.4',
+        method: 'GET',
+        url: '/foo',
+        headers: {
+          referer: 'abc',
+          'content-length': 1337,
+          'user-agent': 'curl',
+          authorization: '42',
+        },
+      }
+      this.sentryManager.captureException({ req }, 'message', 'error')
+      const expectedReq = {
+        remoteAddress: '1.2.3.4',
+        method: 'GET',
+        url: '/foo',
+        headers: {
+          referer: 'abc',
+          'content-length': 1337,
+          'user-agent': 'curl',
+        },
+      }
+      expect(this.Sentry.captureException).to.have.been.calledWith(
+        sinon.match({
+          message: 'message',
+        }),
+        sinon.match({
+          tags: sinon.match.any,
+          level: sinon.match.any,
+          extra: {
+            info: sinon.match.any,
+            req: expectedReq,
+          },
+        })
+      )
+      expect(this.Sentry.captureException.args[0][1].extra.req).to.deep.equal(
+        expectedReq
+      )
+    })
+    it('should sanitize response', function () {
+      const res = {
+        statusCode: 417,
+        body: Buffer.from('foo'),
+        getHeader(key) {
+          expect(key).to.be.oneOf(['content-length'])
+          if (key === 'content-length') return 1337
+        },
+      }
+      this.sentryManager.captureException({ res }, 'message', 'error')
+      const expectedRes = {
+        statusCode: 417,
+        headers: {
+          'content-length': 1337,
+        },
+      }
+      expect(this.Sentry.captureException).to.have.been.calledWith(
+        sinon.match({
+          message: 'message',
+        }),
+        sinon.match({
+          tags: sinon.match.any,
+          level: sinon.match.any,
+          extra: {
+            info: sinon.match.any,
+            res: expectedRes,
+          },
+        })
+      )
+      expect(this.Sentry.captureException.args[0][1].extra.res).to.deep.equal(
+        expectedRes
+      )
+    })
+
+    describe('reportedToSentry', function () {
+      it('should mark the error as reported to sentry', function () {
+        const err = new Error()
+        this.sentryManager.captureException({ err }, 'message')
+        expect(this.Sentry.captureException).to.have.been.called
+        expect(err.reportedToSentry).to.equal(true)
+      })
+
+      it('should mark two errors as reported to sentry', function () {
+        const err1 = new Error()
+        const err2 = new Error()
+        this.sentryManager.captureException({ err: err1, err2 }, 'message')
+        expect(this.Sentry.captureException).to.have.been.called
+        expect(err1.reportedToSentry).to.equal(true)
+        expect(err2.reportedToSentry).to.equal(true)
+      })
+
+      it('should not mark arbitrary objects as reported to sentry', function () {
+        const err = new Error()
+        const ctx = { foo: 'bar' }
+        this.sentryManager.captureException({ err, ctx }, 'message')
+        expect(this.Sentry.captureException).to.have.been.called
+        expect(ctx.reportedToSentry).not.to.exist
+      })
+    })
+  })
+})

libraries/logger/tsconfig.json

@@ -2,7 +2,6 @@
   "extends": "../../tsconfig.backend.json",
   "include": [
     "**/*.js",
-    "**/*.cjs",
-    "**/*.ts"
+    "**/*.cjs"
   ]
 }

libraries/metrics/.dockerignore

@@ -0,0 +1 @@
+node_modules/

libraries/metrics/.gitignore

@@ -0,0 +1,3 @@
+node_modules
+
+.npmrc

libraries/metrics/.mocharc.cjs

@@ -1,13 +0,0 @@
-let reporterOptions = {}
-if (process.env.CI) {
-  reporterOptions = {
-    reporter: '/overleaf/node_modules/mocha-multi-reporters',
-    'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
-  }
-}
-const all = {
-  require: 'test/setup.js',
-  ...reporterOptions,
-}
-
-module.exports = all

libraries/metrics/.nvmrc

@@ -1 +1 @@
-22.18.0
+18.20.2

libraries/metrics/buildscript.txt

@@ -1,9 +1,10 @@
 metrics
 --dependencies=None
+--docker-repos=gcr.io/overleaf-ops
 --env-add=
 --env-pass-through=
 --esmock-loader=False
 --is-library=True
---node-version=22.18.0
---pipeline-owner=🚉 Platform
+--node-version=18.20.2
 --public-repo=False
+--script-version=4.5.0

libraries/metrics/http.js

@@ -75,4 +75,4 @@ class RequestLogger {
   }
 }
 
-module.exports = { monitor, RequestLogger }
+module.exports.monitor = monitor

libraries/metrics/index.js

@@ -97,12 +97,6 @@ function histogram(key, value, buckets, labels = {}) {
 }
 
 class Timer {
-  /**
-   * @param {string} key
-   * @param {number} sampleRate
-   * @param {Record<string, any>} labels
-   * @param {Array<number>} buckets
-   */
   constructor(key, sampleRate = 1, labels = {}, buckets = undefined) {
     if (typeof sampleRate === 'object') {
       // called with (key, labels, buckets)

libraries/metrics/initialize.js

@@ -5,8 +5,6 @@
  * before any other module to support code instrumentation.
  */
 
-const metricsModuleImportStartTime = performance.now()
-
 const APP_NAME = process.env.METRICS_APP_NAME || 'unknown'
 const BUILD_VERSION = process.env.BUILD_VERSION
 const ENABLE_PROFILE_AGENT = process.env.ENABLE_PROFILE_AGENT === 'true'
@@ -90,7 +88,7 @@ function initializeProfileAgent() {
 }
 
 function initializePrometheus() {
-  const os = require('node:os')
+  const os = require('os')
   const promClient = require('prom-client')
   promClient.register.setDefaultLabels({ app: APP_NAME, host: os.hostname() })
   promClient.collectDefaultMetrics({ timeout: 5000, prefix: '' })
@@ -105,5 +103,3 @@ function recordProcessStart() {
   const metrics = require('.')
   metrics.inc('process_startup')
 }
-
-module.exports = { metricsModuleImportStartTime }

libraries/metrics/leaked_sockets.js

@@ -6,8 +6,8 @@
  * logged along with the corresponding information from /proc/net/tcp.
  */
 
-const fs = require('node:fs')
-const diagnosticsChannel = require('node:diagnostics_channel')
+const fs = require('fs')
+const diagnosticsChannel = require('diagnostics_channel')
 
 const SOCKET_MONITOR_INTERVAL = 60 * 1000
 // set the threshold for logging leaked sockets in minutes, defaults to 15

libraries/metrics/open_sockets.js

@@ -11,13 +11,13 @@ const seconds = 1000
 // In Node 0.10 the default is 5, which means only 5 open connections at one.
 // Node 0.12 has a default of Infinity. Make sure we have no limit set,
 // regardless of Node version.
-require('node:http').globalAgent.maxSockets = Infinity
-require('node:https').globalAgent.maxSockets = Infinity
+require('http').globalAgent.maxSockets = Infinity
+require('https').globalAgent.maxSockets = Infinity
 
-const SOCKETS_HTTP = require('node:http').globalAgent.sockets
-const SOCKETS_HTTPS = require('node:https').globalAgent.sockets
-const FREE_SOCKETS_HTTP = require('node:http').globalAgent.freeSockets
-const FREE_SOCKETS_HTTPS = require('node:https').globalAgent.freeSockets
+const SOCKETS_HTTP = require('http').globalAgent.sockets
+const SOCKETS_HTTPS = require('https').globalAgent.sockets
+const FREE_SOCKETS_HTTP = require('http').globalAgent.freeSockets
+const FREE_SOCKETS_HTTPS = require('https').globalAgent.freeSockets
 
 // keep track of set gauges and reset them in the next collection cycle
 const SEEN_HOSTS_HTTP = new Set()

libraries/metrics/package.json

@@ -9,7 +9,7 @@
   "main": "index.js",
   "dependencies": {
     "@google-cloud/opentelemetry-cloud-trace-exporter": "^2.1.0",
-    "@google-cloud/profiler": "^6.0.3",
+    "@google-cloud/profiler": "^6.0.0",
     "@opentelemetry/api": "^1.4.1",
     "@opentelemetry/auto-instrumentations-node": "^0.39.1",
     "@opentelemetry/exporter-trace-otlp-http": "^0.41.2",
@@ -23,18 +23,16 @@
   "devDependencies": {
     "bunyan": "^1.0.0",
     "chai": "^4.3.6",
-    "mocha": "^11.1.0",
-    "mocha-junit-reporter": "^2.2.1",
-    "mocha-multi-reporters": "^1.5.1",
+    "mocha": "^10.2.0",
     "sandboxed-module": "^2.0.4",
     "sinon": "^9.2.4",
     "typescript": "^5.0.4"
   },
   "scripts": {
-    "lint": "eslint --ext .cjs,.js,.jsx,.mjs,.ts --max-warnings 0 --format unix .",
-    "lint:fix": "eslint --fix --ext .cjs,.js,.jsx,.mjs,.ts .",
+    "lint": "eslint --ext .js --ext .cjs --ext .ts --max-warnings 0 --format unix .",
+    "lint:fix": "eslint --fix --ext .js --ext .cjs --ext .ts .",
     "test:unit": "mocha --exit test/**/*.{js,cjs}",
-    "test:acceptance": "mocha --recursive --exit --grep=$MOCHA_GREP test/acceptance",
+    "test:acceptance": "mocha --reporter spec --recursive --exit --grep=$MOCHA_GREP test/acceptance",
     "test": "npm run lint && npm run format && npm run types:check && npm run test:unit",
     "format": "prettier --list-different $PWD/'**/*.{js,cjs,ts}'",
     "format:fix": "prettier --write $PWD/'**/*.{js,cjs,ts}'",

libraries/metrics/test/acceptance/metrics_tests.js

@@ -1,6 +1,6 @@
-const { promisify } = require('node:util')
-const os = require('node:os')
-const http = require('node:http')
+const { promisify } = require('util')
+const os = require('os')
+const http = require('http')
 const { expect } = require('chai')
 const Metrics = require('../..')
 
@@ -316,7 +316,7 @@ async function checkSummaryValues(key, values) {
   for (const quantile of Object.keys(values)) {
     expect(found[quantile]).to.be.within(
       values[quantile] - 5,
-      values[quantile] + 15,
+      values[quantile] + 5,
       `quantile: ${quantile}`
     )
   }

libraries/metrics/test/mocha-multi-reporters.cjs

@@ -1,9 +0,0 @@
-module.exports = {
-  reporterEnabled: 'spec, mocha-junit-reporter',
-  mochaJunitReporterReporterOptions: {
-    mochaFile: `reports/junit-mocha-${process.env.MOCHA_GREP}.xml`,
-    includePending: true,
-    jenkinsMode: true,
-    output: true,
-  },
-}

libraries/metrics/test/setup.js

@@ -1 +0,0 @@
-// no setup yet

libraries/metrics/test/unit/js/event_loop.js

@@ -5,7 +5,7 @@
  */
 const chai = require('chai')
 const { expect } = chai
-const path = require('node:path')
+const path = require('path')
 const modulePath = path.join(__dirname, '../../../event_loop.js')
 const SandboxedModule = require('sandboxed-module')
 const sinon = require('sinon')

libraries/metrics/test/unit/js/http.js

@@ -1,4 +1,4 @@
-const Path = require('node:path')
+const Path = require('path')
 const SandboxedModule = require('sandboxed-module')
 const sinon = require('sinon')
 

libraries/metrics/tsconfig.json

@@ -2,7 +2,6 @@
   "extends": "../../tsconfig.backend.json",
   "include": [
     "**/*.js",
-    "**/*.cjs",
-    "**/*.ts"
+    "**/*.cjs"
   ]
 }

libraries/mongo-utils/.nvmrc

@@ -1 +0,0 @@
-22.18.0

libraries/mongo-utils/batchedUpdate.js

@@ -1,321 +0,0 @@
-// @ts-check
-/* eslint-disable no-console */
-const { ObjectId, ReadPreference } = require('mongodb')
-
-const READ_PREFERENCE_SECONDARY =
-  process.env.MONGO_HAS_SECONDARIES === 'true'
-    ? ReadPreference.secondary.mode
-    : ReadPreference.secondaryPreferred.mode
-
-const ONE_MONTH_IN_MS = 1000 * 60 * 60 * 24 * 31
-let ID_EDGE_PAST
-const ID_EDGE_FUTURE = objectIdFromMs(Date.now() + 1000)
-let BATCH_DESCENDING
-let BATCH_SIZE
-let VERBOSE_LOGGING
-let BATCH_RANGE_START
-let BATCH_RANGE_END
-let BATCH_MAX_TIME_SPAN_IN_MS
-let BATCHED_UPDATE_RUNNING = false
-
-/**
- * @typedef {import("mongodb").Collection} Collection
- * @typedef {import("mongodb-legacy").Collection} LegacyCollection
- * @typedef {import("mongodb").Document} Document
- * @typedef {import("mongodb").FindOptions} FindOptions
- * @typedef {import("mongodb").UpdateFilter<Document>} UpdateDocument
- */
-
-/**
- * @typedef {Object} BatchedUpdateOptions
- * @property {string} [BATCH_DESCENDING]
- * @property {string} [BATCH_LAST_ID]
- * @property {string} [BATCH_MAX_TIME_SPAN_IN_MS]
- * @property {string} [BATCH_RANGE_END]
- * @property {string} [BATCH_RANGE_START]
- * @property {string} [BATCH_SIZE]
- * @property {string} [VERBOSE_LOGGING]
- * @property {(progress: string) => Promise<void>} [trackProgress]
- */
-
-/**
- * @param {BatchedUpdateOptions} options
- */
-function refreshGlobalOptionsForBatchedUpdate(options = {}) {
-  options = Object.assign({}, options, process.env)
-
-  BATCH_DESCENDING = options.BATCH_DESCENDING === 'true'
-  BATCH_SIZE = parseInt(options.BATCH_SIZE || '1000', 10) || 1000
-  VERBOSE_LOGGING = options.VERBOSE_LOGGING === 'true'
-  if (options.BATCH_LAST_ID) {
-    BATCH_RANGE_START = objectIdFromInput(options.BATCH_LAST_ID)
-  } else if (options.BATCH_RANGE_START) {
-    BATCH_RANGE_START = objectIdFromInput(options.BATCH_RANGE_START)
-  } else {
-    if (BATCH_DESCENDING) {
-      BATCH_RANGE_START = ID_EDGE_FUTURE
-    } else {
-      BATCH_RANGE_START = ID_EDGE_PAST
-    }
-  }
-  BATCH_MAX_TIME_SPAN_IN_MS = parseInt(
-    options.BATCH_MAX_TIME_SPAN_IN_MS || ONE_MONTH_IN_MS.toString(),
-    10
-  )
-  if (options.BATCH_RANGE_END) {
-    BATCH_RANGE_END = objectIdFromInput(options.BATCH_RANGE_END)
-  } else {
-    if (BATCH_DESCENDING) {
-      BATCH_RANGE_END = ID_EDGE_PAST
-    } else {
-      BATCH_RANGE_END = ID_EDGE_FUTURE
-    }
-  }
-}
-
-/**
- * @param {Collection | LegacyCollection} collection
- * @param {Document} query
- * @param {ObjectId} start
- * @param {ObjectId} end
- * @param {Document} projection
- * @param {FindOptions} findOptions
- * @return {Promise<Array<Document>>}
- */
-async function getNextBatch(
-  collection,
-  query,
-  start,
-  end,
-  projection,
-  findOptions
-) {
-  if (BATCH_DESCENDING) {
-    query._id = {
-      $gt: end,
-      $lte: start,
-    }
-  } else {
-    query._id = {
-      $gt: start,
-      $lte: end,
-    }
-  }
-  return await collection
-    .find(query, findOptions)
-    .project(projection)
-    .sort({ _id: BATCH_DESCENDING ? -1 : 1 })
-    .limit(BATCH_SIZE)
-    .toArray()
-}
-
-/**
- * @param {Collection | LegacyCollection} collection
- * @param {Array<Document>} nextBatch
- * @param {UpdateDocument} update
- * @return {Promise<void>}
- */
-async function performUpdate(collection, nextBatch, update) {
-  await collection.updateMany(
-    { _id: { $in: nextBatch.map(entry => entry._id) } },
-    update
-  )
-}
-
-/**
- * @param {string} input
- * @return {ObjectId}
- */
-function objectIdFromInput(input) {
-  if (input.includes('T')) {
-    const t = new Date(input).getTime()
-    if (Number.isNaN(t)) throw new Error(`${input} is not a valid date`)
-    return objectIdFromMs(t)
-  } else {
-    return new ObjectId(input)
-  }
-}
-
-/**
- * @param {ObjectId} objectId
- * @return {string}
- */
-function renderObjectId(objectId) {
-  return `${objectId} (${objectId.getTimestamp().toISOString()})`
-}
-
-/**
- * @param {number} ms
- * @return {ObjectId}
- */
-function objectIdFromMs(ms) {
-  return ObjectId.createFromTime(ms / 1000)
-}
-
-/**
- * @param {ObjectId} id
- * @return {number}
- */
-function getMsFromObjectId(id) {
-  return id.getTimestamp().getTime()
-}
-
-/**
- * @param {ObjectId} start
- * @return {ObjectId}
- */
-function getNextEnd(start) {
-  let end
-  if (BATCH_DESCENDING) {
-    end = objectIdFromMs(getMsFromObjectId(start) - BATCH_MAX_TIME_SPAN_IN_MS)
-    if (getMsFromObjectId(end) <= getMsFromObjectId(BATCH_RANGE_END)) {
-      end = BATCH_RANGE_END
-    }
-  } else {
-    end = objectIdFromMs(getMsFromObjectId(start) + BATCH_MAX_TIME_SPAN_IN_MS)
-    if (getMsFromObjectId(end) >= getMsFromObjectId(BATCH_RANGE_END)) {
-      end = BATCH_RANGE_END
-    }
-  }
-  return end
-}
-
-/**
- * @param {Collection | LegacyCollection} collection
- * @return {Promise<ObjectId|null>}
- */
-async function getIdEdgePast(collection) {
-  const [first] = await collection
-    .find({})
-    .project({ _id: 1 })
-    .sort({ _id: 1 })
-    .limit(1)
-    .toArray()
-  if (!first) return null
-  // Go one second further into the past in order to include the first entry via
-  // first._id > ID_EDGE_PAST
-  return objectIdFromMs(Math.max(0, getMsFromObjectId(first._id) - 1000))
-}
-
-/**
- * @param {Collection | LegacyCollection} collection
- * @param {Document} query
- * @param {UpdateDocument | ((batch: Array<Document>) => Promise<void>)} update
- * @param {Document} [projection]
- * @param {FindOptions} [findOptions]
- * @param {BatchedUpdateOptions} [batchedUpdateOptions]
- */
-async function batchedUpdate(
-  collection,
-  query,
-  update,
-  projection,
-  findOptions,
-  batchedUpdateOptions = {}
-) {
-  // only a single batchedUpdate can run at a time due to global variables
-  if (BATCHED_UPDATE_RUNNING) {
-    throw new Error('batchedUpdate is already running')
-  }
-  try {
-    BATCHED_UPDATE_RUNNING = true
-    ID_EDGE_PAST = await getIdEdgePast(collection)
-    if (!ID_EDGE_PAST) {
-      console.warn(
-        `The collection ${collection.collectionName} appears to be empty.`
-      )
-      return 0
-    }
-    refreshGlobalOptionsForBatchedUpdate(batchedUpdateOptions)
-    const { trackProgress = async progress => console.warn(progress) } =
-      batchedUpdateOptions
-
-    findOptions = findOptions || {}
-    findOptions.readPreference = READ_PREFERENCE_SECONDARY
-
-    projection = projection || { _id: 1 }
-    let nextBatch
-    let updated = 0
-    let start = BATCH_RANGE_START
-
-    while (start !== BATCH_RANGE_END) {
-      let end = getNextEnd(start)
-      nextBatch = await getNextBatch(
-        collection,
-        query,
-        start,
-        end,
-        projection,
-        findOptions
-      )
-      if (nextBatch.length > 0) {
-        end = nextBatch[nextBatch.length - 1]._id
-        updated += nextBatch.length
-
-        if (VERBOSE_LOGGING) {
-          console.log(
-            `Running update on batch with ids ${JSON.stringify(
-              nextBatch.map(entry => entry._id)
-            )}`
-          )
-        }
-        await trackProgress(
-          `Running update on batch ending ${renderObjectId(end)}`
-        )
-
-        if (typeof update === 'function') {
-          await update(nextBatch)
-        } else {
-          await performUpdate(collection, nextBatch, update)
-        }
-      }
-      await trackProgress(`Completed batch ending ${renderObjectId(end)}`)
-      start = end
-    }
-    return updated
-  } finally {
-    BATCHED_UPDATE_RUNNING = false
-  }
-}
-
-/**
- * @param {Collection | LegacyCollection} collection
- * @param {Document} query
- * @param {UpdateDocument | ((batch: Array<Object>) => Promise<void>)} update
- * @param {Document} [projection]
- * @param {FindOptions} [findOptions]
- * @param {BatchedUpdateOptions} [batchedUpdateOptions]
- */
-function batchedUpdateWithResultHandling(
-  collection,
-  query,
-  update,
-  projection,
-  findOptions,
-  batchedUpdateOptions
-) {
-  batchedUpdate(
-    collection,
-    query,
-    update,
-    projection,
-    findOptions,
-    batchedUpdateOptions
-  )
-    .then(processed => {
-      console.error({ processed })
-      process.exit(0)
-    })
-    .catch(error => {
-      console.error({ error })
-      process.exit(1)
-    })
-}
-
-module.exports = {
-  READ_PREFERENCE_SECONDARY,
-  objectIdFromInput,
-  renderObjectId,
-  batchedUpdate,
-  batchedUpdateWithResultHandling,
-}

libraries/mongo-utils/buildscript.txt

@@ -1,9 +0,0 @@
-mongo-utils
---dependencies=None
---env-add=
---env-pass-through=
---esmock-loader=False
---is-library=True
---node-version=22.18.0
---pipeline-owner=🚉 Platform
---public-repo=False

libraries/mongo-utils/index.js

@@ -1,5 +0,0 @@
-// @ts-check
-
-const { cleanupTestDatabase, dropTestDatabase } = require('./test-utils')
-
-module.exports = { cleanupTestDatabase, dropTestDatabase }

libraries/mongo-utils/package.json

@@ -1,32 +0,0 @@
-{
-  "name": "@overleaf/mongo-utils",
-  "version": "0.0.1",
-  "description": "utilities to help working with mongo",
-  "main": "index.js",
-  "scripts": {
-    "test": "npm run lint && npm run format && npm run types:check && npm run test:unit",
-    "test:unit": "mocha --exit test/**/*.{js,cjs}",
-    "lint": "eslint --ext .cjs,.js,.jsx,.mjs,.ts --max-warnings 0 --format unix .",
-    "lint:fix": "eslint --fix --ext .cjs,.js,.jsx,.mjs,.ts .",
-    "format": "prettier --list-different $PWD/'**/*.{js,cjs,ts}'",
-    "format:fix": "prettier --write $PWD/'**/*.{js,cjs,ts}'",
-    "test:ci": "npm run test:unit",
-    "types:check": "tsc --noEmit"
-  },
-  "author": "Overleaf (https://www.overleaf.com)",
-  "license": "AGPL-3.0-only",
-  "dependencies": {
-    "mongodb": "6.12.0",
-    "mongodb-legacy": "6.1.3"
-  },
-  "devDependencies": {
-    "chai": "^4.3.6",
-    "mocha": "^11.1.0",
-    "mocha-junit-reporter": "^2.2.1",
-    "mocha-multi-reporters": "^1.5.1",
-    "sandboxed-module": "^2.0.4",
-    "sinon": "^9.2.4",
-    "sinon-chai": "^3.7.0",
-    "typescript": "^5.0.4"
-  }
-}

libraries/mongo-utils/test-utils.js

@@ -1,55 +0,0 @@
-// @ts-check
-
-/**
- * @import { MongoClient } from 'mongodb'
- * @import { MongoClient as LegacyMongoClient } from 'mongodb-legacy'
- */
-
-/**
- * Delete all data from the test Mongo database
- *
- * This doesn't drop the collections, so indexes are preserved.
- *
- * @param {MongoClient | LegacyMongoClient} mongoClient
- */
-async function cleanupTestDatabase(mongoClient) {
-  ensureTestDatabase(mongoClient)
-  const db = mongoClient.db()
-  const allCollections = await db.collections()
-  const collections = allCollections.filter(
-    coll => coll.collectionName !== 'migrations'
-  )
-  await Promise.all(collections.map(coll => coll.deleteMany({})))
-}
-
-/**
- * Drop the test Monto database
- *
- * This drops the whole database, including indexes.
- *
- * @param {MongoClient | LegacyMongoClient } mongoClient
- */
-async function dropTestDatabase(mongoClient) {
-  ensureTestDatabase(mongoClient)
-  await mongoClient.db().dropDatabase()
-}
-/**
- * Ensure that the given client is connected to a test database.
- *
- * This should be called before performing destructive operations on the test
- * database.
- *
- * @param {MongoClient | LegacyMongoClient } mongoClient
- */
-function ensureTestDatabase(mongoClient) {
-  const dbName = mongoClient.db().databaseName
-  const env = process.env.NODE_ENV
-
-  if (dbName !== 'test-overleaf' || env !== 'test') {
-    throw new Error(
-      `Refusing to clear database '${dbName}' in environment '${env}'`
-    )
-  }
-}
-
-module.exports = { cleanupTestDatabase, dropTestDatabase }

libraries/mongo-utils/tsconfig.json

@@ -1,8 +0,0 @@
-{
-  "extends": "../../tsconfig.backend.json",
-  "include": [
-    "**/*.js",
-    "**/*.cjs",
-    "**/*.ts"
-  ]
-}

libraries/o-error/.dockerignore

@@ -0,0 +1 @@
+node_modules/

libraries/o-error/.editorconfig

@@ -0,0 +1,9 @@
+root = true
+
+[*]
+charset = utf-8
+indent_style = space
+indent_size = 2
+end_of_line = lf
+insert_final_newline = true
+trim_trailing_whitespace = true

libraries/o-error/.gitignore

@@ -0,0 +1,5 @@
+.nyc_output
+coverage
+node_modules/
+
+.npmrc

libraries/o-error/.mocharc.cjs

@@ -1,13 +0,0 @@
-let reporterOptions = {}
-if (process.env.CI) {
-  reporterOptions = {
-    reporter: '/overleaf/node_modules/mocha-multi-reporters',
-    'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
-  }
-}
-const all = {
-  require: 'test/setup.js',
-  ...reporterOptions,
-}
-
-module.exports = all

libraries/o-error/.nvmrc

@@ -1 +1 @@
-22.18.0
+18.20.2

libraries/o-error/buildscript.txt

@@ -1,9 +1,10 @@
 o-error
 --dependencies=None
+--docker-repos=gcr.io/overleaf-ops
 --env-add=
 --env-pass-through=
 --esmock-loader=False
 --is-library=True
---node-version=22.18.0
---pipeline-owner=🚉 Platform
+--node-version=18.20.2
 --public-repo=False
+--script-version=4.5.0

libraries/o-error/doc/demo.js

@@ -70,7 +70,7 @@ sayHi3(43, (err, result) => {
   }
 })
 
-const promisify = require('node:util').promisify
+const promisify = require('util').promisify
 demoDatabase.findUserAsync = promisify(demoDatabase.findUser)
 
 async function sayHi4NoHandling(userId) {

libraries/o-error/index.cjs

@@ -1,34 +1,20 @@
-// @ts-check
-
 /**
  * Light-weight helpers for handling JavaScript Errors in node.js and the
  * browser.
  */
 class OError extends Error {
-  /**
-   * The error that is the underlying cause of this error
-   *
-   * @type {unknown}
-   */
-  cause
-
-  /**
-   * List of errors encountered as the callback chain is unwound
-   *
-   * @type {TaggedError[] | undefined}
-   */
-  _oErrorTags
-
   /**
    * @param {string} message as for built-in Error
    * @param {Object} [info] extra data to attach to the error
-   * @param {unknown} [cause] the internal error that caused this error
+   * @param {Error} [cause] the internal error that caused this error
    */
   constructor(message, info, cause) {
     super(message)
     this.name = this.constructor.name
     if (info) this.info = info
     if (cause) this.cause = cause
+    /** @private @type {Array<TaggedError> | undefined} */
+    this._oErrorTags // eslint-disable-line
   }
 
   /**
@@ -45,7 +31,7 @@ class OError extends Error {
   /**
    * Wrap the given error, which caused this error.
    *
-   * @param {unknown} cause the internal error that caused this error
+   * @param {Error} cause the internal error that caused this error
    * @return {this}
    */
   withCause(cause) {
@@ -79,16 +65,13 @@ class OError extends Error {
    *   }
    * }
    *
-   * @template {unknown} E
-   * @param {E} error the error to tag
+   * @param {Error} error the error to tag
    * @param {string} [message] message with which to tag `error`
    * @param {Object} [info] extra data with wich to tag `error`
-   * @return {E} the modified `error` argument
+   * @return {Error} the modified `error` argument
    */
   static tag(error, message, info) {
-    const oError = /** @type {{ _oErrorTags: TaggedError[] | undefined }} */ (
-      error
-    )
+    const oError = /** @type{OError} */ (error)
 
     if (!oError._oErrorTags) oError._oErrorTags = []
 
@@ -119,7 +102,7 @@ class OError extends Error {
    *
    * If an info property is repeated, the last one wins.
    *
-   * @param {unknown} error any error (may or may not be an `OError`)
+   * @param {Error | null | undefined} error any error (may or may not be an `OError`)
    * @return {Object}
    */
   static getFullInfo(error) {
@@ -146,7 +129,7 @@ class OError extends Error {
    * Return the `stack` property from `error`, including the `stack`s for any
    * tagged errors added with `OError.tag` and for any `cause`s.
    *
-   * @param {unknown} error any error (may or may not be an `OError`)
+   * @param {Error | null | undefined} error any error (may or may not be an `OError`)
    * @return {string}
    */
   static getFullStack(error) {
@@ -160,7 +143,7 @@ class OError extends Error {
       stack += `\n${oError._oErrorTags.map(tag => tag.stack).join('\n')}`
     }
 
-    const causeStack = OError.getFullStack(oError.cause)
+    const causeStack = oError.cause && OError.getFullStack(oError.cause)
     if (causeStack) {
       stack += '\ncaused by:\n' + indent(causeStack)
     }

libraries/o-error/package.json

@@ -18,8 +18,8 @@
   ],
   "scripts": {
     "build": "npm run --silent test",
-    "lint": "eslint --ext .cjs,.js,.jsx,.mjs,.ts --max-warnings 0 --format unix .",
-    "lint:fix": "eslint --fix --ext .cjs,.js,.jsx,.mjs,.ts .",
+    "lint": "eslint --ext .js --ext .cjs --ext .ts --max-warnings 0 --format unix .",
+    "lint:fix": "eslint --fix --ext .js --ext .cjs --ext .ts .",
     "test": "npm run lint && npm run format && npm run types:check && npm run test:unit",
     "format": "prettier --list-different $PWD/'**/*.{js,cjs,ts}'",
     "format:fix": "prettier --write $PWD/'**/*.{js,cjs,ts}'",
@@ -32,11 +32,9 @@
   "repository": "github:overleaf/o-error",
   "devDependencies": {
     "@types/chai": "^4.3.0",
-    "@types/node": "^18.19.123",
+    "@types/node": "^18.17.4",
     "chai": "^4.3.6",
-    "mocha": "^11.1.0",
-    "mocha-junit-reporter": "^2.2.1",
-    "mocha-multi-reporters": "^1.5.1",
+    "mocha": "^10.2.0",
     "typescript": "^5.0.4"
   }
 }