Bib Editor: the initial commit (buggy)

- do not show the project owner, email and connections number columns
 - add mailto: all active users of a project when clicking the project title
 - fix error handling

.github/ISSUE_TEMPLATE/bug_report.md

@@ -8,12 +8,8 @@ assignees: ''
 
 <!--
 
-Note: If you are using www.overleaf.com and have a problem,
+Note: This form should only be used to report bugs in the
-      or if you would like to request a new feature please contact
+      Extended CE release of Overleaf.
-      the support team at support@overleaf.com
-
-      This form should only be used to report bugs in the
-      Community Edition release of Overleaf.
 
 -->
 

.github/PULL_REQUEST_TEMPLATE.md

@@ -5,7 +5,3 @@
 ## Related issues / Pull Requests
 
 <!-- Fixes #xyz, Contributes to #xyz, Related to #xyz-->
-
-## Contributor Agreement
-
-- [ ] I confirm I have signed the [Contributor License Agreement](https://github.com/overleaf/overleaf/blob/main/CONTRIBUTING.md#contributor-license-agreement)

.yarn/patches/@node-saml-node-saml-npm-5.1.0-f4eb6913cc.patch

@@ -0,0 +1,23 @@
+diff --git a/lib/saml.js b/lib/saml.js
+index 64180fdd7fe6302bf8da7d1ca65c01bb1ea5e2c1..7015c3b73603f624c15195a0a38c2fbf750a4c60 100644
+--- a/lib/saml.js
++++ b/lib/saml.js
+@@ -327,7 +327,8 @@ class SAML {
+         const requestOrResponse = request || response;
+         (0, utility_1.assertRequired)(requestOrResponse, "either request or response is required");
+         let buffer;
+-        if (this.options.skipRequestCompression) {
++       // logout requestOrResponse must be compressed anyway
++        if (this.options.skipRequestCompression && operation !== "logout") {
+             buffer = Buffer.from(requestOrResponse, "utf8");
+         }
+         else {
+@@ -510,7 +511,7 @@ class SAML {
+         try {
+             xml = Buffer.from(container.SAMLResponse, "base64").toString("utf8");
+             doc = await (0, xml_1.parseDomFromString)(xml);
+-            const inResponseToNodes = xml_1.xpath.selectAttributes(doc, "/*[local-name()='Response']/@InResponseTo");
++            const inResponseToNodes = xml_1.xpath.selectAttributes(doc, "/*[local-name()='Response' or local-name()='LogoutResponse']/@InResponseTo");
+             if (inResponseToNodes) {
+                 inResponseTo = inResponseToNodes.length ? inResponseToNodes[0].nodeValue : null;
+                 await this.validateInResponseTo(inResponseTo);

.yarn/patches/ldapauth-fork-npm-5.0.5-c8b86f5231.patch

@@ -0,0 +1,73 @@
+diff --git a/lib/ldapauth.js b/lib/ldapauth.js
+index 88398ed31afee34c8958ad5e7a53653e1b5f6f82..0334177af8a65c7c5f9de7570924ab0f359e3f38 100644
+--- a/lib/ldapauth.js
++++ b/lib/ldapauth.js
+@@ -87,6 +87,7 @@ function LdapAuth(opts) {
+   this.opts.bindProperty || (this.opts.bindProperty = 'dn');
+   this.opts.groupSearchScope || (this.opts.groupSearchScope = 'sub');
+   this.opts.groupDnProperty || (this.opts.groupDnProperty = 'dn');
++  this.opts.tlsStarted = false;
+ 
+   EventEmitter.call(this);
+ 
+@@ -126,21 +127,7 @@ function LdapAuth(opts) {
+   this._userClient.on('error', this._handleError.bind(this));
+ 
+   var self = this;
+-  if (this.opts.starttls) {
+-    // When starttls is enabled, this callback supplants the 'connect' callback
+-    this._adminClient.starttls(this.opts.tlsOptions, this._adminClient.controls, function (err) {
+-      if (err) {
+-        self._handleError(err);
+-      } else {
+-        self._onConnectAdmin();
+-      }
+-    });
+-    this._userClient.starttls(this.opts.tlsOptions, this._userClient.controls, function (err) {
+-      if (err) {
+-        self._handleError(err);
+-      }
+-    });
+-  } else if (opts.reconnect) {
++  if (opts.reconnect && !this.opts.starttls) {
+     this.once('_installReconnectListener', function () {
+       self.log && self.log.trace('install reconnect listener');
+       self._adminClient.on('connect', function () {
+@@ -401,6 +388,37 @@ LdapAuth.prototype._findGroups = function (user, callback) {
+  */
+ LdapAuth.prototype.authenticate = function (username, password, callback) {
+   var self = this;
++  if (this.opts.starttls && !this.opts.tlsStarted) {
++    // When starttls is enabled, this callback supplants the 'connect' callback
++    this._adminClient.starttls(this.opts.tlsOptions, this._adminClient.controls, function (err) {
++      if (err) {
++        self._handleError(err);
++      } else {
++        self._onConnectAdmin(function(){self._handleAuthenticate(username, password, callback);});
++      }
++    });
++    this._userClient.starttls(this.opts.tlsOptions, this._userClient.controls, function (err) {
++      if (err) {
++        self._handleError(err);
++      }
++    });
++  } else {
++    self._handleAuthenticate(username, password, callback);
++  }
++};
++
++/**
++ * Authenticate given credentials against LDAP server
++ * 
++ * @private
++ * @param {string} username - The username to authenticate
++ * @param {string} password - The password to verify
++ * @param {resultCallback} callback - Result handling callback
++ * @returns {undefined}
++ */
++LdapAuth.prototype._handleAuthenticate = function (username, password, callback) {
++  this.opts.tlsStarted = true;
++  var self = this;
+ 
+   if (typeof password === 'undefined' || password === null || password === '') {
+     return callback(new Error('no password given'));

CONTRIBUTING.md

@@ -1,38 +1,29 @@
-# Contributing to Overleaf
+# Contributing to Overleaf CE+
 
 Thank you for reading this! If you'd like to report a bug or join in the development
-of Overleaf, then here are some notes on how to do that.
+of Overleaf CE+, then here are some notes on how to do that.
 
 ## Reporting bugs and opening issues
 
-If you'd like to report a bug or open an issue, please **[check if there is an existing issue](https://github.com/overleaf/overleaf/issues).**
+If you'd like to report a bug or open an issue, please **[check if there is an existing issue](https://github.com/yu-i-i/overleaf-cep).**
 If there is then please add any more information that you have, or give it a 👍.
 
 When submitting an issue please describe the issue as clearly as possible, including how to
 reproduce the bug, which situations it appears in, what you expected to happen, and what actually happens.
 If you can include a screenshot for front end issues that is very helpful.
 
-**Note**: If you are using [www.overleaf.com](www.overleaf.com) and have a problem, or if you would like to request a new feature, please contact the Support team at support@overleaf.com. Raise an issue here only to report bugs in the Community Edition release of Overleaf.
 
 ## Pull Requests
 
-See [our wiki](https://github.com/overleaf/overleaf/wiki)
+See [Overleaf wiki](https://github.com/overleaf/overleaf/wiki)
 for how to manage the Overleaf development environment and for our developer guidelines.
 
 We love pull requests, so be bold with them! Don't be afraid of going ahead
 and changing something, or adding a new feature. We're very happy to work with you
 to get your changes merged into Overleaf.
 
-If you're looking for something to work on, have a look at the [open issues](https://github.com/overleaf/overleaf/issues).
+If you're looking for something to work on, have a look at the [open issues](https://github.com/yu-i-i/overleaf-cep/issues).
 
 ## Security
 
-Please see [our security policy](https://github.com/overleaf/overleaf/security/policy) if you would like to report a potential security vulnerability.
+Please see [Overleaf security policy](https://github.com/overleaf/overleaf/security/policy) if you would like to report a potential security vulnerability.
-
-## Contributor License Agreement
-
-Before we can accept any contributions of code, we need you to agree to our
-[Contributor License Agreement](https://docs.google.com/forms/d/e/1FAIpQLSef79XH3mb7yIiMzZw-yALEegS-wyFetvjTiNBfZvf_IHD2KA/viewform?usp=sf_link).
-This is to ensure that you own the copyright of your contribution, and that you
-agree to give us a license to use it in both the open source version, and the version
-of Overleaf running at www.overleaf.com, which may have additional changes.

CREDITS.md

@@ -0,0 +1,53 @@
+# CREDITS
+
+This project includes ideas and adapted code from several open-source projects.\
+In most cases the original code has been modified, optimized, or extended.
+
+## Symbol palette
+
+The symbol palette feature is based on the
+[original Overleaf implementation](https://github.com/overleaf/web/tree/master/frontend/js/features/symbol-palette)
+
+The original code was slightly improved, particularly in the parts related to keyboard input.
+
+## LDAP authentication
+
+LDAP authentication was inspired by [this project](https://github.com/smhaller/ldap-overleaf-sl).
+
+The project provided the idea and part of the implementation for adding LDAP users to the user's contacts.
+
+## Real-time track changes and comments
+
+The Track Changes and Comments feature exists largely in the original Overleaf codebase.\
+The missing parts were implemented based on [this code](https://github.com/ertuil/overleaf).
+
+The referenced code was fixed, optimized, and extended.
+
+## Sandboxed compiles
+
+The Sandboxed Compiles feature was until recently largely present in the original Overleaf codebase.\
+The missing parts were implemented in this project.
+
+## Import file from external URL
+
+The "From External URL" feature exists in the original Overleaf code.\
+The missing proxy component was implemented in this project.
+
+## Git integration
+
+The Git integration feature includes parts of the frontend code
+from [here](https://github.com/ayaka-notes/overleaf-pro/tree/feat-git-bridge)
+
+## GitHub synchronization
+
+The GitHub synchronization feature includes parts of the frontend code and the OAuth2 backend implementation
+from [here](https://github.com/ayaka-notes/overleaf-pro/commit/06a30fe9a0ed75e5ab40b50a8a4e94f43161cf71).
+
+## Sign Up page
+
+The Sign Up page is based on [this code](https://github.com/ayaka-notes/overleaf-pro/tree/feat-public-registeration).
+
+# Acknowledgments
+
+Thanks to the users of the project for valuable feedback, suggestions,
+and help in identifying and fixing bugs.

README.md

@@ -6,52 +6,65 @@
 <h4 align="center">An open-source online real-time collaborative LaTeX editor.</h4>
 
 <p align="center">
-  <a href="https://github.com/overleaf/overleaf/wiki">Wiki</a> •
+  <a href="https://github.com/yu-i-i/overleaf-cep/wiki">Wiki</a> •
   <a href="https://www.overleaf.com/for/enterprises">Server Pro</a> •
-  <a href="#contributing">Contributing</a> •
-  <a href="https://mailchi.mp/overleaf.com/community-edition-and-server-pro">Mailing List</a> •
   <a href="#authors">Authors</a> •
   <a href="#license">License</a>
 </p>
 
-<img src="doc/screenshot.png" alt="A screenshot of a project being edited in Overleaf Community Edition">
+<img src="doc/screenshot.png" alt="A screenshot of a project being edited in Overleaf Extended Community Edition">
 <p align="center">
-  Figure 1: A screenshot of a project being edited in Overleaf Community Edition.
+  Figure 1: A screenshot of a project being edited in Overleaf Extended Community Edition.
 </p>
 
 ## Community Edition
 
-[Overleaf](https://www.overleaf.com) is an open-source online real-time collaborative LaTeX editor. We run a hosted version at [www.overleaf.com](https://www.overleaf.com), but you can also run your own local version, and contribute to the development of Overleaf.
+[Overleaf](https://www.overleaf.com) is an open-source online real-time collaborative LaTeX editor. Overleaf runs a hosted version at [www.overleaf.com](https://www.overleaf.com), but you can also run your own local version, and contribute to the development of Overleaf.
+
+## Extended Community Edition (CE+)
+
+The present "extended" version of Overleaf CE includes:
+
+- Sandboxed compiles with TeX Live image selection
+- Sign Up page
+- LDAP authentication
+- SAML authentication
+- OpenID Connect authentication
+- Real-time track changes and comments
+- Symbol palette
+- Template gallery
+- Import file from external URL
+- Git integration
+- GitHub synchronization
+- Zotero integration
+- Reference Search and Pick Tool
+- Document Import (`.docx`, `.md`) and Export (`.docx`, `.md`, `.html`) 
+- Advanced administrator tools for managing user accounts and projects
+- Logo tools
 
 > [!CAUTION]
-> Overleaf Community Edition is intended for use in environments where **all** users are trusted. Community Edition is **not** appropriate for scenarios where isolation of users is required due to Sandbox Compiles not being available. When not using Sandboxed Compiles, users have full read and write access to the `sharelatex` container resources (filesystem, network, environment variables) when running LaTeX compiles.
+> Overleaf Community Edition is intended for use in environments where **all** users are trusted. Community Edition is **not** appropriate for scenarios where isolation of users is required due to Sandbox Compiles not being available. When not using Sandboxed Compiles, users have full read and write access to the `sharelatex` container resources (filesystem, network, environment variables) when running LaTeX compiles. 
+Therefore, in any environment where not all users can be fully trusted, it is strongly recommended to enable the Sandboxed Compiles feature available in the Extended Community Edition.
 
-For more information on Sandbox Compiles check out our [documentation](https://docs.overleaf.com/on-premises/configuration/overleaf-toolkit/server-pro-only-configuration/sandboxed-compiles).
+For more information on Sandbox Compiles check out Overleaf [documentation](https://docs.overleaf.com/on-premises/configuration/overleaf-toolkit/server-pro-only-configuration/sandboxed-compiles).
 
 ## Enterprise
 
-If you want help installing and maintaining Overleaf in your lab or workplace, we offer an officially supported version called [Overleaf Server Pro](https://www.overleaf.com/for/enterprises). It also includes more features for security (SSO with LDAP or SAML), administration and collaboration (e.g. tracked changes). [Find out more!](https://www.overleaf.com/for/enterprises)
+If you want help installing and maintaining Overleaf in your lab or workplace, Overleaf offers an officially supported version called [Overleaf Server Pro](https://www.overleaf.com/for/enterprises).
-
-## Keeping up to date
-
-Sign up to the [mailing list](https://mailchi.mp/overleaf.com/community-edition-and-server-pro) to get updates on Overleaf releases and development.
 
 ## Installation
 
-We have detailed installation instructions in the [Overleaf Toolkit](https://github.com/overleaf/toolkit/).
+Detailed installation instructions can be found in the [Overleaf Toolkit](https://github.com/overleaf/toolkit/).
-
+Configuration details and release history for the Extended Community Edition can be found on the [Extended CE Wiki Page](https://github.com/yu-i-i/overleaf-cep/wiki).
-## Upgrading
-
-If you are upgrading from a previous version of Overleaf, please see the [Release Notes section on the Wiki](https://github.com/overleaf/overleaf/wiki#release-notes) for all of the versions between your current version and the version you are upgrading to.
 
 ## Overleaf Docker Image
 
 This repo contains two dockerfiles, [`Dockerfile-base`](server-ce/Dockerfile-base), which builds the
-`sharelatex/sharelatex-base` image, and [`Dockerfile`](server-ce/Dockerfile) which builds the
+`sharelatex/sharelatex-base:ext-ce` image, and [`Dockerfile`](server-ce/Dockerfile) which builds the
-`sharelatex/sharelatex` (or "community") image.
+`sharelatex/sharelatex:ext-ce` image.
 
 The Base image generally contains the basic dependencies like `wget`, plus `texlive`.
-We split this out because it's a pretty heavy set of
+This is split out because it's a pretty heavy set of
 dependencies, and it's nice to not have to rebuild all of that every time.
 
 The `sharelatex/sharelatex` image extends the base image and adds the actual Overleaf code
@@ -59,22 +72,22 @@ and services.
 
 Use `make build-base` and `make build-community` from `server-ce/` to build these images.
 
-We use the [Phusion base-image](https://github.com/phusion/baseimage-docker)
+The [Phusion base-image](https://github.com/phusion/baseimage-docker)
-(which is extended by our `base` image) to provide us with a VM-like container
+(which is extended by the `base` image) provides a VM-like container
 in which to run the Overleaf services. Baseimage uses the `runit` service
-manager to manage services, and we add our init-scripts from the `server-ce/runit`
+manager to manage services, and init scripts from the `server-ce/runit`
-folder.
+folder are added.
-
-## Contributing
-
-Please see the [CONTRIBUTING](CONTRIBUTING.md) file for information on contributing to the development of Overleaf.
 
 ## Authors
 
-[The Overleaf Team](https://www.overleaf.com/about)
+[The Overleaf Team](https://www.overleaf.com/about)\
+[yu-i-i](https://github.com/yu-i-i), [davrot](https://github.com/davrot) — CE extensions; references to adapted code are listed in [`CREDITS`](CREDITS.md)
 
 ## License
 
 The code in this repository is released under the GNU AFFERO GENERAL PUBLIC LICENSE, version 3. A copy can be found in the [`LICENSE`](LICENSE) file.
 
-Copyright (c) Overleaf, 2014-2025.
+Copyright (c) Overleaf, 2014-2026.\
+Copyright (c) @yu-i-i, 2024-2026, for CE extensions.
+
+Portions of the code are derived from other open-source projects; see [`CREDITS`](CREDITS.md).

develop/README.md

@@ -66,6 +66,7 @@ each service:
 | `real-time`        | 9237 |
 | `history-v1`       | 9239 |
 | `project-history`  | 9240 |
+| `linked-url-proxy` | 9241 |
 
 To attach to a service using Chrome's _remote debugging_, go to
 <chrome://inspect/> and make sure _Discover network targets_ is checked. Next

develop/dev.env

@@ -7,6 +7,7 @@ FILESTORE_HOST=filestore
 GRACEFUL_SHUTDOWN_DELAY_SECONDS=0
 HISTORY_V1_HOST=history-v1
 HISTORY_REDIS_HOST=redis
+LINKED_URL_PROXY_HOST=linked-url-proxy
 LISTEN_ADDRESS=0.0.0.0
 MONGO_HOST=mongo
 MONGO_URL=mongodb://mongo/sharelatex?directConnection=true

develop/docker-compose.dev.yml

@@ -68,6 +68,17 @@ services:
       - ../services/history-v1/knexfile.js:/overleaf/services/history-v1/knexfile.js
       - ../services/history-v1/migrations:/overleaf/services/history-v1/migrations
 
+  linked-url-proxy:
+    command: ["node", "--watch", "app.mjs"]
+    environment:
+      - NODE_OPTIONS=--inspect=0.0.0.0:9229
+    ports:
+      - "127.0.0.1:9241:9229"
+    volumes:
+      - ../services/linked-url-proxy/app:/overleaf/services/linked-url-proxy/app
+      - ../services/linked-url-proxy/config:/overleaf/services/linked-url-proxy/config
+      - ../services/linked-url-proxy/app.mjs:/overleaf/services/linked-url-proxy/app.mjs
+
   notifications:
     command: ["node", "--watch", "app.ts"]
     environment:

develop/docker-compose.yml

@@ -88,6 +88,13 @@ services:
     volumes:
       - history-v1-buckets:/buckets
 
+  linked-url-proxy:
+    build:
+      context: ..
+      dockerfile: services/linked-url-proxy/Dockerfile
+    env_file:
+      - dev.env
+
   mongo:
     image: mongo:8
     command: --replSet overleaf
@@ -141,7 +148,7 @@ services:
       - dev.env
     environment:
       - APP_NAME=Overleaf Community Edition
-      - ENABLED_LINKED_FILE_TYPES=project_file,project_output_file
+      - ENABLED_LINKED_FILE_TYPES=project_file,project_output_file,zotero
       - EMAIL_CONFIRMATION_DISABLED=true
       - NODE_ENV=development
       - OVERLEAF_ALLOW_PUBLIC_ACCESS=true
@@ -158,6 +165,7 @@ services:
       - document-updater
       - filestore
       - history-v1
+      - linked-url-proxy
       - notifications
       - project-history
       - real-time
@@ -173,3 +181,18 @@ services:
       - "127.0.0.1:80:3808"
     volumes:
       - ./webpack.config.dev-env.js:/overleaf/services/web/webpack.config.dev-env.js
+
+  git-bridge:
+    image: quay.io/sharelatex/git-bridge:latest
+    expose:
+      - "8000"
+    ports:
+      - "8000:8000"
+    environment:
+      GIT_BRIDGE_API_BASE_URL: "http://web:3000/api/v0"
+      GIT_BRIDGE_OAUTH2_SERVER: "http://web:3000"
+      GIT_BRIDGE_POSTBACK_BASE_URL: "http://git-bridge:8000"
+      GIT_BRIDGE_ROOT_DIR: "/data/git-bridge"
+      LOG_LEVEL: "DEBUG"
+    volumes:
+      - ./data/git-bridge:/data/git-bridge

develop/webpack.config.dev-env.js

@@ -14,6 +14,11 @@ module.exports = merge(base, {
         target: 'http://real-time:3026',
         ws: true,
       },
+      {
+        context: '/git/**',
+        target: 'http://git-bridge:8000',
+        pathRewrite: { '^/git': '' }
+      },
       {
         context: ['!**/*.js', '!**/*.css', '!**/*.json'],
         target: 'http://web:3000',

docker-compose.yml

@@ -33,7 +33,7 @@ services:
       # IMPORTANT: Required for invite token generation
       # OVERLEAF_INVITE_TOKEN_SECRET: generate with `openssl rand -base64 32`
 
-      ENABLED_LINKED_FILE_TYPES: "project_file,project_output_file"
+      ENABLED_LINKED_FILE_TYPES: "project_file,project_output_file,zotero"
 
       # Enables Thumbnail generation using ImageMagick
       ENABLE_CONVERSIONS: "true"

libraries/access-token-encryptor/Makefile

@@ -0,0 +1,59 @@
+# Run the test commands in the scope of the monorepo.
+# Eslint and prettier (plus some configs) are on the root.
+RUN_LINTING = ../../bin/run -w /overleaf/libraries/access-token-encryptor monorepo yarn run --silent
+RUN_LINTING_MONOREPO = ../../bin/run monorepo yarn run --silent
+
+WITH_CACHE_FOLDER = \
+	lint \
+	lint_ci \
+	lint_fix \
+	format \
+	format_ci \
+	format_fix \
+	typecheck \
+	typecheck_ci \
+
+$(WITH_CACHE_FOLDER): ../../node_modules/.cache/prettier/
+../../node_modules/.cache/prettier/:
+	mkdir -p $@
+
+$(WITH_CACHE_FOLDER): ../../node_modules/.cache/eslint/
+../../node_modules/.cache/eslint/:
+	mkdir -p $@
+
+format:
+	$(RUN_LINTING_MONOREPO) format libraries/access-token-encryptor
+
+format_ci:
+	$(RUN_LINTING_MONOREPO) format libraries/access-token-encryptor
+
+format_fix:
+	$(RUN_LINTING_MONOREPO) format:fix libraries/access-token-encryptor
+
+lint:
+	$(RUN_LINTING) lint
+
+lint_ci:
+	-$(RUN_LINTING) lint --format json --output-file reports/eslint.json
+	sed -i 's_"filePath":"/overleaf_"filePath":"$(MONOREPO)_g' reports/eslint.json
+
+lint_fix:
+	$(RUN_LINTING) lint:fix
+
+typecheck:
+	$(RUN_LINTING) types:check
+
+typecheck_ci:
+	$(RUN_LINTING) types:check
+
+test: test_unit
+test_unit:
+ifneq (,$(wildcard test/unit))
+	$(RUN_LINTING) test:unit
+endif
+
+test: test_acceptance
+test_acceptance:
+ifneq (,$(wildcard test/acceptance))
+	$(RUN_LINTING) test:acceptance
+endif

libraries/access-token-encryptor/buildscript.txt

@@ -5,6 +5,5 @@ access-token-encryptor
 --esmock-loader=False
 --is-library=True
 --node-version=24.14.1
---package-name=@overleaf/access-token-encryptor
 --pipeline-owner=32
 --public-repo=False

libraries/fetch-utils/Makefile

@@ -0,0 +1,59 @@
+# Run the test commands in the scope of the monorepo.
+# Eslint and prettier (plus some configs) are on the root.
+RUN_LINTING = ../../bin/run -w /overleaf/libraries/fetch-utils monorepo yarn run --silent
+RUN_LINTING_MONOREPO = ../../bin/run monorepo yarn run --silent
+
+WITH_CACHE_FOLDER = \
+	lint \
+	lint_ci \
+	lint_fix \
+	format \
+	format_ci \
+	format_fix \
+	typecheck \
+	typecheck_ci \
+
+$(WITH_CACHE_FOLDER): ../../node_modules/.cache/prettier/
+../../node_modules/.cache/prettier/:
+	mkdir -p $@
+
+$(WITH_CACHE_FOLDER): ../../node_modules/.cache/eslint/
+../../node_modules/.cache/eslint/:
+	mkdir -p $@
+
+format:
+	$(RUN_LINTING_MONOREPO) format libraries/fetch-utils
+
+format_ci:
+	$(RUN_LINTING_MONOREPO) format libraries/fetch-utils
+
+format_fix:
+	$(RUN_LINTING_MONOREPO) format:fix libraries/fetch-utils
+
+lint:
+	$(RUN_LINTING) lint
+
+lint_ci:
+	-$(RUN_LINTING) lint --format json --output-file reports/eslint.json
+	sed -i 's_"filePath":"/overleaf_"filePath":"$(MONOREPO)_g' reports/eslint.json
+
+lint_fix:
+	$(RUN_LINTING) lint:fix
+
+typecheck:
+	$(RUN_LINTING) types:check
+
+typecheck_ci:
+	$(RUN_LINTING) types:check
+
+test: test_unit
+test_unit:
+ifneq (,$(wildcard test/unit))
+	$(RUN_LINTING) test:unit
+endif
+
+test: test_acceptance
+test_acceptance:
+ifneq (,$(wildcard test/acceptance))
+	$(RUN_LINTING) test:acceptance
+endif

libraries/fetch-utils/buildscript.txt

@@ -5,6 +5,5 @@ fetch-utils
 --esmock-loader=False
 --is-library=True
 --node-version=24.14.1
---package-name=@overleaf/fetch-utils
 --pipeline-owner=32
 --public-repo=False

libraries/logger/Makefile

@@ -0,0 +1,59 @@
+# Run the test commands in the scope of the monorepo.
+# Eslint and prettier (plus some configs) are on the root.
+RUN_LINTING = ../../bin/run -w /overleaf/libraries/logger monorepo yarn run --silent
+RUN_LINTING_MONOREPO = ../../bin/run monorepo yarn run --silent
+
+WITH_CACHE_FOLDER = \
+	lint \
+	lint_ci \
+	lint_fix \
+	format \
+	format_ci \
+	format_fix \
+	typecheck \
+	typecheck_ci \
+
+$(WITH_CACHE_FOLDER): ../../node_modules/.cache/prettier/
+../../node_modules/.cache/prettier/:
+	mkdir -p $@
+
+$(WITH_CACHE_FOLDER): ../../node_modules/.cache/eslint/
+../../node_modules/.cache/eslint/:
+	mkdir -p $@
+
+format:
+	$(RUN_LINTING_MONOREPO) format libraries/logger
+
+format_ci:
+	$(RUN_LINTING_MONOREPO) format libraries/logger
+
+format_fix:
+	$(RUN_LINTING_MONOREPO) format:fix libraries/logger
+
+lint:
+	$(RUN_LINTING) lint
+
+lint_ci:
+	-$(RUN_LINTING) lint --format json --output-file reports/eslint.json
+	sed -i 's_"filePath":"/overleaf_"filePath":"$(MONOREPO)_g' reports/eslint.json
+
+lint_fix:
+	$(RUN_LINTING) lint:fix
+
+typecheck:
+	$(RUN_LINTING) types:check
+
+typecheck_ci:
+	$(RUN_LINTING) types:check
+
+test: test_unit
+test_unit:
+ifneq (,$(wildcard test/unit))
+	$(RUN_LINTING) test:unit
+endif
+
+test: test_acceptance
+test_acceptance:
+ifneq (,$(wildcard test/acceptance))
+	$(RUN_LINTING) test:acceptance
+endif

libraries/logger/buildscript.txt

@@ -5,6 +5,5 @@ logger
 --esmock-loader=False
 --is-library=True
 --node-version=24.14.1
---package-name=@overleaf/logger
 --pipeline-owner=32
 --public-repo=False

libraries/metrics/Makefile

@@ -0,0 +1,59 @@
+# Run the test commands in the scope of the monorepo.
+# Eslint and prettier (plus some configs) are on the root.
+RUN_LINTING = ../../bin/run -w /overleaf/libraries/metrics monorepo yarn run --silent
+RUN_LINTING_MONOREPO = ../../bin/run monorepo yarn run --silent
+
+WITH_CACHE_FOLDER = \
+	lint \
+	lint_ci \
+	lint_fix \
+	format \
+	format_ci \
+	format_fix \
+	typecheck \
+	typecheck_ci \
+
+$(WITH_CACHE_FOLDER): ../../node_modules/.cache/prettier/
+../../node_modules/.cache/prettier/:
+	mkdir -p $@
+
+$(WITH_CACHE_FOLDER): ../../node_modules/.cache/eslint/
+../../node_modules/.cache/eslint/:
+	mkdir -p $@
+
+format:
+	$(RUN_LINTING_MONOREPO) format libraries/metrics
+
+format_ci:
+	$(RUN_LINTING_MONOREPO) format libraries/metrics
+
+format_fix:
+	$(RUN_LINTING_MONOREPO) format:fix libraries/metrics
+
+lint:
+	$(RUN_LINTING) lint
+
+lint_ci:
+	-$(RUN_LINTING) lint --format json --output-file reports/eslint.json
+	sed -i 's_"filePath":"/overleaf_"filePath":"$(MONOREPO)_g' reports/eslint.json
+
+lint_fix:
+	$(RUN_LINTING) lint:fix
+
+typecheck:
+	$(RUN_LINTING) types:check
+
+typecheck_ci:
+	$(RUN_LINTING) types:check
+
+test: test_unit
+test_unit:
+ifneq (,$(wildcard test/unit))
+	$(RUN_LINTING) test:unit
+endif
+
+test: test_acceptance
+test_acceptance:
+ifneq (,$(wildcard test/acceptance))
+	$(RUN_LINTING) test:acceptance
+endif

libraries/metrics/buildscript.txt

@@ -5,6 +5,5 @@ metrics
 --esmock-loader=False
 --is-library=True
 --node-version=24.14.1
---package-name=@overleaf/metrics
 --pipeline-owner=32
 --public-repo=False

libraries/mongo-utils/Makefile

@@ -0,0 +1,59 @@
+# Run the test commands in the scope of the monorepo.
+# Eslint and prettier (plus some configs) are on the root.
+RUN_LINTING = ../../bin/run -w /overleaf/libraries/mongo-utils monorepo yarn run --silent
+RUN_LINTING_MONOREPO = ../../bin/run monorepo yarn run --silent
+
+WITH_CACHE_FOLDER = \
+	lint \
+	lint_ci \
+	lint_fix \
+	format \
+	format_ci \
+	format_fix \
+	typecheck \
+	typecheck_ci \
+
+$(WITH_CACHE_FOLDER): ../../node_modules/.cache/prettier/
+../../node_modules/.cache/prettier/:
+	mkdir -p $@
+
+$(WITH_CACHE_FOLDER): ../../node_modules/.cache/eslint/
+../../node_modules/.cache/eslint/:
+	mkdir -p $@
+
+format:
+	$(RUN_LINTING_MONOREPO) format libraries/mongo-utils
+
+format_ci:
+	$(RUN_LINTING_MONOREPO) format libraries/mongo-utils
+
+format_fix:
+	$(RUN_LINTING_MONOREPO) format:fix libraries/mongo-utils
+
+lint:
+	$(RUN_LINTING) lint
+
+lint_ci:
+	-$(RUN_LINTING) lint --format json --output-file reports/eslint.json
+	sed -i 's_"filePath":"/overleaf_"filePath":"$(MONOREPO)_g' reports/eslint.json
+
+lint_fix:
+	$(RUN_LINTING) lint:fix
+
+typecheck:
+	$(RUN_LINTING) types:check
+
+typecheck_ci:
+	$(RUN_LINTING) types:check
+
+test: test_unit
+test_unit:
+ifneq (,$(wildcard test/unit))
+	$(RUN_LINTING) test:unit
+endif
+
+test: test_acceptance
+test_acceptance:
+ifneq (,$(wildcard test/acceptance))
+	$(RUN_LINTING) test:acceptance
+endif

libraries/mongo-utils/buildscript.txt

@@ -5,7 +5,6 @@ mongo-utils
 --esmock-loader=False
 --is-library=True
 --node-version=24.14.1
---package-name=@overleaf/mongo-utils
 --pipeline-owner=32
 --public-repo=False
 --tsconfig-no-implicit-any=True

libraries/o-error/Makefile

@@ -0,0 +1,59 @@
+# Run the test commands in the scope of the monorepo.
+# Eslint and prettier (plus some configs) are on the root.
+RUN_LINTING = ../../bin/run -w /overleaf/libraries/o-error monorepo yarn run --silent
+RUN_LINTING_MONOREPO = ../../bin/run monorepo yarn run --silent
+
+WITH_CACHE_FOLDER = \
+	lint \
+	lint_ci \
+	lint_fix \
+	format \
+	format_ci \
+	format_fix \
+	typecheck \
+	typecheck_ci \
+
+$(WITH_CACHE_FOLDER): ../../node_modules/.cache/prettier/
+../../node_modules/.cache/prettier/:
+	mkdir -p $@
+
+$(WITH_CACHE_FOLDER): ../../node_modules/.cache/eslint/
+../../node_modules/.cache/eslint/:
+	mkdir -p $@
+
+format:
+	$(RUN_LINTING_MONOREPO) format libraries/o-error
+
+format_ci:
+	$(RUN_LINTING_MONOREPO) format libraries/o-error
+
+format_fix:
+	$(RUN_LINTING_MONOREPO) format:fix libraries/o-error
+
+lint:
+	$(RUN_LINTING) lint
+
+lint_ci:
+	-$(RUN_LINTING) lint --format json --output-file reports/eslint.json
+	sed -i 's_"filePath":"/overleaf_"filePath":"$(MONOREPO)_g' reports/eslint.json
+
+lint_fix:
+	$(RUN_LINTING) lint:fix
+
+typecheck:
+	$(RUN_LINTING) types:check
+
+typecheck_ci:
+	$(RUN_LINTING) types:check
+
+test: test_unit
+test_unit:
+ifneq (,$(wildcard test/unit))
+	$(RUN_LINTING) test:unit
+endif
+
+test: test_acceptance
+test_acceptance:
+ifneq (,$(wildcard test/acceptance))
+	$(RUN_LINTING) test:acceptance
+endif

libraries/o-error/buildscript.txt

@@ -5,6 +5,5 @@ o-error
 --esmock-loader=False
 --is-library=True
 --node-version=24.14.1
---package-name=@overleaf/o-error
 --pipeline-owner=32
 --public-repo=False

libraries/object-persistor/Makefile

@@ -0,0 +1,59 @@
+# Run the test commands in the scope of the monorepo.
+# Eslint and prettier (plus some configs) are on the root.
+RUN_LINTING = ../../bin/run -w /overleaf/libraries/object-persistor monorepo yarn run --silent
+RUN_LINTING_MONOREPO = ../../bin/run monorepo yarn run --silent
+
+WITH_CACHE_FOLDER = \
+	lint \
+	lint_ci \
+	lint_fix \
+	format \
+	format_ci \
+	format_fix \
+	typecheck \
+	typecheck_ci \
+
+$(WITH_CACHE_FOLDER): ../../node_modules/.cache/prettier/
+../../node_modules/.cache/prettier/:
+	mkdir -p $@
+
+$(WITH_CACHE_FOLDER): ../../node_modules/.cache/eslint/
+../../node_modules/.cache/eslint/:
+	mkdir -p $@
+
+format:
+	$(RUN_LINTING_MONOREPO) format libraries/object-persistor
+
+format_ci:
+	$(RUN_LINTING_MONOREPO) format libraries/object-persistor
+
+format_fix:
+	$(RUN_LINTING_MONOREPO) format:fix libraries/object-persistor
+
+lint:
+	$(RUN_LINTING) lint
+
+lint_ci:
+	-$(RUN_LINTING) lint --format json --output-file reports/eslint.json
+	sed -i 's_"filePath":"/overleaf_"filePath":"$(MONOREPO)_g' reports/eslint.json
+
+lint_fix:
+	$(RUN_LINTING) lint:fix
+
+typecheck:
+	$(RUN_LINTING) types:check
+
+typecheck_ci:
+	$(RUN_LINTING) types:check
+
+test: test_unit
+test_unit:
+ifneq (,$(wildcard test/unit))
+	$(RUN_LINTING) test:unit
+endif
+
+test: test_acceptance
+test_acceptance:
+ifneq (,$(wildcard test/acceptance))
+	$(RUN_LINTING) test:acceptance
+endif

libraries/object-persistor/buildscript.txt

@@ -5,7 +5,6 @@ object-persistor
 --esmock-loader=False
 --is-library=True
 --node-version=24.14.1
---package-name=@overleaf/object-persistor
 --pipeline-owner=32
 --public-repo=False
 --tsconfig-no-implicit-any=True

libraries/overleaf-editor-core/Makefile

@@ -0,0 +1,59 @@
+# Run the test commands in the scope of the monorepo.
+# Eslint and prettier (plus some configs) are on the root.
+RUN_LINTING = ../../bin/run -w /overleaf/libraries/overleaf-editor-core monorepo yarn run --silent
+RUN_LINTING_MONOREPO = ../../bin/run monorepo yarn run --silent
+
+WITH_CACHE_FOLDER = \
+	lint \
+	lint_ci \
+	lint_fix \
+	format \
+	format_ci \
+	format_fix \
+	typecheck \
+	typecheck_ci \
+
+$(WITH_CACHE_FOLDER): ../../node_modules/.cache/prettier/
+../../node_modules/.cache/prettier/:
+	mkdir -p $@
+
+$(WITH_CACHE_FOLDER): ../../node_modules/.cache/eslint/
+../../node_modules/.cache/eslint/:
+	mkdir -p $@
+
+format:
+	$(RUN_LINTING_MONOREPO) format libraries/overleaf-editor-core
+
+format_ci:
+	$(RUN_LINTING_MONOREPO) format libraries/overleaf-editor-core
+
+format_fix:
+	$(RUN_LINTING_MONOREPO) format:fix libraries/overleaf-editor-core
+
+lint:
+	$(RUN_LINTING) lint
+
+lint_ci:
+	-$(RUN_LINTING) lint --format json --output-file reports/eslint.json
+	sed -i 's_"filePath":"/overleaf_"filePath":"$(MONOREPO)_g' reports/eslint.json
+
+lint_fix:
+	$(RUN_LINTING) lint:fix
+
+typecheck:
+	$(RUN_LINTING) types:check
+
+typecheck_ci:
+	$(RUN_LINTING) types:check
+
+test: test_unit
+test_unit:
+ifneq (,$(wildcard test/unit))
+	$(RUN_LINTING) test:unit
+endif
+
+test: test_acceptance
+test_acceptance:
+ifneq (,$(wildcard test/acceptance))
+	$(RUN_LINTING) test:acceptance
+endif

libraries/overleaf-editor-core/buildscript.txt

@@ -5,6 +5,5 @@ overleaf-editor-core
 --esmock-loader=False
 --is-library=True
 --node-version=24.14.1
---package-name=overleaf-editor-core
 --pipeline-owner=44
 --public-repo=False

libraries/overleaf-editor-core/lib/file_data/tracked_change.js

@@ -56,8 +56,7 @@ class TrackedChange {
       return false
     }
     return (
-      this.tracking.type === other.tracking.type &&
+      this.tracking.canMergeWith(other.tracking) &&
-      this.tracking.userId === other.tracking.userId &&
       this.range.touches(other.range) &&
       this.range.canMerge(other.range)
     )
@@ -75,13 +74,7 @@ class TrackedChange {
     }
     return new TrackedChange(
       this.range.merge(other.range),
-      new TrackingProps(
+      this.tracking.mergeWith(other.tracking)
-        this.tracking.type,
-        this.tracking.userId,
-        this.tracking.ts.getTime() > other.tracking.ts.getTime()
-          ? this.tracking.ts
-          : other.tracking.ts
-      )
     )
   }
 

libraries/overleaf-editor-core/test/unit/text_operation.test.js

@@ -405,6 +405,49 @@ describe('TextOperation', function () {
       })
     )
 
+    it(
+      'compose associativity (randomised)',
+      random.test(numTrials, () => {
+        const str = random.string(20)
+        const comments = random.comments(6)
+
+        const a = randomOperation(str, comments.ids)
+        const afterA = new StringFileData(str, comments.comments)
+        a.apply(afterA)
+
+        const b = randomOperation(afterA.getContent(), comments.ids)
+        const afterB = new StringFileData(
+          afterA.getContent(),
+          comments.comments
+        )
+        b.apply(afterB)
+
+        const c = randomOperation(afterB.getContent(), comments.ids)
+
+        const ab = a.compose(b)
+        const ab_c = ab.compose(c)
+
+        const bc = b.compose(c)
+        const a_bc = a.compose(bc)
+
+        const ab_c_file = new StringFileData(str, comments.comments)
+        ab_c.apply(ab_c_file)
+
+        const a_bc_file = new StringFileData(str, comments.comments)
+        a_bc.apply(a_bc_file)
+
+        const fuzzingError = fuzzingErrorMessage({
+          str,
+          comments,
+          a: a.toJSON(),
+          b: b.toJSON(),
+          c: c.toJSON(),
+        })
+
+        expect(ab_c_file.toRaw()).to.deep.equal(a_bc_file.toRaw(), fuzzingError)
+      })
+    )
+
     it('composes two operations with comments', function () {
       expect(
         compose(
@@ -625,11 +668,54 @@ describe('TextOperation', function () {
           a: a.toJSON(),
           b: b.toJSON(),
         })
-        expect(abPrime.equals(baPrime)).to.be.equal(true, fuzzingError)
+        // The composition of ab' and ba' is not guaranteed to be equal, but
+        // should converge to the same file contents + ranges.
         expect(abFile.toRaw()).to.deep.equal(baFile.toRaw(), fuzzingError)
       })
     )
 
+    it('chooses lower tracked change timestamp', function () {
+      const ts1 = '2024-01-01T01:00:00.000Z'
+      const ts2 = '2024-01-01T02:00:00.000Z'
+      const str = 'abcde'
+      const comments = []
+
+      const a = new TextOperation()
+        .retain(2, {
+          tracking: TrackingProps.fromRaw({
+            ts: ts1,
+            type: 'insert',
+            userId: 'user1',
+          }),
+        })
+        .retain(1)
+        .retain(2, {
+          tracking: TrackingProps.fromRaw({
+            ts: ts2,
+            type: 'insert',
+            userId: 'user1',
+          }),
+        })
+
+      const b = new TextOperation().retain(1).remove(3).retain(1)
+
+      const [aPrime, bPrime] = TextOperation.transform(a, b)
+      const aComposeBPrime = a.compose(bPrime)
+      const bComposeAPrime = b.compose(aPrime)
+
+      const aBPFile = new StringFileData(str, comments)
+      aComposeBPrime.apply(aBPFile)
+
+      const bAPFile = new StringFileData(str, comments)
+      bComposeAPrime.apply(bAPFile)
+
+      expect(aBPFile.toRaw()).to.deep.equal(bAPFile.toRaw())
+      expect(aBPFile.trackedChanges.length).to.equal(1)
+      expect(
+        aBPFile.trackedChanges.asSorted()[0].tracking.ts.toISOString()
+      ).to.equal(ts1)
+    })
+
     it('adds a tracked change from operation 1', function () {
       expect(
         transform(

libraries/overleaf-editor-core/test/unit/tracked_change_list.test.js

@@ -8,7 +8,7 @@ const { expect } = require('chai')
 describe('TrackedChangeList', function () {
   describe('applyInsert', function () {
     describe('with same author', function () {
-      it('should merge consecutive tracked changes and use the latest timestamp', function () {
+      it('should merge consecutive tracked changes and use the earliest timestamp', function () {
         const trackedChanges = TrackedChangeList.fromRaw([
           {
             range: { pos: 0, length: 3 },
@@ -33,7 +33,7 @@ describe('TrackedChangeList', function () {
             tracking: {
               type: 'insert',
               userId: 'user1',
-              ts: '2024-01-01T00:00:00.000Z',
+              ts: '2023-01-01T00:00:00.000Z',
             },
           },
         ])
@@ -699,7 +699,7 @@ describe('TrackedChangeList', function () {
           tracking: {
             type: 'insert',
             userId: 'user1',
-            ts: '2024-01-01T00:00:00.000Z',
+            ts: '2023-01-01T00:00:00.000Z',
           },
         },
       ])

libraries/promise-utils/Makefile

@@ -0,0 +1,59 @@
+# Run the test commands in the scope of the monorepo.
+# Eslint and prettier (plus some configs) are on the root.
+RUN_LINTING = ../../bin/run -w /overleaf/libraries/promise-utils monorepo yarn run --silent
+RUN_LINTING_MONOREPO = ../../bin/run monorepo yarn run --silent
+
+WITH_CACHE_FOLDER = \
+	lint \
+	lint_ci \
+	lint_fix \
+	format \
+	format_ci \
+	format_fix \
+	typecheck \
+	typecheck_ci \
+
+$(WITH_CACHE_FOLDER): ../../node_modules/.cache/prettier/
+../../node_modules/.cache/prettier/:
+	mkdir -p $@
+
+$(WITH_CACHE_FOLDER): ../../node_modules/.cache/eslint/
+../../node_modules/.cache/eslint/:
+	mkdir -p $@
+
+format:
+	$(RUN_LINTING_MONOREPO) format libraries/promise-utils
+
+format_ci:
+	$(RUN_LINTING_MONOREPO) format libraries/promise-utils
+
+format_fix:
+	$(RUN_LINTING_MONOREPO) format:fix libraries/promise-utils
+
+lint:
+	$(RUN_LINTING) lint
+
+lint_ci:
+	-$(RUN_LINTING) lint --format json --output-file reports/eslint.json
+	sed -i 's_"filePath":"/overleaf_"filePath":"$(MONOREPO)_g' reports/eslint.json
+
+lint_fix:
+	$(RUN_LINTING) lint:fix
+
+typecheck:
+	$(RUN_LINTING) types:check
+
+typecheck_ci:
+	$(RUN_LINTING) types:check
+
+test: test_unit
+test_unit:
+ifneq (,$(wildcard test/unit))
+	$(RUN_LINTING) test:unit
+endif
+
+test: test_acceptance
+test_acceptance:
+ifneq (,$(wildcard test/acceptance))
+	$(RUN_LINTING) test:acceptance
+endif

libraries/promise-utils/buildscript.txt

@@ -5,6 +5,5 @@ promise-utils
 --esmock-loader=False
 --is-library=True
 --node-version=24.14.1
---package-name=@overleaf/promise-utils
 --pipeline-owner=32
 --public-repo=False

libraries/ranges-tracker/Makefile

@@ -0,0 +1,59 @@
+# Run the test commands in the scope of the monorepo.
+# Eslint and prettier (plus some configs) are on the root.
+RUN_LINTING = ../../bin/run -w /overleaf/libraries/ranges-tracker monorepo yarn run --silent
+RUN_LINTING_MONOREPO = ../../bin/run monorepo yarn run --silent
+
+WITH_CACHE_FOLDER = \
+	lint \
+	lint_ci \
+	lint_fix \
+	format \
+	format_ci \
+	format_fix \
+	typecheck \
+	typecheck_ci \
+
+$(WITH_CACHE_FOLDER): ../../node_modules/.cache/prettier/
+../../node_modules/.cache/prettier/:
+	mkdir -p $@
+
+$(WITH_CACHE_FOLDER): ../../node_modules/.cache/eslint/
+../../node_modules/.cache/eslint/:
+	mkdir -p $@
+
+format:
+	$(RUN_LINTING_MONOREPO) format libraries/ranges-tracker
+
+format_ci:
+	$(RUN_LINTING_MONOREPO) format libraries/ranges-tracker
+
+format_fix:
+	$(RUN_LINTING_MONOREPO) format:fix libraries/ranges-tracker
+
+lint:
+	$(RUN_LINTING) lint
+
+lint_ci:
+	-$(RUN_LINTING) lint --format json --output-file reports/eslint.json
+	sed -i 's_"filePath":"/overleaf_"filePath":"$(MONOREPO)_g' reports/eslint.json
+
+lint_fix:
+	$(RUN_LINTING) lint:fix
+
+typecheck:
+	$(RUN_LINTING) types:check
+
+typecheck_ci:
+	$(RUN_LINTING) types:check
+
+test: test_unit
+test_unit:
+ifneq (,$(wildcard test/unit))
+	$(RUN_LINTING) test:unit
+endif
+
+test: test_acceptance
+test_acceptance:
+ifneq (,$(wildcard test/acceptance))
+	$(RUN_LINTING) test:acceptance
+endif

libraries/ranges-tracker/buildscript.txt

@@ -5,6 +5,5 @@ ranges-tracker
 --esmock-loader=False
 --is-library=True
 --node-version=24.14.1
---package-name=@overleaf/ranges-tracker
 --pipeline-owner=44
 --public-repo=False

libraries/ranges-tracker/index.cjs

@@ -420,7 +420,7 @@ class RangesTracker {
           offset = opStart - changeStart
           change.op.i =
             change.op.i.slice(0, offset) + op.i + change.op.i.slice(offset)
-          change.metadata.ts = metadata.ts
+          change.metadata.ts = pickTimestamp(change.metadata, metadata)
           alreadyMerged = true
           movedChanges.push(change)
         } else if (opStart <= changeStart) {
@@ -728,6 +728,10 @@ class RangesTracker {
         ) {
           removeChanges.push(change)
           previousChange.op.i += change.op.i
+          previousChange.metadata.ts = pickTimestamp(
+            previousChange.metadata,
+            change.metadata
+          )
           movedChanges.push(previousChange)
         }
       } else if (
@@ -790,4 +794,15 @@ class RangesTracker {
   }
 }
 
+function pickTimestamp(oldMetadata, newMetadata) {
+  // Make sure null values don't get treated as 1970-01-01 dates in the
+  // comparison below.
+  if (oldMetadata.ts == null) return newMetadata.ts
+  if (newMetadata.ts == null) return oldMetadata.ts
+
+  return new Date(oldMetadata.ts) < new Date(newMetadata.ts)
+    ? oldMetadata.ts
+    : newMetadata.ts
+}
+
 module.exports = RangesTracker

libraries/ranges-tracker/test/unit/ranges-tracker-test.js

@@ -222,4 +222,114 @@ describe('RangesTracker', function () {
       )
     })
   })
+
+  describe('merging tracked inserts from the same user', function () {
+    beforeEach(function () {
+      this.comments = []
+    })
+
+    it('keeps the earlier timestamp when the existing insert is older', function () {
+      this.rangesTracker = new RangesTracker(
+        [
+          {
+            id: 'id1',
+            op: { p: 10, i: 'foo' },
+            metadata: { user_id: 'user-1', ts: '2024-01-01T00:00:00.000Z' },
+          },
+        ],
+        this.comments
+      )
+      this.rangesTracker.track_changes = true
+      this.rangesTracker.applyOp(
+        { p: 13, i: 'bar' },
+        { user_id: 'user-1', ts: new Date('2024-06-01T00:00:00.000Z') }
+      )
+      expect(this.rangesTracker.changes).to.have.length(1)
+      const [change] = this.rangesTracker.changes
+      expect(change.op).to.deep.equal({ p: 10, i: 'foobar' })
+      expect(change.metadata.user_id).to.equal('user-1')
+      expect(new Date(change.metadata.ts).toISOString()).to.equal(
+        '2024-01-01T00:00:00.000Z'
+      )
+    })
+
+    it('keeps the earlier timestamp when the incoming insert is older', function () {
+      this.rangesTracker = new RangesTracker(
+        [
+          {
+            id: 'id1',
+            op: { p: 10, i: 'foo' },
+            metadata: { user_id: 'user-1', ts: '2024-06-01T00:00:00.000Z' },
+          },
+        ],
+        this.comments
+      )
+      this.rangesTracker.track_changes = true
+      this.rangesTracker.applyOp(
+        { p: 13, i: 'bar' },
+        { user_id: 'user-1', ts: new Date('2024-01-01T00:00:00.000Z') }
+      )
+      expect(this.rangesTracker.changes).to.have.length(1)
+      const [change] = this.rangesTracker.changes
+      expect(change.op).to.deep.equal({ p: 10, i: 'foobar' })
+      expect(change.metadata.user_id).to.equal('user-1')
+      expect(new Date(change.metadata.ts).toISOString()).to.equal(
+        '2024-01-01T00:00:00.000Z'
+      )
+    })
+
+    it('uses the defined timestamp when the existing insert has a null timestamp', function () {
+      this.rangesTracker = new RangesTracker(
+        [
+          {
+            id: 'id1',
+            op: { p: 10, i: 'foo' },
+            metadata: { user_id: 'user-1', ts: null },
+          },
+        ],
+        this.comments
+      )
+      this.rangesTracker.track_changes = true
+      this.rangesTracker.applyOp(
+        { p: 13, i: 'bar' },
+        { user_id: 'user-1', ts: new Date('2024-01-01T00:00:00.000Z') }
+      )
+      expect(this.rangesTracker.changes).to.have.length(1)
+      const [change] = this.rangesTracker.changes
+      expect(new Date(change.metadata.ts).toISOString()).to.equal(
+        '2024-01-01T00:00:00.000Z'
+      )
+    })
+
+    it('keeps the earlier timestamp when re-merging inserts after a deletion', function () {
+      this.rangesTracker = new RangesTracker(
+        [
+          {
+            id: 'id1',
+            op: { p: 10, i: 'aaa' },
+            metadata: { user_id: 'user-1', ts: '2024-03-01T00:00:00.000Z' },
+          },
+          {
+            id: 'id2',
+            op: { p: 13, i: 'bb' },
+            metadata: { user_id: 'user-2', ts: '2024-02-01T00:00:00.000Z' },
+          },
+          {
+            id: 'id3',
+            op: { p: 15, i: 'ccc' },
+            metadata: { user_id: 'user-1', ts: '2024-01-01T00:00:00.000Z' },
+          },
+        ],
+        this.comments
+      )
+      this.rangesTracker.applyOp({ p: 13, d: 'bb' })
+      expect(this.rangesTracker.changes).to.have.length(1)
+      const [change] = this.rangesTracker.changes
+      expect(change.op).to.deep.equal({ p: 10, i: 'aaaccc' })
+      expect(change.metadata.user_id).to.equal('user-1')
+      expect(new Date(change.metadata.ts).toISOString()).to.equal(
+        '2024-01-01T00:00:00.000Z'
+      )
+    })
+  })
 })

libraries/redis-wrapper/Makefile

@@ -0,0 +1,59 @@
+# Run the test commands in the scope of the monorepo.
+# Eslint and prettier (plus some configs) are on the root.
+RUN_LINTING = ../../bin/run -w /overleaf/libraries/redis-wrapper monorepo yarn run --silent
+RUN_LINTING_MONOREPO = ../../bin/run monorepo yarn run --silent
+
+WITH_CACHE_FOLDER = \
+	lint \
+	lint_ci \
+	lint_fix \
+	format \
+	format_ci \
+	format_fix \
+	typecheck \
+	typecheck_ci \
+
+$(WITH_CACHE_FOLDER): ../../node_modules/.cache/prettier/
+../../node_modules/.cache/prettier/:
+	mkdir -p $@
+
+$(WITH_CACHE_FOLDER): ../../node_modules/.cache/eslint/
+../../node_modules/.cache/eslint/:
+	mkdir -p $@
+
+format:
+	$(RUN_LINTING_MONOREPO) format libraries/redis-wrapper
+
+format_ci:
+	$(RUN_LINTING_MONOREPO) format libraries/redis-wrapper
+
+format_fix:
+	$(RUN_LINTING_MONOREPO) format:fix libraries/redis-wrapper
+
+lint:
+	$(RUN_LINTING) lint
+
+lint_ci:
+	-$(RUN_LINTING) lint --format json --output-file reports/eslint.json
+	sed -i 's_"filePath":"/overleaf_"filePath":"$(MONOREPO)_g' reports/eslint.json
+
+lint_fix:
+	$(RUN_LINTING) lint:fix
+
+typecheck:
+	$(RUN_LINTING) types:check
+
+typecheck_ci:
+	$(RUN_LINTING) types:check
+
+test: test_unit
+test_unit:
+ifneq (,$(wildcard test/unit))
+	$(RUN_LINTING) test:unit
+endif
+
+test: test_acceptance
+test_acceptance:
+ifneq (,$(wildcard test/acceptance))
+	$(RUN_LINTING) test:acceptance
+endif

libraries/redis-wrapper/buildscript.txt

@@ -5,6 +5,5 @@ redis-wrapper
 --esmock-loader=False
 --is-library=True
 --node-version=24.14.1
---package-name=@overleaf/redis-wrapper
 --pipeline-owner=32
 --public-repo=False

libraries/settings/Makefile

@@ -0,0 +1,59 @@
+# Run the test commands in the scope of the monorepo.
+# Eslint and prettier (plus some configs) are on the root.
+RUN_LINTING = ../../bin/run -w /overleaf/libraries/settings monorepo yarn run --silent
+RUN_LINTING_MONOREPO = ../../bin/run monorepo yarn run --silent
+
+WITH_CACHE_FOLDER = \
+	lint \
+	lint_ci \
+	lint_fix \
+	format \
+	format_ci \
+	format_fix \
+	typecheck \
+	typecheck_ci \
+
+$(WITH_CACHE_FOLDER): ../../node_modules/.cache/prettier/
+../../node_modules/.cache/prettier/:
+	mkdir -p $@
+
+$(WITH_CACHE_FOLDER): ../../node_modules/.cache/eslint/
+../../node_modules/.cache/eslint/:
+	mkdir -p $@
+
+format:
+	$(RUN_LINTING_MONOREPO) format libraries/settings
+
+format_ci:
+	$(RUN_LINTING_MONOREPO) format libraries/settings
+
+format_fix:
+	$(RUN_LINTING_MONOREPO) format:fix libraries/settings
+
+lint:
+	$(RUN_LINTING) lint
+
+lint_ci:
+	-$(RUN_LINTING) lint --format json --output-file reports/eslint.json
+	sed -i 's_"filePath":"/overleaf_"filePath":"$(MONOREPO)_g' reports/eslint.json
+
+lint_fix:
+	$(RUN_LINTING) lint:fix
+
+typecheck:
+	$(RUN_LINTING) types:check
+
+typecheck_ci:
+	$(RUN_LINTING) types:check
+
+test: test_unit
+test_unit:
+ifneq (,$(wildcard test/unit))
+	$(RUN_LINTING) test:unit
+endif
+
+test: test_acceptance
+test_acceptance:
+ifneq (,$(wildcard test/acceptance))
+	$(RUN_LINTING) test:acceptance
+endif

libraries/settings/buildscript.txt

@@ -5,6 +5,5 @@ settings
 --esmock-loader=False
 --is-library=True
 --node-version=24.14.1
---package-name=@overleaf/settings
 --pipeline-owner=32
 --public-repo=False

libraries/stream-utils/Makefile

@@ -0,0 +1,59 @@
+# Run the test commands in the scope of the monorepo.
+# Eslint and prettier (plus some configs) are on the root.
+RUN_LINTING = ../../bin/run -w /overleaf/libraries/stream-utils monorepo yarn run --silent
+RUN_LINTING_MONOREPO = ../../bin/run monorepo yarn run --silent
+
+WITH_CACHE_FOLDER = \
+	lint \
+	lint_ci \
+	lint_fix \
+	format \
+	format_ci \
+	format_fix \
+	typecheck \
+	typecheck_ci \
+
+$(WITH_CACHE_FOLDER): ../../node_modules/.cache/prettier/
+../../node_modules/.cache/prettier/:
+	mkdir -p $@
+
+$(WITH_CACHE_FOLDER): ../../node_modules/.cache/eslint/
+../../node_modules/.cache/eslint/:
+	mkdir -p $@
+
+format:
+	$(RUN_LINTING_MONOREPO) format libraries/stream-utils
+
+format_ci:
+	$(RUN_LINTING_MONOREPO) format libraries/stream-utils
+
+format_fix:
+	$(RUN_LINTING_MONOREPO) format:fix libraries/stream-utils
+
+lint:
+	$(RUN_LINTING) lint
+
+lint_ci:
+	-$(RUN_LINTING) lint --format json --output-file reports/eslint.json
+	sed -i 's_"filePath":"/overleaf_"filePath":"$(MONOREPO)_g' reports/eslint.json
+
+lint_fix:
+	$(RUN_LINTING) lint:fix
+
+typecheck:
+	$(RUN_LINTING) types:check
+
+typecheck_ci:
+	$(RUN_LINTING) types:check
+
+test: test_unit
+test_unit:
+ifneq (,$(wildcard test/unit))
+	$(RUN_LINTING) test:unit
+endif
+
+test: test_acceptance
+test_acceptance:
+ifneq (,$(wildcard test/acceptance))
+	$(RUN_LINTING) test:acceptance
+endif

libraries/stream-utils/buildscript.txt

@@ -5,6 +5,5 @@ stream-utils
 --esmock-loader=False
 --is-library=True
 --node-version=24.14.1
---package-name=@overleaf/stream-utils
 --pipeline-owner=32
 --public-repo=False

libraries/validation-tools/Makefile

@@ -0,0 +1,59 @@
+# Run the test commands in the scope of the monorepo.
+# Eslint and prettier (plus some configs) are on the root.
+RUN_LINTING = ../../bin/run -w /overleaf/libraries/validation-tools monorepo yarn run --silent
+RUN_LINTING_MONOREPO = ../../bin/run monorepo yarn run --silent
+
+WITH_CACHE_FOLDER = \
+	lint \
+	lint_ci \
+	lint_fix \
+	format \
+	format_ci \
+	format_fix \
+	typecheck \
+	typecheck_ci \
+
+$(WITH_CACHE_FOLDER): ../../node_modules/.cache/prettier/
+../../node_modules/.cache/prettier/:
+	mkdir -p $@
+
+$(WITH_CACHE_FOLDER): ../../node_modules/.cache/eslint/
+../../node_modules/.cache/eslint/:
+	mkdir -p $@
+
+format:
+	$(RUN_LINTING_MONOREPO) format libraries/validation-tools
+
+format_ci:
+	$(RUN_LINTING_MONOREPO) format libraries/validation-tools
+
+format_fix:
+	$(RUN_LINTING_MONOREPO) format:fix libraries/validation-tools
+
+lint:
+	$(RUN_LINTING) lint
+
+lint_ci:
+	-$(RUN_LINTING) lint --format json --output-file reports/eslint.json
+	sed -i 's_"filePath":"/overleaf_"filePath":"$(MONOREPO)_g' reports/eslint.json
+
+lint_fix:
+	$(RUN_LINTING) lint:fix
+
+typecheck:
+	$(RUN_LINTING) types:check
+
+typecheck_ci:
+	$(RUN_LINTING) types:check
+
+test: test_unit
+test_unit:
+ifneq (,$(wildcard test/unit))
+	$(RUN_LINTING) test:unit
+endif
+
+test: test_acceptance
+test_acceptance:
+ifneq (,$(wildcard test/acceptance))
+	$(RUN_LINTING) test:acceptance
+endif

libraries/validation-tools/buildscript.txt

@@ -5,7 +5,6 @@ validation-tools
 --esmock-loader=False
 --is-library=True
 --node-version=24.14.1
---package-name=@overleaf/validation-tools
 --public-repo=False
 --test-acceptance-vitest=True
 --test-unit-vitest=True

package.json

@@ -41,12 +41,13 @@
     "sandboxed-module": "patch:sandboxed-module@npm%3A2.0.4#~/.yarn/patches/sandboxed-module-npm-2.0.4-f8b45aacc9.patch",
     "request/tough-cookie": "5.1.2",
     "request/form-data": "2.5.5",
-    "request/qs": "6.14.1",
+    "request/qs": "6.15.2",
-    "@cypress/request/qs": "6.14.1",
+    "@cypress/request/qs": "6.15.2",
+    "qs": "6.15.2",
     "@opentelemetry/api": "1.9.0",
     "mocha/serialize-javascript": "7.0.5",
-    "pprof/protobufjs": "7.5.5",
+    "protobufjs": "7.6.3",
-    "@google-cloud/profiler/protobufjs": "7.5.5",
+    "@protobufjs/utf8": "1.1.1",
     "mocha-multi-reporters": "patch:mocha-multi-reporters@npm%3A1.5.1#~/.yarn/patches/mocha-multi-reporters-npm-1.5.1-0a1088aed5.patch",
     "pdfjs-dist": "patch:pdfjs-dist@npm%3A5.1.91#~/.yarn/patches/pdfjs-dist-npm-5.1.91.patch",
     "referer-parser": "patch:referer-parser@npm%3A0.0.3#~/.yarn/patches/referer-parser-npm-0.0.3.patch",
@@ -101,11 +102,12 @@
     "cluster-key-slot": "1.1.0",
     "@octokit/request": "9.2.2",
     "randomstring": "1.2.2",
-    "vite": "7.3.1",
+    "vite": "7.3.5",
     "isomorphic-git": "1.33.1",
     "http-status": "1.5.0",
     "knex": "2.4.0",
     "utf-8-validate": "5.0.8",
+    "ws@npm:^8": "8.21.0",
     "samlp": "7.0.2",
     "compression": "1.7.4",
     "cookie-parser": "1.4.6",
@@ -178,6 +180,8 @@
     "fast-xml-builder": "1.1.7",
     "fast-xml-parser": "5.7.3",
     "ip-address": "10.1.1",
+    "ldapauth-fork@npm:^5.0.1": "patch:ldapauth-fork@npm%3A5.0.5#~/.yarn/patches/ldapauth-fork-npm-5.0.5-c8b86f5231.patch",
+    "@node-saml/node-saml@npm:^5.1.0": "patch:@node-saml/node-saml@npm%3A5.1.0#~/.yarn/patches/@node-saml-node-saml-npm-5.1.0-f4eb6913cc.patch",
     "systeminformation": "5.31.6"
   },
   "scripts": {
@@ -196,6 +200,7 @@
   },
   "workspaces": [
     "jobs/mirror-documentation",
+    "jobs/wiki-algolia",
     "libraries/*",
     "server-ce/test",
     "services/analytics",
@@ -212,6 +217,7 @@
     "services/history-v1",
     "services/idp",
     "services/latexqc",
+    "services/linked-url-proxy",
     "services/notifications",
     "services/project-history",
     "services/real-time",

server-ce/Dockerfile

@@ -49,7 +49,7 @@ ADD server-ce/config/env.sh /etc/overleaf/env.sh
 # Configure nginx
 # ---------------
 ADD server-ce/nginx/nginx.conf.template /etc/nginx/templates/nginx.conf.template
-ADD server-ce/nginx/overleaf.conf /etc/nginx/sites-enabled/overleaf.conf
+ADD server-ce/nginx/overleaf.conf.template /etc/nginx/templates/overleaf.conf.template
 ADD server-ce/nginx/clsi-nginx.conf /etc/nginx/sites-enabled/clsi-nginx.conf
 
 
@@ -121,7 +121,9 @@ ENV GRACEFUL_SHUTDOWN_DELAY_SECONDS=1
 ENV NODE_ENV="production"
 ENV LOG_LEVEL="info"
 
-
+ENV GIT_BRIDGE_HOST=git-bridge
+ENV GIT_BRIDGE_PORT=8000
+ENV GIT_BRIDGE_REPOSTORE_MAX_FILE_SIZE=52428800
 EXPOSE 80
 
 ENTRYPOINT ["/sbin/my_init"]

server-ce/Makefile

@@ -37,6 +37,7 @@ build-base:
 	  --cache-from $(OVERLEAF_BASE_BRANCH) \
 	  --tag $(OVERLEAF_BASE_TAG) \
 	  --tag $(OVERLEAF_BASE_BRANCH) \
+	  --network=host \
 	  $(MONOREPO_ROOT)
 
 
@@ -52,6 +53,7 @@ build-community:
 	  --file Dockerfile \
 	  --tag $(OVERLEAF_TAG) \
 	  --tag $(OVERLEAF_BRANCH) \
+	  --network=host \
 	  $(MONOREPO_ROOT)
 
 push:

server-ce/config/env.sh

@@ -5,6 +5,7 @@ export DOCUMENT_UPDATER_HOST=127.0.0.1
 export DOCUPDATER_HOST=127.0.0.1
 export FILESTORE_HOST=127.0.0.1
 export HISTORY_V1_HOST=127.0.0.1
+export LINKED_URL_PROXY_HOST=127.0.0.1
 export NOTIFICATIONS_HOST=127.0.0.1
 export PROJECT_HISTORY_HOST=127.0.0.1
 export REALTIME_HOST=127.0.0.1

server-ce/config/settings.js

@@ -193,6 +193,8 @@ const settings = {
   maintenanceMessage: process.env.OVERLEAF_MAINTENANCE_MESSAGE,
   maintenanceMessageHTML: process.env.OVERLEAF_MAINTENANCE_MESSAGE_HTML,
 
+  enableRegiserPage: process.env.OVERLEAF_ENABLE_REGISTER_PAGE === 'true',
+
   // The name this is used to describe your Overleaf Community Edition Installation
   appName: process.env.OVERLEAF_APP_NAME || 'Overleaf Community Edition',
 
@@ -393,7 +395,16 @@ if (process.env.OVERLEAF_EMAIL_FROM_ADDRESS != null) {
   }
 
   if (
-    process.env.OVERLEAF_EMAIL_SMTP_USER != null ||
+    process.env.OVERLEAF_EMAIL_SMTP_USER != null &&
+    process.env.OVERLEAF_EMAIL_SMTP_PASS_FILE != null
+  ) {
+    settings.email.parameters.auth = {
+      user: process.env.OVERLEAF_EMAIL_SMTP_USER,
+      pass: require("fs").readFileSync(process.env.OVERLEAF_EMAIL_SMTP_PASS_FILE, "utf8")
+    }
+  }
+  else if (
+    process.env.OVERLEAF_EMAIL_SMTP_USER != null &&
     process.env.OVERLEAF_EMAIL_SMTP_PASS != null
   ) {
     settings.email.parameters.auth = {

server-ce/hotfix/5.5.3/saml-ext.patch

@@ -0,0 +1,13 @@
+diff --git a/services/web/modules/authentication/saml/app/src/SAMLModuleManager.mjs b/services/web/modules/authentication/saml/app/src/SAMLModuleManager.mjs
+index 29e9ae52cd..335ace8209 100644
+--- a/services/web/modules/authentication/saml/app/src/SAMLModuleManager.mjs
++++ b/services/web/modules/authentication/saml/app/src/SAMLModuleManager.mjs
+@@ -26,7 +26,7 @@ const SAMLModuleManager = {
+       callbackUrl: `${Settings.siteUrl.replace(/\/+$/, '')}/saml/login/callback`,
+       issuer: process.env.OVERLEAF_SAML_ISSUER,
+       audience: process.env.OVERLEAF_SAML_AUDIENCE,
+-      cert: readFilesContentFromEnv(process.env.OVERLEAF_SAML_IDP_CERT),
++      idpCert: readFilesContentFromEnv(process.env.OVERLEAF_SAML_IDP_CERT),
+       privateKey:  readFilesContentFromEnv(process.env.OVERLEAF_SAML_PRIVATE_KEY),
+       decryptionPvk:  readFilesContentFromEnv(process.env.OVERLEAF_SAML_DECRYPTION_PVK),
+       signatureAlgorithm: process.env.OVERLEAF_SAML_SIGNATURE_ALGORITHM,

server-ce/hotfix/5.5.6-ext-3.3/Dockerfile

@@ -0,0 +1,20 @@
+FROM overleafcep/sharelatex:5.5.6-ext-v3.2
+
+# add normalize and sanitize to linked-url-proxy
+COPY *.patch* .
+RUN bash -ec 'for p in *.patch; do echo "=== Applying $p ==="; patch -p1 < "$p" && rm $p; done'  \
+  &&  npm install als-normalize-urlpath@2.3.0 \
+  && rm -f ./package.json.orig \
+  ./services/history-v1/storage/scripts/back_fill_file_hash.mjs.orig \
+  ./services/history-v1/test/acceptance/js/storage/back_fill_file_hash.test.mjs.orig \
+  ./services/linked-url-proxy/app/js/LinkedUrlProxyController.mjs.orig \
+  ./services/web/config/settings.defaults.js.orig \
+  ./services/web/frontend/js/features/source-editor/components/toolbar/toolbar-items.tsx.orig \
+  ./services/web/test/frontend/features/share-project-modal/components/share-project-modal.test.jsx.orig \
+  ./services/web/package.json.orig \
+  ./services/filestore/app.js.orig \
+  ./package-lock.json.orig \
+  ./package-lock.json.diff \
+  /etc/my_init.d/100_make_overleaf_data_dirs.sh.orig \
+  /etc/overleaf/settings.js.orig
+

server-ce/hotfix/5.5.6-ext-3.3/ext-3.3.patch

@@ -0,0 +1,256 @@
+diff --git a/package.json b/package.json
+index 64fbd258ed..7cc3c7f9b3 100644
+--- a/package.json
++++ b/package.json
+@@ -66,6 +66,7 @@
+     "services/history-v1",
+     "services/idp",
+     "services/latexqc",
++    "services/linked-url-proxy",
+     "services/notifications",
+     "services/project-history",
+     "services/real-time",
+diff --git a/services/linked-url-proxy/Dockerfile b/services/linked-url-proxy/Dockerfile
+index 677006182c..7f2053f614 100644
+--- a/services/linked-url-proxy/Dockerfile
++++ b/services/linked-url-proxy/Dockerfile
+@@ -2,7 +2,7 @@
+ # Instead run bin/update_build_scripts from
+ # https://github.com/overleaf/internal/
+ 
+-FROM node:20.18.2 AS base
++FROM node:22.18.0 AS base
+ 
+ WORKDIR /overleaf/services/linked-url-proxy
+ 
+@@ -13,16 +13,20 @@ RUN mkdir /home/node/.config && chown node:node /home/node/.config
+ FROM base AS app
+ 
+ COPY package.json package-lock.json /overleaf/
++COPY libraries/fetch-utils/package.json /overleaf/libraries/fetch-utils/package.json
+ COPY libraries/logger/package.json /overleaf/libraries/logger/package.json
+ COPY libraries/metrics/package.json /overleaf/libraries/metrics/package.json
++COPY libraries/o-error/package.json /overleaf/libraries/o-error/package.json
+ COPY libraries/settings/package.json /overleaf/libraries/settings/package.json
+ COPY services/linked-url-proxy/package.json /overleaf/services/linked-url-proxy/package.json
+ COPY patches/ /overleaf/patches/
+ 
+ RUN cd /overleaf && npm ci --quiet
+ 
++COPY libraries/fetch-utils/ /overleaf/libraries/fetch-utils/
+ COPY libraries/logger/ /overleaf/libraries/logger/
+ COPY libraries/metrics/ /overleaf/libraries/metrics/
++COPY libraries/o-error/ /overleaf/libraries/o-error/
+ COPY libraries/settings/ /overleaf/libraries/settings/
+ COPY services/linked-url-proxy/ /overleaf/services/linked-url-proxy/
+ 
+diff --git a/services/linked-url-proxy/app/js/LinkedUrlProxyController.mjs b/services/linked-url-proxy/app/js/LinkedUrlProxyController.mjs
+index 1269eede0d..ea82f65d16 100644
+--- a/services/linked-url-proxy/app/js/LinkedUrlProxyController.mjs
++++ b/services/linked-url-proxy/app/js/LinkedUrlProxyController.mjs
+@@ -1,4 +1,6 @@
+ import dns from 'dns/promises'
++import { sanitizeUrl } from 'strict-url-sanitise'
++import normalizeUrlPath from 'als-normalize-urlpath'
+ import ipaddr from 'ipaddr.js'
+ import { URL } from 'node:url'
+ import { Transform } from 'node:stream'
+@@ -18,15 +20,7 @@ function isBlockedIp(ipStr, targetUrl) {
+   }
+ 
+   const range = addr.range()
+-  if ([
+-    'loopback',
+-    'private',
+-    'linkLocal',
+-    'multicast',
+-    'reserved',
+-    'broadcast',
+-    'unspecified'
+-  ].includes(range)) {
++  if (!['unicast'].includes(addr.range())) {
+     return true
+   }
+ 
+@@ -44,7 +38,7 @@ function isBlockedIp(ipStr, targetUrl) {
+   return false
+ }
+ 
+-async function validateSourceUrl(hostname, targetUrl) {
++async function checkUrlAccess(hostname, targetUrl) {
+   const records = await dns.lookup(hostname, { all: true }).catch(() => [])
+   if (!records.length) {
+     const err = new Error(`DNS lookup failed for ${hostname}`)
+@@ -62,22 +56,40 @@ async function validateSourceUrl(hostname, targetUrl) {
+   }
+ }
+ 
+-async function fetchValidated(urlStr, redirectCount = 0) {
++async function validateAndFetch(rawUrl, redirectCount = 0) {
+   if (redirectCount > Settings.maxRedirects) {
+     const err = new Error('Too many redirects')
+     err.info = { status: 421 }
+     throw err
+   }
+ 
+-  const url = new URL(urlStr)
++  const sanitizedUrl = sanitizeUrl(rawUrl)
++  if (!sanitizedUrl) {
++    const err = new Error(`Invalid or unsafe URL: ${rawUrl}`)
++    err.info = { status: 400 }
++    throw err
++  }
++
++  const url = new URL(sanitizedUrl)
++
+   if (!['http:', 'https:'].includes(url.protocol)) {
+     const err = new Error(`${url.protocol} protocol is not allowed`)
+     err.info = { status: 400 }
+     throw err
+   }
+ 
+-  // Validate DNS and blocked IPs
+-  await validateSourceUrl(url.hostname, urlStr)
++  const normalizedPath = normalizeUrlPath(url.pathname).pathname
++
++  if (!normalizedPath) {
++    const err = new Error(`Invalid or unsafe URL path: ${url.pathname}`)
++    err.info = { status: 400 }
++    throw err
++  }
++
++  const normalizedUrl = url.toString()
++
++  // check DNS and allowed resources
++  await checkUrlAccess(url.hostname, normalizedUrl)
+ 
+   const opts = {
+     redirect: 'manual',
+@@ -86,7 +98,7 @@ async function fetchValidated(urlStr, redirectCount = 0) {
+   }
+ 
+   try {
+-    const { stream, response } = await fetchStreamWithResponse(urlStr, opts)
++    const { stream, response } = await fetchStreamWithResponse(normalizedUrl, opts)
+ 
+     const contentLengthHeader = response.headers.get('content-length')
+     if (contentLengthHeader) {
+@@ -112,8 +124,8 @@ async function fetchValidated(urlStr, redirectCount = 0) {
+       if (status >= 300 && status < 400) {
+         const location = err.response.headers.get('Location')
+         if (location) {
+-          const nextUrl = new URL(location, url).toString()
+-          return fetchValidated(nextUrl, redirectCount + 1)
++          const nextUrl = new URL(location, normalizedUrl).toString()
++          return validateAndFetch(nextUrl, redirectCount + 1)
+         } else {
+           const e = new Error('Redirect response missing Location header')
+ 	  e.info = { status: 421 }
+@@ -141,14 +153,14 @@ async function proxy(req, res) {
+       return
+     }
+ 
+-    const { stream: upstreamStream, response, headers } = await fetchValidated(targetUrl)
++    const { stream: upstreamStream, response, headers } = await validateAndFetch(targetUrl)
+ 
+     res.statusCode = response.status || 200
+     res.setHeader('Content-Type', headers['content-type'] || 'application/octet-stream')
+     res.setHeader('Cache-Control', 'no-store')
+ 
+     function onError(err) {
+-      logger.warn({ err, url: req.url }, 'linked-url-proxy request failed')
++      logger.info({ err, url: req.url }, 'linked-url-proxy request failed')
+       try { upstreamStream.destroy() } catch (_) {}
+       if (!res.headersSent) {
+ 	let body = `Error: ${err?.message ?? String(err)}`
+@@ -163,11 +175,9 @@ async function proxy(req, res) {
+     upstreamStream.pipe(res)
+ 
+   } catch (err) {
+-    logger.warn({ err, url: req.url }, 'linked-url-proxy request failed')
+-
+-    let status = err.info.status
++    const status = err.info?.status || 500
++    logger.info({ linkedUrl: err.message, status, url: req.url }, 'linked-url-proxy request failed')
+     let body = `Error: ${err.message || String(err)}`
+-
+     try {
+       res.writeHead(status, { 'Content-Type': 'text/plain' })
+       res.end(body)
+diff --git a/services/linked-url-proxy/package.json b/services/linked-url-proxy/package.json
+index 845551c8ef..8081a80822 100644
+--- a/services/linked-url-proxy/package.json
++++ b/services/linked-url-proxy/package.json
+@@ -2,18 +2,24 @@
+   "name": "@overleaf/linked-url-proxy",
+   "description": "An API for providing linked url proxy",
+   "private": true,
+-  "type": "module",
+   "main": "app.mjs",
+   "scripts": {
+-    "start": "node app.mjs"
++    "start": "node app.mjs",
++    "nodemon": "node --watch app.mjs"
+   },
+-  "version": "0.1.0",
++  "version": "0.1.1",
+   "dependencies": {
+     "@overleaf/settings": "*",
+     "@overleaf/logger": "*",
+     "@overleaf/metrics": "*",
+     "async": "^3.2.5",
+-    "express": "^4.21.2"
+-    "ipaddr.js": "^1.9.1"
++    "express": "^4.21.2",
++    "ipaddr.js": "^1.9.1",
++    "als-normalize-urlpath": "^2.3.0",
++    "strict-url-sanitise": "^0.0.1"
++  },
++  "devDependencies": {
++    "als-normalize-urlpath": "^2.3.0",
++    "strict-url-sanitise": "^0.0.1"
+   }
+ }
+diff --git a/services/linked-url-proxy/tsconfig.json b/services/linked-url-proxy/tsconfig.json
+index c018d6e682..0639496cc8 100644
+--- a/services/linked-url-proxy/tsconfig.json
++++ b/services/linked-url-proxy/tsconfig.json
+@@ -1,7 +1,7 @@
+ {
+   "extends": "../../tsconfig.backend.json",
+   "include": [
+-    "app.js",
++    "app.mjs",
+     "app.ts",
+     "app/js/**/*",
+     "benchmarks/**/*",
+diff --git a/services/web/package.json b/services/web/package.json
+index 21da1188a2..c2c4fcfaba 100644
+--- a/services/web/package.json
++++ b/services/web/package.json
+@@ -69,6 +69,7 @@
+     "last 1 year",
+     "safari > 14"
+   ],
++
+   "dependencies": {
+     "@contentful/rich-text-html-renderer": "^16.0.2",
+     "@contentful/rich-text-types": "^16.0.2",
+@@ -92,6 +93,7 @@
+     "@xmldom/xmldom": "^0.7.13",
+     "accepts": "^1.3.7",
+     "ajv": "^8.12.0",
++    "als-normalize-urlpath": "^2.3.0",
+     "archiver": "^5.3.0",
+     "async": "^3.2.5",
+     "base-x": "^4.0.1",
+@@ -166,7 +168,8 @@
+     "request": "^2.88.2",
+     "requestretry": "^7.1.0",
+     "sanitize-html": "^2.8.1",
+-    "stripe": "^18.1.0",
++    "strict-url-sanitise": "^0.0.1",
++    "stripe": "^18.4.0",
+     "tough-cookie": "^4.0.0",
+     "tsscmp": "^1.0.6",
+     "uid-safe": "^2.1.5",

server-ce/init_scripts/200_nginx_config_template.sh

@@ -16,11 +16,18 @@ fi
 nginx_template_file="${nginx_templates_dir}/nginx.conf.template"
 nginx_config_file="${nginx_dir}/nginx.conf"
 
+overleaf_template_file="${nginx_templates_dir}/overleaf.conf.template"
+overleaf_config_file="${nginx_dir}/sites-enabled/overleaf.conf"
+
 if [ -f "${nginx_template_file}" ]; then
   export NGINX_KEEPALIVE_TIMEOUT="${NGINX_KEEPALIVE_TIMEOUT:-65}"
   export NGINX_WORKER_CONNECTIONS="${NGINX_WORKER_CONNECTIONS:-768}"
   export NGINX_WORKER_PROCESSES="${NGINX_WORKER_PROCESSES:-4}"
+  export MAX_UPLOAD_SIZE="${MAX_UPLOAD_SIZE:-50}"
 
+  export GIT_BRIDGE_HOST="${GIT_BRIDGE_HOST:-git-bridge}"
+  export GIT_BRIDGE_PORT="${GIT_BRIDGE_PORT:-8000}"
+  export GIT_BRIDGE_REPOSTORE_MAX_FILE_SIZE="${GIT_BRIDGE_REPOSTORE_MAX_FILE_SIZE:-52428800}"
   echo "Nginx: generating config file from template"
 
   # Note the single-quotes, they are important.
@@ -31,10 +38,20 @@ if [ -f "${nginx_template_file}" ]; then
     ${NGINX_KEEPALIVE_TIMEOUT}
     ${NGINX_WORKER_CONNECTIONS}
     ${NGINX_WORKER_PROCESSES}
+    ${MAX_UPLOAD_SIZE}
   ' \
     < "${nginx_template_file}" \
     > "${nginx_config_file}"
 
+  # Config git-bridge in nginx
+  envsubst '
+    ${GIT_BRIDGE_HOST}
+    ${GIT_BRIDGE_PORT}
+    ${GIT_BRIDGE_REPOSTORE_MAX_FILE_SIZE}
+  ' \
+    < "${overleaf_template_file}" \
+    > "${overleaf_config_file}"
+
   echo "Checking Nginx config"
   nginx -t
 

server-ce/nginx/clsi-nginx.conf

@@ -34,8 +34,8 @@ server {
     root /var/lib/overleaf/data/output/$1-$2/generated-files/$3/;
   }
   # handle output files for anonymous users
-  location ~ ^/project/([0-9a-f]+)/build/([0-9a-f-]+)/output/(.+)$ {
+  location ~ ^/project/([0-9a-f-]+)/build/([0-9a-f-]+)/output/(.+)$ {
-    rewrite ^/project/([0-9a-f]+)/build/([0-9a-f-]+)/output/(.+)$ /$3 break;
+    rewrite ^/project/([0-9a-f-]+)/build/([0-9a-f-]+)/output/(.+)$ /$3 break;
     root /var/lib/overleaf/data/output/$1/generated-files/$2/;
   }
 

server-ce/nginx/nginx.conf.template

@@ -46,7 +46,7 @@ http {
 	gzip_disable "msie6";
 	gzip_proxied any; # allow upstream server to compress.
 
-	client_max_body_size 50m;
+	client_max_body_size ${MAX_UPLOAD_SIZE}m;
 
 	# gzip_vary on;
 	# gzip_proxied any;

server-ce/nginx/overleaf.conf.template

@@ -21,6 +21,23 @@ server {
 		proxy_send_timeout 10m;
 	}
 
+	location ^~ /git/ {
+		resolver_timeout 2s;
+		resolver 127.0.0.11 valid=10s;
+		set $git_upstream http://${GIT_BRIDGE_HOST}:${GIT_BRIDGE_PORT};
+		rewrite ^/git/(.*)$ /$1 break;
+		client_max_body_size ${GIT_BRIDGE_REPOSTORE_MAX_FILE_SIZE};
+		proxy_pass $git_upstream;
+		proxy_http_version 1.1;
+		proxy_set_header Upgrade $http_upgrade;
+		proxy_set_header Connection "upgrade";
+		proxy_set_header Host $host;
+		proxy_set_header X-Forwarded-Host $host;
+		proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+		proxy_read_timeout 10m;
+		proxy_send_timeout 10m;
+	}
+
 	location /socket.io {
 		proxy_pass http://127.0.0.1:3026;
 		proxy_http_version 1.1;

server-ce/runit/linked-url-proxy-overleaf/run

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+NODE_PARAMS=""
+if [ "$DEBUG_NODE" == "true" ]; then
+    echo "running debug - linked-url-proxy"
+    NODE_PARAMS="--inspect=0.0.0.0:30660"
+fi
+
+source /etc/overleaf/env.sh
+export LISTEN_ADDRESS=127.0.0.1
+
+exec /sbin/setuser www-data /usr/bin/node $NODE_PARAMS /overleaf/services/linked-url-proxy/app.mjs >> /var/log/overleaf/linked-url-proxy.log 2>&1

server-ce/services.js

@@ -29,6 +29,9 @@ module.exports = [
   {
     name: 'history-v1',
   },
+  {
+    name: 'linked-url-proxy',
+  },
 ]
 
 if (require.main === module) {

server-ce/test/docker-compose.yml

@@ -19,7 +19,7 @@ services:
       OVERLEAF_EMAIL_SMTP_HOST: "mailtrap"
       OVERLEAF_EMAIL_SMTP_PORT: "25"
       OVERLEAF_EMAIL_SMTP_IGNORE_TLS: "true"
-      ENABLED_LINKED_FILE_TYPES: "project_file,project_output_file"
+      ENABLED_LINKED_FILE_TYPES: "project_file,project_output_file,zotero"
       ENABLE_CONVERSIONS: "true"
       EMAIL_CONFIRMATION_DISABLED: "true"
       OVERLEAF_INVITE_TOKEN_SECRET: "invite-token-secret"

server-ce/test/editor.spec.ts

@@ -2,6 +2,8 @@ import {
   createNewFile,
   createProjectAndOpenInNewEditor,
   openProjectById,
+  openProjectByName,
+  shareProjectByEmailAndAcceptInviteViaDash,
   testNewFileUpload,
 } from './helpers/project'
 import { isExcludedBySharding, startWith } from './helpers/config'
@@ -321,6 +323,124 @@ describe('editor', function () {
   })
 })
 
+describe('editing mode', function () {
+  if (isExcludedBySharding('PRO_DEFAULT_1')) return
+  startWith({ pro: true })
+
+  const OWNER = USER
+  const EDITOR = COLLABORATOR
+  const REVIEWER = 'reviewer@example.com'
+  const VIEWER = 'viewer@example.com'
+  ensureUserExists({ email: OWNER })
+  ensureUserExists({ email: EDITOR })
+  ensureUserExists({ email: REVIEWER })
+  ensureUserExists({ email: VIEWER })
+
+  let projectName: string
+
+  function openEditingModeMenu() {
+    cy.findByRole('button', { name: 'View' }).click()
+    cy.findByRole('menuitem', { name: 'Editing mode' }).click()
+  }
+
+  beforeWithReRunOnTestRetry(() => {
+    projectName = `project-${uuid()}`
+    login(OWNER)
+    createProjectAndOpenInNewEditor(projectName, { type: 'Example project' })
+
+    shareProjectByEmailAndAcceptInviteViaDash(
+      projectName,
+      EDITOR,
+      'Editor',
+      true
+    )
+    login(OWNER)
+    shareProjectByEmailAndAcceptInviteViaDash(
+      projectName,
+      REVIEWER,
+      'Reviewer',
+      true
+    )
+    login(OWNER)
+    shareProjectByEmailAndAcceptInviteViaDash(
+      projectName,
+      VIEWER,
+      'Viewer',
+      true
+    )
+  })
+
+  it('lets an editor switch between editing and reviewing', function () {
+    login(EDITOR)
+    openProjectByName(projectName, true)
+
+    openEditingModeMenu()
+    cy.findByRole('menuitem', { name: 'Editing' })
+      .should('have.attr', 'aria-selected', 'true')
+      .and('not.have.attr', 'aria-disabled')
+    cy.findByRole('menuitem', { name: 'Reviewing' })
+      .should('have.attr', 'aria-selected', 'false')
+      .and('not.have.attr', 'aria-disabled')
+    cy.findByRole('menuitem', { name: 'Viewing' }).should('not.exist')
+
+    cy.findByRole('menuitem', { name: 'Reviewing' }).click()
+    cy.findByRole('button', { name: 'Reviewing' }).should('exist')
+
+    openEditingModeMenu()
+    cy.findByRole('menuitem', { name: 'Reviewing' }).should(
+      'have.attr',
+      'aria-selected',
+      'true'
+    )
+    cy.findByRole('menuitem', { name: 'Editing' }).should(
+      'have.attr',
+      'aria-selected',
+      'false'
+    )
+
+    cy.findByRole('menuitem', { name: 'Editing' }).click()
+    cy.findByRole('button', { name: 'Editing' }).should('exist')
+  })
+
+  it('shows a reviewer in reviewing mode with editing disabled', function () {
+    login(REVIEWER)
+    openProjectByName(projectName, true)
+
+    openEditingModeMenu()
+    cy.findByRole('menuitem', { name: 'Reviewing' })
+      .should('have.attr', 'aria-selected', 'true')
+      .and('not.have.attr', 'aria-disabled')
+    cy.findByRole('menuitem', { name: 'Editing' }).should(
+      'have.attr',
+      'aria-disabled',
+      'true'
+    )
+    cy.findByRole('menuitem', { name: 'Viewing' }).should('not.exist')
+  })
+
+  it('shows a viewer in viewing mode with editing and reviewing disabled', function () {
+    login(VIEWER)
+    openProjectByName(projectName, true)
+
+    openEditingModeMenu()
+    cy.findByRole('menuitem', { name: 'Viewing' }).should(
+      'have.attr',
+      'aria-selected',
+      'true'
+    )
+    cy.findByRole('menuitem', { name: 'Editing' }).should(
+      'have.attr',
+      'aria-disabled',
+      'true'
+    )
+    cy.findByRole('menuitem', { name: 'Reviewing' }).should(
+      'have.attr',
+      'aria-disabled',
+      'true'
+    )
+  })
+})
+
 function createRandomLetterString() {
   const chars = 'abcdefghijklmnopqrstuvwxyz'
   let result = ''

server-ce/test/filestore-migration.spec.ts

@@ -256,7 +256,12 @@ describe('filestore migration', function () {
     })
   } else {
     // 5.x
-    startWith({ version: '5.5.5', pro: true, withDataDir: true })
+    startWith({
+      version: '5.5.5',
+      pro: true,
+      withDataDir: true,
+      resetData: true,
+    })
     defaultImage = 'frog.jpg'
     ensureUserExists({ email })
     before(function () {

server-ce/test/helpers/project.ts

@@ -206,7 +206,7 @@ export function openProjectViaInviteNotification(
 function shareProjectByEmail(
   projectName: string,
   email: string,
-  level: 'Viewer' | 'Editor',
+  level: 'Viewer' | 'Editor' | 'Reviewer',
   newEditor: boolean = false
 ) {
   openProjectByName(projectName, newEditor)
@@ -231,7 +231,7 @@ function shareProjectByEmail(
 export function shareProjectByEmailAndAcceptInviteViaDash(
   projectName: string,
   email: string,
-  level: 'Viewer' | 'Editor',
+  level: 'Viewer' | 'Editor' | 'Reviewer',
   newEditor: boolean = false
 ) {
   shareProjectByEmail(projectName, email, level, newEditor)

server-ce/test/project-sharing.spec.ts

@@ -21,8 +21,8 @@ import { beforeWithReRunOnTestRetry } from './helpers/beforeWithReRunOnTestRetry
 
 describe('Project Sharing', function () {
   if (isExcludedBySharding('PRO_CUSTOM_4')) return
+  startWith({ withDataDir: true, pro: true, resetData: true })
   ensureUserExists({ email: 'user@example.com' })
-  startWith({ withDataDir: true, pro: true })
 
   let projectName: string
   let recompile: () => void

services/chat/Makefile

@@ -2,10 +2,10 @@
 # Instead run bin/update_build_scripts from
 # https://github.com/overleaf/internal/
 
-BUILD_NUMBER ?= local
+export BUILD_NUMBER ?= local
-BRANCH_NAME ?= $(shell git rev-parse --abbrev-ref HEAD)
+export BRANCH_NAME ?= $(shell git rev-parse --abbrev-ref HEAD)
-BRANCH_NAME_TAG_SAFE = $(shell echo $(BRANCH_NAME) | sed 's/\//\-\-/g' | cut -c1-96)
+export BRANCH_NAME_TAG_SAFE = $(shell echo $(BRANCH_NAME) | sed 's/\//\-\-/g' | cut -c1-96)
-PROJECT_NAME = chat
+export PROJECT_NAME = chat
 BUILD_DIR_NAME = $(shell pwd | xargs basename | tr -cd '[a-zA-Z0-9_.\-]')
 HERE=$(shell pwd)
 export MONOREPO ?= $(shell cd ../../ && pwd)
@@ -28,13 +28,10 @@ IMAGE_CACHE ?= $(IMAGE_REPO):cache-$(shell cat \
 	$(MONOREPO)/tools/migrations/package.json \
 	$(MONOREPO)/.yarn/patches/* \
 | sha256sum | cut -d '-' -f1)
+export MOCHA_GREP ?=
 
 DOCKER_COMPOSE_FLAGS ?= -f docker-compose.yml
-DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \
+DOCKER_COMPOSE := docker compose ${DOCKER_COMPOSE_FLAGS}
-	BRANCH_NAME=$(BRANCH_NAME_TAG_SAFE) \
-	PROJECT_NAME=$(PROJECT_NAME) \
-	MOCHA_GREP=${MOCHA_GREP} \
-	docker compose ${DOCKER_COMPOSE_FLAGS}
 
 # Common flags for docker compose run.
 # --rm           Tear down test runner container

services/chat/buildscript.txt

@@ -5,6 +5,5 @@ chat
 --env-pass-through=
 --esmock-loader=False
 --node-version=24.14.1
---package-name=@overleaf/chat
 --pipeline-owner=44
 --public-repo=False

services/chat/docker-compose.ci.yml

@@ -4,7 +4,7 @@
 
 services:
   test_unit:
-    image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER
+    image: ci/$PROJECT_NAME:$BRANCH_NAME_TAG_SAFE-$BUILD_NUMBER
     user: node
     volumes:
       - ./reports:/overleaf/services/chat/reports
@@ -21,7 +21,7 @@ services:
         condition: service_started
 
   test_acceptance:
-    image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER
+    image: ci/$PROJECT_NAME:$BRANCH_NAME_TAG_SAFE-$BUILD_NUMBER
     environment:
       CI:
       RETRIES:
@@ -42,7 +42,7 @@ services:
 
   tar:
     build: .
-    image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER
+    image: ci/$PROJECT_NAME:$BRANCH_NAME_TAG_SAFE-$BUILD_NUMBER
     volumes:
       - ./:/tmp/build/
     command: tar -czf /tmp/build/build.tar.gz --exclude=build.tar.gz --exclude-vcs .

services/chat/test/acceptance/js/helpers/MongoHelper.js

@@ -5,6 +5,6 @@ before('run migrations', async function () {
   this.timeout(60_000)
 
   await promisify(exec)(
-    `cd ../../tools/migrations && npm run migrations -- migrate -t server-ce`
+    `cd ../../tools/migrations && yarn run migrations migrate -t 'server-ce & !nonblocking'`
   )
 })

services/clsi/.gitignore

@@ -1,4 +1,5 @@
 compiles
 output
 cache
-uploads
+uploads
+sharded

services/clsi/Makefile

@@ -2,10 +2,10 @@
 # Instead run bin/update_build_scripts from
 # https://github.com/overleaf/internal/
 
-BUILD_NUMBER ?= local
+export BUILD_NUMBER ?= local
-BRANCH_NAME ?= $(shell git rev-parse --abbrev-ref HEAD)
+export BRANCH_NAME ?= $(shell git rev-parse --abbrev-ref HEAD)
-BRANCH_NAME_TAG_SAFE = $(shell echo $(BRANCH_NAME) | sed 's/\//\-\-/g' | cut -c1-96)
+export BRANCH_NAME_TAG_SAFE = $(shell echo $(BRANCH_NAME) | sed 's/\//\-\-/g' | cut -c1-96)
-PROJECT_NAME = clsi
+export PROJECT_NAME = clsi
 BUILD_DIR_NAME = $(shell pwd | xargs basename | tr -cd '[a-zA-Z0-9_.\-]')
 HERE=$(shell pwd)
 export MONOREPO ?= $(shell cd ../../ && pwd)
@@ -29,13 +29,10 @@ IMAGE_CACHE ?= $(IMAGE_REPO):cache-$(shell cat \
 	$(MONOREPO)/services/clsi/package.json \
 	$(MONOREPO)/.yarn/patches/* \
 | sha256sum | cut -d '-' -f1)
+export MOCHA_GREP ?=
 
 DOCKER_COMPOSE_FLAGS ?= -f docker-compose.yml
-DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \
+DOCKER_COMPOSE := docker compose ${DOCKER_COMPOSE_FLAGS}
-	BRANCH_NAME=$(BRANCH_NAME_TAG_SAFE) \
-	PROJECT_NAME=$(PROJECT_NAME) \
-	MOCHA_GREP=${MOCHA_GREP} \
-	docker compose ${DOCKER_COMPOSE_FLAGS}
 
 # Common flags for docker compose run.
 # --rm           Tear down test runner container

services/clsi/README.md

@@ -29,7 +29,7 @@ The CLSI can be configured through the following environment variables:
 - `PROCESS_LIFE_SPAN_LIMIT_MS` - Process life span limit in milliseconds
 - `SMOKE_TEST` - Whether to run smoke tests
 - `TEXLIVE_IMAGE` - The TeX Live Docker image to use for sibling containers, e.g. `us-east1-docker.pkg.dev/overleaf-ops/ol-docker/texlive-full:2025.1`
-- `TEX_LIVE_IMAGE_NAME_OVERRIDE` - The name of the registry for the Docker image e.g. `us-east1-docker.pkg.dev/overleaf-ops/ol-docker`
+- `TEX_LIVE_DOCKER_IMAGE_ROOT` - The name of the registry for the Docker image e.g. `us-east1-docker.pkg.dev/overleaf-ops/ol-docker`
 - `TEXLIVE_IMAGE_USER` - When using sibling containers, the user to run as in the TeX Live image. Defaults to `tex`
 - `TEXLIVE_OPENOUT_ANY` - Sets the `openout_any` environment variable for TeX Live (see the `\openout` primitive [documentation](http://tug.org/texinfohtml/web2c.html#tex-invocation))
 

services/clsi/app/js/CommandRunner.js

@@ -10,7 +10,7 @@ import logger from '@overleaf/logger'
 let commandRunnerPath
 
 if ((Settings.clsi != null ? Settings.clsi.dockerRunner : undefined) === true) {
-  commandRunnerPath = './DockerRunner.js'
+  commandRunnerPath = './DockerRunner.mjs'
 } else {
   commandRunnerPath = './LocalCommandRunner.js'
 }

services/clsi/app/js/CompileManager.js

@@ -192,6 +192,11 @@ async function doCompile(request, stats, timings) {
     }
   }
 
+  // Pass through checkpoint setting
+  if (request.enableCheckpoint) {
+    env.ENABLE_CHECKPOINT = '1'
+  }
+
   const compileStart = Date.now()
 
   const compileName = getCompileName(request.project_id, request.user_id)

services/clsi/app/js/ConversionManager.js

@@ -173,6 +173,7 @@ const LATEX_EXPORT_CONFIGS = {
       '--to',
       'html',
       '--standalone',
+      '--mathml',
     ],
   },
 }

services/clsi/app/js/DockerLockManager.js

@@ -0,0 +1,110 @@
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS101: Remove unnecessary use of Array.from
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+import logger from '@overleaf/logger'
+let LockManager
+
+const LockState = {} // locks for docker container operations, by container name
+
+export default LockManager = {
+  MAX_LOCK_HOLD_TIME: 15000, // how long we can keep a lock
+  MAX_LOCK_WAIT_TIME: 10000, // how long we wait for a lock
+  LOCK_TEST_INTERVAL: 1000, // retry time
+
+  tryLock(key, callback) {
+    let lockValue
+    if (callback == null) {
+      callback = function () {}
+    }
+    const existingLock = LockState[key]
+    if (existingLock != null) {
+      // the lock is already taken, check how old it is
+      const lockAge = Date.now() - existingLock.created
+      if (lockAge < LockManager.MAX_LOCK_HOLD_TIME) {
+        return callback(null, false) // we didn't get the lock, bail out
+      } else {
+        logger.error(
+          { key, lock: existingLock, age: lockAge },
+          'taking old lock by force'
+        )
+      }
+    }
+    // take the lock
+    LockState[key] = lockValue = { created: Date.now() }
+    return callback(null, true, lockValue)
+  },
+
+  getLock(key, callback) {
+    let attempt
+    if (callback == null) {
+      callback = function () {}
+    }
+    const startTime = Date.now()
+    return (attempt = () =>
+      LockManager.tryLock(key, function (error, gotLock, lockValue) {
+        if (error != null) {
+          return callback(error)
+        }
+        if (gotLock) {
+          return callback(null, lockValue)
+        } else if (Date.now() - startTime > LockManager.MAX_LOCK_WAIT_TIME) {
+          const e = new Error('Lock timeout')
+          e.key = key
+          return callback(e)
+        } else {
+          return setTimeout(attempt, LockManager.LOCK_TEST_INTERVAL)
+        }
+      }))()
+  },
+
+  releaseLock(key, lockValue, callback) {
+    if (callback == null) {
+      callback = function () {}
+    }
+    const existingLock = LockState[key]
+    if (existingLock === lockValue) {
+      // lockValue is an object, so we can test by reference
+      delete LockState[key] // our lock, so we can free it
+      return callback()
+    } else if (existingLock != null) {
+      // lock exists but doesn't match ours
+      logger.error(
+        { key, lock: existingLock },
+        'tried to release lock taken by force'
+      )
+      return callback()
+    } else {
+      logger.error(
+        { key, lock: existingLock },
+        'tried to release lock that has gone'
+      )
+      return callback()
+    }
+  },
+
+  runWithLock(key, runner, callback) {
+    if (callback == null) {
+      callback = function () {}
+    }
+    return LockManager.getLock(key, function (error, lockValue) {
+      if (error != null) {
+        return callback(error)
+      }
+      return runner((error1, ...args) =>
+        LockManager.releaseLock(key, lockValue, function (error2) {
+          error = error1 || error2
+          if (error != null) {
+            return callback(error)
+          }
+          return callback(null, ...Array.from(args))
+        })
+      )
+    })
+  },
+}

services/clsi/app/js/DockerRunner.mjs

@@ -0,0 +1,630 @@
+import { promisify } from 'node:util'
+import Settings from '@overleaf/settings'
+import logger from '@overleaf/logger'
+import Docker from 'dockerode'
+import crypto from 'node:crypto'
+import async from 'async'
+import LockManager from './DockerLockManager.js'
+import Path from 'node:path'
+import _ from 'lodash'
+import { getLastProjectAccessTime } from './LastProjectAccess.js'
+
+const dockerode = new Docker()
+
+const ONE_HOUR_IN_MS = 60 * 60 * 1000
+logger.debug('using docker runner')
+
+let containerMonitorTimeout
+let containerMonitorInterval
+
+const DockerRunner = {
+  run(
+    projectId,
+    command,
+    directory,
+    image,
+    timeout,
+    environment,
+    compileGroup,
+    cwd,
+    callback
+  ) {
+    command = command.map(arg =>
+      arg.toString().replace('$COMPILE_DIR', '/compile')
+    )
+    if (image == null) {
+      image = Settings.clsi.docker.image
+    }
+
+    if (
+      Settings.clsi.docker.allowedImages &&
+      !Settings.clsi.docker.allowedImages.includes(image)
+    ) {
+      return callback(new Error('image not allowed'))
+    }
+
+    if (Settings.texliveImageNameOveride != null) {
+      const img = Path.basename(image)
+      image = `${Settings.texliveImageNameOveride}/${img}`
+    }
+
+    if (compileGroup === 'synctex-output') {
+      // In: directory = '/overleaf/services/clsi/output/projectId-userId/generated-files/buildId'
+      //             directory.split('/').slice(-3) === 'projectId-userId/generated-files/buildId'
+      //  sandboxedCompilesHostDirOutput = '/host/output'
+      // Out:                  directory = '/host/output/projectId-userId/generated-files/buildId'
+      directory = Path.join(
+        Settings.path.sandboxedCompilesHostDirOutput,
+        ...directory.split('/').slice(-3)
+      )
+    } else {
+      // In:   directory = '/overleaf/services/clsi/compiles/projectId-userId'
+      //                       Path.basename(directory) === 'projectId-userId'
+      //  sandboxedCompilesHostDirCompiles = '/host/compiles'
+      // Out:                    directory = '/host/compiles/projectId-userId'
+      directory = Path.join(
+        Settings.path.sandboxedCompilesHostDirCompiles,
+        Path.basename(directory)
+      )
+    }
+
+    const volumes = { [directory]: '/compile' }
+    if (
+      compileGroup === 'synctex' ||
+      compileGroup === 'synctex-output' ||
+      compileGroup === 'wordcount'
+    ) {
+      volumes[directory] += ':ro'
+    }
+
+    const options = DockerRunner._getContainerOptions(
+      command,
+      image,
+      volumes,
+      timeout,
+      environment,
+      compileGroup,
+      cwd
+    )
+    const fingerprint = DockerRunner._fingerprintContainer(options)
+    const name = `project-${projectId}-${fingerprint}`
+    options.name = name
+
+    // logOptions = _.clone(options)
+    // logOptions?.HostConfig?.SecurityOpt = "secomp used, removed in logging"
+    logger.debug({ projectId }, 'running docker container')
+    DockerRunner._runAndWaitForContainer(
+      options,
+      volumes,
+      timeout,
+      (error, output) => {
+        if (error && error.statusCode === 500) {
+          logger.debug(
+            { err: error, projectId },
+            'error running container so destroying and retrying'
+          )
+          DockerRunner.destroyContainer(name, null, true, error => {
+            if (error != null) {
+              return callback(error)
+            }
+            DockerRunner._runAndWaitForContainer(
+              options,
+              volumes,
+              timeout,
+              callback
+            )
+          })
+        } else {
+          callback(error, output)
+        }
+      }
+    )
+
+    // pass back the container name to allow it to be killed
+    return name
+  },
+
+  kill(containerId, callback) {
+    logger.debug({ containerId }, 'sending kill signal to container')
+    const container = dockerode.getContainer(containerId)
+    container.kill(error => {
+      if (
+        error != null &&
+        error.message != null &&
+        error.message.match(/Cannot kill container .* is not running/)
+      ) {
+        logger.warn(
+          { err: error, containerId },
+          'container not running, continuing'
+        )
+        error = null
+      }
+      if (error != null) {
+        logger.error({ err: error, containerId }, 'error killing container')
+        callback(error)
+      } else {
+        callback()
+      }
+    })
+  },
+
+  _runAndWaitForContainer(options, volumes, timeout, _callback) {
+    const callback = _.once(_callback)
+    const { name } = options
+
+    let streamEnded = false
+    let containerReturned = false
+    let output = {}
+
+    function callbackIfFinished() {
+      if (streamEnded && containerReturned) {
+        callback(null, output)
+      }
+    }
+
+    function attachStreamHandler(error, _output) {
+      if (error != null) {
+        return callback(error)
+      }
+      output = _output
+      streamEnded = true
+      callbackIfFinished()
+    }
+
+    DockerRunner.startContainer(
+      options,
+      volumes,
+      attachStreamHandler,
+      (error, containerId) => {
+        if (error != null) {
+          return callback(error)
+        }
+
+        DockerRunner.waitForContainer(
+          name,
+          timeout,
+          options,
+          (error, exitCode) => {
+            if (error != null) {
+              return callback(error)
+            }
+            if (exitCode === 137) {
+              // exit status from kill -9
+              const err = new Error('terminated')
+              err.terminated = true
+              return callback(err)
+            }
+            if (exitCode === 1) {
+              // exit status from chktex
+              const err = new Error('exited')
+              err.code = exitCode
+              return callback(err)
+            }
+            output.exitCode = exitCode
+            containerReturned = true
+            logger.debug(
+              // The seccomp policy is very large. Avoid logging it. _.omit deep clones.
+              { exitCode, options: _.omit(options, 'HostConfig.SecurityOpt') },
+              'docker container has exited'
+            )
+            callbackIfFinished()
+          }
+        )
+      }
+    )
+  },
+
+  _getContainerOptions(
+    command,
+    image,
+    volumes,
+    timeout,
+    environment,
+    compileGroup,
+    cwd
+  ) {
+    const timeoutInSeconds = timeout / 1000
+
+    for (const hostVol in volumes) {
+      const dockerVol = volumes[hostVol]
+      if (volumes[hostVol].slice(-3).indexOf(':r') === -1) {
+        volumes[hostVol] = `${dockerVol}:rw`
+      }
+    }
+
+    // merge settings and environment parameter
+    const env = {}
+    for (const src of [Settings.clsi.docker.env, environment || {}]) {
+      for (const key in src) {
+        const value = src[key]
+        env[key] = value
+      }
+    }
+    // set the path based on the image year
+    const match = image.match(/:([0-9]+)\.[0-9]+|:TL([0-9]+)/)
+    // the rolling build does not follow our <year>.<version>.<patch> convention
+    const year = match ? match[1] || match[2] : 'rolling'
+
+    env.PATH = `/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/texlive/${year}/bin/x86_64-linux/`
+    const options = {
+      Cmd: command,
+      Image: image,
+      WorkingDir: cwd
+        ? Path.posix.join('/compile', cwd)
+        : '/compile',
+      NetworkDisabled: true,
+      Memory: 1024 * 1024 * 1024 * 1024, // 1 Gb
+      User: Settings.clsi.docker.user,
+      Env: Object.entries(env).map(([key, value]) => `${key}=${value}`),
+      HostConfig: {
+        Binds: Object.entries(volumes).map(
+          ([hostVol, dockerVol]) => `${hostVol}:${dockerVol}`
+        ),
+        LogConfig: { Type: 'none', Config: {} },
+        Ulimits: [
+          {
+            Name: 'cpu',
+            Soft: timeoutInSeconds + 5,
+            Hard: timeoutInSeconds + 10,
+          },
+        ],
+        CapDrop: ['ALL'],
+        SecurityOpt: ['no-new-privileges'],
+      },
+    }
+
+    if (Settings.clsi.docker.seccomp_profile != null) {
+      options.HostConfig.SecurityOpt.push(
+        `seccomp=${Settings.clsi.docker.seccomp_profile}`
+      )
+    }
+
+    if (Settings.clsi.docker.apparmor_profile != null) {
+      options.HostConfig.SecurityOpt.push(
+        `apparmor=${Settings.clsi.docker.apparmor_profile}`
+      )
+    }
+
+    if (Settings.clsi.docker.runtime) {
+      options.HostConfig.Runtime = Settings.clsi.docker.runtime
+    }
+
+    if (Settings.clsi.docker.Readonly) {
+      options.HostConfig.ReadonlyRootfs = true
+      options.HostConfig.Tmpfs = { '/tmp': 'rw,noexec,nosuid,size=65536k' }
+      options.Volumes['/home/tex'] = {}
+    }
+
+    // Allow per-compile group overriding of individual settings
+    if (
+      Settings.clsi.docker.compileGroupConfig &&
+      Settings.clsi.docker.compileGroupConfig[compileGroup]
+    ) {
+      const override = Settings.clsi.docker.compileGroupConfig[compileGroup]
+      for (const key in override) {
+        _.set(options, key, override[key])
+      }
+    }
+
+    return options
+  },
+
+  _fingerprintContainer(containerOptions) {
+    // Yay, Hashing!
+    const json = JSON.stringify(containerOptions)
+    return crypto.createHash('md5').update(json).digest('hex')
+  },
+
+  startContainer(options, volumes, attachStreamHandler, callback) {
+    LockManager.runWithLock(
+      options.name,
+      releaseLock =>
+        DockerRunner._startContainer(
+          options,
+          volumes,
+          attachStreamHandler,
+          releaseLock
+        ),
+      callback
+    )
+  },
+
+  // Check that volumes exist and are directories
+  _startContainer(options, volumes, attachStreamHandler, callback) {
+    callback = _.once(callback)
+    const { name } = options
+
+    logger.debug({ containerName: name }, 'starting container')
+    const container = dockerode.getContainer(name)
+
+    function createAndStartContainer() {
+      dockerode.createContainer(options, (error, container) => {
+        if (error != null) {
+          return callback(error)
+        }
+        startExistingContainer()
+      })
+    }
+
+    function startExistingContainer() {
+      DockerRunner.attachToContainer(
+        options.name,
+        attachStreamHandler,
+        error => {
+          if (error != null) {
+            return callback(error)
+          }
+          container.start(error => {
+            if (error != null && error.statusCode !== 304) {
+              callback(error)
+            } else {
+              // already running
+              callback()
+            }
+          })
+        }
+      )
+    }
+
+    container.inspect((error, stats) => {
+      if (error != null && error.statusCode === 404) {
+        createAndStartContainer()
+      } else if (error != null) {
+        logger.err(
+          { containerName: name, error },
+          'unable to inspect container to start'
+        )
+        callback(error)
+      } else {
+        startExistingContainer()
+      }
+    })
+  },
+
+  attachToContainer(containerId, attachStreamHandler, attachStartCallback) {
+    const container = dockerode.getContainer(containerId)
+    container.attach({ stdout: 1, stderr: 1, stream: 1 }, (error, stream) => {
+      if (error != null) {
+        logger.error(
+          { err: error, containerId },
+          'error attaching to container'
+        )
+        return attachStartCallback(error)
+      } else {
+        attachStartCallback()
+      }
+
+      logger.debug({ containerId }, 'attached to container')
+
+      const MAX_OUTPUT = 1024 * 1024 * 2 // limit output to 2MB
+      function createStringOutputStream(name) {
+        return {
+          data: '',
+          overflowed: false,
+          write(data) {
+            if (this.overflowed) {
+              return
+            }
+            if (this.data.length < MAX_OUTPUT) {
+              this.data += data
+            } else {
+              logger.info(
+                {
+                  containerId,
+                  length: this.data.length,
+                  maxLen: MAX_OUTPUT,
+                },
+                `${name} exceeds max size`
+              )
+              this.data += `(...truncated at ${MAX_OUTPUT} chars...)`
+              this.overflowed = true
+            }
+          },
+          // kill container if too much output
+          // docker.containers.kill(containerId, () ->)
+        }
+      }
+
+      const stdout = createStringOutputStream('stdout')
+      const stderr = createStringOutputStream('stderr')
+
+      container.modem.demuxStream(stream, stdout, stderr)
+
+      stream.on('error', err =>
+        logger.error(
+          { err, containerId },
+          'error reading from container stream'
+        )
+      )
+
+      stream.on('end', () =>
+        attachStreamHandler(null, { stdout: stdout.data, stderr: stderr.data })
+      )
+    })
+  },
+
+  waitForContainer(containerId, timeout, options, _callback) {
+    const callback = _.once(_callback)
+
+    const container = dockerode.getContainer(containerId)
+
+    let timedOut = false
+    const timeoutId = setTimeout(() => {
+      timedOut = true
+      logger.debug({ containerId }, 'timeout reached, killing container')
+      container.kill(err => {
+        logger.warn({ err, containerId }, 'failed to kill container')
+      })
+    }, timeout)
+
+    logger.debug({ containerId }, 'waiting for docker container')
+    container.wait((error, res) => {
+      if (error?.statusCode === 404 && options.HostConfig.AutoRemove) {
+        logger.debug(
+          { containerId },
+          'auto-destroy container destroyed before starting to wait'
+        )
+        clearTimeout(timeoutId)
+        return callback(null, 0)
+      }
+      if (error != null) {
+        clearTimeout(timeoutId)
+        logger.warn({ err: error, containerId }, 'error waiting for container')
+        return callback(error)
+      }
+      if (timedOut) {
+        logger.debug({ containerId }, 'docker container timed out')
+        error = new Error('container timed out')
+        error.timedout = true
+        callback(error)
+      } else {
+        clearTimeout(timeoutId)
+        logger.debug(
+          { containerId, exitCode: res.StatusCode },
+          'docker container returned'
+        )
+        callback(null, res.StatusCode)
+      }
+    })
+  },
+
+  destroyContainer(containerName, containerId, shouldForce, callback) {
+    // We want the containerName for the lock and, ideally, the
+    // containerId to delete.  There is a bug in the docker.io module
+    // where if you delete by name and there is an error, it throws an
+    // async exception, but if you delete by id it just does a normal
+    // error callback. We fall back to deleting by name if no id is
+    // supplied.
+    LockManager.runWithLock(
+      containerName,
+      releaseLock =>
+        DockerRunner._destroyContainer(
+          containerId || containerName,
+          shouldForce,
+          releaseLock
+        ),
+      callback
+    )
+  },
+
+  _destroyContainer(containerId, shouldForce, callback) {
+    logger.debug({ containerId }, 'destroying docker container')
+    const container = dockerode.getContainer(containerId)
+    container.remove({ force: shouldForce === true, v: true }, error => {
+      if (error != null && error.statusCode === 404) {
+        logger.warn(
+          { err: error, containerId },
+          'container not found, continuing'
+        )
+        error = null
+      }
+      if (error != null) {
+        logger.error({ err: error, containerId }, 'error destroying container')
+      } else {
+        logger.debug({ containerId }, 'destroyed container')
+      }
+      callback(error)
+    })
+  },
+
+  // handle expiry of docker containers
+
+  MAX_CONTAINER_AGE: Settings.clsi.docker.maxContainerAge || ONE_HOUR_IN_MS,
+
+  examineOldContainer(container, callback) {
+    const name = container.Name || (container.Names && container.Names[0])
+    const created = container.Created * 1000 // creation time is returned in seconds
+    const now = Date.now()
+    const age = now - created
+    const maxAge = DockerRunner.MAX_CONTAINER_AGE
+    const ttl = maxAge - age
+    logger.debug(
+      { containerName: name, created, now, age, maxAge, ttl },
+      'checking whether to destroy container'
+    )
+    return { name, id: container.Id, ttl }
+  },
+
+  destroyOldContainers(callback) {
+    dockerode.listContainers({ all: true }, (error, containers) => {
+      if (error != null) {
+        return callback(error)
+      }
+      const jobs = []
+      for (const container of containers) {
+        const { name, id, ttl } = DockerRunner.examineOldContainer(container)
+        if (name.slice(0, 9) === '/project-' && ttl <= 0) {
+          // extract projectId: /project-<projectId>-...
+          const match = name.match(/^\/project-([a-f0-9]{24})-/)
+          const projectId = match && match[1]
+
+          if (projectId != null) {
+            const lastAccess = getLastProjectAccessTime(projectId)
+            // if last access time is missing, lastAccess === 0, container will be destroyed
+            // provided that MAX_CONTAINER_AGE is less that 56 years, but anyway, let's check it
+            if (lastAccess && lastAccess > Date.now() - DockerRunner.MAX_CONTAINER_AGE) {
+              continue
+            }
+          }
+
+          // strip the / prefix
+          // the LockManager uses the plain container name
+          const plainName = name.slice(1)
+          jobs.push(cb =>
+            DockerRunner.destroyContainer(plainName, id, false, () => cb())
+          )
+        }
+      }
+      // Ignore errors because some containers get stuck but
+      // will be destroyed next time
+      async.series(jobs, callback)
+    })
+  },
+
+  startContainerMonitor() {
+    logger.debug(
+      { maxAge: DockerRunner.MAX_CONTAINER_AGE },
+      'starting container expiry'
+    )
+
+    // guarantee only one monitor is running
+    DockerRunner.stopContainerMonitor()
+
+    // randomise the start time
+    const randomDelay = Math.floor(Math.random() * 5 * 60 * 1000)
+    containerMonitorTimeout = setTimeout(() => {
+      containerMonitorInterval = setInterval(
+        () =>
+          DockerRunner.destroyOldContainers(err => {
+            if (err) {
+              logger.error({ err }, 'failed to destroy old containers')
+            }
+          }),
+        ONE_HOUR_IN_MS
+      )
+    }, randomDelay)
+  },
+
+  stopContainerMonitor() {
+    if (containerMonitorTimeout) {
+      clearTimeout(containerMonitorTimeout)
+      containerMonitorTimeout = undefined
+    }
+    if (containerMonitorInterval) {
+      clearInterval(containerMonitorInterval)
+      containerMonitorInterval = undefined
+    }
+  },
+
+  canRunSyncTeXInOutputDir() {
+    return Boolean(Settings.path.sandboxedCompilesHostDirOutput)
+  },
+}
+
+DockerRunner.startContainerMonitor()
+
+DockerRunner.promises = {
+  run: promisify(DockerRunner.run),
+  kill: promisify(DockerRunner.kill),
+}
+export default DockerRunner

services/clsi/app/js/RequestParser.js

@@ -65,6 +65,11 @@ function parse(body, callback) {
         type: 'number',
       }
     )
+    response.enableCheckpoint = _parseAttribute(
+      'enableCheckpoint',
+      compile.options.enableCheckpoint,
+      { default: false, type: 'boolean' }
+    )
     response.timeout = _parseAttribute('timeout', compile.options.timeout, {
       default: MAX_TIMEOUT,
       type: 'number',

services/clsi/app/js/ResourceWriter.js

@@ -23,9 +23,15 @@ import Metrics from '@overleaf/metrics'
 import logger from '@overleaf/logger'
 import settings from '@overleaf/settings'
 import ClsiMetrics from './Metrics.js'
+import { Minimatch } from 'minimatch'
 
 const { shouldSkipMetrics } = ClsiMetrics
 
+// Additional file patterns to keep between compiles
+const preciousFileMatcher = new Minimatch(settings.preciousFilePattern, {
+  dot: true,
+})
+
 let ResourceWriter
 
 const parallelFileDownloads = settings.parallelFileDownloads || 1
@@ -265,6 +271,10 @@ export default ResourceWriter = {
       // Epstopdf generated files
       shouldDelete = false
     }
+    // Keep additional precious files and directories
+    if (shouldDelete && preciousFileMatcher.match(path)) {
+      shouldDelete = false
+    }
     if (
       path === 'output.tar.gz' ||
       path === 'output.synctex.gz' ||

services/clsi/buildscript.txt

@@ -1,11 +1,10 @@
 clsi
 --data-dirs=cache,compiles,output
 --dependencies=
---env-add=DOWNLOAD_HOST=http://clsi-nginx:8080,ALLOWED_COMPILE_GROUPS=clsi-perf simple-latex-file,ENABLE_PDF_CACHING=true,PDF_CACHING_ENABLE_WORKER_POOL=true,ALLOWED_IMAGES=quay.io/sharelatex/texlive-full:2017.1 quay.io/sharelatex/texlive-full:2025.1 quay.io/sharelatex/pandoc:3.9 quay.io/sharelatex/pdftocairo:24.02,TEXLIVE_IMAGE=quay.io/sharelatex/texlive-full:2025.1,TEX_LIVE_IMAGE_NAME_OVERRIDE=us-east1-docker.pkg.dev/overleaf-ops/ol-docker,TEXLIVE_IMAGE_USER=tex,SANDBOXED_COMPILES=true,SANDBOXED_COMPILES_HOST_DIR_COMPILES=$PWD/compiles,SANDBOXED_COMPILES_HOST_DIR_OUTPUT=$PWD/output,ENABLE_PANDOC_CONVERSIONS=true,ENABLE_PDF_CONVERSIONS=true
+--env-add=DOWNLOAD_HOST=http://clsi-nginx:8080,ALLOWED_COMPILE_GROUPS=clsi-perf simple-latex-file,ENABLE_PDF_CACHING=true,PDF_CACHING_ENABLE_WORKER_POOL=true,ALLOWED_IMAGES=quay.io/sharelatex/texlive-full:2017.1 quay.io/sharelatex/texlive-full:2025.1 quay.io/sharelatex/pandoc:3.9 quay.io/sharelatex/pdftocairo:24.02,TEXLIVE_IMAGE=quay.io/sharelatex/texlive-full:2025.1,TEX_LIVE_DOCKER_IMAGE_ROOT=us-east1-docker.pkg.dev/overleaf-ops/ol-docker,TEXLIVE_IMAGE_USER=tex,SANDBOXED_COMPILES=true,SANDBOXED_COMPILES_HOST_DIR_COMPILES=$PWD/compiles,SANDBOXED_COMPILES_HOST_DIR_OUTPUT=$PWD/output,ENABLE_PANDOC_CONVERSIONS=true,ENABLE_PDF_CONVERSIONS=true
 --env-pass-through=
 --esmock-loader=False
 --node-version=24.14.1
---package-name=@overleaf/clsi
 --pipeline-owner=32
 --public-repo=True
 --test-unit-vitest=True

services/clsi/config/settings.defaults.cjs

@@ -35,6 +35,7 @@ module.exports = {
     process.env.PDFTOCAIRO_IMAGE || 'quay.io/sharelatex/pdftocairo:24.02',
   enablePdfConversions: process.env.ENABLE_PDF_CONVERSIONS === 'true',
   maxUploadSize: 50 * 1024 * 1024,
+  preciousFilePattern: process.env.PRECIOUS_FILE_PATTERN || '',
 
   internal: {
     clsi: {
@@ -92,7 +93,7 @@ module.exports = {
   parallelFileDownloads:
     parseInt(process.env.FILESTORE_PARALLEL_FILE_DOWNLOADS, 10) || 1,
   filestoreDomainOveride: process.env.FILESTORE_DOMAIN_OVERRIDE,
-  texliveImageNameOveride: process.env.TEX_LIVE_IMAGE_NAME_OVERRIDE,
+  texliveImageNameOveride: process.env.TEX_LIVE_DOCKER_IMAGE_ROOT,
   texliveOpenoutAny: process.env.TEXLIVE_OPENOUT_ANY,
   texliveMaxPrintLine: process.env.TEXLIVE_MAX_PRINT_LINE,
   enablePdfCaching: process.env.ENABLE_PDF_CACHING === 'true',
@@ -124,7 +125,7 @@ if (process.env.ALLOWED_COMPILE_GROUPS) {
 
 if ((process.env.DOCKER_RUNNER || process.env.SANDBOXED_COMPILES) === 'true') {
   if (
-    !fs.existsSync(Path.join(__dirname, '..', 'app', 'js', 'DockerRunner.js'))
+    !fs.existsSync(Path.join(__dirname, '..', 'app', 'js', 'DockerRunner.mjs'))
   ) {
     console.error(
       'Sandboxed compiles are only available with Overleaf Server Pro. Compare Server Pro with Community Edition here: https://docs.overleaf.com/on-premises/welcome/server-pro-vs.-community-edition'
@@ -139,13 +140,13 @@ if ((process.env.DOCKER_RUNNER || process.env.SANDBOXED_COMPILES) === 'true') {
       image:
         process.env.TEXLIVE_IMAGE ||
         process.env.TEX_LIVE_DOCKER_IMAGE ||
-        'quay.io/sharelatex/texlive-full:2017.1',
+        process.env.ALL_TEX_LIVE_DOCKER_IMAGES.split(',')[0].trim(),
       env: {
         HOME: '/tmp',
         CLSI: 1,
       },
       socketPath: '/var/run/docker.sock',
-      user: process.env.TEXLIVE_IMAGE_USER || 'tex',
+      user: process.env.TEXLIVE_IMAGE_USER || 'www-data',
     },
     optimiseInDocker: true,
     expireProjectAfterIdleMs: 24 * 60 * 60 * 1000,
@@ -202,6 +203,16 @@ if ((process.env.DOCKER_RUNNER || process.env.SANDBOXED_COMPILES) === 'true') {
     }
   }
 
+  if (process.env.NEW_APPARMOR_PROFILE) {
+    try {
+      module.exports.clsi.docker.new_apparmor_profile =
+        process.env.NEW_APPARMOR_PROFILE
+    } catch (error) {
+      console.error(error, 'could not apply new apparmor profile setting')
+      process.exit(1)
+    }
+  }
+
   if (process.env.ALLOWED_IMAGES) {
     try {
       module.exports.clsi.docker.allowedImages =

services/clsi/docker-compose.ci.yml

@@ -4,7 +4,7 @@
 
 services:
   test_unit:
-    image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER
+    image: ci/$PROJECT_NAME:$BRANCH_NAME_TAG_SAFE-$BUILD_NUMBER
     volumes:
       - ./reports:/overleaf/services/clsi/reports
       - ../../tsconfig.backend.json:/overleaf/tsconfig.backend.json
@@ -17,7 +17,7 @@ services:
       VITEST_NO_CACHE: true
 
   test_acceptance:
-    image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER
+    image: ci/$PROJECT_NAME:$BRANCH_NAME_TAG_SAFE-$BUILD_NUMBER
     environment:
       CI:
       RETRIES:
@@ -32,7 +32,7 @@ services:
       PDF_CACHING_ENABLE_WORKER_POOL: true
       ALLOWED_IMAGES: quay.io/sharelatex/texlive-full:2017.1 quay.io/sharelatex/texlive-full:2025.1 quay.io/sharelatex/pandoc:3.9 quay.io/sharelatex/pdftocairo:24.02
       TEXLIVE_IMAGE: quay.io/sharelatex/texlive-full:2025.1
-      TEX_LIVE_IMAGE_NAME_OVERRIDE: us-east1-docker.pkg.dev/overleaf-ops/ol-docker
+      TEX_LIVE_DOCKER_IMAGE_ROOT: us-east1-docker.pkg.dev/overleaf-ops/ol-docker
       TEXLIVE_IMAGE_USER: tex
       SANDBOXED_COMPILES: true
       SANDBOXED_COMPILES_HOST_DIR_COMPILES: $PWD/compiles
@@ -51,7 +51,7 @@ services:
 
   tar:
     build: .
-    image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER
+    image: ci/$PROJECT_NAME:$BRANCH_NAME_TAG_SAFE-$BUILD_NUMBER
     volumes:
       - ./:/tmp/build/
     command: tar -czf /tmp/build/build.tar.gz --exclude=build.tar.gz --exclude-vcs .

services/clsi/docker-compose.yml

@@ -55,7 +55,7 @@ services:
       PDF_CACHING_ENABLE_WORKER_POOL: true
       ALLOWED_IMAGES: quay.io/sharelatex/texlive-full:2017.1 quay.io/sharelatex/texlive-full:2025.1 quay.io/sharelatex/pandoc:3.9 quay.io/sharelatex/pdftocairo:24.02
       TEXLIVE_IMAGE: quay.io/sharelatex/texlive-full:2025.1
-      TEX_LIVE_IMAGE_NAME_OVERRIDE: us-east1-docker.pkg.dev/overleaf-ops/ol-docker
+      TEX_LIVE_DOCKER_IMAGE_ROOT: us-east1-docker.pkg.dev/overleaf-ops/ol-docker
       TEXLIVE_IMAGE_USER: tex
       SANDBOXED_COMPILES: true
       SANDBOXED_COMPILES_HOST_DIR_COMPILES: $PWD/compiles

services/clsi/package.json

@@ -31,6 +31,7 @@
     "dockerode": "^4.0.9",
     "express": "4.22.1",
     "lodash": "^4.18.1",
+    "minimatch": "^10.2.5",
     "multer": "2.1.1",
     "overleaf-editor-core": "workspace:*",
     "p-limit": "^3.1.0",

services/clsi/seccomp/clsi-profile.json

@@ -0,0 +1,914 @@
+{
+    "defaultAction": "SCMP_ACT_ERRNO",
+    "architectures": [
+        "SCMP_ARCH_X86_64",
+        "SCMP_ARCH_X86",
+        "SCMP_ARCH_X32",
+        "SCMP_ARCH_AARCH64",
+        "SCMP_ARCH_ARM"
+    ],
+    "syscalls": [
+        {
+          "name": "getrandom",
+          "action": "SCMP_ACT_ALLOW",
+          "args": []
+        },
+        {
+            "name": "access",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "arch_prctl",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "brk",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "chdir",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "chmod",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "clock_getres",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "clock_gettime",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "clock_nanosleep",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "clone",
+            "action": "SCMP_ACT_ALLOW",
+            "args": [
+                {
+                    "index": 0,
+                    "value": 2080505856,
+                    "valueTwo": 0,
+                    "op": "SCMP_CMP_MASKED_EQ"
+                }
+            ]
+        },
+        {
+            "name": "clone3",
+            "action": "SCMP_ACT_ERRNO",
+            "errnoRet": 38
+        },
+        {
+            "name": "close",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "copy_file_range",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "creat",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "dup",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "dup2",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "dup3",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "execve",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "execveat",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "exit",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "exit_group",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "faccessat",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+             "name": "faccessat2",
+             "action": "SCMP_ACT_ALLOW",
+             "args": []
+        },
+        {
+            "name": "fadvise64",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "fadvise64_64",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "fallocate",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "fchdir",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "fchmod",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "fchmodat",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "fcntl",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "fcntl64",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "fdatasync",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "fork",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "fstat",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "fstat64",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "fstatat64",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "fstatfs",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "fstatfs64",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "fsync",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "ftruncate",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "ftruncate64",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "futex",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "futimesat",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "getcpu",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "getcwd",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "getdents",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "getdents64",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "getegid",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "getegid32",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "geteuid",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "geteuid32",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "getgid",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "getgid32",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "getgroups",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "getgroups32",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "getpgid",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "getpgrp",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "getpid",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "getppid",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "getpriority",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "getresgid",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "getresgid32",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "getresuid",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "getresuid32",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "getrlimit",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "get_robust_list",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "getrusage",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "getsid",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "gettid",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "getuid",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "getuid32",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "ioctl",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "kill",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "_llseek",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "lseek",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "lstat",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "lstat64",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "madvise",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "mkdir",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "mkdirat",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "mmap",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "mmap2",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "mprotect",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "mremap",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "munmap",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "newfstatat",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "open",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "openat",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "openat2",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "pause",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "pipe",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "pipe2",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "prctl",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "pread64",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "preadv",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "prlimit64",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "pwrite64",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "pwritev",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "read",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "readlink",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "readlinkat",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "readv",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "rename",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "renameat",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "renameat2",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "restart_syscall",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "rmdir",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "rt_sigaction",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "rt_sigpending",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "rt_sigprocmask",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "rt_sigqueueinfo",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "rt_sigreturn",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "rt_sigsuspend",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "rt_sigtimedwait",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "rt_tgsigqueueinfo",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "sched_getaffinity",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "sched_getparam",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "sched_get_priority_max",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "sched_get_priority_min",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "sched_getscheduler",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "sched_rr_get_interval",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "sched_yield",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "sendfile",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "sendfile64",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "setgroups",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "setgroups32",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "set_robust_list",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "set_tid_address",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "sigaltstack",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "stat",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "statx",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "stat64",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "statfs",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "statfs64",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "sync",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "sync_file_range",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "syncfs",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "sysinfo",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "tgkill",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "timer_create",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "timer_delete",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "timer_getoverrun",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "timer_gettime",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "timer_settime",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "times",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "tkill",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "truncate",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "truncate64",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "umask",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "uname",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "unlink",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "unlinkat",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "utime",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "utimensat",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "utimes",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "vfork",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "vhangup",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "wait4",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "waitid",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "write",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "writev",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "pread",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "setgid",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "setuid",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "capget",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "capset",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "fchown",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "gettimeofday",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "epoll_wait",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "epoll_pwait",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "epoll_pwait2",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "poll",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "ppoll",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "epoll_create",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "epoll_create1",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "epoll_ctl",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "timerfd_create",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "timerfd_settime",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        },
+        {
+            "name": "eventfd2",
+            "action": "SCMP_ACT_ALLOW",
+            "args": []
+        }
+    ]
+}

services/clsi/test/unit/js/ConversionManager.test.js

@@ -91,6 +91,7 @@ const LATEX_TO_DOCUMENT_CASES = [
       '--to',
       'html',
       '--standalone',
+      '--mathml',
       '--resource-path=..',
       '--extract-media=.',
     ],

services/clsi/test/unit/js/DockerLockManager.test.js

@@ -0,0 +1,259 @@
+import { vi, describe, beforeEach, it } from 'vitest'
+
+import sinon from 'sinon'
+import path from 'node:path'
+const modulePath = path.join(
+  import.meta.dirname,
+  '../../../app/js/DockerLockManager'
+)
+
+describe('DockerLockManager', function () {
+  beforeEach(async function (ctx) {
+    vi.doMock('@overleaf/settings', () => ({
+      default: (ctx.Settings = { clsi: { docker: {} } }),
+    }))
+
+    return (ctx.LockManager = (await import(modulePath)).default)
+  })
+
+  return describe('runWithLock', function () {
+    describe('with a single lock', function () {
+      beforeEach(async function (ctx) {
+        await new Promise((resolve, reject) => {
+          ctx.callback = sinon.stub()
+          return ctx.LockManager.runWithLock(
+            'lock-one',
+            releaseLock =>
+              setTimeout(() => releaseLock(null, 'hello', 'world'), 100),
+
+            (err, ...args) => {
+              ctx.callback(err, ...Array.from(args))
+              return resolve()
+            }
+          )
+        })
+      })
+
+      return it('should call the callback', function (ctx) {
+        return ctx.callback
+          .calledWith(null, 'hello', 'world')
+          .should.equal(true)
+      })
+    })
+
+    describe('with two locks', function () {
+      beforeEach(async function (ctx) {
+        await new Promise((resolve, reject) => {
+          ctx.callback1 = sinon.stub()
+          ctx.callback2 = sinon.stub()
+          ctx.LockManager.runWithLock(
+            'lock-one',
+            releaseLock =>
+              setTimeout(() => releaseLock(null, 'hello', 'world', 'one'), 100),
+
+            (err, ...args) => {
+              return ctx.callback1(err, ...Array.from(args))
+            }
+          )
+          return ctx.LockManager.runWithLock(
+            'lock-two',
+            releaseLock =>
+              setTimeout(() => releaseLock(null, 'hello', 'world', 'two'), 200),
+
+            (err, ...args) => {
+              ctx.callback2(err, ...Array.from(args))
+              return resolve()
+            }
+          )
+        })
+      })
+
+      it('should call the first callback', function (ctx) {
+        return ctx.callback1
+          .calledWith(null, 'hello', 'world', 'one')
+          .should.equal(true)
+      })
+
+      return it('should call the second callback', function (ctx) {
+        return ctx.callback2
+          .calledWith(null, 'hello', 'world', 'two')
+          .should.equal(true)
+      })
+    })
+
+    return describe('with lock contention', function () {
+      describe('where the first lock is released quickly', function () {
+        beforeEach(async function (ctx) {
+          await new Promise((resolve, reject) => {
+            ctx.LockManager.MAX_LOCK_WAIT_TIME = 1000
+            ctx.LockManager.LOCK_TEST_INTERVAL = 100
+            ctx.callback1 = sinon.stub()
+            ctx.callback2 = sinon.stub()
+            ctx.LockManager.runWithLock(
+              'lock',
+              releaseLock =>
+                setTimeout(
+                  () => releaseLock(null, 'hello', 'world', 'one'),
+                  100
+                ),
+
+              (err, ...args) => {
+                return ctx.callback1(err, ...Array.from(args))
+              }
+            )
+            return ctx.LockManager.runWithLock(
+              'lock',
+              releaseLock =>
+                setTimeout(
+                  () => releaseLock(null, 'hello', 'world', 'two'),
+                  200
+                ),
+
+              (err, ...args) => {
+                ctx.callback2(err, ...Array.from(args))
+                return resolve()
+              }
+            )
+          })
+        })
+
+        it('should call the first callback', function (ctx) {
+          return ctx.callback1
+            .calledWith(null, 'hello', 'world', 'one')
+            .should.equal(true)
+        })
+
+        return it('should call the second callback', function (ctx) {
+          return ctx.callback2
+            .calledWith(null, 'hello', 'world', 'two')
+            .should.equal(true)
+        })
+      })
+
+      describe('where the first lock is held longer than the waiting time', function () {
+        beforeEach(async function (ctx) {
+          await new Promise((resolve, reject) => {
+            let doneTwo
+            ctx.LockManager.MAX_LOCK_HOLD_TIME = 10000
+            ctx.LockManager.MAX_LOCK_WAIT_TIME = 1000
+            ctx.LockManager.LOCK_TEST_INTERVAL = 100
+            ctx.callback1 = sinon.stub()
+            ctx.callback2 = sinon.stub()
+            let doneOne = (doneTwo = false)
+            const finish = function (key) {
+              if (key === 1) {
+                doneOne = true
+              }
+              if (key === 2) {
+                doneTwo = true
+              }
+              if (doneOne && doneTwo) {
+                return resolve()
+              }
+            }
+            ctx.LockManager.runWithLock(
+              'lock',
+              releaseLock =>
+                setTimeout(
+                  () => releaseLock(null, 'hello', 'world', 'one'),
+                  1100
+                ),
+
+              (err, ...args) => {
+                ctx.callback1(err, ...Array.from(args))
+                return finish(1)
+              }
+            )
+            return ctx.LockManager.runWithLock(
+              'lock',
+              releaseLock =>
+                setTimeout(
+                  () => releaseLock(null, 'hello', 'world', 'two'),
+                  100
+                ),
+
+              (err, ...args) => {
+                ctx.callback2(err, ...Array.from(args))
+                return finish(2)
+              }
+            )
+          })
+        })
+
+        it('should call the first callback', function (ctx) {
+          return ctx.callback1
+            .calledWith(null, 'hello', 'world', 'one')
+            .should.equal(true)
+        })
+
+        return it('should call the second callback with an error', function (ctx) {
+          const error = sinon.match.instanceOf(Error)
+          return ctx.callback2.calledWith(error).should.equal(true)
+        })
+      })
+
+      return describe('where the first lock is held longer than the max holding time', function () {
+        beforeEach(async function (ctx) {
+          await new Promise((resolve, reject) => {
+            let doneTwo
+            ctx.LockManager.MAX_LOCK_HOLD_TIME = 1000
+            ctx.LockManager.MAX_LOCK_WAIT_TIME = 2000
+            ctx.LockManager.LOCK_TEST_INTERVAL = 100
+            ctx.callback1 = sinon.stub()
+            ctx.callback2 = sinon.stub()
+            let doneOne = (doneTwo = false)
+            const finish = function (key) {
+              if (key === 1) {
+                doneOne = true
+              }
+              if (key === 2) {
+                doneTwo = true
+              }
+              if (doneOne && doneTwo) {
+                return resolve()
+              }
+            }
+            ctx.LockManager.runWithLock(
+              'lock',
+              releaseLock =>
+                setTimeout(
+                  () => releaseLock(null, 'hello', 'world', 'one'),
+                  1500
+                ),
+
+              (err, ...args) => {
+                ctx.callback1(err, ...Array.from(args))
+                return finish(1)
+              }
+            )
+            return ctx.LockManager.runWithLock(
+              'lock',
+              releaseLock =>
+                setTimeout(
+                  () => releaseLock(null, 'hello', 'world', 'two'),
+                  100
+                ),
+
+              (err, ...args) => {
+                ctx.callback2(err, ...Array.from(args))
+                return finish(2)
+              }
+            )
+          })
+        })
+
+        it('should call the first callback', function (ctx) {
+          return ctx.callback1
+            .calledWith(null, 'hello', 'world', 'one')
+            .should.equal(true)
+        })
+
+        return it('should call the second callback', function (ctx) {
+          return ctx.callback2
+            .calledWith(null, 'hello', 'world', 'two')
+            .should.equal(true)
+        })
+      })
+    })
+  })
+})

services/clsi/test/unit/js/DockerRunner.test.js

@@ -5,7 +5,7 @@ import Path from 'node:path'
 
 const modulePath = Path.join(
   import.meta.dirname,
-  '../../../app/js/DockerRunner'
+  '../../../app/js/DockerRunner.mjs'
 )
 
 describe('DockerRunner', () => {

services/clsi/test/unit/js/ResourceWriter.test.js

@@ -19,6 +19,12 @@ describe('ResourceWriter', () => {
       }),
     }))
 
+    vi.doMock('@overleaf/settings', () => ({
+      default: (ctx.settings = {
+        preciousFilePattern: '{keepdir/**,.other/keepdir/**}',
+      }),
+    }))
+
     vi.doMock('../../../app/js/ResourceStateManager', () => ({
       default: (ctx.ResourceStateManager = {}),
     }))
@@ -246,6 +252,12 @@ describe('ResourceWriter', () => {
         {
           path: 'output.stderr',
         },
+        {
+          path: 'keepdir/file',
+        },
+        {
+          path: '.other/keepdir/.subdir/file',
+        },
       ]
       ctx.resources = 'mock-resources'
       ctx.request = {
@@ -297,6 +309,18 @@ describe('ResourceWriter', () => {
         .should.equal(true)
     })
 
+    it('should not delete additionally specified files', ctx => {
+      return ctx.ResourceWriter._deleteFileIfNotDirectory
+        .calledWith(path.join(ctx.basePath, 'keepdir/file'))
+        .should.equal(false)
+    })
+
+    it('should not delete additionally specified hidden files', ctx => {
+      return ctx.ResourceWriter._deleteFileIfNotDirectory
+        .calledWith(path.join(ctx.basePath, '.other/keepdir/.subdir/file'))
+        .should.equal(false)
+    })
+
     it('should not delete the extra aux files', ctx => {
       return ctx.ResourceWriter._deleteFileIfNotDirectory
         .calledWith(path.join(ctx.basePath, 'extra.aux'))

services/docstore/Dockerfile

@@ -29,6 +29,7 @@ COPY libraries/object-persistor/package.json /overleaf/libraries/object-persisto
 COPY libraries/promise-utils/package.json /overleaf/libraries/promise-utils/package.json
 COPY libraries/settings/package.json /overleaf/libraries/settings/package.json
 COPY libraries/stream-utils/package.json /overleaf/libraries/stream-utils/package.json
+COPY libraries/validation-tools/package.json /overleaf/libraries/validation-tools/package.json
 COPY services/docstore/package.json /overleaf/services/docstore/package.json
 COPY tools/migrations/package.json /overleaf/tools/migrations/package.json
 COPY .yarn/patches/ /overleaf/.yarn/patches/
@@ -45,6 +46,7 @@ COPY libraries/object-persistor/ /overleaf/libraries/object-persistor/
 COPY libraries/promise-utils/ /overleaf/libraries/promise-utils/
 COPY libraries/settings/ /overleaf/libraries/settings/
 COPY libraries/stream-utils/ /overleaf/libraries/stream-utils/
+COPY libraries/validation-tools/ /overleaf/libraries/validation-tools/
 COPY services/docstore/ /overleaf/services/docstore/
 COPY tools/migrations/ /overleaf/tools/migrations/
 

services/docstore/Makefile

@@ -2,10 +2,10 @@
 # Instead run bin/update_build_scripts from
 # https://github.com/overleaf/internal/
 
-BUILD_NUMBER ?= local
+export BUILD_NUMBER ?= local
-BRANCH_NAME ?= $(shell git rev-parse --abbrev-ref HEAD)
+export BRANCH_NAME ?= $(shell git rev-parse --abbrev-ref HEAD)
-BRANCH_NAME_TAG_SAFE = $(shell echo $(BRANCH_NAME) | sed 's/\//\-\-/g' | cut -c1-96)
+export BRANCH_NAME_TAG_SAFE = $(shell echo $(BRANCH_NAME) | sed 's/\//\-\-/g' | cut -c1-96)
-PROJECT_NAME = docstore
+export PROJECT_NAME = docstore
 BUILD_DIR_NAME = $(shell pwd | xargs basename | tr -cd '[a-zA-Z0-9_.\-]')
 HERE=$(shell pwd)
 export MONOREPO ?= $(shell cd ../../ && pwd)
@@ -26,17 +26,15 @@ IMAGE_CACHE ?= $(IMAGE_REPO):cache-$(shell cat \
 	$(MONOREPO)/libraries/promise-utils/package.json \
 	$(MONOREPO)/libraries/settings/package.json \
 	$(MONOREPO)/libraries/stream-utils/package.json \
+	$(MONOREPO)/libraries/validation-tools/package.json \
 	$(MONOREPO)/services/docstore/package.json \
 	$(MONOREPO)/tools/migrations/package.json \
 	$(MONOREPO)/.yarn/patches/* \
 | sha256sum | cut -d '-' -f1)
+export MOCHA_GREP ?=
 
 DOCKER_COMPOSE_FLAGS ?= -f docker-compose.yml
-DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \
+DOCKER_COMPOSE := docker compose ${DOCKER_COMPOSE_FLAGS}
-	BRANCH_NAME=$(BRANCH_NAME_TAG_SAFE) \
-	PROJECT_NAME=$(PROJECT_NAME) \
-	MOCHA_GREP=${MOCHA_GREP} \
-	docker compose ${DOCKER_COMPOSE_FLAGS}
 
 # Common flags for docker compose run.
 # --rm           Tear down test runner container

services/docstore/app.js

@@ -7,11 +7,7 @@ import Settings from '@overleaf/settings'
 import logger from '@overleaf/logger'
 import express from 'express'
 import bodyParser from 'body-parser'
-import {
+import { handleValidationError } from '@overleaf/validation-tools'
-  celebrate as validate,
-  Joi,
-  errors as handleValidationErrors,
-} from 'celebrate'
 import mongodb from './app/js/mongodb.js'
 import Errors from './app/js/Errors.js'
 import HttpController from './app/js/HttpController.js'
@@ -79,13 +75,6 @@ app.post(
 app.patch(
   '/project/:project_id/doc/:doc_id',
   bodyParser.json(),
-  validate({
-    body: {
-      deleted: Joi.boolean(),
-      name: Joi.string().when('deleted', { is: true, then: Joi.required() }),
-      deletedAt: Joi.date().when('deleted', { is: true, then: Joi.required() }),
-    },
-  }),
   HttpController.patchDoc
 )
 app.delete('/project/:project_id/doc/:doc_id', (req, res) => {
@@ -101,7 +90,7 @@ app.get('/health_check', HttpController.healthCheck)
 
 app.get('/status', (req, res) => res.send('docstore is alive'))
 
-app.use(handleValidationErrors())
+app.use(handleValidationError)
 app.use(function (error, req, res, next) {
   if (error instanceof Errors.NotFoundError) {
     logger.warn({ req }, 'not found')

services/docstore/app/js/HttpController.js

@@ -5,6 +5,17 @@ import HealthChecker from './HealthChecker.js'
 import Errors from './Errors.js'
 import Settings from '@overleaf/settings'
 import { expressify } from '@overleaf/promise-utils'
+import { parseReq, z } from '@overleaf/validation-tools'
+
+const patchDocSchema = z.object({
+  body: z
+    .object({
+      deleted: z.literal(true),
+      deletedAt: z.coerce.date(),
+      name: z.string(),
+    })
+    .strict(),
+})
 
 async function getDoc(req, res) {
   const { doc_id: docId, project_id: projectId } = req.params
@@ -182,18 +193,9 @@ async function updateDoc(req, res) {
 }
 
 async function patchDoc(req, res) {
+  const { body: meta } = parseReq(req, patchDocSchema)
   const { doc_id: docId, project_id: projectId } = req.params
   logger.debug({ projectId, docId }, 'patching doc')
-
-  const allowedFields = ['deleted', 'deletedAt', 'name']
-  const meta = {}
-  Object.entries(req.body).forEach(([field, value]) => {
-    if (allowedFields.includes(field)) {
-      meta[field] = value
-    } else {
-      logger.fatal({ field }, 'joi validation for pathDoc is broken')
-    }
-  })
   await DocManager.patchDoc(projectId, docId, meta)
   res.sendStatus(204)
 }

services/docstore/buildscript.txt

@@ -5,7 +5,6 @@ docstore
 --env-pass-through=
 --esmock-loader=False
 --node-version=24.14.1
---package-name=@overleaf/docstore
 --pipeline-owner=32
 --public-repo=True
 --test-unit-vitest=True

services/docstore/docker-compose.ci.yml

@@ -4,7 +4,7 @@
 
 services:
   test_unit:
-    image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER
+    image: ci/$PROJECT_NAME:$BRANCH_NAME_TAG_SAFE-$BUILD_NUMBER
     user: node
     volumes:
       - ./reports:/overleaf/services/docstore/reports
@@ -23,7 +23,7 @@ services:
         condition: service_started
 
   test_acceptance:
-    image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER
+    image: ci/$PROJECT_NAME:$BRANCH_NAME_TAG_SAFE-$BUILD_NUMBER
     environment:
       CI:
       RETRIES:
@@ -49,7 +49,7 @@ services:
 
   tar:
     build: .
-    image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER
+    image: ci/$PROJECT_NAME:$BRANCH_NAME_TAG_SAFE-$BUILD_NUMBER
     volumes:
       - ./:/tmp/build/
     command: tar -czf /tmp/build/build.tar.gz --exclude=build.tar.gz --exclude-vcs .

services/docstore/package.json

@@ -25,10 +25,10 @@
     "@overleaf/promise-utils": "workspace:*",
     "@overleaf/settings": "workspace:*",
     "@overleaf/stream-utils": "^0.1.0",
+    "@overleaf/validation-tools": "workspace:*",
     "async": "^3.2.5",
     "body-parser": "1.20.4",
     "bunyan": "^1.8.15",
-    "celebrate": "^15.0.3",
     "express": "4.22.1",
     "lodash": "^4.18.1",
     "mongodb-legacy": "6.1.3",

services/docstore/test/acceptance/js/helpers/MongoHelper.js

@@ -5,6 +5,6 @@ before('run migrations', async function () {
   this.timeout(60_000)
 
   await promisify(exec)(
-    `cd ../../tools/migrations && npm run migrations -- migrate -t server-ce`
+    `cd ../../tools/migrations && yarn run migrations migrate -t 'server-ce & !nonblocking'`
   )
 })

services/docstore/test/unit/js/HttpController.test.js

@@ -486,7 +486,11 @@ describe('HttpController', () => {
         project_id: ctx.projectId,
         doc_id: ctx.docId,
       }
-      ctx.req.body = { name: 'foo.tex' }
+      ctx.req.body = {
+        deleted: true,
+        deletedAt: '2026-06-15T00:00:00Z',
+        name: 'foo.tex',
+      }
       ctx.DocManager.patchDoc = sinon.stub().resolves()
       await ctx.HttpController.patchDoc(ctx.req, ctx.res, ctx.next)
     })
@@ -494,7 +498,12 @@ describe('HttpController', () => {
     it('should delete the document', ctx => {
       expect(ctx.DocManager.patchDoc).to.have.been.calledWith(
         ctx.projectId,
-        ctx.docId
+        ctx.docId,
+        {
+          deleted: true,
+          deletedAt: new Date('2026-06-15T00:00:00Z'),
+          name: 'foo.tex',
+        }
       )
     })
 
@@ -510,19 +519,13 @@ describe('HttpController', () => {
         await ctx.HttpController.patchDoc(ctx.req, ctx.res, ctx.next)
       })
 
-      it('should log a message', ctx => {
+      it('should pass a validation error to next', ctx => {
-        expect(ctx.logger.fatal).to.have.been.calledWith(
+        expect(ctx.next).to.have.been.calledOnce
-          { field: 'cannot' },
+        expect(ctx.next.firstCall.args[0].name).to.equal('InvalidRequestError')
-          'joi validation for pathDoc is broken'
-        )
       })
 
-      it('should not pass the invalid field along', ctx => {
+      it('should not patch the document', ctx => {
-        expect(ctx.DocManager.patchDoc).to.have.been.calledWith(
+        expect(ctx.DocManager.patchDoc).not.to.have.been.called
-          ctx.projectId,
-          ctx.docId,
-          {}
-        )
       })
     })
   })

services/document-updater/Makefile

@@ -2,10 +2,10 @@
 # Instead run bin/update_build_scripts from
 # https://github.com/overleaf/internal/
 
-BUILD_NUMBER ?= local
+export BUILD_NUMBER ?= local
-BRANCH_NAME ?= $(shell git rev-parse --abbrev-ref HEAD)
+export BRANCH_NAME ?= $(shell git rev-parse --abbrev-ref HEAD)
-BRANCH_NAME_TAG_SAFE = $(shell echo $(BRANCH_NAME) | sed 's/\//\-\-/g' | cut -c1-96)
+export BRANCH_NAME_TAG_SAFE = $(shell echo $(BRANCH_NAME) | sed 's/\//\-\-/g' | cut -c1-96)
-PROJECT_NAME = document-updater
+export PROJECT_NAME = document-updater
 BUILD_DIR_NAME = $(shell pwd | xargs basename | tr -cd '[a-zA-Z0-9_.\-]')
 HERE=$(shell pwd)
 export MONOREPO ?= $(shell cd ../../ && pwd)
@@ -31,13 +31,10 @@ IMAGE_CACHE ?= $(IMAGE_REPO):cache-$(shell cat \
 	$(MONOREPO)/tools/migrations/package.json \
 	$(MONOREPO)/.yarn/patches/* \
 | sha256sum | cut -d '-' -f1)
+export MOCHA_GREP ?=
 
 DOCKER_COMPOSE_FLAGS ?= -f docker-compose.yml
-DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \
+DOCKER_COMPOSE := docker compose ${DOCKER_COMPOSE_FLAGS}
-	BRANCH_NAME=$(BRANCH_NAME_TAG_SAFE) \
-	PROJECT_NAME=$(PROJECT_NAME) \
-	MOCHA_GREP=${MOCHA_GREP} \
-	docker compose ${DOCKER_COMPOSE_FLAGS}
 
 # Common flags for docker compose run.
 # --rm           Tear down test runner container