mirror of
https://github.com/yu-i-i/overleaf-cep.git
synced 2026-11-12 21:23:45 +01:00
Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Miguel Serrano
|
f0022392ac |
@@ -1,25 +0,0 @@
|
||||
root = true
|
||||
|
||||
[*]
|
||||
charset = utf-8
|
||||
indent_style = space
|
||||
indent_size = 2
|
||||
end_of_line = lf
|
||||
insert_final_newline = true
|
||||
trim_trailing_whitespace = true
|
||||
|
||||
[Makefile]
|
||||
indent_style = tab
|
||||
|
||||
[*.go]
|
||||
indent_style = tab
|
||||
|
||||
[*.{pug,coffee}]
|
||||
indent_style = tab
|
||||
|
||||
[*.{pug,patch}]
|
||||
trim_trailing_whitespace = false
|
||||
|
||||
[Jenkinsfile]
|
||||
insert_final_newline = false
|
||||
max_line_length = off
|
||||
@@ -0,0 +1,44 @@
|
||||
<!--
|
||||
|
||||
Note: If you are using www.overleaf.com and have a problem,
|
||||
or if you would like to request a new feature please contact
|
||||
the support team at support@overleaf.com
|
||||
|
||||
This form should only be used to report bugs in the
|
||||
Community Edition release of Overleaf.
|
||||
|
||||
-->
|
||||
|
||||
|
||||
|
||||
<!-- BUG REPORT TEMPLATE -->
|
||||
|
||||
## Steps to Reproduce
|
||||
<!-- Describe the steps leading up to when / where you found the bug. -->
|
||||
<!-- Screenshots may be helpful here. -->
|
||||
|
||||
1.
|
||||
2.
|
||||
3.
|
||||
|
||||
## Expected Behaviour
|
||||
<!-- What should have happened when you completed the steps above? -->
|
||||
|
||||
## Observed Behaviour
|
||||
<!-- What actually happened when you completed the steps above? -->
|
||||
<!-- Screenshots may be helpful here. -->
|
||||
|
||||
## Context
|
||||
<!-- How has this issue affected you? What were you trying to accomplish? -->
|
||||
|
||||
## Technical Info
|
||||
<!-- Provide any technical details that may be applicable (or N/A if not applicable). -->
|
||||
|
||||
* URL:
|
||||
* Browser Name and version:
|
||||
* Operating System and version (desktop or mobile):
|
||||
* Signed in as:
|
||||
* Project and/or file:
|
||||
|
||||
## Analysis
|
||||
<!--- Optionally, document investigation of / suggest a fix for the bug, e.g. 'comes from this line / commit' -->
|
||||
@@ -1,49 +0,0 @@
|
||||
---
|
||||
name: Bug report
|
||||
about: Report a bug
|
||||
title: ''
|
||||
labels: type:bug
|
||||
assignees: ''
|
||||
|
||||
---
|
||||
|
||||
<!--
|
||||
|
||||
Note: This form should only be used to report bugs in the
|
||||
Extended CE release of Overleaf.
|
||||
|
||||
-->
|
||||
|
||||
|
||||
|
||||
<!-- BUG REPORT TEMPLATE -->
|
||||
|
||||
## Steps to Reproduce
|
||||
<!-- Describe the steps leading up to when / where you found the bug. -->
|
||||
<!-- Screenshots may be helpful here. -->
|
||||
|
||||
1.
|
||||
2.
|
||||
3.
|
||||
|
||||
## Expected Behaviour
|
||||
<!-- What should have happened when you completed the steps above? -->
|
||||
|
||||
## Observed Behaviour
|
||||
<!-- What actually happened when you completed the steps above? -->
|
||||
<!-- Screenshots may be helpful here. -->
|
||||
|
||||
## Context
|
||||
<!-- How has this issue affected you? What were you trying to accomplish? -->
|
||||
|
||||
## Technical Info
|
||||
<!-- Provide any technical details that may be applicable (or N/A if not applicable). -->
|
||||
|
||||
* URL:
|
||||
* Browser Name and version:
|
||||
* Operating System and version (desktop or mobile):
|
||||
* Signed in as:
|
||||
* Project and/or file:
|
||||
|
||||
## Analysis
|
||||
<!--- Optionally, document investigation of / suggest a fix for the bug, e.g. 'comes from this line / commit' -->
|
||||
@@ -4,3 +4,8 @@
|
||||
|
||||
## Related issues / Pull Requests
|
||||
<!-- Fixes #xyz, Contributes to #xyz, Related to #xyz-->
|
||||
|
||||
|
||||
## Contributor Agreement
|
||||
|
||||
- [ ] I confirm I have signed the [Contributor License Agreement](https://github.com/overleaf/overleaf/blob/master/CONTRIBUTING.md#contributor-license-agreement)
|
||||
|
||||
+16
-8
@@ -1,33 +1,41 @@
|
||||
Contributing to Overleaf CE+
|
||||
============================
|
||||
Contributing to Overleaf
|
||||
==========================
|
||||
|
||||
Thank you for reading this! If you'd like to report a bug or join in the development
|
||||
of Overleaf CE+, then here are some notes on how to do that.
|
||||
of Overleaf, then here are some notes on how to do that.
|
||||
|
||||
Reporting bugs and opening issues
|
||||
---------------------------------
|
||||
|
||||
If you'd like to report a bug or open an issue, please **[check if there is an existing issue](https://github.com/yu-i-i/overleaf-cep).**
|
||||
If you'd like to report a bug or open an issue, please **[check if there is an existing issue](https://github.com/overleaf/overleaf/issues).**
|
||||
If there is then please add any more information that you have, or give it a 👍.
|
||||
|
||||
When submitting an issue please describe the issue as clearly as possible, including how to
|
||||
reproduce the bug, which situations it appears in, what you expected to happen, and what actually happens.
|
||||
If you can include a screenshot for front end issues that is very helpful.
|
||||
|
||||
|
||||
Pull Requests
|
||||
-------------
|
||||
|
||||
See [Overleaf wiki](https://github.com/overleaf/overleaf/wiki)
|
||||
See [our wiki](https://github.com/overleaf/overleaf/wiki)
|
||||
for how to manage the Overleaf development environment and for our developer guidelines.
|
||||
|
||||
We love pull requests, so be bold with them! Don't be afraid of going ahead
|
||||
and changing something, or adding a new feature. We're very happy to work with you
|
||||
to get your changes merged into Overleaf.
|
||||
|
||||
If you're looking for something to work on, have a look at the [open issues](https://github.com/yu-i-i/overleaf-cep/issues).
|
||||
If you're looking for something to work on, have a look at the [open issues](https://github.com/overleaf/overleaf/issues).
|
||||
|
||||
Security
|
||||
--------
|
||||
|
||||
Please see [Overleaf security policy](https://github.com/overleaf/overleaf/security/policy) if you would like to report a potential security vulnerability.
|
||||
Please see [our security policy](https://github.com/overleaf/overleaf/security/policy) if you would like to report a potential security vulnerability.
|
||||
|
||||
Contributor License Agreement
|
||||
-----------------------------
|
||||
|
||||
Before we can accept any contributions of code, we need you to agree to our
|
||||
[Contributor License Agreement](https://docs.google.com/forms/d/e/1FAIpQLSef79XH3mb7yIiMzZw-yALEegS-wyFetvjTiNBfZvf_IHD2KA/viewform?usp=sf_link).
|
||||
This is to ensure that you own the copyright of your contribution, and that you
|
||||
agree to give us a license to use it in both the open source version, and the version
|
||||
of Overleaf running at www.overleaf.com, which may have additional changes.
|
||||
|
||||
-70
@@ -1,70 +0,0 @@
|
||||
# CREDITS
|
||||
|
||||
This project includes ideas and adapted code from several open-source projects.\
|
||||
In most cases the original code has been modified, optimized, or extended.
|
||||
|
||||
## Symbol palette
|
||||
|
||||
The symbol palette feature is based on the original Overleaf implementation:
|
||||
|
||||
https://github.com/overleaf/web/tree/master/frontend/js/features/symbol-palette
|
||||
|
||||
The original code was slightly improved, particularly in the parts related to keyboard input.
|
||||
|
||||
## LDAP authentication
|
||||
|
||||
LDAP authentication was inspired by:
|
||||
|
||||
https://github.com/smhaller/ldap-overleaf-sl
|
||||
|
||||
The project provided the idea and part of the implementation for adding LDAP users to the user's contacts.
|
||||
|
||||
## Autocomplete of reference keys
|
||||
|
||||
Autocomplete of reference keys is based on:
|
||||
|
||||
https://github.com/lcpu-club/overleaf
|
||||
|
||||
The implementation largely follows the referenced code, except that a different `.bib` file parser is used.
|
||||
|
||||
## Real-time track changes and comments
|
||||
|
||||
The Track Changes and Comments feature exists largely in the original Overleaf codebase.\
|
||||
The missing parts were implemented based on:
|
||||
|
||||
https://github.com/ertuil/overleaf
|
||||
|
||||
The referenced code was fixed, optimized, and extended.
|
||||
|
||||
## Sandboxed compiles
|
||||
|
||||
The Sandboxed Compiles feature was until recently largely present in the original Overleaf codebase.\
|
||||
The missing parts were implemented in this project.
|
||||
|
||||
## Import file from external URL
|
||||
|
||||
The "From External URL" feature exists in the original Overleaf code.\
|
||||
The missing proxy component was implemented in this project.
|
||||
|
||||
## Git integration
|
||||
|
||||
Git integration is based on:
|
||||
|
||||
https://github.com/ayaka-notes/overleaf-pro/tree/feat-git-bridge
|
||||
|
||||
The backend was completely rewritten.\
|
||||
The frontend uses the referenced code, which was optimized and improved.
|
||||
|
||||
## Original Features
|
||||
|
||||
The following features are original work implemented in this project:
|
||||
|
||||
- Template Gallery
|
||||
- Advanced administrator tools for managing user accounts and projects
|
||||
- SAML authentication
|
||||
- OpenID Connect authentication
|
||||
|
||||
## Acknowledgments
|
||||
|
||||
Thanks to the users of the project for valuable feedback, suggestions,
|
||||
and help in identifying and fixing bugs.
|
||||
@@ -6,60 +6,47 @@
|
||||
<h4 align="center">An open-source online real-time collaborative LaTeX editor.</h4>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://github.com/yu-i-i/overleaf-cep/wiki">Wiki</a> •
|
||||
<a href="https://github.com/overleaf/overleaf/wiki">Wiki</a> •
|
||||
<a href="https://www.overleaf.com/for/enterprises">Server Pro</a> •
|
||||
<a href="#contributing">Contributing</a> •
|
||||
<a href="https://mailchi.mp/overleaf.com/community-edition-and-server-pro">Mailing List</a> •
|
||||
<a href="#authors">Authors</a> •
|
||||
<a href="#license">License</a>
|
||||
</p>
|
||||
|
||||
<img src="doc/screenshot.png" alt="A screenshot of a project being edited in Overleaf Extended Community Edition">
|
||||
<img src="doc/screenshot.png" alt="A screenshot of a project being edited in Overleaf Community Edition">
|
||||
<p align="center">
|
||||
Figure 1: A screenshot of a project being edited in Overleaf Extended Community Edition.
|
||||
Figure 1: A screenshot of a project being edited in Overleaf Community Edition.
|
||||
</p>
|
||||
|
||||
## Community Edition
|
||||
|
||||
[Overleaf](https://www.overleaf.com) is an open-source online real-time collaborative LaTeX editor. Overleaf runs a hosted version at [www.overleaf.com](https://www.overleaf.com), but you can also run your own local version, and contribute to the development of Overleaf.
|
||||
|
||||
## Extended Community Edition (CE+)
|
||||
|
||||
The present "extended" version of Overleaf CE includes:
|
||||
|
||||
- Template gallery
|
||||
- Sandboxed compiles with TeX Live image selection
|
||||
- LDAP authentication
|
||||
- SAML authentication
|
||||
- OpenID Connect authentication
|
||||
- Real-time track changes and comments
|
||||
- Autocomplete of reference keys
|
||||
- Symbol palette
|
||||
- Import file from external URL
|
||||
- Advanced administrator tools for managing user accounts and projects
|
||||
- Git integration
|
||||
|
||||
> [!CAUTION]
|
||||
> Overleaf Community Edition is intended for use in environments where **all** users are trusted. Community Edition is **not** appropriate for scenarios where isolation of users is required due to Sandbox Compiles not being available. When not using Sandboxed Compiles, users have full read and write access to the `sharelatex` container resources (filesystem, network, environment variables) when running LaTeX compiles.
|
||||
Therefore, in any environment where not all users can be fully trusted, it is strongly recommended to enable the Sandboxed Compiles feature available in the Extended Community Edition.
|
||||
|
||||
For more information on Sandbox Compiles check out Overleaf [documentation](https://docs.overleaf.com/on-premises/configuration/overleaf-toolkit/server-pro-only-configuration/sandboxed-compiles).
|
||||
[Overleaf](https://www.overleaf.com) is an open-source online real-time collaborative LaTeX editor. We run a hosted version at [www.overleaf.com](https://www.overleaf.com), but you can also run your own local version, and contribute to the development of Overleaf.
|
||||
|
||||
## Enterprise
|
||||
|
||||
If you want help installing and maintaining Overleaf in your lab or workplace, Overleaf offers an officially supported version called [Overleaf Server Pro](https://www.overleaf.com/for/enterprises).
|
||||
If you want help installing and maintaining Overleaf in your lab or workplace, we offer an officially supported version called [Overleaf Server Pro](https://www.overleaf.com/for/enterprises). It also includes more features for security (SSO with LDAP or SAML), administration and collaboration (e.g. tracked changes). [Find out more!](https://www.overleaf.com/for/enterprises)
|
||||
|
||||
## Keeping up to date
|
||||
|
||||
Sign up to the [mailing list](https://mailchi.mp/overleaf.com/community-edition-and-server-pro) to get updates on Overleaf releases and development.
|
||||
|
||||
## Installation
|
||||
|
||||
Detailed installation instructions can be found in the [Overleaf Toolkit](https://github.com/overleaf/toolkit/).
|
||||
Configuration details and release history for the Extended Community Edition can be found on the [Extended CE Wiki Page](https://github.com/yu-i-i/overleaf-cep/wiki).
|
||||
We have detailed installation instructions in the [Overleaf Toolkit](https://github.com/overleaf/toolkit/).
|
||||
|
||||
## Upgrading
|
||||
|
||||
If you are upgrading from a previous version of Overleaf, please see the [Release Notes section on the Wiki](https://github.com/overleaf/overleaf/wiki#release-notes) for all of the versions between your current version and the version you are upgrading to.
|
||||
|
||||
## Overleaf Docker Image
|
||||
|
||||
This repo contains two dockerfiles, [`Dockerfile-base`](server-ce/Dockerfile-base), which builds the
|
||||
`sharelatex/sharelatex-base:ext-ce` image, and [`Dockerfile`](server-ce/Dockerfile) which builds the
|
||||
`sharelatex/sharelatex:ext-ce` image.
|
||||
`sharelatex/sharelatex-base` image, and [`Dockerfile`](server-ce/Dockerfile) which builds the
|
||||
`sharelatex/sharelatex` (or "community") image.
|
||||
|
||||
The Base image generally contains the basic dependencies like `wget`, plus `texlive`.
|
||||
This is split out because it's a pretty heavy set of
|
||||
The Base image generally contains the basic dependencies like `wget` and
|
||||
`aspell`, plus `texlive`. We split this out because it's a pretty heavy set of
|
||||
dependencies, and it's nice to not have to rebuild all of that every time.
|
||||
|
||||
The `sharelatex/sharelatex` image extends the base image and adds the actual Overleaf code
|
||||
@@ -67,22 +54,23 @@ and services.
|
||||
|
||||
Use `make build-base` and `make build-community` from `server-ce/` to build these images.
|
||||
|
||||
The [Phusion base-image](https://github.com/phusion/baseimage-docker)
|
||||
(which is extended by the `base` image) provides a VM-like container
|
||||
We use the [Phusion base-image](https://github.com/phusion/baseimage-docker)
|
||||
(which is extended by our `base` image) to provide us with a VM-like container
|
||||
in which to run the Overleaf services. Baseimage uses the `runit` service
|
||||
manager to manage services, and init scripts from the `server-ce/runit`
|
||||
folder are added.
|
||||
manager to manage services, and we add our init-scripts from the `server-ce/runit`
|
||||
folder.
|
||||
|
||||
|
||||
## Contributing
|
||||
|
||||
Please see the [CONTRIBUTING](CONTRIBUTING.md) file for information on contributing to the development of Overleaf.
|
||||
|
||||
## Authors
|
||||
|
||||
[The Overleaf Team](https://www.overleaf.com/about)\
|
||||
[yu-i-i](https://github.com/yu-i-i/overleaf-cep) — CE extensions; references to adapted code are listed in [`CREDITS`](CREDITS.md)
|
||||
[The Overleaf Team](https://www.overleaf.com/about)
|
||||
|
||||
## License
|
||||
|
||||
The code in this repository is released under the GNU AFFERO GENERAL PUBLIC LICENSE, version 3. A copy can be found in the [`LICENSE`](LICENSE) file.
|
||||
|
||||
Copyright (c) Overleaf, 2014-2026.\
|
||||
Copyright (c) yu-i-i, 2024-2026, for CE extensions.
|
||||
|
||||
Portions of the code are derived from other open-source projects; see [`CREDITS`](CREDITS.md).
|
||||
Copyright (c) Overleaf, 2014-2024.
|
||||
|
||||
@@ -1,3 +0,0 @@
|
||||
/* eslint-disable no-undef */
|
||||
|
||||
rs.initiate({ _id: 'overleaf', members: [{ _id: 0, host: 'mongo:27017' }] })
|
||||
+7
-3
@@ -11,6 +11,12 @@ bin/build
|
||||
> [!NOTE]
|
||||
> If Docker is running out of RAM while building the services in parallel, create a `.env` file in this directory containing `COMPOSE_PARALLEL_LIMIT=1`.
|
||||
|
||||
Next, initialize the database:
|
||||
|
||||
```shell
|
||||
bin/init
|
||||
```
|
||||
|
||||
Then start the services:
|
||||
|
||||
```shell
|
||||
@@ -42,7 +48,7 @@ To do this, use the included `bin/dev` script:
|
||||
bin/dev
|
||||
```
|
||||
|
||||
This will start all services using `node --watch`, which will automatically monitor the code and restart the services as necessary.
|
||||
This will start all services using `nodemon`, which will automatically monitor the code and restart the services as necessary.
|
||||
|
||||
To improve performance, you can start only a subset of the services in development mode by providing a space-separated list to the `bin/dev` script:
|
||||
|
||||
@@ -77,10 +83,8 @@ each service:
|
||||
| `filestore` | 9235 |
|
||||
| `notifications` | 9236 |
|
||||
| `real-time` | 9237 |
|
||||
| `references` | 9238 |
|
||||
| `history-v1` | 9239 |
|
||||
| `project-history` | 9240 |
|
||||
| `linked-url-proxy` | 9241 |
|
||||
|
||||
To attach to a service using Chrome's _remote debugging_, go to
|
||||
<chrome://inspect/> and make sure _Discover network targets_ is checked. Next
|
||||
|
||||
Executable
+6
@@ -0,0 +1,6 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
docker compose up --detach mongo
|
||||
curl --max-time 10 --retry 5 --retry-delay 5 --retry-all-errors --silent --output /dev/null localhost:27017
|
||||
docker compose exec mongo mongosh --eval "rs.initiate({ _id: 'overleaf', members: [{ _id: 0, host: 'mongo:27017' }] })"
|
||||
docker compose down mongo
|
||||
+1
-7
@@ -6,20 +6,14 @@ DOCUMENT_UPDATER_HOST=document-updater
|
||||
FILESTORE_HOST=filestore
|
||||
GRACEFUL_SHUTDOWN_DELAY_SECONDS=0
|
||||
HISTORY_V1_HOST=history-v1
|
||||
HISTORY_REDIS_HOST=redis
|
||||
LINKED_URL_PROXY_HOST=linked-url-proxy
|
||||
LISTEN_ADDRESS=0.0.0.0
|
||||
MONGO_HOST=mongo
|
||||
MONGO_URL=mongodb://mongo/sharelatex?directConnection=true
|
||||
NOTIFICATIONS_HOST=notifications
|
||||
PROJECT_HISTORY_HOST=project-history
|
||||
QUEUES_REDIS_HOST=redis
|
||||
DSMP_REDIS_HOST=redis
|
||||
REALTIME_HOST=real-time
|
||||
REDIS_HOST=redis
|
||||
REFERENCES_HOST=references
|
||||
SESSION_SECRET=foo
|
||||
V1_HISTORY_HOST=history-v1
|
||||
SPELLING_HOST=spelling
|
||||
WEBPACK_HOST=webpack
|
||||
WEB_API_PASSWORD=overleaf
|
||||
WEB_API_USER=overleaf
|
||||
|
||||
@@ -79,26 +79,15 @@ services:
|
||||
- ../services/history-v1/knexfile.js:/overleaf/services/history-v1/knexfile.js
|
||||
- ../services/history-v1/migrations:/overleaf/services/history-v1/migrations
|
||||
|
||||
linked-url-proxy:
|
||||
command: ["node", "--watch", "app.mjs"]
|
||||
environment:
|
||||
- NODE_OPTIONS=--inspect=0.0.0.0:9229
|
||||
ports:
|
||||
- "127.0.0.1:9241:9229"
|
||||
volumes:
|
||||
- ../services/linked-url-proxy/app:/overleaf/services/linked-url-proxy/app
|
||||
- ../services/linked-url-proxy/config:/overleaf/services/linked-url-proxy/config
|
||||
- ../services/linked-url-proxy/app.mjs:/overleaf/services/linked-url-proxy/app.mjs
|
||||
|
||||
notifications:
|
||||
command: ["node", "--watch", "app.ts"]
|
||||
command: ["node", "--watch", "app.js"]
|
||||
environment:
|
||||
- NODE_OPTIONS=--inspect=0.0.0.0:9229
|
||||
ports:
|
||||
- "127.0.0.1:9236:9229"
|
||||
volumes:
|
||||
- ../services/notifications/app:/overleaf/services/notifications/app
|
||||
- ../services/notifications/app.ts:/overleaf/services/notifications/app.ts
|
||||
- ../services/notifications/app.js:/overleaf/services/notifications/app.js
|
||||
- ../services/notifications/config:/overleaf/services/notifications/config
|
||||
|
||||
project-history:
|
||||
@@ -123,19 +112,8 @@ services:
|
||||
- ../services/real-time/app.js:/overleaf/services/real-time/app.js
|
||||
- ../services/real-time/config:/overleaf/services/real-time/config
|
||||
|
||||
references:
|
||||
command: ["node", "--watch", "app.js"]
|
||||
environment:
|
||||
- NODE_OPTIONS=--inspect=0.0.0.0:9229
|
||||
ports:
|
||||
- "127.0.0.1:9238:9229"
|
||||
volumes:
|
||||
- ../services/references/app:/overleaf/services/references/app
|
||||
- ../services/references/config:/overleaf/services/references/config
|
||||
- ../services/references/app.js:/overleaf/services/references/app.js
|
||||
|
||||
web:
|
||||
command: ["node", "--watch", "app.mjs", "--watch-locales"]
|
||||
command: ["node", "--watch", "app.js", "--watch-locales"]
|
||||
environment:
|
||||
- NODE_OPTIONS=--inspect=0.0.0.0:9229
|
||||
ports:
|
||||
|
||||
+14
-41
@@ -1,5 +1,6 @@
|
||||
volumes:
|
||||
clsi-cache:
|
||||
clsi-output:
|
||||
filestore-public-files:
|
||||
filestore-template-files:
|
||||
filestore-uploads:
|
||||
@@ -7,6 +8,7 @@ volumes:
|
||||
mongo-data:
|
||||
redis-data:
|
||||
sharelatex-data:
|
||||
spelling-cache:
|
||||
web-data:
|
||||
history-v1-buckets:
|
||||
|
||||
@@ -25,16 +27,15 @@ services:
|
||||
env_file:
|
||||
- dev.env
|
||||
environment:
|
||||
- DOCKER_RUNNER=true
|
||||
- TEXLIVE_IMAGE=texlive-full # docker build texlive -t texlive-full
|
||||
- SANDBOXED_COMPILES=true
|
||||
- SANDBOXED_COMPILES_HOST_DIR_COMPILES=${PWD}/compiles
|
||||
- SANDBOXED_COMPILES_HOST_DIR_OUTPUT=${PWD}/output
|
||||
- COMPILES_HOST_DIR=${PWD}/compiles
|
||||
user: root
|
||||
volumes:
|
||||
- ${PWD}/compiles:/overleaf/services/clsi/compiles
|
||||
- ${PWD}/output:/overleaf/services/clsi/output
|
||||
- ${DOCKER_SOCKET_PATH:-/var/run/docker.sock}:/var/run/docker.sock
|
||||
- clsi-cache:/overleaf/services/clsi/cache
|
||||
- clsi-output:/overleaf/services/clsi/output
|
||||
|
||||
contacts:
|
||||
build:
|
||||
@@ -69,6 +70,7 @@ services:
|
||||
- filestore-public-files:/overleaf/services/filestore/public_files
|
||||
- filestore-template-files:/overleaf/services/filestore/template_files
|
||||
- filestore-uploads:/overleaf/services/filestore/uploads
|
||||
- filestore-user-files:/overleaf/services/filestore/user_files
|
||||
|
||||
history-v1:
|
||||
build:
|
||||
@@ -86,28 +88,13 @@ services:
|
||||
volumes:
|
||||
- history-v1-buckets:/buckets
|
||||
|
||||
linked-url-proxy:
|
||||
build:
|
||||
context: ..
|
||||
dockerfile: services/linked-url-proxy/Dockerfile
|
||||
env_file:
|
||||
- dev.env
|
||||
|
||||
mongo:
|
||||
image: mongo:6.0
|
||||
image: mongo:5
|
||||
command: --replSet overleaf
|
||||
ports:
|
||||
- "127.0.0.1:27017:27017" # for debugging
|
||||
volumes:
|
||||
- mongo-data:/data/db
|
||||
- ../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js
|
||||
environment:
|
||||
MONGO_INITDB_DATABASE: sharelatex
|
||||
extra_hosts:
|
||||
# Required when using the automatic database setup for initializing the
|
||||
# replica set. This override is not needed when running the setup after
|
||||
# starting up mongo.
|
||||
- mongo:127.0.0.1
|
||||
|
||||
notifications:
|
||||
build:
|
||||
@@ -137,12 +124,14 @@ services:
|
||||
volumes:
|
||||
- redis-data:/data
|
||||
|
||||
references:
|
||||
spelling:
|
||||
build:
|
||||
context: ..
|
||||
dockerfile: services/references/Dockerfile
|
||||
dockerfile: services/spelling/Dockerfile
|
||||
env_file:
|
||||
- dev.env
|
||||
volumes:
|
||||
- spelling-cache:/overleaf/services/spelling/cache
|
||||
|
||||
web:
|
||||
build:
|
||||
@@ -157,7 +146,7 @@ services:
|
||||
- EMAIL_CONFIRMATION_DISABLED=true
|
||||
- NODE_ENV=development
|
||||
- OVERLEAF_ALLOW_PUBLIC_ACCESS=true
|
||||
command: ["node", "app.mjs"]
|
||||
command: ["node", "app.js"]
|
||||
volumes:
|
||||
- sharelatex-data:/var/lib/overleaf
|
||||
- web-data:/overleaf/services/web/data
|
||||
@@ -171,34 +160,18 @@ services:
|
||||
- document-updater
|
||||
- filestore
|
||||
- history-v1
|
||||
- linked-url-proxy
|
||||
- notifications
|
||||
- project-history
|
||||
- real-time
|
||||
- references
|
||||
- spelling
|
||||
|
||||
webpack:
|
||||
build:
|
||||
context: ..
|
||||
dockerfile: services/web/Dockerfile
|
||||
target: webpack
|
||||
target: dev
|
||||
command: ["npx", "webpack", "serve", "--config", "webpack.config.dev-env.js"]
|
||||
ports:
|
||||
- "127.0.0.1:80:3808"
|
||||
volumes:
|
||||
- ./webpack.config.dev-env.js:/overleaf/services/web/webpack.config.dev-env.js
|
||||
|
||||
git-bridge:
|
||||
image: quay.io/sharelatex/git-bridge:latest
|
||||
expose:
|
||||
- "8000"
|
||||
ports:
|
||||
- "8000:8000"
|
||||
environment:
|
||||
GIT_BRIDGE_API_BASE_URL: "http://web:3000/api/v0"
|
||||
GIT_BRIDGE_OAUTH2_SERVER: "http://web:3000"
|
||||
GIT_BRIDGE_POSTBACK_BASE_URL: "http://git-bridge:8000"
|
||||
GIT_BRIDGE_ROOT_DIR: "/data/git-bridge"
|
||||
LOG_LEVEL: "DEBUG"
|
||||
volumes:
|
||||
- ./data/git-bridge:/data/git-bridge
|
||||
|
||||
@@ -14,11 +14,6 @@ module.exports = merge(base, {
|
||||
target: 'http://real-time:3026',
|
||||
ws: true,
|
||||
},
|
||||
{
|
||||
context: '/git/**',
|
||||
target: 'http://git-bridge:8000',
|
||||
pathRewrite: { '^/git': '' }
|
||||
},
|
||||
{
|
||||
context: ['!**/*.js', '!**/*.css', '!**/*.json'],
|
||||
target: 'http://web:3000',
|
||||
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 13 KiB After Width: | Height: | Size: 71 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 525 KiB After Width: | Height: | Size: 587 KiB |
+11
-14
@@ -1,3 +1,4 @@
|
||||
version: '2.2'
|
||||
services:
|
||||
sharelatex:
|
||||
restart: always
|
||||
@@ -39,6 +40,10 @@ services:
|
||||
# Disables email confirmation requirement
|
||||
EMAIL_CONFIRMATION_DISABLED: 'true'
|
||||
|
||||
# temporary fix for LuaLaTex compiles
|
||||
# see https://github.com/overleaf/overleaf/issues/695
|
||||
TEXMFVAR: /var/lib/overleaf/tmp/texmf-var
|
||||
|
||||
## Set for SSL via nginx-proxy
|
||||
#VIRTUAL_HOST: 103.112.212.22
|
||||
|
||||
@@ -72,19 +77,11 @@ services:
|
||||
## Server Pro ##
|
||||
################
|
||||
|
||||
## The Community Edition is intended for use in environments where all users are trusted and is not appropriate for
|
||||
## scenarios where isolation of users is required. Sandboxed Compiles are not available in the Community Edition,
|
||||
## so the following environment variables must be commented out to avoid compile issues.
|
||||
##
|
||||
## Sandboxed Compiles: https://docs.overleaf.com/on-premises/configuration/overleaf-toolkit/server-pro-only-configuration/sandboxed-compiles
|
||||
## Sandboxed Compiles: https://github.com/overleaf/overleaf/wiki/Server-Pro:-Sandboxed-Compiles
|
||||
SANDBOXED_COMPILES: 'true'
|
||||
### Bind-mount source for /var/lib/overleaf/data/compiles inside the container.
|
||||
SANDBOXED_COMPILES_HOST_DIR_COMPILES: '/home/user/sharelatex_data/data/compiles'
|
||||
### Bind-mount source for /var/lib/overleaf/data/output inside the container.
|
||||
SANDBOXED_COMPILES_HOST_DIR_OUTPUT: '/home/user/sharelatex_data/data/output'
|
||||
### Backwards compatibility (before Server Pro 5.5)
|
||||
DOCKER_RUNNER: 'true'
|
||||
SANDBOXED_COMPILES_SIBLING_CONTAINERS: 'true'
|
||||
### Bind-mount source for /var/lib/overleaf/data/compiles inside the container.
|
||||
SANDBOXED_COMPILES_HOST_DIR: '/home/user/sharelatex_data/data/compiles'
|
||||
|
||||
## Works with test LDAP server shown at bottom of docker compose
|
||||
# OVERLEAF_LDAP_URL: 'ldap://ldap:389'
|
||||
@@ -105,12 +102,12 @@ services:
|
||||
|
||||
mongo:
|
||||
restart: always
|
||||
image: mongo:6.0
|
||||
image: mongo:5.0
|
||||
container_name: mongo
|
||||
command: '--replSet overleaf'
|
||||
volumes:
|
||||
- ~/mongo_data:/data/db
|
||||
- ./bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js
|
||||
- ./mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js
|
||||
environment:
|
||||
MONGO_INITDB_DATABASE: sharelatex
|
||||
extra_hosts:
|
||||
@@ -118,7 +115,7 @@ services:
|
||||
# This override is not needed when running the setup after starting up mongo.
|
||||
- mongo:127.0.0.1
|
||||
healthcheck:
|
||||
test: echo 'db.stats().ok' | mongosh localhost:27017/test --quiet
|
||||
test: echo 'db.stats().ok' | mongo localhost:27017/test --quiet
|
||||
interval: 10s
|
||||
timeout: 10s
|
||||
retries: 5
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
node_modules/
|
||||
@@ -0,0 +1,46 @@
|
||||
compileFolder
|
||||
|
||||
Compiled source #
|
||||
###################
|
||||
*.com
|
||||
*.class
|
||||
*.dll
|
||||
*.exe
|
||||
*.o
|
||||
*.so
|
||||
|
||||
# Packages #
|
||||
############
|
||||
# it's better to unpack these files and commit the raw source
|
||||
# git has its own built in compression methods
|
||||
*.7z
|
||||
*.dmg
|
||||
*.gz
|
||||
*.iso
|
||||
*.jar
|
||||
*.rar
|
||||
*.tar
|
||||
*.zip
|
||||
|
||||
# Logs and databases #
|
||||
######################
|
||||
*.log
|
||||
*.sql
|
||||
*.sqlite
|
||||
|
||||
# OS generated files #
|
||||
######################
|
||||
.DS_Store?
|
||||
ehthumbs.db
|
||||
Icon?
|
||||
Thumbs.db
|
||||
|
||||
/node_modules/*
|
||||
data/*/*
|
||||
|
||||
**.swp
|
||||
|
||||
/log.json
|
||||
hash_folder
|
||||
|
||||
.npmrc
|
||||
@@ -1,13 +0,0 @@
|
||||
let reporterOptions = {}
|
||||
if (process.env.CI) {
|
||||
reporterOptions = {
|
||||
reporter: '/overleaf/node_modules/mocha-multi-reporters',
|
||||
'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
|
||||
}
|
||||
}
|
||||
const all = {
|
||||
require: 'test/setup.js',
|
||||
...reporterOptions,
|
||||
}
|
||||
|
||||
module.exports = all
|
||||
@@ -0,0 +1,6 @@
|
||||
{
|
||||
"ui": "bdd",
|
||||
"recursive": "true",
|
||||
"reporter": "spec",
|
||||
"require": "test/setup.js"
|
||||
}
|
||||
@@ -1 +1 @@
|
||||
22.18.0
|
||||
18.20.2
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
access-token-encryptor
|
||||
--dependencies=None
|
||||
--docker-repos=gcr.io/overleaf-ops
|
||||
--env-add=
|
||||
--env-pass-through=
|
||||
--esmock-loader=False
|
||||
--is-library=True
|
||||
--node-version=22.18.0
|
||||
--pipeline-owner=🚉 Platform
|
||||
--node-version=18.20.2
|
||||
--public-repo=False
|
||||
--script-version=4.5.0
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
const { promisify } = require('node:util')
|
||||
const crypto = require('node:crypto')
|
||||
const { promisify } = require('util')
|
||||
const crypto = require('crypto')
|
||||
|
||||
const ALGORITHM = 'aes-256-ctr'
|
||||
|
||||
|
||||
@@ -5,8 +5,8 @@
|
||||
"main": "index.js",
|
||||
"scripts": {
|
||||
"test": "npm run lint && npm run format && npm run types:check && npm run test:unit",
|
||||
"lint": "eslint --ext .cjs,.js,.jsx,.mjs,.ts --max-warnings 0 --format unix .",
|
||||
"lint:fix": "eslint --fix --ext .cjs,.js,.jsx,.mjs,.ts .",
|
||||
"lint": "eslint --ext .js --ext .cjs --ext .ts --max-warnings 0 --format unix .",
|
||||
"lint:fix": "eslint --fix --ext .js --ext .cjs --ext .ts .",
|
||||
"format": "prettier --list-different $PWD/'**/*.{js,cjs,ts}'",
|
||||
"format:fix": "prettier --write $PWD/'**/*.{js,cjs,ts}'",
|
||||
"test:ci": "npm run test:unit",
|
||||
@@ -21,9 +21,7 @@
|
||||
"devDependencies": {
|
||||
"chai": "^4.3.6",
|
||||
"chai-as-promised": "^7.1.1",
|
||||
"mocha": "^11.1.0",
|
||||
"mocha-junit-reporter": "^2.2.1",
|
||||
"mocha-multi-reporters": "^1.5.1",
|
||||
"mocha": "^10.2.0",
|
||||
"sandboxed-module": "^2.0.4",
|
||||
"typescript": "^5.0.4"
|
||||
}
|
||||
|
||||
@@ -1,9 +0,0 @@
|
||||
module.exports = {
|
||||
reporterEnabled: 'spec, mocha-junit-reporter',
|
||||
mochaJunitReporterReporterOptions: {
|
||||
mochaFile: `reports/junit-mocha-${process.env.MOCHA_GREP}.xml`,
|
||||
includePending: true,
|
||||
jenkinsMode: true,
|
||||
output: true,
|
||||
},
|
||||
}
|
||||
@@ -1,13 +1,4 @@
|
||||
const chai = require('chai')
|
||||
const chaiAsPromised = require('chai-as-promised')
|
||||
const SandboxedModule = require('sandboxed-module')
|
||||
|
||||
chai.use(chaiAsPromised)
|
||||
|
||||
SandboxedModule.configure({
|
||||
sourceTransformers: {
|
||||
removeNodePrefix: function (source) {
|
||||
return source.replace(/require\(['"]node:/g, "require('")
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
@@ -2,7 +2,6 @@
|
||||
"extends": "../../tsconfig.backend.json",
|
||||
"include": [
|
||||
"**/*.js",
|
||||
"**/*.cjs",
|
||||
"**/*.ts"
|
||||
"**/*.cjs"
|
||||
]
|
||||
}
|
||||
|
||||
@@ -1,11 +0,0 @@
|
||||
module.exports = {
|
||||
rules: {
|
||||
'no-unnecessary-trans': require('./no-unnecessary-trans'),
|
||||
'prefer-kebab-url': require('./prefer-kebab-url'),
|
||||
'should-unescape-trans': require('./should-unescape-trans'),
|
||||
'no-generated-editor-themes': require('./no-generated-editor-themes'),
|
||||
'require-script-runner': require('./require-script-runner'),
|
||||
'require-vi-doMock-valid-path': require('./require-vi-doMock-valid-path'),
|
||||
'require-loading-label': require('./require-loading-label'),
|
||||
},
|
||||
}
|
||||
@@ -1,21 +0,0 @@
|
||||
module.exports = {
|
||||
meta: {
|
||||
type: 'error',
|
||||
docs: {
|
||||
description:
|
||||
'Prohibit CodeMirror themes that are generated in a function',
|
||||
},
|
||||
},
|
||||
create(context) {
|
||||
return {
|
||||
':matches(ArrowFunctionExpression, FunctionDeclaration, FunctionExpression) CallExpression > MemberExpression[object.name="EditorView"]:matches([property.name="theme"],[property.name="baseTheme"])'(
|
||||
node
|
||||
) {
|
||||
context.report({
|
||||
node,
|
||||
message: `EditorView.theme and EditorView.baseTheme each add CSS to the page for every instance of the theme. Store the theme in a variable and reuse it instead.`,
|
||||
})
|
||||
},
|
||||
}
|
||||
},
|
||||
}
|
||||
@@ -1,43 +0,0 @@
|
||||
module.exports = {
|
||||
meta: {
|
||||
type: 'problem',
|
||||
fixable: 'code',
|
||||
docs: {
|
||||
description: 'Prohibit Trans with no components or values',
|
||||
},
|
||||
},
|
||||
create(context) {
|
||||
return {
|
||||
'JSXOpeningElement[name.name="Trans"]'(node) {
|
||||
const attributes = new Map(
|
||||
node.attributes.map(attr => [attr.name.name, attr])
|
||||
)
|
||||
|
||||
if (!attributes.has('components')) {
|
||||
if (node.parent.children.length > 0) {
|
||||
context.report({
|
||||
node,
|
||||
message: `Trans components must not have child elements`,
|
||||
})
|
||||
} else if (attributes.has('values')) {
|
||||
context.report({
|
||||
node,
|
||||
message: `Use t('…') when there are no components`,
|
||||
})
|
||||
} else {
|
||||
context.report({
|
||||
node,
|
||||
message: `Use t('…') when there are no components`,
|
||||
fix(fixer) {
|
||||
const i18nKey = attributes.get('i18nKey').value.value
|
||||
|
||||
// Note: Prettier can fix indentation
|
||||
return fixer.replaceText(node.parent, `{t('${i18nKey}')}`)
|
||||
},
|
||||
})
|
||||
}
|
||||
}
|
||||
},
|
||||
}
|
||||
},
|
||||
}
|
||||
@@ -1,17 +0,0 @@
|
||||
{
|
||||
"name": "@overleaf/eslint-plugin",
|
||||
"version": "0.1.0",
|
||||
"author": "Overleaf (https://www.overleaf.com)",
|
||||
"license": "AGPL-3.0-only",
|
||||
"main": "index.js",
|
||||
"dependencies": {
|
||||
"eslint": "^8.51.0",
|
||||
"lodash": "^4.17.21"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@typescript-eslint/parser": "^8.50.0"
|
||||
},
|
||||
"scripts": {
|
||||
"test": "node rules.test.js"
|
||||
}
|
||||
}
|
||||
@@ -1,83 +0,0 @@
|
||||
// URL parts should be kebab-case, but we didn't have this rule in the past.
|
||||
// The ESLint rule `prefer-kebab-url` will ignore these "legacy" URL parts.
|
||||
|
||||
const ignoreWords = {
|
||||
snake: new Set([
|
||||
'clear_saml_data',
|
||||
'confirm_link',
|
||||
'confirm_university_domain',
|
||||
'create_recurly_account',
|
||||
'current_history_content',
|
||||
'current_user',
|
||||
'default_email',
|
||||
'disable_managed_users',
|
||||
'doc_snapshot',
|
||||
'enable_history_ranges_support',
|
||||
'features_override',
|
||||
'generate_password_reset_url',
|
||||
'get_assignment',
|
||||
'get_clone',
|
||||
'health_check',
|
||||
'institutional_emails',
|
||||
'latest_template',
|
||||
'link_after_saml_response',
|
||||
'linked_file',
|
||||
'metrics_segmentation',
|
||||
'new_users',
|
||||
'no_autostart_post_gateway',
|
||||
'personal_info',
|
||||
'planned_maintenance',
|
||||
'refresh_features',
|
||||
'register_admin',
|
||||
'register_ldap_admin',
|
||||
'register_saml_admin',
|
||||
'restore_file',
|
||||
'revert_file',
|
||||
'saved_vers',
|
||||
'send_test_email',
|
||||
'session_maintenance',
|
||||
'set_in_session',
|
||||
'sign_in_to_link',
|
||||
'split_test',
|
||||
'sso_configuration_test',
|
||||
'sso_email',
|
||||
'sso_enrollment',
|
||||
'track_changes',
|
||||
'update_admin',
|
||||
'user_details',
|
||||
]),
|
||||
camel: new Set([
|
||||
'addWorkflowScope',
|
||||
'aiErrorAssistant',
|
||||
'beginAuth',
|
||||
'brandVariationId',
|
||||
'closeEditor',
|
||||
'completeRegistration',
|
||||
'deactivateOldProjects',
|
||||
'deletedSubscription',
|
||||
'disconnectAllUsers',
|
||||
'editingSession',
|
||||
'emailSubscription',
|
||||
'enableManagedUsers',
|
||||
'externalCollaboration',
|
||||
'flushProjectToTpds',
|
||||
'indexAll',
|
||||
'offboardManagedUser',
|
||||
'openEditor',
|
||||
'perfTest',
|
||||
'pollDropboxForUser',
|
||||
'resendInvite',
|
||||
'resendManagedUserInvite',
|
||||
'salesContactForm',
|
||||
'showSupport',
|
||||
]),
|
||||
other: new Set([
|
||||
'Project',
|
||||
'disableSSO',
|
||||
'enableSSO',
|
||||
'resendSSOLinkInvite',
|
||||
'usersCSV',
|
||||
]),
|
||||
}
|
||||
|
||||
module.exports = { ignoreWords }
|
||||
@@ -1,91 +0,0 @@
|
||||
const _ = require('lodash')
|
||||
const { ignoreWords } = require('./prefer-kebab-url-ignore')
|
||||
|
||||
const removeTextBetweenBrackets = text => {
|
||||
while (text.includes('[') || text.includes('(')) {
|
||||
text = text.replaceAll(/\[[^[\]]*]/g, '')
|
||||
text = text.replaceAll(/\([^()]*\)/g, '')
|
||||
}
|
||||
return text
|
||||
}
|
||||
|
||||
const shouldIgnoreWord = str =>
|
||||
str.includes(':') ||
|
||||
str.includes('(') ||
|
||||
str === '*' ||
|
||||
str.match(/^[a-z0-9.]+$/) ||
|
||||
ignoreWords.snake.has(str) ||
|
||||
ignoreWords.camel.has(str) ||
|
||||
ignoreWords.other.has(str)
|
||||
|
||||
const getSuggestion = routePath => {
|
||||
if (typeof routePath === 'string') {
|
||||
const kebabed = routePath
|
||||
.split('/')
|
||||
.map(word => (shouldIgnoreWord(word) ? word : _.kebabCase(word)))
|
||||
.join('/')
|
||||
return kebabed === routePath ? null : `'${kebabed}'`
|
||||
}
|
||||
|
||||
if (routePath instanceof RegExp) {
|
||||
const words = removeTextBetweenBrackets(routePath.source).match(/[\w-]+/g)
|
||||
if (!words) return routePath
|
||||
|
||||
let newSource = routePath.source
|
||||
for (const word of words) {
|
||||
if (!shouldIgnoreWord(word)) {
|
||||
newSource = newSource.replaceAll(
|
||||
new RegExp(`\\b${word}\\b`, 'g'),
|
||||
_.kebabCase(word)
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
const kebabed = new RegExp(newSource, routePath.flags)
|
||||
return kebabed.source.toString() === routePath.source.toString()
|
||||
? null
|
||||
: kebabed
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
meta: {
|
||||
type: 'problem',
|
||||
fixable: 'code',
|
||||
hasSuggestions: true,
|
||||
docs: {
|
||||
description: 'Enforce using kebab-case for URL paths',
|
||||
},
|
||||
},
|
||||
create: context => ({
|
||||
CallExpression(node) {
|
||||
if (
|
||||
node.callee.type === 'MemberExpression' &&
|
||||
node.arguments[0]?.type === 'Literal' &&
|
||||
[/app/i, /router/i].some(callee =>
|
||||
typeof callee === 'string'
|
||||
? node.callee.object.name === callee
|
||||
: callee.test(node.callee.object.name)
|
||||
) &&
|
||||
['get', 'post', 'put', 'delete'].includes(node.callee.property.name)
|
||||
) {
|
||||
const routePath = node.arguments[0].value
|
||||
|
||||
const suggestion = getSuggestion(routePath)
|
||||
|
||||
if (suggestion) {
|
||||
context.report({
|
||||
node: node.arguments[0],
|
||||
message: 'Route path should be in kebab-case.',
|
||||
suggest: [
|
||||
{
|
||||
desc: `Change to kebab-case: ${suggestion}`,
|
||||
fix: fixer => fixer.replaceText(node.arguments[0], suggestion),
|
||||
},
|
||||
],
|
||||
})
|
||||
}
|
||||
}
|
||||
},
|
||||
}),
|
||||
}
|
||||
@@ -1,49 +0,0 @@
|
||||
module.exports = {
|
||||
meta: {
|
||||
type: 'problem',
|
||||
fixable: null,
|
||||
docs: {
|
||||
description:
|
||||
'Require loadingLabel prop when isLoading is specified on OLButton',
|
||||
},
|
||||
schema: [],
|
||||
},
|
||||
create(context) {
|
||||
return {
|
||||
'JSXOpeningElement[name.name="OLButton"]'(node) {
|
||||
const attributes = new Map(
|
||||
node.attributes.map(attr => [attr.name?.name, attr])
|
||||
)
|
||||
|
||||
const isLoadingAttr = attributes.get('isLoading')
|
||||
const loadingLabelAttr = attributes.get('loadingLabel')
|
||||
|
||||
if (isLoadingAttr && !loadingLabelAttr) {
|
||||
const isLoadingValue = isLoadingAttr.value
|
||||
|
||||
if (
|
||||
!isLoadingValue ||
|
||||
(isLoadingValue.type === 'JSXExpressionContainer' &&
|
||||
isLoadingValue.expression.type === 'Literal' &&
|
||||
isLoadingValue.expression.value === true)
|
||||
) {
|
||||
context.report({
|
||||
node: isLoadingAttr,
|
||||
message:
|
||||
'Button with isLoading prop must also specify loadingLabel',
|
||||
})
|
||||
} else if (
|
||||
isLoadingValue.type === 'JSXExpressionContainer' &&
|
||||
isLoadingValue.expression.type !== 'Literal'
|
||||
) {
|
||||
context.report({
|
||||
node: isLoadingAttr,
|
||||
message:
|
||||
'Button with isLoading prop must also specify loadingLabel',
|
||||
})
|
||||
}
|
||||
}
|
||||
},
|
||||
}
|
||||
},
|
||||
}
|
||||
@@ -1,28 +0,0 @@
|
||||
module.exports = {
|
||||
meta: {
|
||||
type: 'suggestion',
|
||||
docs: {
|
||||
description: 'Require Script Runner for scripts',
|
||||
},
|
||||
},
|
||||
create(context) {
|
||||
let hasImport = false
|
||||
|
||||
return {
|
||||
ImportDeclaration(node) {
|
||||
if (node.source.value.endsWith('lib/ScriptRunner.mjs')) {
|
||||
hasImport = true
|
||||
}
|
||||
},
|
||||
'Program:exit'() {
|
||||
if (!hasImport) {
|
||||
context.report({
|
||||
loc: { line: 1, column: 0 },
|
||||
message:
|
||||
'Please use Script Runner for scripts. Refer to the developer manual (https://manual.dev-overleaf.com/development/code/web_scripts/#monitor-script-execution-and-usage-with-script-runner) for more information.',
|
||||
})
|
||||
}
|
||||
},
|
||||
}
|
||||
},
|
||||
}
|
||||
@@ -1,138 +0,0 @@
|
||||
const path = require('node:path')
|
||||
const fs = require('node:fs')
|
||||
|
||||
module.exports = {
|
||||
meta: {
|
||||
type: 'problem',
|
||||
docs: {
|
||||
description: 'Ensure vi.doMock first argument is a resolvable path.',
|
||||
category: 'Best Practices',
|
||||
recommended: false,
|
||||
url: '',
|
||||
},
|
||||
fixable: 'code',
|
||||
hasSuggestions: true,
|
||||
schema: [],
|
||||
messages: {
|
||||
unresolvablePath:
|
||||
'The path "{{pathValue}}" in vi.doMock() cannot be resolved relative to the current file.',
|
||||
notAStringLiteral:
|
||||
'The first argument of vi.doMock() must be (or resolve to) a string literal representing a path.',
|
||||
noArguments: 'vi.doMock() called with no arguments.',
|
||||
},
|
||||
},
|
||||
create(context) {
|
||||
const currentFilePath = context.getFilename()
|
||||
// ESLint can sometimes pass <text> or <input> for snippets not in a file
|
||||
if (currentFilePath === '<text>' || currentFilePath === '<input>') {
|
||||
return {}
|
||||
}
|
||||
const currentDirectory = path.dirname(currentFilePath)
|
||||
|
||||
function canResolve(modulePath) {
|
||||
try {
|
||||
require.resolve(path.resolve(currentDirectory, modulePath))
|
||||
return true
|
||||
} catch (e) {
|
||||
const absolutePath = path.resolve(currentDirectory, modulePath)
|
||||
const extensions = [
|
||||
'',
|
||||
'.js',
|
||||
'.mjs',
|
||||
'.ts',
|
||||
'.jsx',
|
||||
'.tsx',
|
||||
'.json',
|
||||
'.node',
|
||||
'/index.js',
|
||||
'/index.ts',
|
||||
] // Add common extensions
|
||||
for (const ext of extensions) {
|
||||
if (fs.existsSync(absolutePath + ext)) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
CallExpression(node) {
|
||||
if (
|
||||
node.callee.type === 'MemberExpression' &&
|
||||
node.callee.object.type === 'Identifier' &&
|
||||
node.callee.object.name === 'vi' &&
|
||||
node.callee.property.type === 'Identifier' &&
|
||||
node.callee.property.name === 'doMock'
|
||||
) {
|
||||
if (node.arguments.length === 0) {
|
||||
context.report({
|
||||
node,
|
||||
messageId: 'noArguments',
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
const firstArg = node.arguments[0]
|
||||
let pathValue = firstArg.value
|
||||
|
||||
if (
|
||||
firstArg.type !== 'Literal' ||
|
||||
typeof firstArg.value !== 'string'
|
||||
) {
|
||||
if (firstArg.type === 'Identifier') {
|
||||
const variable = context
|
||||
.getScope()
|
||||
.variables.find(v => v.name === firstArg.name)
|
||||
if (
|
||||
variable &&
|
||||
variable.defs.length > 0 &&
|
||||
variable.defs[0].node.init &&
|
||||
variable.defs[0].node.init.type === 'Literal' &&
|
||||
typeof variable.defs[0].node.init.value === 'string'
|
||||
) {
|
||||
pathValue = variable.defs[0].node.init.value
|
||||
if (canResolve(pathValue)) {
|
||||
return
|
||||
}
|
||||
// If the first argument was a variable that didn't resolve then we can't auto-fix it
|
||||
}
|
||||
}
|
||||
context.report({
|
||||
node: firstArg,
|
||||
messageId: 'notAStringLiteral',
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
if (!pathValue.startsWith('.')) {
|
||||
return
|
||||
}
|
||||
|
||||
if (!canResolve(pathValue)) {
|
||||
const mjsPath = pathValue.replace('.js', '.mjs')
|
||||
const additionalReportOptions = {}
|
||||
if (canResolve(mjsPath)) {
|
||||
additionalReportOptions.fix = fixer =>
|
||||
fixer.replaceText(firstArg, `'${mjsPath}'`)
|
||||
additionalReportOptions.suggest = [
|
||||
{
|
||||
desc: `Replace with "${pathValue.replace('.js', '.mjs')}"`,
|
||||
fix: fixer => fixer.replaceText(firstArg, `'${mjsPath}'`),
|
||||
},
|
||||
]
|
||||
}
|
||||
context.report({
|
||||
node: firstArg,
|
||||
messageId: 'unresolvablePath',
|
||||
data: {
|
||||
pathValue,
|
||||
},
|
||||
...additionalReportOptions,
|
||||
})
|
||||
}
|
||||
}
|
||||
},
|
||||
}
|
||||
},
|
||||
}
|
||||
@@ -1,168 +0,0 @@
|
||||
const { RuleTester } = require('eslint')
|
||||
const preferKebabUrl = require('./prefer-kebab-url')
|
||||
const noUnnecessaryTrans = require('./no-unnecessary-trans')
|
||||
const shouldUnescapeTrans = require('./should-unescape-trans')
|
||||
const noGeneratedEditorThemes = require('./no-generated-editor-themes')
|
||||
const viDoMockValidPath = require('./require-vi-doMock-valid-path')
|
||||
|
||||
const ruleTester = new RuleTester({
|
||||
parser: require.resolve('@typescript-eslint/parser'),
|
||||
parserOptions: {
|
||||
ecmaVersion: 'latest',
|
||||
ecmaFeatures: { jsx: true },
|
||||
},
|
||||
})
|
||||
|
||||
ruleTester.run('prefer-kebab-url', preferKebabUrl, {
|
||||
valid: [
|
||||
{ code: `app.get('/foo-bar')` },
|
||||
{ code: `app.get('/foo-bar/:id')` },
|
||||
{ code: `router.post('/foo-bar')` },
|
||||
{ code: `router.get('/foo-bar/:id/:name/:age')` },
|
||||
{ code: `webRouter.get('/foo-bar/:user_id/(ProjectName)/get-info')` },
|
||||
{ code: `webApp.post('/foo-bar/:user_id/(ProjectName)/get-info')` },
|
||||
{
|
||||
code: `router.get(/^\\/download\\/project\\/([^/]*)\\/output\\/output\\.pdf$/)`,
|
||||
},
|
||||
{
|
||||
code: `webRouter.get(/^\\/project\\/([^/]*)\\/user\\/([0-9a-f]+)\\/build\\/([0-9a-f-]+)\\/output\\/(.*)$/)`,
|
||||
},
|
||||
],
|
||||
invalid: [
|
||||
{
|
||||
code: `app.get('/fooBar')`,
|
||||
errors: [{ message: 'Route path should be in kebab-case.' }],
|
||||
},
|
||||
{
|
||||
code: `app.get('/fooBar/:id')`,
|
||||
errors: [{ message: 'Route path should be in kebab-case.' }],
|
||||
},
|
||||
{
|
||||
code: `webRouter.get('/foo_bar/:id/FooBar/:name/fooBar')`,
|
||||
errors: [{ message: 'Route path should be in kebab-case.' }],
|
||||
},
|
||||
{
|
||||
code: `router.get(/^\\/downLoad\\/pro-ject\\/([^/]*)\\/OutPut\\/out-put\\.pdf$/)`,
|
||||
errors: [{ message: 'Route path should be in kebab-case.' }],
|
||||
},
|
||||
],
|
||||
})
|
||||
|
||||
ruleTester.run('no-unnecessary-trans', noUnnecessaryTrans, {
|
||||
valid: [
|
||||
{ code: `<Trans i18nKey="test" components={{ strong: <strong/> }}/>` },
|
||||
],
|
||||
invalid: [
|
||||
{
|
||||
code: `<Trans i18nKey="test" values={{ test: 'foo '}}/>`,
|
||||
errors: [{ message: `Use t('…') when there are no components` }],
|
||||
},
|
||||
{
|
||||
code: `<Trans i18nKey="test" />`,
|
||||
errors: [{ message: `Use t('…') when there are no components` }],
|
||||
output: `{t('test')}`,
|
||||
},
|
||||
],
|
||||
})
|
||||
|
||||
ruleTester.run('should-unescape-trans', shouldUnescapeTrans, {
|
||||
valid: [
|
||||
{
|
||||
code: `<Trans i18nKey="test" components={{ strong: <strong/> }}/>`,
|
||||
},
|
||||
{
|
||||
code: `<Trans i18nKey="test" values={{ foo: 'bar' }} components={{ strong: <strong/> }} shouldUnescape tOptions={{ interpolation: { escapeValue: true } }}/>`,
|
||||
},
|
||||
],
|
||||
invalid: [
|
||||
{
|
||||
code: `<Trans i18nKey="test" values={{ foo: 'bar' }} components={{ strong: <strong/> }} />`,
|
||||
errors: [{ message: 'Trans with values must have shouldUnescape' }],
|
||||
output: `<Trans i18nKey="test" values={{ foo: 'bar' }}\nshouldUnescape components={{ strong: <strong/> }} />`,
|
||||
},
|
||||
{
|
||||
code: `<Trans i18nKey="test" values={{ foo: 'bar' }} components={{ strong: <strong/> }} shouldUnescape />`,
|
||||
errors: [
|
||||
{
|
||||
message:
|
||||
'Trans with shouldUnescape must have tOptions.interpolation.escapeValue',
|
||||
},
|
||||
],
|
||||
output: `<Trans i18nKey="test" values={{ foo: 'bar' }} components={{ strong: <strong/> }} shouldUnescape\ntOptions={{ interpolation: { escapeValue: true } }} />`,
|
||||
},
|
||||
],
|
||||
})
|
||||
|
||||
const noGeneratedEditorThemesError =
|
||||
'EditorView.theme and EditorView.baseTheme each add CSS to the page for every instance of the theme. Store the theme in a variable and reuse it instead.'
|
||||
ruleTester.run('no-generated-editor-themes', noGeneratedEditorThemes, {
|
||||
valid: [
|
||||
{
|
||||
code: `EditorView.theme({ '.cm-editor': { color: 'black' } })`,
|
||||
},
|
||||
{
|
||||
code: `const theme = EditorView.theme({ '.cm-editor': { color: 'black' } })`,
|
||||
},
|
||||
],
|
||||
invalid: [
|
||||
{
|
||||
code: `function createTheme() { return EditorView.theme({ '.cm-editor': { color: 'black' } }) }`,
|
||||
errors: [
|
||||
{
|
||||
message: noGeneratedEditorThemesError,
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
code: `() => EditorView.theme({ '.cm-editor': { color: 'black' } })`,
|
||||
errors: [
|
||||
{
|
||||
message: noGeneratedEditorThemesError,
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
code: `class Foo { createTheme() { return EditorView.theme({ '.cm-editor': { color: 'black' } }) } }`,
|
||||
errors: [
|
||||
{
|
||||
message: noGeneratedEditorThemesError,
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
})
|
||||
|
||||
ruleTester.run('domock-require-valid-path', viDoMockValidPath, {
|
||||
valid: [
|
||||
{
|
||||
code: 'vi.doMock("./require-vi-doMock-valid-path.js")',
|
||||
filename: __filename,
|
||||
},
|
||||
{
|
||||
code: 'const filename = "./require-vi-doMock-valid-path.js"; vi.doMock(filename);',
|
||||
filename: __filename,
|
||||
},
|
||||
],
|
||||
invalid: [
|
||||
{
|
||||
code: "vi.doMock('./require-vi-doMock-valid-path2')",
|
||||
filename: __filename,
|
||||
errors: [
|
||||
{
|
||||
message:
|
||||
'The path "./require-vi-doMock-valid-path2" in vi.doMock() cannot be resolved relative to the current file.',
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
code: 'const filename = "./require-vi-doMock-valid-path2.js"; vi.doMock(filename);',
|
||||
filename: __filename,
|
||||
errors: [
|
||||
{
|
||||
message:
|
||||
'The first argument of vi.doMock() must be (or resolve to) a string literal representing a path.',
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
})
|
||||
@@ -1,60 +0,0 @@
|
||||
module.exports = {
|
||||
meta: {
|
||||
type: 'problem',
|
||||
fixable: 'code',
|
||||
docs: {
|
||||
description: 'Ensure that Trans with values has shouldUnescape',
|
||||
},
|
||||
},
|
||||
create(context) {
|
||||
return {
|
||||
'JSXOpeningElement[name.name="Trans"]'(node) {
|
||||
const attributes = new Map(
|
||||
node.attributes.map(attr => [attr.name.name, attr])
|
||||
)
|
||||
|
||||
if (attributes.has('values') && !attributes.has('shouldUnescape')) {
|
||||
context.report({
|
||||
node,
|
||||
message: 'Trans with values must have shouldUnescape',
|
||||
fix(fixer) {
|
||||
return fixer.insertTextAfter(
|
||||
attributes.get('values'),
|
||||
'\nshouldUnescape' // Note: Prettier can fix indentation
|
||||
)
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
if (attributes.has('values') && attributes.has('shouldUnescape')) {
|
||||
const tOptions = attributes.get('tOptions')
|
||||
if (!tOptions) {
|
||||
context.report({
|
||||
node,
|
||||
message:
|
||||
'Trans with shouldUnescape must have tOptions.interpolation.escapeValue',
|
||||
fix(fixer) {
|
||||
return fixer.insertTextAfter(
|
||||
attributes.get('shouldUnescape'),
|
||||
'\ntOptions={{ interpolation: { escapeValue: true } }}' // Note: Prettier can fix indentation
|
||||
)
|
||||
},
|
||||
})
|
||||
} else {
|
||||
const property = tOptions.value.expression.properties
|
||||
.find(p => p.key.name === 'interpolation')
|
||||
?.value.properties.find(p => p.key.name === 'escapeValue')
|
||||
|
||||
if (property?.value.value !== true) {
|
||||
context.report({
|
||||
node,
|
||||
message:
|
||||
'Trans with shouldUnescape must have tOptions.interpolation.escapeValue set to true',
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
}
|
||||
},
|
||||
}
|
||||
@@ -1,6 +0,0 @@
|
||||
{
|
||||
"extends": "../../tsconfig.backend.json",
|
||||
"include": [
|
||||
"**/*.js",
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
node_modules/
|
||||
@@ -0,0 +1,3 @@
|
||||
|
||||
# managed by monorepo$ bin/update_build_scripts
|
||||
.npmrc
|
||||
@@ -1,13 +0,0 @@
|
||||
let reporterOptions = {}
|
||||
if (process.env.CI) {
|
||||
reporterOptions = {
|
||||
reporter: '/overleaf/node_modules/mocha-multi-reporters',
|
||||
'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
|
||||
}
|
||||
}
|
||||
const all = {
|
||||
require: 'test/setup.js',
|
||||
...reporterOptions,
|
||||
}
|
||||
|
||||
module.exports = all
|
||||
@@ -0,0 +1,6 @@
|
||||
{
|
||||
"ui": "bdd",
|
||||
"recursive": "true",
|
||||
"reporter": "spec",
|
||||
"require": "test/setup.js"
|
||||
}
|
||||
@@ -1 +1 @@
|
||||
22.18.0
|
||||
18.20.2
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
fetch-utils
|
||||
--dependencies=None
|
||||
--docker-repos=gcr.io/overleaf-ops
|
||||
--env-add=
|
||||
--env-pass-through=
|
||||
--esmock-loader=False
|
||||
--is-library=True
|
||||
--node-version=22.18.0
|
||||
--pipeline-owner=🚉 Platform
|
||||
--node-version=18.20.2
|
||||
--public-repo=False
|
||||
--script-version=4.5.0
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
const _ = require('lodash')
|
||||
const { Readable } = require('node:stream')
|
||||
const { Readable } = require('stream')
|
||||
const OError = require('@overleaf/o-error')
|
||||
const fetch = require('node-fetch')
|
||||
const http = require('node:http')
|
||||
const https = require('node:https')
|
||||
const http = require('http')
|
||||
const https = require('https')
|
||||
|
||||
/**
|
||||
* @import { Response } from 'node-fetch'
|
||||
@@ -23,11 +23,11 @@ async function fetchJson(url, opts = {}) {
|
||||
}
|
||||
|
||||
async function fetchJsonWithResponse(url, opts = {}) {
|
||||
const { fetchOpts, detachSignal } = parseOpts(opts)
|
||||
const { fetchOpts } = parseOpts(opts)
|
||||
fetchOpts.headers = fetchOpts.headers ?? {}
|
||||
fetchOpts.headers.Accept = fetchOpts.headers.Accept ?? 'application/json'
|
||||
|
||||
const response = await performRequest(url, fetchOpts, detachSignal)
|
||||
const response = await performRequest(url, fetchOpts)
|
||||
if (!response.ok) {
|
||||
const body = await maybeGetResponseBody(response)
|
||||
throw new RequestFailedError(url, opts, response, body)
|
||||
@@ -53,8 +53,8 @@ async function fetchStream(url, opts = {}) {
|
||||
}
|
||||
|
||||
async function fetchStreamWithResponse(url, opts = {}) {
|
||||
const { fetchOpts, abortController, detachSignal } = parseOpts(opts)
|
||||
const response = await performRequest(url, fetchOpts, detachSignal)
|
||||
const { fetchOpts, abortController } = parseOpts(opts)
|
||||
const response = await performRequest(url, fetchOpts)
|
||||
|
||||
if (!response.ok) {
|
||||
const body = await maybeGetResponseBody(response)
|
||||
@@ -76,8 +76,8 @@ async function fetchStreamWithResponse(url, opts = {}) {
|
||||
* @throws {RequestFailedError} if the response has a failure status code
|
||||
*/
|
||||
async function fetchNothing(url, opts = {}) {
|
||||
const { fetchOpts, detachSignal } = parseOpts(opts)
|
||||
const response = await performRequest(url, fetchOpts, detachSignal)
|
||||
const { fetchOpts } = parseOpts(opts)
|
||||
const response = await performRequest(url, fetchOpts)
|
||||
if (!response.ok) {
|
||||
const body = await maybeGetResponseBody(response)
|
||||
throw new RequestFailedError(url, opts, response, body)
|
||||
@@ -95,22 +95,9 @@ async function fetchNothing(url, opts = {}) {
|
||||
* @throws {RequestFailedError} if the response has a non redirect status code or missing Location header
|
||||
*/
|
||||
async function fetchRedirect(url, opts = {}) {
|
||||
const { location } = await fetchRedirectWithResponse(url, opts)
|
||||
return location
|
||||
}
|
||||
|
||||
/**
|
||||
* Make a request and extract the redirect from the response.
|
||||
*
|
||||
* @param {string | URL} url - request URL
|
||||
* @param {object} opts - fetch options
|
||||
* @return {Promise<{location: string, response: Response}>}
|
||||
* @throws {RequestFailedError} if the response has a non redirect status code or missing Location header
|
||||
*/
|
||||
async function fetchRedirectWithResponse(url, opts = {}) {
|
||||
const { fetchOpts, detachSignal } = parseOpts(opts)
|
||||
const { fetchOpts } = parseOpts(opts)
|
||||
fetchOpts.redirect = 'manual'
|
||||
const response = await performRequest(url, fetchOpts, detachSignal)
|
||||
const response = await performRequest(url, fetchOpts)
|
||||
if (response.status < 300 || response.status >= 400) {
|
||||
const body = await maybeGetResponseBody(response)
|
||||
throw new RequestFailedError(url, opts, response, body)
|
||||
@@ -125,7 +112,7 @@ async function fetchRedirectWithResponse(url, opts = {}) {
|
||||
)
|
||||
}
|
||||
await discardResponseBody(response)
|
||||
return { location, response }
|
||||
return location
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -142,8 +129,8 @@ async function fetchString(url, opts = {}) {
|
||||
}
|
||||
|
||||
async function fetchStringWithResponse(url, opts = {}) {
|
||||
const { fetchOpts, detachSignal } = parseOpts(opts)
|
||||
const response = await performRequest(url, fetchOpts, detachSignal)
|
||||
const { fetchOpts } = parseOpts(opts)
|
||||
const response = await performRequest(url, fetchOpts)
|
||||
if (!response.ok) {
|
||||
const body = await maybeGetResponseBody(response)
|
||||
throw new RequestFailedError(url, opts, response, body)
|
||||
@@ -178,14 +165,13 @@ function parseOpts(opts) {
|
||||
|
||||
const abortController = new AbortController()
|
||||
fetchOpts.signal = abortController.signal
|
||||
let detachSignal = () => {}
|
||||
if (opts.signal) {
|
||||
detachSignal = abortOnSignal(abortController, opts.signal)
|
||||
abortOnSignal(abortController, opts.signal)
|
||||
}
|
||||
if (opts.body instanceof Readable) {
|
||||
abortOnDestroyedRequest(abortController, fetchOpts.body)
|
||||
}
|
||||
return { fetchOpts, abortController, detachSignal }
|
||||
return { fetchOpts, abortController }
|
||||
}
|
||||
|
||||
function setupJsonBody(fetchOpts, json) {
|
||||
@@ -209,9 +195,6 @@ function abortOnSignal(abortController, signal) {
|
||||
abortController.abort(signal.reason)
|
||||
}
|
||||
signal.addEventListener('abort', listener)
|
||||
return () => {
|
||||
signal.removeEventListener('abort', listener)
|
||||
}
|
||||
}
|
||||
|
||||
function abortOnDestroyedRequest(abortController, stream) {
|
||||
@@ -230,12 +213,11 @@ function abortOnDestroyedResponse(abortController, response) {
|
||||
})
|
||||
}
|
||||
|
||||
async function performRequest(url, fetchOpts, detachSignal) {
|
||||
async function performRequest(url, fetchOpts) {
|
||||
let response
|
||||
try {
|
||||
response = await fetch(url, fetchOpts)
|
||||
} catch (err) {
|
||||
detachSignal()
|
||||
if (fetchOpts.body instanceof Readable) {
|
||||
fetchOpts.body.destroy()
|
||||
}
|
||||
@@ -244,7 +226,6 @@ async function performRequest(url, fetchOpts, detachSignal) {
|
||||
method: fetchOpts.method ?? 'GET',
|
||||
})
|
||||
}
|
||||
response.body.on('close', detachSignal)
|
||||
if (fetchOpts.body instanceof Readable) {
|
||||
response.body.on('close', () => {
|
||||
if (!fetchOpts.body.readableEnded) {
|
||||
@@ -316,7 +297,6 @@ module.exports = {
|
||||
fetchStreamWithResponse,
|
||||
fetchNothing,
|
||||
fetchRedirect,
|
||||
fetchRedirectWithResponse,
|
||||
fetchString,
|
||||
fetchStringWithResponse,
|
||||
RequestFailedError,
|
||||
|
||||
@@ -5,8 +5,8 @@
|
||||
"main": "index.js",
|
||||
"scripts": {
|
||||
"test": "npm run lint && npm run format && npm run types:check && npm run test:unit",
|
||||
"lint": "eslint --ext .cjs,.js,.jsx,.mjs,.ts --max-warnings 0 --format unix .",
|
||||
"lint:fix": "eslint --fix --ext .cjs,.js,.jsx,.mjs,.ts .",
|
||||
"lint": "eslint --ext .js --ext .cjs --ext .ts --max-warnings 0 --format unix .",
|
||||
"lint:fix": "eslint --fix --ext .js --ext .cjs --ext .ts .",
|
||||
"format": "prettier --list-different $PWD/'**/*.{js,cjs,ts}'",
|
||||
"format:fix": "prettier --write $PWD/'**/*.{js,cjs,ts}'",
|
||||
"test:ci": "npm run test:unit",
|
||||
@@ -16,14 +16,12 @@
|
||||
"author": "Overleaf (https://www.overleaf.com)",
|
||||
"license": "AGPL-3.0-only",
|
||||
"devDependencies": {
|
||||
"@types/node-fetch": "^2.6.13",
|
||||
"body-parser": "1.20.4",
|
||||
"@types/node-fetch": "^2.6.11",
|
||||
"body-parser": "^1.20.3",
|
||||
"chai": "^4.3.6",
|
||||
"chai-as-promised": "^7.1.1",
|
||||
"express": "4.22.1",
|
||||
"mocha": "^11.1.0",
|
||||
"mocha-junit-reporter": "^2.2.1",
|
||||
"mocha-multi-reporters": "^1.5.1",
|
||||
"express": "^4.21.0",
|
||||
"mocha": "^10.2.0",
|
||||
"typescript": "^5.0.4"
|
||||
},
|
||||
"dependencies": {
|
||||
|
||||
@@ -1,9 +0,0 @@
|
||||
module.exports = {
|
||||
reporterEnabled: 'spec, mocha-junit-reporter',
|
||||
mochaJunitReporterReporterOptions: {
|
||||
mochaFile: `reports/junit-mocha-${process.env.MOCHA_GREP}.xml`,
|
||||
includePending: true,
|
||||
jenkinsMode: true,
|
||||
output: true,
|
||||
},
|
||||
}
|
||||
@@ -1,10 +1,7 @@
|
||||
const { expect } = require('chai')
|
||||
const fs = require('node:fs')
|
||||
const events = require('node:events')
|
||||
const { FetchError, AbortError } = require('node-fetch')
|
||||
const { Readable } = require('node:stream')
|
||||
const { pipeline } = require('node:stream/promises')
|
||||
const { once } = require('node:events')
|
||||
const { Readable } = require('stream')
|
||||
const { once } = require('events')
|
||||
const { TestServer } = require('./helpers/TestServer')
|
||||
const selfsigned = require('selfsigned')
|
||||
const {
|
||||
@@ -27,17 +24,13 @@ const pems = selfsigned.generate(attrs, { days: 365 })
|
||||
const PRIVATE_KEY = pems.private
|
||||
const PUBLIC_CERT = pems.cert
|
||||
|
||||
const dns = require('node:dns')
|
||||
const dns = require('dns')
|
||||
const _originalLookup = dns.lookup
|
||||
// Custom DNS resolver function
|
||||
dns.lookup = (hostname, options, callback) => {
|
||||
if (hostname === 'example.com') {
|
||||
// If the hostname is our test case, return the ip address for the test server
|
||||
if (options?.all) {
|
||||
callback(null, [{ address: '127.0.0.1', family: 4 }])
|
||||
} else {
|
||||
callback(null, '127.0.0.1', 4)
|
||||
}
|
||||
callback(null, '127.0.0.1', 4)
|
||||
} else {
|
||||
// Otherwise, use the default lookup
|
||||
_originalLookup(hostname, options, callback)
|
||||
@@ -206,31 +199,6 @@ describe('fetch-utils', function () {
|
||||
).to.be.rejectedWith(AbortError)
|
||||
expect(stream.destroyed).to.be.true
|
||||
})
|
||||
|
||||
it('detaches from signal on success', async function () {
|
||||
const signal = AbortSignal.timeout(10_000)
|
||||
for (let i = 0; i < 20; i++) {
|
||||
const s = await fetchStream(this.url('/hello'), { signal })
|
||||
expect(events.getEventListeners(signal, 'abort')).to.have.length(1)
|
||||
await pipeline(s, fs.createWriteStream('/dev/null'))
|
||||
expect(events.getEventListeners(signal, 'abort')).to.have.length(0)
|
||||
}
|
||||
})
|
||||
|
||||
it('detaches from signal on error', async function () {
|
||||
const signal = AbortSignal.timeout(10_000)
|
||||
for (let i = 0; i < 20; i++) {
|
||||
try {
|
||||
await fetchStream(this.url('/500'), { signal })
|
||||
} catch (err) {
|
||||
if (err instanceof RequestFailedError && err.response.status === 500)
|
||||
continue
|
||||
throw err
|
||||
} finally {
|
||||
expect(events.getEventListeners(signal, 'abort')).to.have.length(0)
|
||||
}
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
describe('fetchNothing', function () {
|
||||
@@ -419,16 +387,9 @@ async function* infiniteIterator() {
|
||||
async function abortOnceReceived(func, server) {
|
||||
const controller = new AbortController()
|
||||
const promise = func(controller.signal)
|
||||
expect(events.getEventListeners(controller.signal, 'abort')).to.have.length(1)
|
||||
await once(server.events, 'request-received')
|
||||
controller.abort()
|
||||
try {
|
||||
return await promise
|
||||
} finally {
|
||||
expect(events.getEventListeners(controller.signal, 'abort')).to.have.length(
|
||||
0
|
||||
)
|
||||
}
|
||||
return await promise
|
||||
}
|
||||
|
||||
async function expectRequestAborted(req) {
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
const express = require('express')
|
||||
const bodyParser = require('body-parser')
|
||||
const { EventEmitter } = require('node:events')
|
||||
const http = require('node:http')
|
||||
const https = require('node:https')
|
||||
const { promisify } = require('node:util')
|
||||
const { EventEmitter } = require('events')
|
||||
const http = require('http')
|
||||
const https = require('https')
|
||||
const { promisify } = require('util')
|
||||
|
||||
class TestServer {
|
||||
constructor() {
|
||||
|
||||
@@ -2,7 +2,6 @@
|
||||
"extends": "../../tsconfig.backend.json",
|
||||
"include": [
|
||||
"**/*.js",
|
||||
"**/*.cjs",
|
||||
"**/*.ts"
|
||||
"**/*.cjs"
|
||||
]
|
||||
}
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
node_modules/
|
||||
@@ -0,0 +1,3 @@
|
||||
node_modules
|
||||
|
||||
.npmrc
|
||||
@@ -1,13 +0,0 @@
|
||||
let reporterOptions = {}
|
||||
if (process.env.CI) {
|
||||
reporterOptions = {
|
||||
reporter: '/overleaf/node_modules/mocha-multi-reporters',
|
||||
'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
|
||||
}
|
||||
}
|
||||
const all = {
|
||||
require: 'test/setup.js',
|
||||
...reporterOptions,
|
||||
}
|
||||
|
||||
module.exports = all
|
||||
@@ -0,0 +1,3 @@
|
||||
{
|
||||
"require": "test/setup.js"
|
||||
}
|
||||
@@ -1 +1 @@
|
||||
22.18.0
|
||||
18.20.2
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
logger
|
||||
--dependencies=None
|
||||
--docker-repos=gcr.io/overleaf-ops
|
||||
--env-add=
|
||||
--env-pass-through=
|
||||
--esmock-loader=False
|
||||
--is-library=True
|
||||
--node-version=22.18.0
|
||||
--pipeline-owner=🚉 Platform
|
||||
--node-version=18.20.2
|
||||
--public-repo=False
|
||||
--script-version=4.5.0
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
const { fetchString } = require('@overleaf/fetch-utils')
|
||||
const fs = require('node:fs')
|
||||
const fs = require('fs')
|
||||
|
||||
class LogLevelChecker {
|
||||
constructor(logger, defaultLevel) {
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
const Stream = require('node:stream')
|
||||
const Stream = require('stream')
|
||||
const bunyan = require('bunyan')
|
||||
const GCPManager = require('./gcp-manager')
|
||||
const SentryManager = require('./sentry-manager')
|
||||
const Serializers = require('./serializers')
|
||||
const {
|
||||
FileLogLevelChecker,
|
||||
@@ -11,13 +12,11 @@ const LoggingManager = {
|
||||
/**
|
||||
* @param {string} name - The name of the logger
|
||||
*/
|
||||
initialize(name, options = {}) {
|
||||
initialize(name) {
|
||||
this.isProduction =
|
||||
(process.env.NODE_ENV || '').toLowerCase() === 'production'
|
||||
const isTest = (process.env.NODE_ENV || '').toLowerCase() === 'test'
|
||||
this.defaultLevel =
|
||||
process.env.LOG_LEVEL ||
|
||||
(this.isProduction ? 'info' : isTest ? 'fatal' : 'debug')
|
||||
process.env.LOG_LEVEL || (this.isProduction ? 'info' : 'debug')
|
||||
this.loggerName = name
|
||||
this.logger = bunyan.createLogger({
|
||||
name,
|
||||
@@ -27,13 +26,17 @@ const LoggingManager = {
|
||||
req: Serializers.req,
|
||||
res: Serializers.res,
|
||||
},
|
||||
streams: options.streams ?? [this._getOutputStreamConfig()],
|
||||
streams: [this._getOutputStreamConfig()],
|
||||
})
|
||||
this._setupRingBuffer()
|
||||
this._setupLogLevelChecker()
|
||||
return this
|
||||
},
|
||||
|
||||
initializeErrorReporting(dsn, options) {
|
||||
this.sentryManager = new SentryManager()
|
||||
},
|
||||
|
||||
/**
|
||||
* @param {Record<string, any>|string} attributes - Attributes to log (nice serialization for err, req, res)
|
||||
* @param {string} [message] - Optional message
|
||||
@@ -65,6 +68,9 @@ const LoggingManager = {
|
||||
})
|
||||
}
|
||||
this.logger.error(attributes, message, ...Array.from(args))
|
||||
if (this.sentryManager) {
|
||||
this.sentryManager.captureExceptionRateLimited(attributes, message)
|
||||
}
|
||||
},
|
||||
|
||||
/**
|
||||
@@ -92,6 +98,9 @@ const LoggingManager = {
|
||||
*/
|
||||
fatal(attributes, message) {
|
||||
this.logger.fatal(attributes, message)
|
||||
if (this.sentryManager) {
|
||||
this.sentryManager.captureException(attributes, message, 'fatal')
|
||||
}
|
||||
},
|
||||
|
||||
_getOutputStreamConfig() {
|
||||
|
||||
@@ -13,8 +13,8 @@
|
||||
"test": "npm run lint && npm run format && npm run types:check && npm run test:unit",
|
||||
"format": "prettier --list-different $PWD/'**/*.{js,cjs,ts}'",
|
||||
"format:fix": "prettier --write $PWD/'**/*.{js,cjs,ts}'",
|
||||
"lint": "eslint --ext .cjs,.js,.jsx,.mjs,.ts --max-warnings 0 --format unix .",
|
||||
"lint:fix": "eslint --fix --ext .cjs,.js,.jsx,.mjs,.ts .",
|
||||
"lint": "eslint --ext .js --ext .cjs --ext .ts --max-warnings 0 --format unix .",
|
||||
"lint:fix": "eslint --fix --ext .js --ext .cjs --ext .ts .",
|
||||
"test:ci": "npm run test:unit",
|
||||
"test:unit": "mocha --exit test/**/*.{js,cjs}",
|
||||
"types:check": "tsc --noEmit"
|
||||
@@ -23,13 +23,12 @@
|
||||
"@google-cloud/logging-bunyan": "^5.1.0",
|
||||
"@overleaf/fetch-utils": "*",
|
||||
"@overleaf/o-error": "*",
|
||||
"@sentry/node": "^6.13.2",
|
||||
"bunyan": "^1.8.14"
|
||||
},
|
||||
"devDependencies": {
|
||||
"chai": "^4.3.6",
|
||||
"mocha": "^11.1.0",
|
||||
"mocha-junit-reporter": "^2.2.1",
|
||||
"mocha-multi-reporters": "^1.5.1",
|
||||
"mocha": "^10.2.0",
|
||||
"sandboxed-module": "^2.0.4",
|
||||
"sinon": "^9.2.4",
|
||||
"sinon-chai": "^3.7.0",
|
||||
|
||||
@@ -0,0 +1,106 @@
|
||||
const Serializers = require('./serializers')
|
||||
const RATE_LIMIT_MAX_ERRORS = 5
|
||||
const RATE_LIMIT_INTERVAL_MS = 60000
|
||||
|
||||
class SentryManager {
|
||||
constructor(dsn, options) {
|
||||
this.Sentry = require('@sentry/node')
|
||||
this.Sentry.init({ dsn, ...options })
|
||||
// for rate limiting on sentry reporting
|
||||
this.lastErrorTimeStamp = 0
|
||||
this.lastErrorCount = 0
|
||||
}
|
||||
|
||||
captureExceptionRateLimited(attributes, message) {
|
||||
const now = Date.now()
|
||||
// have we recently reported an error?
|
||||
const recentSentryReport =
|
||||
now - this.lastErrorTimeStamp < RATE_LIMIT_INTERVAL_MS
|
||||
// if so, increment the error count
|
||||
if (recentSentryReport) {
|
||||
this.lastErrorCount++
|
||||
} else {
|
||||
this.lastErrorCount = 0
|
||||
this.lastErrorTimeStamp = now
|
||||
}
|
||||
// only report 5 errors every minute to avoid overload
|
||||
if (this.lastErrorCount < RATE_LIMIT_MAX_ERRORS) {
|
||||
// add a note if the rate limit has been hit
|
||||
const note =
|
||||
this.lastErrorCount + 1 === RATE_LIMIT_MAX_ERRORS
|
||||
? '(rate limited)'
|
||||
: ''
|
||||
// report the exception
|
||||
this.captureException(attributes, message, `error${note}`)
|
||||
}
|
||||
}
|
||||
|
||||
captureException(attributes, message, level) {
|
||||
// handle case of logger.error "message"
|
||||
if (typeof attributes === 'string') {
|
||||
attributes = { err: new Error(attributes) }
|
||||
}
|
||||
|
||||
// extract any error object
|
||||
let error = Serializers.err(attributes.err || attributes.error)
|
||||
|
||||
// avoid reporting errors twice
|
||||
for (const key in attributes) {
|
||||
const value = attributes[key]
|
||||
if (value instanceof Error && value.reportedToSentry) {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// include our log message in the error report
|
||||
if (error == null) {
|
||||
if (typeof message === 'string') {
|
||||
error = { message }
|
||||
}
|
||||
} else if (message != null) {
|
||||
attributes.description = message
|
||||
}
|
||||
|
||||
// report the error
|
||||
if (error != null) {
|
||||
// capture attributes and use *_id objects as tags
|
||||
const tags = {}
|
||||
const extra = {}
|
||||
for (const key in attributes) {
|
||||
let value = attributes[key]
|
||||
if (Serializers[key]) {
|
||||
value = Serializers[key](value)
|
||||
}
|
||||
if (key.match(/_id/) && typeof value === 'string') {
|
||||
tags[key] = value
|
||||
}
|
||||
extra[key] = value
|
||||
}
|
||||
|
||||
// OError integration
|
||||
extra.info = error.info
|
||||
delete error.info
|
||||
|
||||
// Sentry wants to receive an Error instance.
|
||||
const errInstance = new Error(error.message)
|
||||
Object.assign(errInstance, error)
|
||||
|
||||
try {
|
||||
// send the error to sentry
|
||||
this.Sentry.captureException(errInstance, { tags, extra, level })
|
||||
|
||||
// put a flag on the errors to avoid reporting them multiple times
|
||||
for (const key in attributes) {
|
||||
const value = attributes[key]
|
||||
if (value instanceof Error) {
|
||||
value.reportedToSentry = true
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
// ignore Sentry errors
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = SentryManager
|
||||
@@ -1,20 +0,0 @@
|
||||
const logger = require('./')
|
||||
const bunyan = require('bunyan')
|
||||
const serializers = require('./serializers')
|
||||
|
||||
function testLogRecorder() {
|
||||
const currentTest = this.currentTest
|
||||
for (const level of ['error', 'fatal']) {
|
||||
logger[level] = (info, msg) => {
|
||||
const entry = { level, ...info, msg }
|
||||
for (const [name, fn] of Object.entries(serializers)) {
|
||||
if (name in entry) entry[name] = fn(entry[name])
|
||||
}
|
||||
currentTest.consoleErrors = (currentTest.consoleErrors || []).concat(
|
||||
JSON.stringify(info, bunyan.safeCycles())
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = testLogRecorder
|
||||
@@ -1,9 +0,0 @@
|
||||
module.exports = {
|
||||
reporterEnabled: 'spec, mocha-junit-reporter',
|
||||
mochaJunitReporterReporterOptions: {
|
||||
mochaFile: `reports/junit-mocha-${process.env.MOCHA_GREP}.xml`,
|
||||
includePending: true,
|
||||
jenkinsMode: true,
|
||||
output: true,
|
||||
},
|
||||
}
|
||||
@@ -8,9 +8,4 @@ chai.use(sinonChai)
|
||||
|
||||
SandboxedModule.configure({
|
||||
globals: { Buffer, JSON, console, process },
|
||||
sourceTransformers: {
|
||||
removeNodePrefix: function (source) {
|
||||
return source.replace(/require\(['"]node:/g, "require('")
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
const Path = require('node:path')
|
||||
const { promisify } = require('node:util')
|
||||
const Path = require('path')
|
||||
const { promisify } = require('util')
|
||||
const SandboxedModule = require('sandboxed-module')
|
||||
const sinon = require('sinon')
|
||||
const { expect } = require('chai')
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
const SandboxedModule = require('sandboxed-module')
|
||||
const bunyan = require('bunyan')
|
||||
const { expect } = require('chai')
|
||||
const path = require('node:path')
|
||||
const path = require('path')
|
||||
const sinon = require('sinon')
|
||||
|
||||
const MODULE_PATH = path.join(__dirname, '../../logging-manager.js')
|
||||
@@ -43,14 +43,20 @@ describe('LoggingManager', function () {
|
||||
.stub()
|
||||
.returns(this.GCEMetadataLogLevelChecker),
|
||||
}
|
||||
this.SentryManager = {
|
||||
captureException: sinon.stub(),
|
||||
captureExceptionRateLimited: sinon.stub(),
|
||||
}
|
||||
this.LoggingManager = SandboxedModule.require(MODULE_PATH, {
|
||||
requires: {
|
||||
bunyan: this.Bunyan,
|
||||
'./log-level-checker': this.LogLevelChecker,
|
||||
'./sentry-manager': sinon.stub().returns(this.SentryManager),
|
||||
},
|
||||
})
|
||||
this.loggerName = 'test'
|
||||
this.logger = this.LoggingManager.initialize(this.loggerName)
|
||||
this.logger.initializeErrorReporting('test_dsn')
|
||||
})
|
||||
|
||||
afterEach(function () {
|
||||
@@ -154,6 +160,13 @@ describe('LoggingManager', function () {
|
||||
})
|
||||
})
|
||||
|
||||
describe('logger.error', function () {
|
||||
it('should report errors to Sentry', function () {
|
||||
this.logger.error({ foo: 'bar' }, 'message')
|
||||
expect(this.SentryManager.captureExceptionRateLimited).to.have.been.called
|
||||
})
|
||||
})
|
||||
|
||||
describe('ringbuffer', function () {
|
||||
beforeEach(function () {
|
||||
this.logBufferMock = [
|
||||
|
||||
@@ -0,0 +1,247 @@
|
||||
const Path = require('path')
|
||||
const SandboxedModule = require('sandboxed-module')
|
||||
const { expect } = require('chai')
|
||||
const sinon = require('sinon')
|
||||
|
||||
const MODULE_PATH = Path.join(__dirname, '../../sentry-manager.js')
|
||||
|
||||
describe('SentryManager', function () {
|
||||
beforeEach(function () {
|
||||
this.clock = sinon.useFakeTimers(Date.now())
|
||||
this.Sentry = {
|
||||
init: sinon.stub(),
|
||||
captureException: sinon.stub(),
|
||||
}
|
||||
this.SentryManager = SandboxedModule.require(MODULE_PATH, {
|
||||
requires: {
|
||||
'@sentry/node': this.Sentry,
|
||||
},
|
||||
})
|
||||
this.sentryManager = new this.SentryManager('test_dsn')
|
||||
})
|
||||
|
||||
afterEach(function () {
|
||||
this.clock.restore()
|
||||
})
|
||||
|
||||
describe('captureExceptionRateLimited', function () {
|
||||
it('should report a single error to sentry', function () {
|
||||
this.sentryManager.captureExceptionRateLimited({ foo: 'bar' }, 'message')
|
||||
expect(this.Sentry.captureException).to.have.been.calledOnce
|
||||
})
|
||||
|
||||
it('should report the same error to sentry only once', function () {
|
||||
const error1 = new Error('this is the error')
|
||||
this.sentryManager.captureExceptionRateLimited(
|
||||
{ foo: error1 },
|
||||
'first message'
|
||||
)
|
||||
this.sentryManager.captureExceptionRateLimited(
|
||||
{ bar: error1 },
|
||||
'second message'
|
||||
)
|
||||
expect(this.Sentry.captureException).to.have.been.calledOnce
|
||||
})
|
||||
|
||||
it('should report two different errors to sentry individually', function () {
|
||||
const error1 = new Error('this is the error')
|
||||
const error2 = new Error('this is the error')
|
||||
this.sentryManager.captureExceptionRateLimited(
|
||||
{ foo: error1 },
|
||||
'first message'
|
||||
)
|
||||
this.sentryManager.captureExceptionRateLimited(
|
||||
{ bar: error2 },
|
||||
'second message'
|
||||
)
|
||||
expect(this.Sentry.captureException).to.have.been.calledTwice
|
||||
})
|
||||
|
||||
it('for multiple errors should only report a maximum of 5 errors to sentry', function () {
|
||||
for (let i = 0; i < 10; i++) {
|
||||
this.sentryManager.captureExceptionRateLimited(
|
||||
{ foo: 'bar' },
|
||||
'message'
|
||||
)
|
||||
}
|
||||
expect(this.Sentry.captureException).to.have.callCount(5)
|
||||
})
|
||||
|
||||
it('for multiple errors with a minute delay should report 10 errors to sentry', function () {
|
||||
for (let i = 0; i < 10; i++) {
|
||||
this.sentryManager.captureExceptionRateLimited(
|
||||
{ foo: 'bar' },
|
||||
'message'
|
||||
)
|
||||
}
|
||||
expect(this.Sentry.captureException).to.have.callCount(5)
|
||||
|
||||
// allow a minute to pass
|
||||
this.clock.tick(61 * 1000)
|
||||
|
||||
for (let i = 0; i < 10; i++) {
|
||||
this.sentryManager.captureExceptionRateLimited(
|
||||
{ foo: 'bar' },
|
||||
'message'
|
||||
)
|
||||
}
|
||||
expect(this.Sentry.captureException).to.have.callCount(10)
|
||||
})
|
||||
})
|
||||
|
||||
describe('captureException', function () {
|
||||
it('should remove the path from fs errors', function () {
|
||||
const fsError = new Error(
|
||||
"Error: ENOENT: no such file or directory, stat '/tmp/3279b8d0-da10-11e8-8255-efd98985942b'"
|
||||
)
|
||||
fsError.path = '/tmp/3279b8d0-da10-11e8-8255-efd98985942b'
|
||||
this.sentryManager.captureException({ err: fsError }, 'message', 'error')
|
||||
expect(this.Sentry.captureException).to.have.been.calledWith(
|
||||
sinon.match.has(
|
||||
'message',
|
||||
'Error: ENOENT: no such file or directory, stat'
|
||||
)
|
||||
)
|
||||
})
|
||||
|
||||
it('should sanitize error', function () {
|
||||
const err = {
|
||||
name: 'CustomError',
|
||||
message: 'hello',
|
||||
_oErrorTags: [{ stack: 'here:1', info: { one: 1 } }],
|
||||
stack: 'here:0',
|
||||
info: { key: 'value' },
|
||||
code: 42,
|
||||
signal: 9,
|
||||
path: '/foo',
|
||||
}
|
||||
this.sentryManager.captureException({ err }, 'message', 'error')
|
||||
const expectedErr = {
|
||||
name: 'CustomError',
|
||||
message: 'hello',
|
||||
stack: 'here:0\nhere:1',
|
||||
code: 42,
|
||||
signal: 9,
|
||||
path: '/foo',
|
||||
}
|
||||
expect(this.Sentry.captureException).to.have.been.calledWith(
|
||||
sinon.match(expectedErr),
|
||||
sinon.match({
|
||||
tags: sinon.match.any,
|
||||
level: sinon.match.any,
|
||||
extra: {
|
||||
description: 'message',
|
||||
info: sinon.match({
|
||||
one: 1,
|
||||
key: 'value',
|
||||
}),
|
||||
},
|
||||
})
|
||||
)
|
||||
// Chai is very picky with comparing Error instances. Go the long way of comparing all the fields manually.
|
||||
const gotErr = this.Sentry.captureException.args[0][0]
|
||||
for (const [key, wanted] of Object.entries(expectedErr)) {
|
||||
expect(gotErr).to.have.property(key, wanted)
|
||||
}
|
||||
})
|
||||
it('should sanitize request', function () {
|
||||
const req = {
|
||||
ip: '1.2.3.4',
|
||||
method: 'GET',
|
||||
url: '/foo',
|
||||
headers: {
|
||||
referer: 'abc',
|
||||
'content-length': 1337,
|
||||
'user-agent': 'curl',
|
||||
authorization: '42',
|
||||
},
|
||||
}
|
||||
this.sentryManager.captureException({ req }, 'message', 'error')
|
||||
const expectedReq = {
|
||||
remoteAddress: '1.2.3.4',
|
||||
method: 'GET',
|
||||
url: '/foo',
|
||||
headers: {
|
||||
referer: 'abc',
|
||||
'content-length': 1337,
|
||||
'user-agent': 'curl',
|
||||
},
|
||||
}
|
||||
expect(this.Sentry.captureException).to.have.been.calledWith(
|
||||
sinon.match({
|
||||
message: 'message',
|
||||
}),
|
||||
sinon.match({
|
||||
tags: sinon.match.any,
|
||||
level: sinon.match.any,
|
||||
extra: {
|
||||
info: sinon.match.any,
|
||||
req: expectedReq,
|
||||
},
|
||||
})
|
||||
)
|
||||
expect(this.Sentry.captureException.args[0][1].extra.req).to.deep.equal(
|
||||
expectedReq
|
||||
)
|
||||
})
|
||||
it('should sanitize response', function () {
|
||||
const res = {
|
||||
statusCode: 417,
|
||||
body: Buffer.from('foo'),
|
||||
getHeader(key) {
|
||||
expect(key).to.be.oneOf(['content-length'])
|
||||
if (key === 'content-length') return 1337
|
||||
},
|
||||
}
|
||||
this.sentryManager.captureException({ res }, 'message', 'error')
|
||||
const expectedRes = {
|
||||
statusCode: 417,
|
||||
headers: {
|
||||
'content-length': 1337,
|
||||
},
|
||||
}
|
||||
expect(this.Sentry.captureException).to.have.been.calledWith(
|
||||
sinon.match({
|
||||
message: 'message',
|
||||
}),
|
||||
sinon.match({
|
||||
tags: sinon.match.any,
|
||||
level: sinon.match.any,
|
||||
extra: {
|
||||
info: sinon.match.any,
|
||||
res: expectedRes,
|
||||
},
|
||||
})
|
||||
)
|
||||
expect(this.Sentry.captureException.args[0][1].extra.res).to.deep.equal(
|
||||
expectedRes
|
||||
)
|
||||
})
|
||||
|
||||
describe('reportedToSentry', function () {
|
||||
it('should mark the error as reported to sentry', function () {
|
||||
const err = new Error()
|
||||
this.sentryManager.captureException({ err }, 'message')
|
||||
expect(this.Sentry.captureException).to.have.been.called
|
||||
expect(err.reportedToSentry).to.equal(true)
|
||||
})
|
||||
|
||||
it('should mark two errors as reported to sentry', function () {
|
||||
const err1 = new Error()
|
||||
const err2 = new Error()
|
||||
this.sentryManager.captureException({ err: err1, err2 }, 'message')
|
||||
expect(this.Sentry.captureException).to.have.been.called
|
||||
expect(err1.reportedToSentry).to.equal(true)
|
||||
expect(err2.reportedToSentry).to.equal(true)
|
||||
})
|
||||
|
||||
it('should not mark arbitrary objects as reported to sentry', function () {
|
||||
const err = new Error()
|
||||
const ctx = { foo: 'bar' }
|
||||
this.sentryManager.captureException({ err, ctx }, 'message')
|
||||
expect(this.Sentry.captureException).to.have.been.called
|
||||
expect(ctx.reportedToSentry).not.to.exist
|
||||
})
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -2,7 +2,6 @@
|
||||
"extends": "../../tsconfig.backend.json",
|
||||
"include": [
|
||||
"**/*.js",
|
||||
"**/*.cjs",
|
||||
"**/*.ts"
|
||||
"**/*.cjs"
|
||||
]
|
||||
}
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
node_modules/
|
||||
@@ -0,0 +1,3 @@
|
||||
node_modules
|
||||
|
||||
.npmrc
|
||||
@@ -1,13 +0,0 @@
|
||||
let reporterOptions = {}
|
||||
if (process.env.CI) {
|
||||
reporterOptions = {
|
||||
reporter: '/overleaf/node_modules/mocha-multi-reporters',
|
||||
'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
|
||||
}
|
||||
}
|
||||
const all = {
|
||||
require: 'test/setup.js',
|
||||
...reporterOptions,
|
||||
}
|
||||
|
||||
module.exports = all
|
||||
@@ -1 +1 @@
|
||||
22.18.0
|
||||
18.20.2
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
metrics
|
||||
--dependencies=None
|
||||
--docker-repos=gcr.io/overleaf-ops
|
||||
--env-add=
|
||||
--env-pass-through=
|
||||
--esmock-loader=False
|
||||
--is-library=True
|
||||
--node-version=22.18.0
|
||||
--pipeline-owner=🚉 Platform
|
||||
--node-version=18.20.2
|
||||
--public-repo=False
|
||||
--script-version=4.5.0
|
||||
|
||||
@@ -75,4 +75,4 @@ class RequestLogger {
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { monitor, RequestLogger }
|
||||
module.exports.monitor = monitor
|
||||
|
||||
@@ -97,12 +97,6 @@ function histogram(key, value, buckets, labels = {}) {
|
||||
}
|
||||
|
||||
class Timer {
|
||||
/**
|
||||
* @param {string} key
|
||||
* @param {number} sampleRate
|
||||
* @param {Record<string, any>} labels
|
||||
* @param {Array<number>} buckets
|
||||
*/
|
||||
constructor(key, sampleRate = 1, labels = {}, buckets = undefined) {
|
||||
if (typeof sampleRate === 'object') {
|
||||
// called with (key, labels, buckets)
|
||||
|
||||
@@ -5,8 +5,6 @@
|
||||
* before any other module to support code instrumentation.
|
||||
*/
|
||||
|
||||
const metricsModuleImportStartTime = performance.now()
|
||||
|
||||
const APP_NAME = process.env.METRICS_APP_NAME || 'unknown'
|
||||
const BUILD_VERSION = process.env.BUILD_VERSION
|
||||
const ENABLE_PROFILE_AGENT = process.env.ENABLE_PROFILE_AGENT === 'true'
|
||||
@@ -90,7 +88,7 @@ function initializeProfileAgent() {
|
||||
}
|
||||
|
||||
function initializePrometheus() {
|
||||
const os = require('node:os')
|
||||
const os = require('os')
|
||||
const promClient = require('prom-client')
|
||||
promClient.register.setDefaultLabels({ app: APP_NAME, host: os.hostname() })
|
||||
promClient.collectDefaultMetrics({ timeout: 5000, prefix: '' })
|
||||
@@ -105,5 +103,3 @@ function recordProcessStart() {
|
||||
const metrics = require('.')
|
||||
metrics.inc('process_startup')
|
||||
}
|
||||
|
||||
module.exports = { metricsModuleImportStartTime }
|
||||
|
||||
@@ -6,8 +6,8 @@
|
||||
* logged along with the corresponding information from /proc/net/tcp.
|
||||
*/
|
||||
|
||||
const fs = require('node:fs')
|
||||
const diagnosticsChannel = require('node:diagnostics_channel')
|
||||
const fs = require('fs')
|
||||
const diagnosticsChannel = require('diagnostics_channel')
|
||||
|
||||
const SOCKET_MONITOR_INTERVAL = 60 * 1000
|
||||
// set the threshold for logging leaked sockets in minutes, defaults to 15
|
||||
|
||||
@@ -11,13 +11,13 @@ const seconds = 1000
|
||||
// In Node 0.10 the default is 5, which means only 5 open connections at one.
|
||||
// Node 0.12 has a default of Infinity. Make sure we have no limit set,
|
||||
// regardless of Node version.
|
||||
require('node:http').globalAgent.maxSockets = Infinity
|
||||
require('node:https').globalAgent.maxSockets = Infinity
|
||||
require('http').globalAgent.maxSockets = Infinity
|
||||
require('https').globalAgent.maxSockets = Infinity
|
||||
|
||||
const SOCKETS_HTTP = require('node:http').globalAgent.sockets
|
||||
const SOCKETS_HTTPS = require('node:https').globalAgent.sockets
|
||||
const FREE_SOCKETS_HTTP = require('node:http').globalAgent.freeSockets
|
||||
const FREE_SOCKETS_HTTPS = require('node:https').globalAgent.freeSockets
|
||||
const SOCKETS_HTTP = require('http').globalAgent.sockets
|
||||
const SOCKETS_HTTPS = require('https').globalAgent.sockets
|
||||
const FREE_SOCKETS_HTTP = require('http').globalAgent.freeSockets
|
||||
const FREE_SOCKETS_HTTPS = require('https').globalAgent.freeSockets
|
||||
|
||||
// keep track of set gauges and reset them in the next collection cycle
|
||||
const SEEN_HOSTS_HTTP = new Set()
|
||||
|
||||
@@ -9,7 +9,7 @@
|
||||
"main": "index.js",
|
||||
"dependencies": {
|
||||
"@google-cloud/opentelemetry-cloud-trace-exporter": "^2.1.0",
|
||||
"@google-cloud/profiler": "^6.0.3",
|
||||
"@google-cloud/profiler": "^6.0.0",
|
||||
"@opentelemetry/api": "^1.4.1",
|
||||
"@opentelemetry/auto-instrumentations-node": "^0.39.1",
|
||||
"@opentelemetry/exporter-trace-otlp-http": "^0.41.2",
|
||||
@@ -23,18 +23,16 @@
|
||||
"devDependencies": {
|
||||
"bunyan": "^1.0.0",
|
||||
"chai": "^4.3.6",
|
||||
"mocha": "^11.1.0",
|
||||
"mocha-junit-reporter": "^2.2.1",
|
||||
"mocha-multi-reporters": "^1.5.1",
|
||||
"mocha": "^10.2.0",
|
||||
"sandboxed-module": "^2.0.4",
|
||||
"sinon": "^9.2.4",
|
||||
"typescript": "^5.0.4"
|
||||
},
|
||||
"scripts": {
|
||||
"lint": "eslint --ext .cjs,.js,.jsx,.mjs,.ts --max-warnings 0 --format unix .",
|
||||
"lint:fix": "eslint --fix --ext .cjs,.js,.jsx,.mjs,.ts .",
|
||||
"lint": "eslint --ext .js --ext .cjs --ext .ts --max-warnings 0 --format unix .",
|
||||
"lint:fix": "eslint --fix --ext .js --ext .cjs --ext .ts .",
|
||||
"test:unit": "mocha --exit test/**/*.{js,cjs}",
|
||||
"test:acceptance": "mocha --recursive --exit --grep=$MOCHA_GREP test/acceptance",
|
||||
"test:acceptance": "mocha --reporter spec --recursive --exit --grep=$MOCHA_GREP test/acceptance",
|
||||
"test": "npm run lint && npm run format && npm run types:check && npm run test:unit",
|
||||
"format": "prettier --list-different $PWD/'**/*.{js,cjs,ts}'",
|
||||
"format:fix": "prettier --write $PWD/'**/*.{js,cjs,ts}'",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
const { promisify } = require('node:util')
|
||||
const os = require('node:os')
|
||||
const http = require('node:http')
|
||||
const { promisify } = require('util')
|
||||
const os = require('os')
|
||||
const http = require('http')
|
||||
const { expect } = require('chai')
|
||||
const Metrics = require('../..')
|
||||
|
||||
@@ -316,7 +316,7 @@ async function checkSummaryValues(key, values) {
|
||||
for (const quantile of Object.keys(values)) {
|
||||
expect(found[quantile]).to.be.within(
|
||||
values[quantile] - 5,
|
||||
values[quantile] + 15,
|
||||
values[quantile] + 5,
|
||||
`quantile: ${quantile}`
|
||||
)
|
||||
}
|
||||
|
||||
@@ -1,9 +0,0 @@
|
||||
module.exports = {
|
||||
reporterEnabled: 'spec, mocha-junit-reporter',
|
||||
mochaJunitReporterReporterOptions: {
|
||||
mochaFile: `reports/junit-mocha-${process.env.MOCHA_GREP}.xml`,
|
||||
includePending: true,
|
||||
jenkinsMode: true,
|
||||
output: true,
|
||||
},
|
||||
}
|
||||
@@ -1 +0,0 @@
|
||||
// no setup yet
|
||||
@@ -5,7 +5,7 @@
|
||||
*/
|
||||
const chai = require('chai')
|
||||
const { expect } = chai
|
||||
const path = require('node:path')
|
||||
const path = require('path')
|
||||
const modulePath = path.join(__dirname, '../../../event_loop.js')
|
||||
const SandboxedModule = require('sandboxed-module')
|
||||
const sinon = require('sinon')
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
const Path = require('node:path')
|
||||
const Path = require('path')
|
||||
const SandboxedModule = require('sandboxed-module')
|
||||
const sinon = require('sinon')
|
||||
|
||||
|
||||
@@ -2,7 +2,6 @@
|
||||
"extends": "../../tsconfig.backend.json",
|
||||
"include": [
|
||||
"**/*.js",
|
||||
"**/*.cjs",
|
||||
"**/*.ts"
|
||||
"**/*.cjs"
|
||||
]
|
||||
}
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
22.18.0
|
||||
@@ -1,321 +0,0 @@
|
||||
// @ts-check
|
||||
/* eslint-disable no-console */
|
||||
const { ObjectId, ReadPreference } = require('mongodb')
|
||||
|
||||
const READ_PREFERENCE_SECONDARY =
|
||||
process.env.MONGO_HAS_SECONDARIES === 'true'
|
||||
? ReadPreference.secondary.mode
|
||||
: ReadPreference.secondaryPreferred.mode
|
||||
|
||||
const ONE_MONTH_IN_MS = 1000 * 60 * 60 * 24 * 31
|
||||
let ID_EDGE_PAST
|
||||
const ID_EDGE_FUTURE = objectIdFromMs(Date.now() + 1000)
|
||||
let BATCH_DESCENDING
|
||||
let BATCH_SIZE
|
||||
let VERBOSE_LOGGING
|
||||
let BATCH_RANGE_START
|
||||
let BATCH_RANGE_END
|
||||
let BATCH_MAX_TIME_SPAN_IN_MS
|
||||
let BATCHED_UPDATE_RUNNING = false
|
||||
|
||||
/**
|
||||
* @typedef {import("mongodb").Collection} Collection
|
||||
* @typedef {import("mongodb-legacy").Collection} LegacyCollection
|
||||
* @typedef {import("mongodb").Document} Document
|
||||
* @typedef {import("mongodb").FindOptions} FindOptions
|
||||
* @typedef {import("mongodb").UpdateFilter<Document>} UpdateDocument
|
||||
*/
|
||||
|
||||
/**
|
||||
* @typedef {Object} BatchedUpdateOptions
|
||||
* @property {string} [BATCH_DESCENDING]
|
||||
* @property {string} [BATCH_LAST_ID]
|
||||
* @property {string} [BATCH_MAX_TIME_SPAN_IN_MS]
|
||||
* @property {string} [BATCH_RANGE_END]
|
||||
* @property {string} [BATCH_RANGE_START]
|
||||
* @property {string} [BATCH_SIZE]
|
||||
* @property {string} [VERBOSE_LOGGING]
|
||||
* @property {(progress: string) => Promise<void>} [trackProgress]
|
||||
*/
|
||||
|
||||
/**
|
||||
* @param {BatchedUpdateOptions} options
|
||||
*/
|
||||
function refreshGlobalOptionsForBatchedUpdate(options = {}) {
|
||||
options = Object.assign({}, options, process.env)
|
||||
|
||||
BATCH_DESCENDING = options.BATCH_DESCENDING === 'true'
|
||||
BATCH_SIZE = parseInt(options.BATCH_SIZE || '1000', 10) || 1000
|
||||
VERBOSE_LOGGING = options.VERBOSE_LOGGING === 'true'
|
||||
if (options.BATCH_LAST_ID) {
|
||||
BATCH_RANGE_START = objectIdFromInput(options.BATCH_LAST_ID)
|
||||
} else if (options.BATCH_RANGE_START) {
|
||||
BATCH_RANGE_START = objectIdFromInput(options.BATCH_RANGE_START)
|
||||
} else {
|
||||
if (BATCH_DESCENDING) {
|
||||
BATCH_RANGE_START = ID_EDGE_FUTURE
|
||||
} else {
|
||||
BATCH_RANGE_START = ID_EDGE_PAST
|
||||
}
|
||||
}
|
||||
BATCH_MAX_TIME_SPAN_IN_MS = parseInt(
|
||||
options.BATCH_MAX_TIME_SPAN_IN_MS || ONE_MONTH_IN_MS.toString(),
|
||||
10
|
||||
)
|
||||
if (options.BATCH_RANGE_END) {
|
||||
BATCH_RANGE_END = objectIdFromInput(options.BATCH_RANGE_END)
|
||||
} else {
|
||||
if (BATCH_DESCENDING) {
|
||||
BATCH_RANGE_END = ID_EDGE_PAST
|
||||
} else {
|
||||
BATCH_RANGE_END = ID_EDGE_FUTURE
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {Collection | LegacyCollection} collection
|
||||
* @param {Document} query
|
||||
* @param {ObjectId} start
|
||||
* @param {ObjectId} end
|
||||
* @param {Document} projection
|
||||
* @param {FindOptions} findOptions
|
||||
* @return {Promise<Array<Document>>}
|
||||
*/
|
||||
async function getNextBatch(
|
||||
collection,
|
||||
query,
|
||||
start,
|
||||
end,
|
||||
projection,
|
||||
findOptions
|
||||
) {
|
||||
if (BATCH_DESCENDING) {
|
||||
query._id = {
|
||||
$gt: end,
|
||||
$lte: start,
|
||||
}
|
||||
} else {
|
||||
query._id = {
|
||||
$gt: start,
|
||||
$lte: end,
|
||||
}
|
||||
}
|
||||
return await collection
|
||||
.find(query, findOptions)
|
||||
.project(projection)
|
||||
.sort({ _id: BATCH_DESCENDING ? -1 : 1 })
|
||||
.limit(BATCH_SIZE)
|
||||
.toArray()
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {Collection | LegacyCollection} collection
|
||||
* @param {Array<Document>} nextBatch
|
||||
* @param {UpdateDocument} update
|
||||
* @return {Promise<void>}
|
||||
*/
|
||||
async function performUpdate(collection, nextBatch, update) {
|
||||
await collection.updateMany(
|
||||
{ _id: { $in: nextBatch.map(entry => entry._id) } },
|
||||
update
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} input
|
||||
* @return {ObjectId}
|
||||
*/
|
||||
function objectIdFromInput(input) {
|
||||
if (input.includes('T')) {
|
||||
const t = new Date(input).getTime()
|
||||
if (Number.isNaN(t)) throw new Error(`${input} is not a valid date`)
|
||||
return objectIdFromMs(t)
|
||||
} else {
|
||||
return new ObjectId(input)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {ObjectId} objectId
|
||||
* @return {string}
|
||||
*/
|
||||
function renderObjectId(objectId) {
|
||||
return `${objectId} (${objectId.getTimestamp().toISOString()})`
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {number} ms
|
||||
* @return {ObjectId}
|
||||
*/
|
||||
function objectIdFromMs(ms) {
|
||||
return ObjectId.createFromTime(ms / 1000)
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {ObjectId} id
|
||||
* @return {number}
|
||||
*/
|
||||
function getMsFromObjectId(id) {
|
||||
return id.getTimestamp().getTime()
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {ObjectId} start
|
||||
* @return {ObjectId}
|
||||
*/
|
||||
function getNextEnd(start) {
|
||||
let end
|
||||
if (BATCH_DESCENDING) {
|
||||
end = objectIdFromMs(getMsFromObjectId(start) - BATCH_MAX_TIME_SPAN_IN_MS)
|
||||
if (getMsFromObjectId(end) <= getMsFromObjectId(BATCH_RANGE_END)) {
|
||||
end = BATCH_RANGE_END
|
||||
}
|
||||
} else {
|
||||
end = objectIdFromMs(getMsFromObjectId(start) + BATCH_MAX_TIME_SPAN_IN_MS)
|
||||
if (getMsFromObjectId(end) >= getMsFromObjectId(BATCH_RANGE_END)) {
|
||||
end = BATCH_RANGE_END
|
||||
}
|
||||
}
|
||||
return end
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {Collection | LegacyCollection} collection
|
||||
* @return {Promise<ObjectId|null>}
|
||||
*/
|
||||
async function getIdEdgePast(collection) {
|
||||
const [first] = await collection
|
||||
.find({})
|
||||
.project({ _id: 1 })
|
||||
.sort({ _id: 1 })
|
||||
.limit(1)
|
||||
.toArray()
|
||||
if (!first) return null
|
||||
// Go one second further into the past in order to include the first entry via
|
||||
// first._id > ID_EDGE_PAST
|
||||
return objectIdFromMs(Math.max(0, getMsFromObjectId(first._id) - 1000))
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {Collection | LegacyCollection} collection
|
||||
* @param {Document} query
|
||||
* @param {UpdateDocument | ((batch: Array<Document>) => Promise<void>)} update
|
||||
* @param {Document} [projection]
|
||||
* @param {FindOptions} [findOptions]
|
||||
* @param {BatchedUpdateOptions} [batchedUpdateOptions]
|
||||
*/
|
||||
async function batchedUpdate(
|
||||
collection,
|
||||
query,
|
||||
update,
|
||||
projection,
|
||||
findOptions,
|
||||
batchedUpdateOptions = {}
|
||||
) {
|
||||
// only a single batchedUpdate can run at a time due to global variables
|
||||
if (BATCHED_UPDATE_RUNNING) {
|
||||
throw new Error('batchedUpdate is already running')
|
||||
}
|
||||
try {
|
||||
BATCHED_UPDATE_RUNNING = true
|
||||
ID_EDGE_PAST = await getIdEdgePast(collection)
|
||||
if (!ID_EDGE_PAST) {
|
||||
console.warn(
|
||||
`The collection ${collection.collectionName} appears to be empty.`
|
||||
)
|
||||
return 0
|
||||
}
|
||||
refreshGlobalOptionsForBatchedUpdate(batchedUpdateOptions)
|
||||
const { trackProgress = async progress => console.warn(progress) } =
|
||||
batchedUpdateOptions
|
||||
|
||||
findOptions = findOptions || {}
|
||||
findOptions.readPreference = READ_PREFERENCE_SECONDARY
|
||||
|
||||
projection = projection || { _id: 1 }
|
||||
let nextBatch
|
||||
let updated = 0
|
||||
let start = BATCH_RANGE_START
|
||||
|
||||
while (start !== BATCH_RANGE_END) {
|
||||
let end = getNextEnd(start)
|
||||
nextBatch = await getNextBatch(
|
||||
collection,
|
||||
query,
|
||||
start,
|
||||
end,
|
||||
projection,
|
||||
findOptions
|
||||
)
|
||||
if (nextBatch.length > 0) {
|
||||
end = nextBatch[nextBatch.length - 1]._id
|
||||
updated += nextBatch.length
|
||||
|
||||
if (VERBOSE_LOGGING) {
|
||||
console.log(
|
||||
`Running update on batch with ids ${JSON.stringify(
|
||||
nextBatch.map(entry => entry._id)
|
||||
)}`
|
||||
)
|
||||
}
|
||||
await trackProgress(
|
||||
`Running update on batch ending ${renderObjectId(end)}`
|
||||
)
|
||||
|
||||
if (typeof update === 'function') {
|
||||
await update(nextBatch)
|
||||
} else {
|
||||
await performUpdate(collection, nextBatch, update)
|
||||
}
|
||||
}
|
||||
await trackProgress(`Completed batch ending ${renderObjectId(end)}`)
|
||||
start = end
|
||||
}
|
||||
return updated
|
||||
} finally {
|
||||
BATCHED_UPDATE_RUNNING = false
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {Collection | LegacyCollection} collection
|
||||
* @param {Document} query
|
||||
* @param {UpdateDocument | ((batch: Array<Object>) => Promise<void>)} update
|
||||
* @param {Document} [projection]
|
||||
* @param {FindOptions} [findOptions]
|
||||
* @param {BatchedUpdateOptions} [batchedUpdateOptions]
|
||||
*/
|
||||
function batchedUpdateWithResultHandling(
|
||||
collection,
|
||||
query,
|
||||
update,
|
||||
projection,
|
||||
findOptions,
|
||||
batchedUpdateOptions
|
||||
) {
|
||||
batchedUpdate(
|
||||
collection,
|
||||
query,
|
||||
update,
|
||||
projection,
|
||||
findOptions,
|
||||
batchedUpdateOptions
|
||||
)
|
||||
.then(processed => {
|
||||
console.error({ processed })
|
||||
process.exit(0)
|
||||
})
|
||||
.catch(error => {
|
||||
console.error({ error })
|
||||
process.exit(1)
|
||||
})
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
READ_PREFERENCE_SECONDARY,
|
||||
objectIdFromInput,
|
||||
renderObjectId,
|
||||
batchedUpdate,
|
||||
batchedUpdateWithResultHandling,
|
||||
}
|
||||
@@ -1,9 +0,0 @@
|
||||
mongo-utils
|
||||
--dependencies=None
|
||||
--env-add=
|
||||
--env-pass-through=
|
||||
--esmock-loader=False
|
||||
--is-library=True
|
||||
--node-version=22.18.0
|
||||
--pipeline-owner=🚉 Platform
|
||||
--public-repo=False
|
||||
@@ -1,5 +0,0 @@
|
||||
// @ts-check
|
||||
|
||||
const { cleanupTestDatabase, dropTestDatabase } = require('./test-utils')
|
||||
|
||||
module.exports = { cleanupTestDatabase, dropTestDatabase }
|
||||
@@ -1,32 +0,0 @@
|
||||
{
|
||||
"name": "@overleaf/mongo-utils",
|
||||
"version": "0.0.1",
|
||||
"description": "utilities to help working with mongo",
|
||||
"main": "index.js",
|
||||
"scripts": {
|
||||
"test": "npm run lint && npm run format && npm run types:check && npm run test:unit",
|
||||
"test:unit": "mocha --exit test/**/*.{js,cjs}",
|
||||
"lint": "eslint --ext .cjs,.js,.jsx,.mjs,.ts --max-warnings 0 --format unix .",
|
||||
"lint:fix": "eslint --fix --ext .cjs,.js,.jsx,.mjs,.ts .",
|
||||
"format": "prettier --list-different $PWD/'**/*.{js,cjs,ts}'",
|
||||
"format:fix": "prettier --write $PWD/'**/*.{js,cjs,ts}'",
|
||||
"test:ci": "npm run test:unit",
|
||||
"types:check": "tsc --noEmit"
|
||||
},
|
||||
"author": "Overleaf (https://www.overleaf.com)",
|
||||
"license": "AGPL-3.0-only",
|
||||
"dependencies": {
|
||||
"mongodb": "6.12.0",
|
||||
"mongodb-legacy": "6.1.3"
|
||||
},
|
||||
"devDependencies": {
|
||||
"chai": "^4.3.6",
|
||||
"mocha": "^11.1.0",
|
||||
"mocha-junit-reporter": "^2.2.1",
|
||||
"mocha-multi-reporters": "^1.5.1",
|
||||
"sandboxed-module": "^2.0.4",
|
||||
"sinon": "^9.2.4",
|
||||
"sinon-chai": "^3.7.0",
|
||||
"typescript": "^5.0.4"
|
||||
}
|
||||
}
|
||||
@@ -1,55 +0,0 @@
|
||||
// @ts-check
|
||||
|
||||
/**
|
||||
* @import { MongoClient } from 'mongodb'
|
||||
* @import { MongoClient as LegacyMongoClient } from 'mongodb-legacy'
|
||||
*/
|
||||
|
||||
/**
|
||||
* Delete all data from the test Mongo database
|
||||
*
|
||||
* This doesn't drop the collections, so indexes are preserved.
|
||||
*
|
||||
* @param {MongoClient | LegacyMongoClient} mongoClient
|
||||
*/
|
||||
async function cleanupTestDatabase(mongoClient) {
|
||||
ensureTestDatabase(mongoClient)
|
||||
const db = mongoClient.db()
|
||||
const allCollections = await db.collections()
|
||||
const collections = allCollections.filter(
|
||||
coll => coll.collectionName !== 'migrations'
|
||||
)
|
||||
await Promise.all(collections.map(coll => coll.deleteMany({})))
|
||||
}
|
||||
|
||||
/**
|
||||
* Drop the test Monto database
|
||||
*
|
||||
* This drops the whole database, including indexes.
|
||||
*
|
||||
* @param {MongoClient | LegacyMongoClient } mongoClient
|
||||
*/
|
||||
async function dropTestDatabase(mongoClient) {
|
||||
ensureTestDatabase(mongoClient)
|
||||
await mongoClient.db().dropDatabase()
|
||||
}
|
||||
/**
|
||||
* Ensure that the given client is connected to a test database.
|
||||
*
|
||||
* This should be called before performing destructive operations on the test
|
||||
* database.
|
||||
*
|
||||
* @param {MongoClient | LegacyMongoClient } mongoClient
|
||||
*/
|
||||
function ensureTestDatabase(mongoClient) {
|
||||
const dbName = mongoClient.db().databaseName
|
||||
const env = process.env.NODE_ENV
|
||||
|
||||
if (dbName !== 'test-overleaf' || env !== 'test') {
|
||||
throw new Error(
|
||||
`Refusing to clear database '${dbName}' in environment '${env}'`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { cleanupTestDatabase, dropTestDatabase }
|
||||
@@ -1,8 +0,0 @@
|
||||
{
|
||||
"extends": "../../tsconfig.backend.json",
|
||||
"include": [
|
||||
"**/*.js",
|
||||
"**/*.cjs",
|
||||
"**/*.ts"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
node_modules/
|
||||
@@ -0,0 +1,9 @@
|
||||
root = true
|
||||
|
||||
[*]
|
||||
charset = utf-8
|
||||
indent_style = space
|
||||
indent_size = 2
|
||||
end_of_line = lf
|
||||
insert_final_newline = true
|
||||
trim_trailing_whitespace = true
|
||||
@@ -0,0 +1,5 @@
|
||||
.nyc_output
|
||||
coverage
|
||||
node_modules/
|
||||
|
||||
.npmrc
|
||||
@@ -1,13 +0,0 @@
|
||||
let reporterOptions = {}
|
||||
if (process.env.CI) {
|
||||
reporterOptions = {
|
||||
reporter: '/overleaf/node_modules/mocha-multi-reporters',
|
||||
'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
|
||||
}
|
||||
}
|
||||
const all = {
|
||||
require: 'test/setup.js',
|
||||
...reporterOptions,
|
||||
}
|
||||
|
||||
module.exports = all
|
||||
@@ -1 +1 @@
|
||||
22.18.0
|
||||
18.20.2
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
o-error
|
||||
--dependencies=None
|
||||
--docker-repos=gcr.io/overleaf-ops
|
||||
--env-add=
|
||||
--env-pass-through=
|
||||
--esmock-loader=False
|
||||
--is-library=True
|
||||
--node-version=22.18.0
|
||||
--pipeline-owner=🚉 Platform
|
||||
--node-version=18.20.2
|
||||
--public-repo=False
|
||||
--script-version=4.5.0
|
||||
|
||||
@@ -70,7 +70,7 @@ sayHi3(43, (err, result) => {
|
||||
}
|
||||
})
|
||||
|
||||
const promisify = require('node:util').promisify
|
||||
const promisify = require('util').promisify
|
||||
demoDatabase.findUserAsync = promisify(demoDatabase.findUser)
|
||||
|
||||
async function sayHi4NoHandling(userId) {
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user