mirror of
https://github.com/yu-i-i/overleaf-cep.git
synced 2026-11-12 13:13:45 +01:00
[web] optimize process notifications and add monitoring metrics (#34699)
GitOrigin-RevId: 411355cbdbb25d871b9cfb2afbb0ad235041337f
This commit is contained in:
Kristina
@@ -1 +0,0 @@
|
||||
24.14.1
|
||||
@@ -1,51 +0,0 @@
|
||||
# This file was auto-generated, do not edit it directly.
|
||||
# Instead run bin/update_build_scripts from
|
||||
# https://github.com/overleaf/internal/
|
||||
|
||||
FROM node:24.14.1 AS base
|
||||
|
||||
# Corepack setup, shared between all the images.
|
||||
ENV PATH="/overleaf/node_modules/.bin:$PATH"
|
||||
ENV COREPACK_HOME=/opt/corepack
|
||||
RUN corepack enable && corepack install -g yarn@4.14.1
|
||||
ENV COREPACK_ENABLE_NETWORK=0
|
||||
|
||||
WORKDIR /overleaf/services/filestore
|
||||
COPY services/filestore/install_deps.sh /overleaf/services/filestore/
|
||||
RUN chmod 0755 ./install_deps.sh && ./install_deps.sh
|
||||
|
||||
# Google Cloud Storage needs a writable $HOME/.config for resumable uploads
|
||||
# (see https://googleapis.dev/nodejs/storage/latest/File.html#createWriteStream)
|
||||
RUN mkdir /home/node/.config && chown node:node /home/node/.config
|
||||
|
||||
FROM base AS app
|
||||
|
||||
COPY package.json yarn.lock .yarnrc.yml /overleaf/
|
||||
COPY libraries/eslint-plugin/package.json /overleaf/libraries/eslint-plugin/package.json
|
||||
COPY libraries/fetch-utils/package.json /overleaf/libraries/fetch-utils/package.json
|
||||
COPY libraries/logger/package.json /overleaf/libraries/logger/package.json
|
||||
COPY libraries/metrics/package.json /overleaf/libraries/metrics/package.json
|
||||
COPY libraries/o-error/package.json /overleaf/libraries/o-error/package.json
|
||||
COPY libraries/object-persistor/package.json /overleaf/libraries/object-persistor/package.json
|
||||
COPY libraries/settings/package.json /overleaf/libraries/settings/package.json
|
||||
COPY libraries/stream-utils/package.json /overleaf/libraries/stream-utils/package.json
|
||||
COPY services/filestore/package.json /overleaf/services/filestore/package.json
|
||||
COPY .yarn/patches/ /overleaf/.yarn/patches/
|
||||
|
||||
RUN cd /overleaf && yarn workspaces focus @overleaf/filestore overleaf
|
||||
COPY libraries/eslint-plugin/ /overleaf/libraries/eslint-plugin/
|
||||
COPY libraries/fetch-utils/ /overleaf/libraries/fetch-utils/
|
||||
COPY libraries/logger/ /overleaf/libraries/logger/
|
||||
COPY libraries/metrics/ /overleaf/libraries/metrics/
|
||||
COPY libraries/o-error/ /overleaf/libraries/o-error/
|
||||
COPY libraries/object-persistor/ /overleaf/libraries/object-persistor/
|
||||
COPY libraries/settings/ /overleaf/libraries/settings/
|
||||
COPY libraries/stream-utils/ /overleaf/libraries/stream-utils/
|
||||
COPY services/filestore/ /overleaf/services/filestore/
|
||||
|
||||
FROM app
|
||||
RUN mkdir -p uploads template_files \
|
||||
&& chown node:node uploads template_files
|
||||
USER node
|
||||
|
||||
CMD ["node", "--expose-gc", "app.js"]
|
||||
@@ -1,205 +0,0 @@
|
||||
# This file was auto-generated, do not edit it directly.
|
||||
# Instead run bin/update_build_scripts from
|
||||
# https://github.com/overleaf/internal/
|
||||
|
||||
export BUILD_NUMBER ?= local
|
||||
export BRANCH_NAME ?= $(shell git rev-parse --abbrev-ref HEAD)
|
||||
export BRANCH_NAME_TAG_SAFE = $(shell echo $(BRANCH_NAME) | sed 's/\//\-\-/g' | cut -c1-96)
|
||||
export PROJECT_NAME = filestore
|
||||
BUILD_DIR_NAME = $(shell pwd | xargs basename | tr -cd '[a-zA-Z0-9_.\-]')
|
||||
HERE=$(shell pwd)
|
||||
export MONOREPO ?= $(shell cd ../../ && pwd)
|
||||
export RETRIES ?= $(if $(CI),3,0)
|
||||
IMAGE_CI ?= ci/$(PROJECT_NAME):$(BRANCH_NAME_TAG_SAFE)-$(BUILD_NUMBER)
|
||||
IMAGE_REPO ?= us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME)
|
||||
IMAGE_REPO_FINAL ?= $(IMAGE_REPO):$(BRANCH_NAME_TAG_SAFE)-$(BUILD_NUMBER)
|
||||
IMAGE_CACHE ?= $(IMAGE_REPO):cache-$(shell cat \
|
||||
$(MONOREPO)/package.json \
|
||||
$(MONOREPO)/yarn.lock \
|
||||
$(MONOREPO)/libraries/eslint-plugin/package.json \
|
||||
$(MONOREPO)/libraries/fetch-utils/package.json \
|
||||
$(MONOREPO)/libraries/logger/package.json \
|
||||
$(MONOREPO)/libraries/metrics/package.json \
|
||||
$(MONOREPO)/libraries/o-error/package.json \
|
||||
$(MONOREPO)/libraries/object-persistor/package.json \
|
||||
$(MONOREPO)/libraries/settings/package.json \
|
||||
$(MONOREPO)/libraries/stream-utils/package.json \
|
||||
$(MONOREPO)/services/filestore/package.json \
|
||||
$(MONOREPO)/.yarn/patches/* \
|
||||
| sha256sum | cut -d '-' -f1)
|
||||
export MOCHA_GREP ?=
|
||||
|
||||
DOCKER_COMPOSE_FLAGS ?= -f docker-compose.yml
|
||||
DOCKER_COMPOSE := docker compose ${DOCKER_COMPOSE_FLAGS}
|
||||
|
||||
# Common flags for docker compose run.
|
||||
# --rm Tear down test runner container
|
||||
# --build Make sure that we use the latest base image
|
||||
# --quiet-build Hide build output
|
||||
ifeq ($(CI),true)
|
||||
DC_RUN_FLAGS := --rm
|
||||
else
|
||||
DC_RUN_FLAGS := --rm --build --quiet-build
|
||||
endif
|
||||
|
||||
COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE ?= test_acceptance_$(BUILD_DIR_NAME)
|
||||
DOCKER_COMPOSE_TEST_ACCEPTANCE = \
|
||||
COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE) $(DOCKER_COMPOSE)
|
||||
|
||||
COMPOSE_PROJECT_NAME_TEST_UNIT ?= test_unit_$(BUILD_DIR_NAME)
|
||||
DOCKER_COMPOSE_TEST_UNIT = \
|
||||
COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_UNIT) $(DOCKER_COMPOSE)
|
||||
|
||||
.PHONY: print-branch-tag-safe
|
||||
print-branch-tag-safe:
|
||||
@echo $(BRANCH_NAME_TAG_SAFE)
|
||||
|
||||
clean:
|
||||
-docker rmi $(IMAGE_CI)
|
||||
-docker rmi $(IMAGE_REPO_FINAL)
|
||||
-$(DOCKER_COMPOSE_TEST_UNIT) down --remove-orphans --rmi local --timeout 0 --volumes
|
||||
-$(DOCKER_COMPOSE_TEST_ACCEPTANCE) down --remove-orphans --rmi local --timeout 0 --volumes
|
||||
-rm -rf reports/
|
||||
-git clean -dfX uploads template_files
|
||||
|
||||
# Run the linting commands in the scope of the monorepo.
|
||||
# Eslint and prettier (plus some configs) are on the root.
|
||||
RUN_LINTING = ../../bin/run -w /overleaf/services/$(PROJECT_NAME) monorepo yarn run --silent
|
||||
RUN_LINTING_MONOREPO = ../../bin/run monorepo yarn run --silent
|
||||
|
||||
RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/eslint.config.mjs:/overleaf/eslint.config.mjs --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc.cjs:/overleaf/.prettierrc.cjs --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/filestore/reports:/overleaf/services/filestore/reports --volume $(MONOREPO)/.cache:/overleaf/.cache $(IMAGE_CI) yarn run --silent
|
||||
RUN_LINTING_CI_MONOREPO = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/eslint.config.mjs:/overleaf/eslint.config.mjs --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc.cjs:/overleaf/.prettierrc.cjs --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json --volume $(MONOREPO)/services/filestore/reports:/overleaf/services/filestore/reports --volume $(MONOREPO)/.cache:/overleaf/.cache -w /overleaf $(IMAGE_CI) yarn run --silent
|
||||
|
||||
SHELLCHECK_OPTS = \
|
||||
--shell=bash \
|
||||
--external-sources
|
||||
SHELLCHECK_COLOR := $(if $(CI),--color=never,--color)
|
||||
SHELLCHECK_FILES := { git ls-files "*.sh" -z; git grep -Plz "\A\#\!.*bash"; } | sort -zu
|
||||
|
||||
shellcheck:
|
||||
@$(SHELLCHECK_FILES) | xargs -0 -r docker run --rm -v $(HERE):/mnt -w /mnt \
|
||||
koalaman/shellcheck:stable $(SHELLCHECK_OPTS) $(SHELLCHECK_COLOR)
|
||||
|
||||
shellcheck_fix:
|
||||
@$(SHELLCHECK_FILES) | while IFS= read -r -d '' file; do \
|
||||
diff=$$(docker run --rm -v $(HERE):/mnt -w /mnt koalaman/shellcheck:stable $(SHELLCHECK_OPTS) --format=diff "$$file" 2>/dev/null); \
|
||||
if [ -n "$$diff" ] && ! echo "$$diff" | patch -p1 >/dev/null 2>&1; then echo "\033[31m$$file\033[0m"; \
|
||||
elif [ -n "$$diff" ]; then echo "$$file"; \
|
||||
else echo "\033[2m$$file\033[0m"; fi \
|
||||
done
|
||||
|
||||
WITH_CACHE_FOLDER = \
|
||||
lint \
|
||||
lint_ci \
|
||||
lint_fix \
|
||||
format \
|
||||
format_ci \
|
||||
format_fix \
|
||||
typecheck \
|
||||
typecheck_ci \
|
||||
|
||||
$(WITH_CACHE_FOLDER): ../../.cache/prettier/
|
||||
../../.cache/prettier/:
|
||||
mkdir -p $@
|
||||
|
||||
$(WITH_CACHE_FOLDER): ../../.cache/eslint/
|
||||
../../.cache/eslint/:
|
||||
mkdir -p $@
|
||||
|
||||
format:
|
||||
$(RUN_LINTING_MONOREPO) format services/filestore
|
||||
|
||||
format_ci:
|
||||
$(RUN_LINTING_CI_MONOREPO) format services/filestore
|
||||
|
||||
format_fix:
|
||||
$(RUN_LINTING_MONOREPO) format:fix services/filestore
|
||||
|
||||
lint:
|
||||
$(RUN_LINTING) lint
|
||||
|
||||
lint_ci:
|
||||
-$(RUN_LINTING_CI) lint --format json --output-file reports/eslint.json
|
||||
sed -i 's_"filePath":"/overleaf_"filePath":"$(MONOREPO)_g' reports/eslint.json
|
||||
|
||||
lint_fix:
|
||||
$(RUN_LINTING) lint:fix
|
||||
|
||||
typecheck:
|
||||
$(RUN_LINTING) types:check
|
||||
|
||||
typecheck_ci:
|
||||
$(RUN_LINTING_CI) types:check
|
||||
|
||||
test: format lint typecheck shellcheck test_unit test_acceptance
|
||||
|
||||
test_unit:
|
||||
ifneq (,$(wildcard test/unit))
|
||||
$(DOCKER_COMPOSE_TEST_UNIT) run $(DC_RUN_FLAGS) test_unit
|
||||
endif
|
||||
|
||||
test_clean: test_unit_clean
|
||||
test_unit_clean:
|
||||
ifneq (,$(wildcard test/unit))
|
||||
$(DOCKER_COMPOSE_TEST_UNIT) down -v -t 0
|
||||
endif
|
||||
|
||||
test_acceptance: test_acceptance_clean test_acceptance_pre_run test_acceptance_run
|
||||
$(MAKE) test_acceptance_clean
|
||||
|
||||
test_acceptance_debug: test_acceptance_clean test_acceptance_pre_run test_acceptance_run_debug
|
||||
$(MAKE) test_acceptance_clean
|
||||
|
||||
test_acceptance_run:
|
||||
ifneq (,$(wildcard test/acceptance))
|
||||
$(DOCKER_COMPOSE_TEST_ACCEPTANCE) run $(DC_RUN_FLAGS) test_acceptance
|
||||
endif
|
||||
|
||||
test_acceptance_run_debug:
|
||||
ifneq (,$(wildcard test/acceptance))
|
||||
$(DOCKER_COMPOSE_TEST_ACCEPTANCE) run -p 127.0.0.9:19999:19999 $(DC_RUN_FLAGS) test_acceptance yarn run --inspect=0.0.0.0:19999 --inspect-brk test:acceptance
|
||||
endif
|
||||
|
||||
test_clean: test_acceptance_clean
|
||||
test_acceptance_clean:
|
||||
$(DOCKER_COMPOSE_TEST_ACCEPTANCE) down -v -t 0
|
||||
|
||||
test_acceptance_pre_run:
|
||||
ifneq (,$(wildcard test/acceptance/js/scripts/pre-run))
|
||||
$(DOCKER_COMPOSE_TEST_ACCEPTANCE) run $(DC_RUN_FLAGS) test_acceptance test/acceptance/js/scripts/pre-run
|
||||
endif
|
||||
|
||||
benchmarks:
|
||||
$(DOCKER_COMPOSE_TEST_ACCEPTANCE) run $(DC_RUN_FLAGS) test_acceptance yarn run benchmarks
|
||||
|
||||
build:
|
||||
docker build \
|
||||
--pull \
|
||||
--build-arg BUILDKIT_INLINE_CACHE=1 \
|
||||
--tag $(IMAGE_CI) \
|
||||
--tag $(IMAGE_CACHE) \
|
||||
--tag $(IMAGE_REPO_FINAL) \
|
||||
--cache-from $(IMAGE_CACHE) \
|
||||
--file Dockerfile \
|
||||
../..
|
||||
|
||||
tar:
|
||||
$(DOCKER_COMPOSE) up tar
|
||||
|
||||
push:
|
||||
docker push $(IMAGE_REPO_FINAL)
|
||||
|
||||
push_branch:
|
||||
docker push $(IMAGE_CACHE)
|
||||
|
||||
.PHONY: clean \
|
||||
format format_fix \
|
||||
lint lint_fix \
|
||||
build_types typecheck \
|
||||
lint_ci format_ci typecheck_ci \
|
||||
shellcheck shellcheck_fix \
|
||||
test test_clean test_unit test_unit_clean \
|
||||
test_acceptance test_acceptance_debug test_acceptance_pre_run \
|
||||
test_acceptance_run test_acceptance_run_debug test_acceptance_clean \
|
||||
benchmarks \
|
||||
build tar publish \
|
||||
@@ -1,14 +0,0 @@
|
||||
filestore
|
||||
--data-dirs=uploads,template_files
|
||||
--dependencies=s3,gcs
|
||||
--deploy-pipeline=filestore-readonly
|
||||
--env-add=ENABLE_CONVERSIONS=true,USE_PROM_METRICS=true,AWS_S3_USER_FILES_STORAGE_CLASS=REDUCED_REDUNDANCY,AWS_S3_USER_FILES_BUCKET_NAME=fake-user-files,AWS_S3_USER_FILES_DEK_BUCKET_NAME=fake-user-files-dek,AWS_S3_TEMPLATE_FILES_BUCKET_NAME=fake-template-files,GCS_USER_FILES_BUCKET_NAME=fake-gcs-user-files,GCS_TEMPLATE_FILES_BUCKET_NAME=fake-gcs-template-files
|
||||
--env-pass-through=
|
||||
--esmock-loader=False
|
||||
--node-version=24.14.1
|
||||
--pipeline-owner=32
|
||||
--public-repo=True
|
||||
--test-acceptance-shards=SHARD_01_,SHARD_02_,SHARD_03_
|
||||
--test-unit-vitest=True
|
||||
--tsconfig-extra-includes=vitest.config.unit.cjs,vitest.config.acceptance.cjs
|
||||
--use-large-ci-runner=True
|
||||
@@ -1,227 +0,0 @@
|
||||
# This file was auto-generated, do not edit it directly.
|
||||
# Instead run bin/update_build_scripts from
|
||||
# https://github.com/overleaf/internal/
|
||||
|
||||
volumes:
|
||||
minio-certs:
|
||||
|
||||
services:
|
||||
test_unit:
|
||||
image: ci/$PROJECT_NAME:$BRANCH_NAME_TAG_SAFE-$BUILD_NUMBER
|
||||
user: node
|
||||
volumes:
|
||||
- ./reports:/overleaf/services/filestore/reports
|
||||
- ../../tsconfig.backend.json:/overleaf/tsconfig.backend.json
|
||||
command: yarn run test:unit:_run
|
||||
environment:
|
||||
CI:
|
||||
MONGO_CONNECTION_STRING: mongodb://mongo/test-overleaf
|
||||
NODE_ENV: test
|
||||
NODE_OPTIONS: "--unhandled-rejections=strict"
|
||||
VITEST_NO_CACHE: true
|
||||
|
||||
test_acceptance:
|
||||
image: ci/$PROJECT_NAME:$BRANCH_NAME_TAG_SAFE-$BUILD_NUMBER
|
||||
environment:
|
||||
CI:
|
||||
RETRIES:
|
||||
MONGO_HOST: mongo
|
||||
POSTGRES_HOST: postgres
|
||||
AWS_S3_ENDPOINT: https://minio:9000
|
||||
AWS_S3_PATH_STYLE: "true"
|
||||
DELETE_OBJECTS_MD5_FALLBACK: true
|
||||
AWS_ACCESS_KEY_ID: OVERLEAF_FILESTORE_S3_ACCESS_KEY_ID
|
||||
AWS_SECRET_ACCESS_KEY: OVERLEAF_FILESTORE_S3_SECRET_ACCESS_KEY
|
||||
MINIO_ROOT_USER: MINIO_ROOT_USER
|
||||
MINIO_ROOT_PASSWORD: MINIO_ROOT_PASSWORD
|
||||
GCS_API_ENDPOINT: http://gcs:9090
|
||||
GCS_PROJECT_ID: fake
|
||||
STORAGE_EMULATOR_HOST: http://gcs:9090/storage/v1
|
||||
MOCHA_GREP: ${MOCHA_GREP}
|
||||
NODE_ENV: test
|
||||
NODE_OPTIONS: "--unhandled-rejections=strict"
|
||||
ENABLE_CONVERSIONS: true
|
||||
USE_PROM_METRICS: true
|
||||
AWS_S3_USER_FILES_STORAGE_CLASS: REDUCED_REDUNDANCY
|
||||
AWS_S3_USER_FILES_BUCKET_NAME: fake-user-files
|
||||
AWS_S3_USER_FILES_DEK_BUCKET_NAME: fake-user-files-dek
|
||||
AWS_S3_TEMPLATE_FILES_BUCKET_NAME: fake-template-files
|
||||
GCS_USER_FILES_BUCKET_NAME: fake-gcs-user-files
|
||||
GCS_TEMPLATE_FILES_BUCKET_NAME: fake-gcs-template-files
|
||||
volumes:
|
||||
- ./reports:/overleaf/services/filestore/reports
|
||||
- minio-certs:/certs
|
||||
depends_on:
|
||||
certs:
|
||||
condition: service_completed_successfully
|
||||
minio:
|
||||
condition: service_started
|
||||
minio_setup:
|
||||
condition: service_completed_successfully
|
||||
gcs:
|
||||
condition: service_healthy
|
||||
user: node
|
||||
command: yarn run test:acceptance
|
||||
|
||||
tar:
|
||||
build: .
|
||||
image: ci/$PROJECT_NAME:$BRANCH_NAME_TAG_SAFE-$BUILD_NUMBER
|
||||
volumes:
|
||||
- ./:/tmp/build/
|
||||
command: tar -czf /tmp/build/build.tar.gz --exclude=build.tar.gz --exclude-vcs .
|
||||
user: root
|
||||
|
||||
certs:
|
||||
build:
|
||||
dockerfile_inline: |
|
||||
FROM node:24.14.1
|
||||
RUN wget -O /certgen "https://github.com/minio/certgen/releases/download/v1.3.0/certgen-linux-"`dpkg --print-architecture`
|
||||
RUN chmod +x /certgen
|
||||
volumes:
|
||||
- minio-certs:/certs
|
||||
working_dir: /certs
|
||||
entrypoint: sh
|
||||
command:
|
||||
- "-cex"
|
||||
- |
|
||||
if [ ! -f private.key ] || [ ! -f public.crt ]; then
|
||||
/certgen -host minio
|
||||
fi
|
||||
if ! openssl x509 -checkend 864000 -noout -in public.crt > /dev/null; then
|
||||
/certgen -host minio
|
||||
fi
|
||||
|
||||
minio:
|
||||
image: minio/minio:RELEASE.2024-10-13T13-34-11Z
|
||||
command: server /data
|
||||
volumes:
|
||||
- minio-certs:/root/.minio/certs
|
||||
environment:
|
||||
MINIO_ROOT_USER: MINIO_ROOT_USER
|
||||
MINIO_ROOT_PASSWORD: MINIO_ROOT_PASSWORD
|
||||
depends_on:
|
||||
certs:
|
||||
condition: service_completed_successfully
|
||||
|
||||
minio_setup:
|
||||
depends_on:
|
||||
certs:
|
||||
condition: service_completed_successfully
|
||||
minio:
|
||||
condition: service_started
|
||||
image: minio/mc:RELEASE.2024-10-08T09-37-26Z
|
||||
volumes:
|
||||
- minio-certs:/root/.mc/certs/CAs
|
||||
entrypoint: sh
|
||||
command:
|
||||
- "-cex"
|
||||
- |
|
||||
sleep 1
|
||||
mc alias set s3 https://minio:9000 MINIO_ROOT_USER MINIO_ROOT_PASSWORD \
|
||||
|| sleep 3 && \
|
||||
mc alias set s3 https://minio:9000 MINIO_ROOT_USER MINIO_ROOT_PASSWORD \
|
||||
|| sleep 3 && \
|
||||
mc alias set s3 https://minio:9000 MINIO_ROOT_USER MINIO_ROOT_PASSWORD \
|
||||
|| sleep 3 && \
|
||||
mc alias set s3 https://minio:9000 MINIO_ROOT_USER MINIO_ROOT_PASSWORD
|
||||
mc mb --ignore-existing s3/fake-user-files
|
||||
mc mb --ignore-existing s3/fake-user-files-dek
|
||||
mc mb --ignore-existing s3/fake-template-files
|
||||
mc admin user add s3 \
|
||||
OVERLEAF_FILESTORE_S3_ACCESS_KEY_ID \
|
||||
OVERLEAF_FILESTORE_S3_SECRET_ACCESS_KEY
|
||||
|
||||
echo '
|
||||
{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"s3:ListBucket"
|
||||
],
|
||||
"Resource": "arn:aws:s3:::fake-user-files"
|
||||
},
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"s3:PutObject",
|
||||
"s3:GetObject",
|
||||
"s3:DeleteObject",
|
||||
"s3:AbortMultipartUpload",
|
||||
"s3:ListMultipartUploadParts",
|
||||
"s3:ListBucketMultipartUploads"
|
||||
],
|
||||
"Resource": "arn:aws:s3:::fake-user-files/*"
|
||||
},
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"s3:ListBucket"
|
||||
],
|
||||
"Resource": "arn:aws:s3:::fake-user-files-dek"
|
||||
},
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"s3:PutObject",
|
||||
"s3:GetObject",
|
||||
"s3:DeleteObject",
|
||||
"s3:AbortMultipartUpload",
|
||||
"s3:ListMultipartUploadParts",
|
||||
"s3:ListBucketMultipartUploads"
|
||||
],
|
||||
"Resource": "arn:aws:s3:::fake-user-files-dek/*"
|
||||
},
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"s3:ListBucket"
|
||||
],
|
||||
"Resource": "arn:aws:s3:::fake-template-files"
|
||||
},
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"s3:PutObject",
|
||||
"s3:GetObject",
|
||||
"s3:DeleteObject",
|
||||
"s3:AbortMultipartUpload",
|
||||
"s3:ListMultipartUploadParts",
|
||||
"s3:ListBucketMultipartUploads"
|
||||
],
|
||||
"Resource": "arn:aws:s3:::fake-template-files/*"
|
||||
},
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"s3:ListBucket"
|
||||
],
|
||||
"Resource": "arn:aws:s3:::random-bucket-*"
|
||||
},
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"s3:PutObject",
|
||||
"s3:GetObject",
|
||||
"s3:DeleteObject",
|
||||
"s3:AbortMultipartUpload",
|
||||
"s3:ListMultipartUploadParts",
|
||||
"s3:ListBucketMultipartUploads"
|
||||
],
|
||||
"Resource": "arn:aws:s3:::random-bucket-*"
|
||||
}
|
||||
]
|
||||
}' > policy-filestore.json
|
||||
|
||||
mc admin policy create s3 overleaf-filestore policy-filestore.json
|
||||
mc admin policy attach s3 overleaf-filestore \
|
||||
--user=OVERLEAF_FILESTORE_S3_ACCESS_KEY_ID
|
||||
|
||||
gcs:
|
||||
image: fsouza/fake-gcs-server:1.52.3
|
||||
command: ["--port=9090", "--scheme=http", "--external-url=http://gcs:9090"]
|
||||
healthcheck:
|
||||
test: wget --quiet --output-document=/dev/null http://localhost:9090/storage/v1/b
|
||||
interval: 1s
|
||||
retries: 20
|
||||
@@ -1,239 +0,0 @@
|
||||
# This file was auto-generated, do not edit it directly.
|
||||
# Instead run bin/update_build_scripts from
|
||||
# https://github.com/overleaf/internal/
|
||||
|
||||
volumes:
|
||||
minio-certs:
|
||||
|
||||
services:
|
||||
test_unit:
|
||||
build:
|
||||
context: ../..
|
||||
dockerfile: services/filestore/Dockerfile
|
||||
target: base
|
||||
volumes:
|
||||
- .:/overleaf/services/filestore
|
||||
- ../../node_modules:/overleaf/node_modules
|
||||
- ../../libraries:/overleaf/libraries
|
||||
- ../../package.json:/overleaf/package.json
|
||||
- ../../.yarnrc.yml:/overleaf/.yarnrc.yml
|
||||
- ../../yarn.lock:/overleaf/yarn.lock
|
||||
- ../../.yarn:/overleaf/.yarn
|
||||
- ../../tsconfig.backend.json:/overleaf/tsconfig.backend.json
|
||||
working_dir: /overleaf/services/filestore
|
||||
environment:
|
||||
MOCHA_GREP: ${MOCHA_GREP}
|
||||
LOG_LEVEL: ${LOG_LEVEL:-}
|
||||
MONGO_CONNECTION_STRING: mongodb://mongo/test-overleaf
|
||||
NODE_ENV: test
|
||||
NODE_OPTIONS: "--unhandled-rejections=strict"
|
||||
command: yarn run --silent test:unit
|
||||
user: node
|
||||
|
||||
test_acceptance:
|
||||
build:
|
||||
context: ../..
|
||||
dockerfile: services/filestore/Dockerfile
|
||||
target: base
|
||||
volumes:
|
||||
- .:/overleaf/services/filestore
|
||||
- ../../node_modules:/overleaf/node_modules
|
||||
- ../../libraries:/overleaf/libraries
|
||||
- ../../package.json:/overleaf/package.json
|
||||
- ../../.yarnrc.yml:/overleaf/.yarnrc.yml
|
||||
- ../../yarn.lock:/overleaf/yarn.lock
|
||||
- ../../.yarn:/overleaf/.yarn
|
||||
- minio-certs:/certs
|
||||
working_dir: /overleaf/services/filestore
|
||||
environment:
|
||||
RETRIES:
|
||||
MONGO_CONNECTION_STRING: mongodb://mongo/test-overleaf
|
||||
POSTGRES_HOST: postgres
|
||||
AWS_S3_ENDPOINT: https://minio:9000
|
||||
AWS_S3_PATH_STYLE: "true"
|
||||
DELETE_OBJECTS_MD5_FALLBACK: true
|
||||
AWS_ACCESS_KEY_ID: OVERLEAF_FILESTORE_S3_ACCESS_KEY_ID
|
||||
AWS_SECRET_ACCESS_KEY: OVERLEAF_FILESTORE_S3_SECRET_ACCESS_KEY
|
||||
MINIO_ROOT_USER: MINIO_ROOT_USER
|
||||
MINIO_ROOT_PASSWORD: MINIO_ROOT_PASSWORD
|
||||
GCS_API_ENDPOINT: http://gcs:9090
|
||||
GCS_PROJECT_ID: fake
|
||||
STORAGE_EMULATOR_HOST: http://gcs:9090/storage/v1
|
||||
MOCHA_GREP: ${MOCHA_GREP}
|
||||
LOG_LEVEL: ${LOG_LEVEL:-}
|
||||
NODE_ENV: test
|
||||
NODE_OPTIONS: "--unhandled-rejections=strict"
|
||||
ENABLE_CONVERSIONS: true
|
||||
USE_PROM_METRICS: true
|
||||
AWS_S3_USER_FILES_STORAGE_CLASS: REDUCED_REDUNDANCY
|
||||
AWS_S3_USER_FILES_BUCKET_NAME: fake-user-files
|
||||
AWS_S3_USER_FILES_DEK_BUCKET_NAME: fake-user-files-dek
|
||||
AWS_S3_TEMPLATE_FILES_BUCKET_NAME: fake-template-files
|
||||
GCS_USER_FILES_BUCKET_NAME: fake-gcs-user-files
|
||||
GCS_TEMPLATE_FILES_BUCKET_NAME: fake-gcs-template-files
|
||||
user: node
|
||||
depends_on:
|
||||
certs:
|
||||
condition: service_completed_successfully
|
||||
minio:
|
||||
condition: service_started
|
||||
minio_setup:
|
||||
condition: service_completed_successfully
|
||||
gcs:
|
||||
condition: service_healthy
|
||||
command: yarn run --silent test:acceptance
|
||||
|
||||
certs:
|
||||
build:
|
||||
dockerfile_inline: |
|
||||
FROM node:24.14.1
|
||||
RUN wget -O /certgen "https://github.com/minio/certgen/releases/download/v1.3.0/certgen-linux-"`dpkg --print-architecture`
|
||||
RUN chmod +x /certgen
|
||||
volumes:
|
||||
- minio-certs:/certs
|
||||
working_dir: /certs
|
||||
entrypoint: sh
|
||||
command:
|
||||
- "-cex"
|
||||
- |
|
||||
if [ ! -f private.key ] || [ ! -f public.crt ]; then
|
||||
/certgen -host minio
|
||||
fi
|
||||
if ! openssl x509 -checkend 864000 -noout -in public.crt > /dev/null; then
|
||||
/certgen -host minio
|
||||
fi
|
||||
|
||||
minio:
|
||||
image: minio/minio:RELEASE.2024-10-13T13-34-11Z
|
||||
command: server /data
|
||||
volumes:
|
||||
- minio-certs:/root/.minio/certs
|
||||
environment:
|
||||
MINIO_ROOT_USER: MINIO_ROOT_USER
|
||||
MINIO_ROOT_PASSWORD: MINIO_ROOT_PASSWORD
|
||||
depends_on:
|
||||
certs:
|
||||
condition: service_completed_successfully
|
||||
|
||||
minio_setup:
|
||||
depends_on:
|
||||
certs:
|
||||
condition: service_completed_successfully
|
||||
minio:
|
||||
condition: service_started
|
||||
image: minio/mc:RELEASE.2024-10-08T09-37-26Z
|
||||
volumes:
|
||||
- minio-certs:/root/.mc/certs/CAs
|
||||
entrypoint: sh
|
||||
command:
|
||||
- "-cex"
|
||||
- |
|
||||
sleep 1
|
||||
mc alias set s3 https://minio:9000 MINIO_ROOT_USER MINIO_ROOT_PASSWORD \
|
||||
|| sleep 3 && \
|
||||
mc alias set s3 https://minio:9000 MINIO_ROOT_USER MINIO_ROOT_PASSWORD \
|
||||
|| sleep 3 && \
|
||||
mc alias set s3 https://minio:9000 MINIO_ROOT_USER MINIO_ROOT_PASSWORD \
|
||||
|| sleep 3 && \
|
||||
mc alias set s3 https://minio:9000 MINIO_ROOT_USER MINIO_ROOT_PASSWORD
|
||||
mc mb --ignore-existing s3/fake-user-files
|
||||
mc mb --ignore-existing s3/fake-user-files-dek
|
||||
mc mb --ignore-existing s3/fake-template-files
|
||||
mc admin user add s3 \
|
||||
OVERLEAF_FILESTORE_S3_ACCESS_KEY_ID \
|
||||
OVERLEAF_FILESTORE_S3_SECRET_ACCESS_KEY
|
||||
|
||||
echo '
|
||||
{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"s3:ListBucket"
|
||||
],
|
||||
"Resource": "arn:aws:s3:::fake-user-files"
|
||||
},
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"s3:PutObject",
|
||||
"s3:GetObject",
|
||||
"s3:DeleteObject",
|
||||
"s3:AbortMultipartUpload",
|
||||
"s3:ListMultipartUploadParts",
|
||||
"s3:ListBucketMultipartUploads"
|
||||
],
|
||||
"Resource": "arn:aws:s3:::fake-user-files/*"
|
||||
},
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"s3:ListBucket"
|
||||
],
|
||||
"Resource": "arn:aws:s3:::fake-user-files-dek"
|
||||
},
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"s3:PutObject",
|
||||
"s3:GetObject",
|
||||
"s3:DeleteObject",
|
||||
"s3:AbortMultipartUpload",
|
||||
"s3:ListMultipartUploadParts",
|
||||
"s3:ListBucketMultipartUploads"
|
||||
],
|
||||
"Resource": "arn:aws:s3:::fake-user-files-dek/*"
|
||||
},
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"s3:ListBucket"
|
||||
],
|
||||
"Resource": "arn:aws:s3:::fake-template-files"
|
||||
},
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"s3:PutObject",
|
||||
"s3:GetObject",
|
||||
"s3:DeleteObject",
|
||||
"s3:AbortMultipartUpload",
|
||||
"s3:ListMultipartUploadParts",
|
||||
"s3:ListBucketMultipartUploads"
|
||||
],
|
||||
"Resource": "arn:aws:s3:::fake-template-files/*"
|
||||
},
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"s3:ListBucket"
|
||||
],
|
||||
"Resource": "arn:aws:s3:::random-bucket-*"
|
||||
},
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"s3:PutObject",
|
||||
"s3:GetObject",
|
||||
"s3:DeleteObject",
|
||||
"s3:AbortMultipartUpload",
|
||||
"s3:ListMultipartUploadParts",
|
||||
"s3:ListBucketMultipartUploads"
|
||||
],
|
||||
"Resource": "arn:aws:s3:::random-bucket-*"
|
||||
}
|
||||
]
|
||||
}' > policy-filestore.json
|
||||
|
||||
mc admin policy create s3 overleaf-filestore policy-filestore.json
|
||||
mc admin policy attach s3 overleaf-filestore \
|
||||
--user=OVERLEAF_FILESTORE_S3_ACCESS_KEY_ID
|
||||
|
||||
gcs:
|
||||
image: fsouza/fake-gcs-server:1.52.3
|
||||
command: ["--port=9090", "--scheme=http", "--external-url=http://gcs:9090"]
|
||||
healthcheck:
|
||||
test: wget --quiet --output-document=/dev/null http://localhost:9090/storage/v1/b
|
||||
interval: 1s
|
||||
retries: 20
|
||||
Reference in New Issue
Block a user