From 399e834e36c6b39bf49dc470760377bb16b50834 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Tue, 6 Aug 2024 11:08:18 +0200 Subject: [PATCH] Merge pull request #19764 from overleaf/jpa-server-pro-cache [server-pro] faster local build GitOrigin-RevId: d71abde37253e4c0398afa6935290af79a1204be --- server-ce/Dockerfile | 13 +++++++------ server-ce/Dockerfile-base | 14 ++++++++++---- server-ce/Makefile | 2 ++ server-ce/genScript.js | 7 ------- server-ce/test/package-lock.json | 2 +- .../macros/invalidate-babel-cache-if-needed.js | 11 +++++++++-- 6 files changed, 29 insertions(+), 20 deletions(-) diff --git a/server-ce/Dockerfile b/server-ce/Dockerfile index 6802f657aa..cf3f41cb71 100644 --- a/server-ce/Dockerfile +++ b/server-ce/Dockerfile @@ -19,13 +19,14 @@ ADD services/ /overleaf/services/ # ----------------------- ADD patches/ /overleaf/patches -# Install npm dependencies +# Install npm dependencies and build webpack assets # ------------------------ -RUN node genScript install | bash - -# Compile -# -------------------- -RUN node genScript compile | bash +RUN --mount=type=cache,target=/root/.cache \ + --mount=type=cache,target=/root/.npm \ + --mount=type=cache,target=/overleaf/services/web/node_modules/.cache,id=server-ce-webpack-cache \ + --mount=type=tmpfs,target=/tmp true \ +&& node genScript install | bash \ +&& node genScript compile | bash # Copy runit service startup scripts to its location # -------------------------------------------------- diff --git a/server-ce/Dockerfile-base b/server-ce/Dockerfile-base index f381000652..bbbbad230e 100644 --- a/server-ce/Dockerfile-base +++ b/server-ce/Dockerfile-base @@ -14,13 +14,20 @@ ENV REBUILT_AFTER="2024-07-09" # Install dependencies # -------------------- -RUN apt-get update \ +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ +# Technically, we are using potentially stale package-lists with the below line. +# Practically, apt refreshes the lists as needed and release builds run in fresh CI VMs without the cache. + --mount=type=cache,target=/var/lib/apt/lists,sharing=locked true \ +# Enable caching: https://docs.docker.com/reference/dockerfile/#example-cache-apt-packages +&& rm -f /etc/apt/apt.conf.d/docker-clean && echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache \ +&& apt-get update \ && apt-get install -y \ unattended-upgrades \ build-essential wget net-tools unzip time imagemagick optipng strace nginx git python3 python-is-python3 zlib1g-dev libpcre3-dev gettext-base libwww-perl ca-certificates curl gnupg \ qpdf \ aspell aspell-en aspell-af aspell-am aspell-ar aspell-ar-large aspell-bg aspell-bn aspell-br aspell-ca aspell-cs aspell-cy aspell-da aspell-de aspell-de-1901 aspell-el aspell-eo aspell-es aspell-et aspell-eu-es aspell-fa aspell-fo aspell-fr aspell-ga aspell-gl-minimos aspell-gu aspell-he aspell-hi aspell-hr aspell-hsb aspell-hu aspell-hy aspell-id aspell-is aspell-it aspell-kk aspell-kn aspell-ku aspell-lt aspell-lv aspell-ml aspell-mr aspell-nl aspell-no aspell-nr aspell-ns aspell-pa aspell-pl aspell-pt aspell-pt-br aspell-ro aspell-ru aspell-sk aspell-sl aspell-ss aspell-st aspell-sv aspell-tl aspell-tn aspell-ts aspell-uk aspell-uz aspell-xh aspell-zu \ -&& unattended-upgrade -v \ +# upgrade base-image, batch all the upgrades together, rather than installing them on-by-one (which is slow!) +&& unattended-upgrade --verbose --no-minimal-upgrade-steps \ # install Node.js https://github.com/nodesource/distributions#nodejs && mkdir -p /etc/apt/keyrings \ && curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg \ @@ -31,8 +38,7 @@ RUN apt-get update \ && rm -rf \ # We are adding a custom nginx config in the main Dockerfile. /etc/nginx/nginx.conf \ - /etc/nginx/sites-enabled/default \ - /var/lib/apt/lists/* + /etc/nginx/sites-enabled/default # Install TexLive # --------------- diff --git a/server-ce/Makefile b/server-ce/Makefile index 8847252fb5..6a70a8e027 100644 --- a/server-ce/Makefile +++ b/server-ce/Makefile @@ -16,6 +16,7 @@ build-base: cp .dockerignore $(MONOREPO_ROOT) docker build \ --build-arg BUILDKIT_INLINE_CACHE=1 \ + --progress=plain \ --file Dockerfile-base \ --pull \ --cache-from $(OVERLEAF_BASE_LATEST) \ @@ -29,6 +30,7 @@ build-community: cp .dockerignore $(MONOREPO_ROOT) docker build \ --build-arg BUILDKIT_INLINE_CACHE=1 \ + --progress=plain \ --build-arg OVERLEAF_BASE_TAG \ --build-arg MONOREPO_REVISION \ --cache-from $(OVERLEAF_LATEST) \ diff --git a/server-ce/genScript.js b/server-ce/genScript.js index dfc3ea77f2..81fa0e176c 100644 --- a/server-ce/genScript.js +++ b/server-ce/genScript.js @@ -19,8 +19,6 @@ switch (process.argv.pop()) { console.log('npm install --include=dev') // run webpack console.log('npm run webpack:production') - // drop webpack/babel cache - console.log('rm -rf node_modules/.cache') // uninstall webpack and frontend dependencies console.log('npm install --omit=dev') // precompile pug @@ -37,8 +35,3 @@ switch (process.argv.pop()) { console.log('exit 101') process.exit(101) } - -console.log('set +x') -console.log( - 'rm -rf /root/.cache /root/.npm $(find /tmp/ -mindepth 1 -maxdepth 1)' -) diff --git a/server-ce/test/package-lock.json b/server-ce/test/package-lock.json index 8f347d3935..6e84524ae0 100644 --- a/server-ce/test/package-lock.json +++ b/server-ce/test/package-lock.json @@ -14,7 +14,7 @@ "adm-zip": "^0.5.12", "body-parser": "^1.20.2", "celebrate": "^15.0.3", - "cypress": "^13.13.2", + "cypress": "13.13.2", "express": "^4.19.2", "isomorphic-git": "^1.25.10", "js-yaml": "^4.1.0", diff --git a/services/web/frontend/macros/invalidate-babel-cache-if-needed.js b/services/web/frontend/macros/invalidate-babel-cache-if-needed.js index 12ebcadbb0..df9d00eb63 100644 --- a/services/web/frontend/macros/invalidate-babel-cache-if-needed.js +++ b/services/web/frontend/macros/invalidate-babel-cache-if-needed.js @@ -16,8 +16,15 @@ module.exports = function invalidateBabelCacheIfNeeded() { console.warn( 'Detected change in overleafModuleImports, purging babel cache!' ) - fs.rmSync(cachePath, { recursive: true, force: true, maxRetries: 5 }) - fs.mkdirSync(cachePath) + // Gracefully handle cache mount in Server Pro build, only purge nested folders and keep .cache/ folder. + fs.mkdirSync(cachePath, { recursive: true }) + for (const name of fs.readdirSync(cachePath)) { + fs.rmSync(Path.join(cachePath, name), { + recursive: true, + force: true, + maxRetries: 5, + }) + } fs.writeFileSync(statePath, newState) } }