[server-ce] tests: migrate host-admin to ESM, zod and npm-workspaces (#28838)

* [server-ce] tests: migrate host-admin to ESM, zod and npm-workspaces

* [server-ce] test: use import.meta.dirname

Co-authored-by: Eric Mc Sween <eric.mcsween@overleaf.com>

* [server-ce] test: fix zod schema for docker compose endpoint

---------

Co-authored-by: Eric Mc Sween <eric.mcsween@overleaf.com>
GitOrigin-RevId: d490948693f341210c0ab5c2947db7c9a17775ef

server-ce/test/Jenkinsfile

@@ -64,8 +64,7 @@ pipeline {
       parallel {
         stage('Install deps') {
           steps {
-            sh 'make install -j10'
-            sh 'make -C server-ce/test npm_install_in_docker'
+            sh 'make monorepo_setup'
             script {
               job_npm_install_done = true
             }
@@ -85,9 +84,7 @@ pipeline {
                 return job_npm_install_done
               }
             }
-            dir('server-ce/test') {
-              sh 'make format_in_docker'
-            }
+            sh 'bin/run -w /overleaf/server-ce/test monorepo npm run format'
           }
         }
         stage('Copybara') {

server-ce/test/Makefile

@@ -5,6 +5,7 @@ all: test-e2e
 # We need to have both file-system layouts agree on the path for the docker compose project.
 # Notable the container labels com.docker.compose.project.working_dir and com.docker.compose.project.config_files need to match when creating containers from the docker host (how you started things) and from host-admin (how tests reconfigure the instance).
 export PWD = $(shell pwd)
+export MONOREPO = $(shell cd ../../ && pwd)
 
 export TEX_LIVE_DOCKER_IMAGE ?= us-east1-docker.pkg.dev/overleaf-ops/ol-docker/texlive-full:2023.1
 export ALL_TEX_LIVE_DOCKER_IMAGES ?= us-east1-docker.pkg.dev/overleaf-ops/ol-docker/texlive-full:2023.1,us-east1-docker.pkg.dev/overleaf-ops/ol-docker/texlive-full:2022.1
@@ -96,14 +97,4 @@ build_mailtrap:
 	git clone https://github.com/dbck/docker-mailtrap.git || true && cd docker-mailtrap && git checkout v1.5.0
 	docker build -t mailtrap docker-mailtrap/build
 
-npm_install_in_docker: export COMPOSE_PROJECT_NAME=
-npm_install_in_docker:
-	$(MAKE) -C ../../ .metadata/docker-image/monorepo
-	cd ../../ && bin/run --no-deps --workdir /overleaf/server-ce/test monorepo npm --no-dry-run install
-
-format_in_docker: export COMPOSE_PROJECT_NAME=
-format_in_docker:
-	$(MAKE) -C ../../ .metadata/docker-image/monorepo
-	cd ../../ && bin/run --no-deps --workdir /overleaf/server-ce/test monorepo npm run format
-
 .PHONY: test-e2e test-e2e-open

server-ce/test/cypress.config.ts

@@ -1,6 +1,6 @@
-const { defineConfig } = require('cypress')
-const { readPdf, readFileInZip } = require('./helpers/read-file')
-const fs = require('node:fs')
+import { defineConfig } from 'cypress'
+import { readFileInZip, readPdf } from './helpers/read-file'
+import fs from 'node:fs'
 
 if (process.env.CYPRESS_SHARD && !process.env.SPEC_PATTERN) {
   // Running Cypress on all the specs is wasteful (~1min) when only few of them
@@ -36,20 +36,19 @@ const specPattern = process.env.SPEC_PATTERN || './**/*.spec.ts'
 let reporterOptions = {}
 if (process.env.CI) {
   reporterOptions = {
-    reporter: '/overleaf/server-ce/test/node_modules/cypress-multi-reporters',
+    reporter: `${process.env.MONOREPO}/node_modules/cypress-multi-reporters`,
     reporterOptions: {
       configFile: 'cypress/cypress-multi-reporters.json',
     },
   }
 }
 
-module.exports = defineConfig({
+export default defineConfig({
   defaultCommandTimeout: 10_000,
   fixturesFolder: 'cypress/fixtures',
   video: process.env.CYPRESS_VIDEO === 'true',
   screenshotsFolder: 'cypress/results',
   videosFolder: 'cypress/results',
-  videoUploadOnPasses: false,
   viewportHeight: 768,
   viewportWidth: 1024,
   e2e: {

server-ce/test/docker-compose.yml

@@ -70,11 +70,15 @@ services:
     stop_grace_period: 0s
     entrypoint: npm
     command: run cypress:run
-    working_dir: /overleaf/server-ce/test
+    # See comment in Makefile regarding matching file paths
+    working_dir: $PWD
     volumes:
-      - ./:/overleaf/server-ce/test
+      - $PWD:$PWD
+      - $MONOREPO/libraries:$MONOREPO/libraries:ro
+      - $MONOREPO/node_modules:$MONOREPO/node_modules:ro
     user: "${DOCKER_USER:-1000:1000}"
     environment:
+      MONOREPO:
       CYPRESS_SHARD:
       CYPRESS_BASE_URL: http://sharelatex
       CYPRESS_FULL_FILESTORE_MIGRATION:
@@ -101,11 +105,14 @@ services:
     working_dir: $PWD
     volumes:
       - $PWD:$PWD
+      - $MONOREPO/libraries:$MONOREPO/libraries:ro
+      - $MONOREPO/node_modules:$MONOREPO/node_modules:ro
       - /var/run/docker.sock:/var/run/docker.sock
     stop_grace_period: 0s
     environment:
       CI:
       PWD:
+      MONOREPO:
       CYPRESS_SHARD:
       COMPOSE_PROJECT_NAME:
       TEX_LIVE_DOCKER_IMAGE:

server-ce/test/helpers/read-file.ts

@@ -1,10 +1,9 @@
 import fs from 'fs'
 import path from 'path'
-import pdf from 'pdf-parse'
+// @ts-ignore broken package entrypoint
+import pdf from 'pdf-parse/lib/pdf-parse.js'
 import AdmZip from 'adm-zip'
-import { promisify } from 'util'
-
-const sleep = promisify(setTimeout)
+import { setTimeout } from 'timers/promises'
 
 const MAX_ATTEMPTS = 15
 const POLL_INTERVAL = 500
@@ -31,7 +30,7 @@ export async function readFileInZip({
         throw new Error(`${fileToRead} not found in ${pathToZip}`)
       }
     }
-    await sleep(POLL_INTERVAL)
+    await setTimeout(POLL_INTERVAL)
     attempt++
   }
   throw new Error(`${pathToZip} not found`)
@@ -45,7 +44,7 @@ export async function readPdf(file: string) {
       const { text } = await pdf(dataBuffer)
       return text
     }
-    await sleep(POLL_INTERVAL)
+    await setTimeout(POLL_INTERVAL)
     attempt++
   }
   throw new Error(`${file} not found`)

server-ce/test/host-admin.js

@@ -1,17 +1,14 @@
-const fs = require('fs')
-const Path = require('path')
-const { execFile } = require('child_process')
-const express = require('express')
-const bodyParser = require('body-parser')
-const {
-  celebrate: validate,
-  Joi,
-  errors: handleValidationErrors,
-} = require('celebrate')
-const YAML = require('js-yaml')
+import fs from 'node:fs'
+import Path from 'node:path'
+import { execFile } from 'node:child_process'
+import bodyParser from 'body-parser'
+import express from 'express'
+import YAML from 'js-yaml'
+import { isZodErrorLike } from 'zod-validation-error'
+import { ParamsError, validateReq, z } from '@overleaf/validation-tools'
 
 const DATA_DIR = Path.join(
-  __dirname,
+  import.meta.dirname,
   'data',
   // Give each shard their own data dir.
   process.env.CYPRESS_SHARD || 'default'
@@ -108,84 +105,80 @@ app.use((req, res, next) => {
   next()
 })
 
-app.post(
-  '/run/script',
-  validate(
-    {
-      body: {
-        cwd: Joi.string().required(),
-        script: Joi.string().required(),
-        args: Joi.array().items(Joi.string()),
-        user: Joi.string().required(),
-        hasOverleafEnv: Joi.boolean().required(),
-      },
-    },
-    { allowUnknown: false }
-  ),
-  (req, res) => {
-    const { cwd, script, args, user, hasOverleafEnv } = req.body
+app.post('/run/script', (req, res) => {
+  const {
+    body: { cwd, script, args, user, hasOverleafEnv },
+  } = validateReq(
+    req,
+    z.object({
+      body: z.object({
+        cwd: z.string(),
+        script: z.string(),
+        args: z.array(z.string()),
+        user: z.string(),
+        hasOverleafEnv: z.boolean(),
+      }),
+    })
+  )
 
-    const env = hasOverleafEnv
-      ? 'source /etc/overleaf/env.sh || source /etc/sharelatex/env.sh'
-      : 'true'
+  const env = hasOverleafEnv
+    ? 'source /etc/overleaf/env.sh || source /etc/sharelatex/env.sh'
+    : 'true'
 
-    runDockerCompose(
-      'exec',
-      [
-        '--workdir',
-        `/overleaf/${cwd}`,
-        'sharelatex',
-        'bash',
-        '-c',
-        `source /etc/container_environment.sh && ${env} && /sbin/setuser ${user} node ${script} ${args.map(a => JSON.stringify(a)).join(' ')}`,
-      ],
-      (error, stdout, stderr) => {
-        res.json({
-          error,
-          stdout,
-          stderr,
-        })
-      }
-    )
-  }
-)
+  runDockerCompose(
+    'exec',
+    [
+      '--workdir',
+      `/overleaf/${cwd}`,
+      'sharelatex',
+      'bash',
+      '-c',
+      `source /etc/container_environment.sh && ${env} && /sbin/setuser ${user} node ${script} ${args.map(a => JSON.stringify(a)).join(' ')}`,
+    ],
+    (error, stdout, stderr) => {
+      res.json({
+        error,
+        stdout,
+        stderr,
+      })
+    }
+  )
+})
 
-app.post(
-  '/run/gruntTask',
-  validate(
-    {
-      body: {
-        task: Joi.string().required(),
-        args: Joi.array().items(Joi.string()),
-      },
-    },
-    { allowUnknown: false }
-  ),
-  (req, res) => {
-    const { task, args } = req.body
+app.post('/run/gruntTask', (req, res) => {
+  const {
+    body: { task, args },
+  } = validateReq(
+    req,
+    z.object({
+      body: z.object({
+        task: z.string(),
+        args: z.array(z.string()),
+      }),
+    })
+  )
 
-    runDockerCompose(
-      'exec',
-      [
-        '--workdir',
-        '/var/www/sharelatex',
-        'sharelatex',
-        'bash',
-        '-c',
-        `source /etc/container_environment.sh && /sbin/setuser www-data grunt ${JSON.stringify(task)} ${args.map(a => JSON.stringify(a)).join(' ')}`,
-      ],
-      (error, stdout, stderr) => {
-        res.json({
-          error,
-          stdout,
-          stderr,
-        })
-      }
-    )
-  }
-)
+  runDockerCompose(
+    'exec',
+    [
+      '--workdir',
+      '/var/www/sharelatex',
+      'sharelatex',
+      'bash',
+      '-c',
+      `source /etc/container_environment.sh && /sbin/setuser www-data grunt ${JSON.stringify(task)} ${args.map(a => JSON.stringify(a)).join(' ')}`,
+    ],
+    (error, stdout, stderr) => {
+      res.json({
+        error,
+        stdout,
+        stderr,
+      })
+    }
+  )
+})
 
-const allowedVars = Joi.object(
+const allowedVars = z.object(
   Object.fromEntries(
     [
       'OVERLEAF_APP_NAME',
@@ -227,7 +220,7 @@ const allowedVars = Joi.object(
       'SHARELATEX_SITE_URL',
       'SHARELATEX_MONGO_URL',
       'SHARELATEX_REDIS_HOST',
-    ].map(name => [name, Joi.string()])
+    ].map(name => [name, z.string().optional()])
   )
 )
 
@@ -296,36 +289,37 @@ function setVarsDockerCompose({
   writeDockerComposeOverride(cfg)
 }
 
-app.post(
-  '/docker/compose/:cmd',
-  validate(
-    {
-      body: {
-        args: Joi.array().allow(
-          '--detach',
-          '--wait',
-          '--volumes',
-          '--timeout=60',
-          'sharelatex',
-          'git-bridge',
-          'mongo',
-          'redis'
+app.post('/docker/compose/:cmd', (req, res) => {
+  const {
+    params: { cmd },
+    body: { args },
+  } = validateReq(
+    req,
+    z.object({
+      params: z.object({
+        cmd: z.literal(['up', 'stop', 'down', 'ps', 'logs']),
+      }),
+      body: z.object({
+        args: z.array(
+          z.literal([
+            '--detach',
+            '--wait',
+            '--volumes',
+            '--timeout=60',
+            'sharelatex',
+            'git-bridge',
+            'mongo',
+            'redis',
+          ])
         ),
-      },
-      params: {
-        cmd: Joi.allow('up', 'stop', 'down', 'ps', 'logs'),
-      },
-    },
-    { allowUnknown: false }
-  ),
-  (req, res) => {
-    const { cmd } = req.params
-    const { args } = req.body
-    runDockerCompose(cmd, args, (error, stdout, stderr) => {
-      res.json({ error, stdout, stderr })
+      }),
     })
-  }
-)
+  )
+
+  runDockerCompose(cmd, args, (error, stdout, stderr) => {
+    res.json({ error, stdout, stderr })
+  })
+})
 
 function maybeResetData(resetData, callback) {
   if (!resetData) return callback()
@@ -347,88 +341,78 @@ function maybeResetData(resetData, callback) {
   )
 }
 
-app.post(
-  '/reconfigure',
-  validate(
-    {
-      body: {
-        pro: Joi.boolean().required(),
-        mongoVersion: Joi.string().allow('').optional(),
-        version: Joi.string().required(),
+app.post('/reconfigure', (req, res) => {
+  const {
+    body: { pro, version, vars, withDataDir, resetData, mongoVersion },
+  } = validateReq(
+    req,
+    z.object({
+      body: z.object({
+        pro: z.boolean(),
+        version: z.string(),
         vars: allowedVars,
-        withDataDir: Joi.boolean().optional(),
-        resetData: Joi.boolean().optional(),
-      },
-    },
-    { allowUnknown: false }
-  ),
-  (req, res) => {
-    const { pro, version, vars, withDataDir, resetData, mongoVersion } =
-      req.body
-    maybeResetData(resetData, (error, stdout, stderr) => {
-      if (error) return res.json({ error, stdout, stderr })
-
-      const previousConfigServer = previousConfig
-      const newConfig = JSON.stringify(req.body)
-      if (previousConfig === newConfig) {
-        return res.json({ previousConfigServer })
-      }
-
-      try {
-        setVarsDockerCompose({ pro, version, vars, withDataDir, mongoVersion })
-      } catch (error) {
-        return res.json({ error })
-      }
-
-      if (error) return res.json({ error, stdout, stderr })
-      runDockerCompose(
-        'up',
-        ['--detach', '--wait', 'sharelatex'],
-        (error, stdout, stderr) => {
-          previousConfig = newConfig
-          res.json({ error, stdout, stderr, previousConfigServer })
-        }
-      )
+        withDataDir: z.boolean(),
+        resetData: z.boolean(),
+        mongoVersion: z.string(),
+      }),
     })
-  }
-)
+  )
+  maybeResetData(resetData, (error, stdout, stderr) => {
+    if (error) return res.json({ error, stdout, stderr })
 
-app.post(
-  '/mongo/setFeatureCompatibilityVersion',
-  validate(
-    {
-      body: {
-        mongoVersion: Joi.string().required(),
-      },
-    },
-    { allowUnknown: false }
-  ),
-  (req, res) => {
-    const { mongoVersion } = req.body
-    const mongosh = mongoVersion > '5' ? 'mongosh' : 'mongo'
-    const params = {
-      setFeatureCompatibilityVersion: mongoVersion,
+    const previousConfigServer = previousConfig
+    const newConfig = JSON.stringify(req.body)
+    if (previousConfig === newConfig) {
+      return res.json({ previousConfigServer })
     }
-    if (mongoVersion >= '7.0') {
-      // MongoServerError: Once you have upgraded to 7.0, you will not be able to downgrade FCV and binary version without support assistance. Please re-run this command with 'confirm: true' to acknowledge this and continue with the FCV upgrade.
-      // NOTE: 6.0 does not know about this flag. So conditionally add it.
-      // MongoServerError: BSON field 'setFeatureCompatibilityVersion.confirm' is an unknown field.
-      params.confirm = true
+
+    try {
+      setVarsDockerCompose({ pro, version, vars, withDataDir, mongoVersion })
+    } catch (error) {
+      return res.json({ error })
     }
+
+    if (error) return res.json({ error, stdout, stderr })
     runDockerCompose(
-      'exec',
-      [
-        'mongo',
-        mongosh,
-        '--eval',
-        `db.adminCommand(${JSON.stringify(params)})`,
-      ],
+      'up',
+      ['--detach', '--wait', 'sharelatex'],
       (error, stdout, stderr) => {
-        res.json({ error, stdout, stderr })
+        previousConfig = newConfig
+        res.json({ error, stdout, stderr, previousConfigServer })
       }
     )
+  })
+})
+
+app.post('/mongo/setFeatureCompatibilityVersion', (req, res) => {
+  const {
+    body: { mongoVersion },
+  } = validateReq(
+    req,
+    z.object({
+      body: z.object({
+        mongoVersion: z.string(),
+      }),
+    })
+  )
+  const mongosh = mongoVersion > '5' ? 'mongosh' : 'mongo'
+  const params = {
+    setFeatureCompatibilityVersion: mongoVersion,
   }
-)
+  if (mongoVersion >= '7.0') {
+    // MongoServerError: Once you have upgraded to 7.0, you will not be able to downgrade FCV and binary version without support assistance. Please re-run this command with 'confirm: true' to acknowledge this and continue with the FCV upgrade.
+    // NOTE: 6.0 does not know about this flag. So conditionally add it.
+    // MongoServerError: BSON field 'setFeatureCompatibilityVersion.confirm' is an unknown field.
+    params.confirm = true
+  }
+  runDockerCompose(
+    'exec',
+    ['mongo', mongosh, '--eval', `db.adminCommand(${JSON.stringify(params)})`],
+    (error, stdout, stderr) => {
+      res.json({ error, stdout, stderr })
+    }
+  )
+})
 
 app.get('/redis/keys', (req, res) => {
   runDockerCompose(
@@ -450,7 +434,14 @@ app.delete('/data/user_files', (req, res) => {
   )
 })
 
-app.use(handleValidationErrors())
+app.use((error, req, res, next) => {
+  if (error instanceof ParamsError) {
+    res.status(404).json({ error })
+  } else if (isZodErrorLike(error)) {
+    res.status(400).json({ error })
+  }
+  next(error)
+})
 
 purgeDataDir()
 writeDockerComposeOverride(defaultDockerComposeOverride())

server-ce/test/package.json

@@ -1,22 +1,23 @@
 {
-  "name": "@overleaf/server-ce/test",
+  "name": "@overleaf/server-ce-test",
   "description": "e2e tests for Overleaf Community Edition",
   "private": true,
+  "type": "module",
   "scripts": {
     "cypress:open": "cypress open --e2e --browser chrome",
     "cypress:run": "cypress run --e2e --browser chrome",
     "format": "prettier --list-different $PWD/'**/*.{js,mjs,ts,tsx,json}'",
     "format:fix": "prettier --write $PWD/'**/*.{js,mjs,ts,tsx,json}'"
   },
-  "dependencies": {
+  "devDependencies": {
     "@isomorphic-git/lightning-fs": "^4.6.0",
+    "@overleaf/validation-tools": "*",
     "@testing-library/cypress": "^10.0.3",
     "@types/adm-zip": "^0.5.7",
     "@types/pdf-parse": "^1.1.5",
     "@types/uuid": "^9.0.8",
     "adm-zip": "^0.5.12",
     "body-parser": "^1.20.3",
-    "celebrate": "^15.0.3",
     "cypress": "13.13.2",
     "cypress-multi-reporters": "^2.0.5",
     "express": "^4.21.2",
@@ -24,7 +25,7 @@
     "js-yaml": "^4.1.0",
     "mocha-junit-reporter": "^2.2.1",
     "pdf-parse": "^1.1.1",
-    "typescript": "^5.0.4",
-    "uuid": "^9.0.1"
+    "uuid": "^9.0.1",
+    "zod-validation-error": "^4.0.1"
   }
 }