Allow ESM OL modules to be loaded

Lots of changes to async/await required to allow this. We have to make
some changes to handle the fact that modules are loaded async in stages
rather than sync (so we can't control when top-level functionality is
run in a fine grained way)

GitOrigin-RevId: 0127b15bfc4f228a267df3af8519c675e900654e

services/web/app/src/infrastructure/Modules.js

@@ -4,6 +4,7 @@ const async = require('async')
 const { promisify } = require('util')
 const Settings = require('@overleaf/settings')
 const Views = require('./Views')
+const _ = require('lodash')
 
 const MODULE_BASE_PATH = Path.join(__dirname, '/../../../modules')
 
@@ -13,27 +14,33 @@ const _hooks = {}
 const _middleware = {}
 let _viewIncludes = {}
 
-function modules() {
+async function modules() {
   if (!_modulesLoaded) {
-    loadModules()
+    await loadModules()
   }
   return _modules
 }
 
-function loadModules() {
+async function loadModulesImpl() {
   const settingsCheckModule = Path.join(
     MODULE_BASE_PATH,
     'settings-check',
     'index.js'
   )
   if (fs.existsSync(settingsCheckModule)) {
-    require(settingsCheckModule)
+    await import(settingsCheckModule)
   }
 
   for (const moduleName of Settings.moduleImportSequence || []) {
-    const loadedModule = require(
-      Path.join(MODULE_BASE_PATH, moduleName, 'index.js')
-    )
+    let path
+    if (fs.existsSync(Path.join(MODULE_BASE_PATH, moduleName, 'index.mjs'))) {
+      path = Path.join(MODULE_BASE_PATH, moduleName, 'index.mjs')
+    } else {
+      path = Path.join(MODULE_BASE_PATH, moduleName, 'index.js')
+    }
+    const module = await import(path)
+    const loadedModule = module.default || module
+
     loadedModule.name = moduleName
     _modules.push(loadedModule)
     if (loadedModule.viewIncludes) {
@@ -52,20 +59,26 @@ function loadModules() {
     }
   }
   _modulesLoaded = true
-  attachHooks()
-  attachMiddleware()
+  await attachHooks()
+  await attachMiddleware()
 }
 
-function applyRouter(webRouter, privateApiRouter, publicApiRouter) {
-  for (const module of modules()) {
+const loadModules = _.memoize(loadModulesImpl)
+
+async function applyRouter(webRouter, privateApiRouter, publicApiRouter) {
+  for (const module of await modules()) {
     if (module.router && module.router.apply) {
       module.router.apply(webRouter, privateApiRouter, publicApiRouter)
     }
   }
 }
 
-function applyNonCsrfRouter(webRouter, privateApiRouter, publicApiRouter) {
-  for (const module of modules()) {
+async function applyNonCsrfRouter(
+  webRouter,
+  privateApiRouter,
+  publicApiRouter
+) {
+  for (const module of await modules()) {
     if (module.nonCsrfRouter != null) {
       module.nonCsrfRouter.apply(webRouter, privateApiRouter, publicApiRouter)
     }
@@ -80,7 +93,7 @@ function applyNonCsrfRouter(webRouter, privateApiRouter, publicApiRouter) {
 }
 
 async function start() {
-  for (const module of modules()) {
+  for (const module of await modules()) {
     await module.start?.()
   }
 }
@@ -89,13 +102,13 @@ function loadViewIncludes(app) {
   _viewIncludes = Views.compileViewIncludes(app)
 }
 
-function applyMiddleware(appOrRouter, middlewareName, options) {
+async function applyMiddleware(appOrRouter, middlewareName, options) {
   if (!middlewareName) {
     throw new Error(
       'middleware name must be provided to register module middleware'
     )
   }
-  for (const module of modules()) {
+  for (const module of await modules()) {
     if (module[middlewareName]) {
       module[middlewareName](appOrRouter, options)
     }
@@ -115,9 +128,9 @@ function moduleIncludesAvailable(view) {
   return (_viewIncludes[view] || []).length > 0
 }
 
-function linkedFileAgentsIncludes() {
+async function linkedFileAgentsIncludes() {
   const agents = {}
-  for (const module of modules()) {
+  for (const module of await modules()) {
     for (const name in module.linkedFileAgents) {
       const agentFunction = module.linkedFileAgents[name]
       agents[name] = agentFunction()
@@ -126,8 +139,8 @@ function linkedFileAgentsIncludes() {
   return agents
 }
 
-function attachHooks() {
-  for (const module of modules()) {
+async function attachHooks() {
+  for (const module of await modules()) {
     for (const hook in module.hooks || {}) {
       const method = module.hooks[hook]
       attachHook(hook, method)
@@ -142,8 +155,8 @@ function attachHook(name, method) {
   _hooks[name].push(method)
 }
 
-function attachMiddleware() {
-  for (const module of modules()) {
+async function attachMiddleware() {
+  for (const module of await modules()) {
     for (const middleware in module.middleware || {}) {
       const method = module.middleware[middleware]
       if (_middleware[middleware] == null) {
@@ -155,28 +168,37 @@ function attachMiddleware() {
 }
 
 function fireHook(name, ...rest) {
+  const adjustedLength = Math.max(rest.length, 1)
+  const args = rest.slice(0, adjustedLength - 1)
+  const callback = rest[adjustedLength - 1]
+  function fire() {
+    const methods = _hooks[name] || []
+    const callMethods = methods.map(method => cb => method(...args, cb))
+    async.series(callMethods, function (error, results) {
+      if (error) {
+        return callback(error)
+      }
+      callback(null, results)
+    })
+  }
+
   // ensure that modules are loaded if we need to fire a hook
   // this can happen if a script calls a method that fires a hook
   if (!_modulesLoaded) {
     loadModules()
+      .then(() => {
+        fire()
+      })
+      .catch(err => callback(err))
+  } else {
+    fire()
   }
-  const adjustedLength = Math.max(rest.length, 1)
-  const args = rest.slice(0, adjustedLength - 1)
-  const callback = rest[adjustedLength - 1]
-  const methods = _hooks[name] || []
-  const callMethods = methods.map(method => cb => method(...args, cb))
-  async.series(callMethods, function (error, results) {
-    if (error) {
-      return callback(error)
-    }
-    callback(null, results)
-  })
 }
 
-function getMiddleware(name) {
+async function getMiddleware(name) {
   // ensure that modules are loaded if we need to call a middleware
   if (!_modulesLoaded) {
-    loadModules()
+    await loadModules()
   }
   return _middleware[name] || []
 }