overleaf-cep/services/web/app/src/Features/Compile/CompileManager.js

let CompileManager
const Settings = require('@overleaf/settings')
const RedisWrapper = require('../../infrastructure/RedisWrapper')
const rclient = RedisWrapper.client('clsi_recently_compiled')
const ProjectGetter = require('../Project/ProjectGetter')
const ProjectRootDocManager = require('../Project/ProjectRootDocManager')
const UserGetter = require('../User/UserGetter')
const ClsiManager = require('./ClsiManager')
const Metrics = require('@overleaf/metrics')
const { RateLimiter } = require('../../infrastructure/RateLimiter')
const SplitTestHandler = require('../SplitTests/SplitTestHandler')
const UserAnalyticsIdCache = require('../Analytics/UserAnalyticsIdCache')

const NEW_COMPILE_TIMEOUT_ENFORCED_CUTOFF = new Date('2023-09-18T11:00:00.000Z')
const NEW_COMPILE_TIMEOUT_ENFORCED_CUTOFF_DEFAULT_BASELINE = new Date(
  '2023-10-10T11:00:00.000Z'
)

module.exports = CompileManager = {
  NEW_COMPILE_TIMEOUT_ENFORCED_CUTOFF,
  NEW_COMPILE_TIMEOUT_ENFORCED_CUTOFF_DEFAULT_BASELINE,

  compile(projectId, userId, options = {}, _callback) {
    const timer = new Metrics.Timer('editor.compile')
    const callback = function (...args) {
      timer.done()
      _callback(...args)
    }

    CompileManager._checkIfRecentlyCompiled(
      projectId,
      userId,
      function (error, recentlyCompiled) {
        if (error) {
          return callback(error)
        }
        if (recentlyCompiled) {
          return callback(null, 'too-recently-compiled', [])
        }

        CompileManager._checkIfAutoCompileLimitHasBeenHit(
          options.isAutoCompile,
          'everyone',
          function (err, canCompile) {
            if (err || !canCompile) {
              return callback(null, 'autocompile-backoff', [])
            }

            ProjectRootDocManager.ensureRootDocumentIsSet(
              projectId,
              function (error) {
                if (error) {
                  return callback(error)
                }
                CompileManager.getProjectCompileLimits(
                  projectId,
                  function (error, limits) {
                    if (error) {
                      return callback(error)
                    }
                    for (const key in limits) {
                      const value = limits[key]
                      options[key] = value
                    }
                    if (options.timeout !== 20) {
                      // temporary override to force the new compile timeout
                      if (options.forceNewCompileTimeout === 'active') {
                        options.timeout = 20
                      } else if (
                        options.forceNewCompileTimeout === 'changing'
                      ) {
                        options.timeout = 60
                      }
                    }
                    // Put a lower limit on autocompiles for free users, based on compileGroup
                    CompileManager._checkCompileGroupAutoCompileLimit(
                      options.isAutoCompile,
                      limits.compileGroup,
                      function (err, canCompile) {
                        if (err || !canCompile) {
                          return callback(null, 'autocompile-backoff', [])
                        }
                        // only pass userId down to clsi if this is a per-user compile
                        const compileAsUser = Settings.disablePerUserCompiles
                          ? undefined
                          : userId
                        ClsiManager.sendRequest(
                          projectId,
                          compileAsUser,
                          options,
                          function (
                            error,
                            status,
                            outputFiles,
                            clsiServerId,
                            validationProblems,
                            stats,
                            timings,
                            outputUrlPrefix
                          ) {
                            if (error) {
                              return callback(error)
                            }
                            callback(
                              null,
                              status,
                              outputFiles,
                              clsiServerId,
                              limits,
                              validationProblems,
                              stats,
                              timings,
                              outputUrlPrefix
                            )
                          }
                        )
                      }
                    )
                  }
                )
              }
            )
          }
        )
      }
    )
  },

  stopCompile(projectId, userId, callback) {
    CompileManager.getProjectCompileLimits(projectId, function (error, limits) {
      if (error) {
        return callback(error)
      }
      ClsiManager.stopCompile(projectId, userId, limits, callback)
    })
  },

  deleteAuxFiles(projectId, userId, clsiserverid, callback) {
    CompileManager.getProjectCompileLimits(projectId, function (error, limits) {
      if (error) {
        return callback(error)
      }
      ClsiManager.deleteAuxFiles(
        projectId,
        userId,
        limits,
        clsiserverid,
        callback
      )
    })
  },

  getProjectCompileLimits(projectId, callback) {
    ProjectGetter.getProject(
      projectId,
      { owner_ref: 1 },
      function (error, project) {
        if (error) {
          return callback(error)
        }
        UserGetter.getUser(
          project.owner_ref,
          {
            _id: 1,
            alphaProgram: 1,
            analyticsId: 1,
            betaProgram: 1,
            features: 1,
            splitTests: 1,
            signUpDate: 1, // for compile-timeout-20s
          },
          function (err, owner) {
            if (err) {
              return callback(err)
            }
            const ownerFeatures = (owner && owner.features) || {}
            // put alpha users into their own compile group
            if (owner && owner.alphaProgram) {
              ownerFeatures.compileGroup = 'alpha'
            }
            UserAnalyticsIdCache.callbacks.get(
              owner._id,
              function (err, analyticsId) {
                if (err) {
                  return callback(err)
                }
                const limits = {
                  timeout:
                    ownerFeatures.compileTimeout ||
                    Settings.defaultFeatures.compileTimeout,
                  compileGroup:
                    ownerFeatures.compileGroup ||
                    Settings.defaultFeatures.compileGroup,
                  ownerAnalyticsId: analyticsId,
                }
                CompileManager._getCompileBackendClassDetails(
                  owner,
                  limits.compileGroup,
                  (
                    err,
                    { compileBackendClass, showFasterCompilesFeedbackUI }
                  ) => {
                    if (err) return callback(err)
                    limits.compileBackendClass = compileBackendClass
                    limits.showFasterCompilesFeedbackUI =
                      showFasterCompilesFeedbackUI
                    if (compileBackendClass === 'n2d' && limits.timeout <= 60) {
                      // project owners with faster compiles but with <= 60 compile timeout (default)
                      // will have a 20s compile timeout
                      // The compile-timeout-20s split test exists to enable a gradual rollout
                      SplitTestHandler.getAssignmentForMongoUser(
                        owner,
                        'compile-timeout-20s',
                        (err, assignment) => {
                          if (err) return callback(err)
                          // users who were on the 'default' servers at time of original rollout
                          // will have a later cutoff date for the 20s timeout in the next phase
                          // we check the backend class at version 8 (baseline)
                          const backendClassHistory =
                            owner.splitTests?.['compile-backend-class-n2d'] ||
                            []
                          const backendClassBaselineVariant =
                            backendClassHistory.find(version => {
                              return version.versionNumber === 8
                            })?.variantName
                          const timeoutEnforcedCutoff =
                            backendClassBaselineVariant === 'default'
                              ? NEW_COMPILE_TIMEOUT_ENFORCED_CUTOFF_DEFAULT_BASELINE
                              : NEW_COMPILE_TIMEOUT_ENFORCED_CUTOFF
                          if (assignment?.variant === '20s') {
                            if (owner.signUpDate > timeoutEnforcedCutoff) {
                              limits.timeout = 20
                              callback(null, limits)
                            } else {
                              SplitTestHandler.getAssignmentForMongoUser(
                                owner,
                                'compile-timeout-20s-existing-users',
                                (err, assignmentExistingUsers) => {
                                  if (err) return callback(err)
                                  if (
                                    assignmentExistingUsers?.variant === '20s'
                                  ) {
                                    limits.timeout = 20
                                  }
                                  callback(null, limits)
                                }
                              )
                            }
                          } else {
                            callback(null, limits)
                          }
                        }
                      )
                    } else {
                      callback(null, limits)
                    }
                  }
                )
              }
            )
          }
        )
      }
    )
  },

  COMPILE_DELAY: 1, // seconds
  _checkIfRecentlyCompiled(projectId, userId, callback) {
    const key = `compile:${projectId}:${userId}`
    rclient.set(
      key,
      true,
      'EX',
      this.COMPILE_DELAY,
      'NX',
      function (error, ok) {
        if (error) {
          return callback(error)
        }
        if (ok === 'OK') {
          callback(null, false)
        } else {
          callback(null, true)
        }
      }
    )
  },

  _checkCompileGroupAutoCompileLimit(isAutoCompile, compileGroup, callback) {
    if (!isAutoCompile) {
      return callback(null, true)
    }
    if (compileGroup === 'standard') {
      // apply extra limits to the standard compile group
      CompileManager._checkIfAutoCompileLimitHasBeenHit(
        isAutoCompile,
        compileGroup,
        callback
      )
    } else {
      Metrics.inc(`auto-compile-${compileGroup}`)
      callback(null, true)
    }
  }, // always allow priority group users to compile

  _checkIfAutoCompileLimitHasBeenHit(isAutoCompile, compileGroup, callback) {
    if (!isAutoCompile) {
      return callback(null, true)
    }
    Metrics.inc(`auto-compile-${compileGroup}`)
    const rateLimiter = getAutoCompileRateLimiter(compileGroup)
    rateLimiter
      .consume('global', 1, { method: 'global' })
      .then(() => {
        callback(null, true)
      })
      .catch(() => {
        // Don't differentiate between errors and rate limits. Silently trigger
        // the rate limit if there's an error consuming the points.
        Metrics.inc(`auto-compile-${compileGroup}-limited`)
        callback(null, false)
      })
  },

  _getCompileBackendClassDetails(owner, compileGroup, callback) {
    const { defaultBackendClass } = Settings.apis.clsi
    if (compileGroup === 'standard') {
      return SplitTestHandler.getAssignmentForMongoUser(
        owner,
        'compile-backend-class-n2d',
        (err, assignment) => {
          if (err) return callback(err, {})
          const { variant } = assignment
          callback(null, {
            compileBackendClass:
              variant === 'default' ? defaultBackendClass : variant,
            showFasterCompilesFeedbackUI: false,
          })
        }
      )
    }
    SplitTestHandler.getAssignmentForMongoUser(
      owner,
      'compile-backend-class',
      (err, assignment) => {
        if (err) return callback(err, {})
        const { analytics, variant } = assignment
        const activeForUser = analytics?.segmentation?.splitTest != null
        callback(null, {
          compileBackendClass:
            variant === 'default' ? defaultBackendClass : variant,
          showFasterCompilesFeedbackUI: activeForUser,
        })
      }
    )
  },

  wordCount(projectId, userId, file, clsiserverid, callback) {
    CompileManager.getProjectCompileLimits(projectId, function (error, limits) {
      if (error) {
        return callback(error)
      }
      ClsiManager.wordCount(
        projectId,
        userId,
        file,
        limits,
        clsiserverid,
        callback
      )
    })
  },
}

const autoCompileRateLimiters = new Map()
function getAutoCompileRateLimiter(compileGroup) {
  let rateLimiter = autoCompileRateLimiters.get(compileGroup)
  if (rateLimiter == null) {
    rateLimiter = new RateLimiter(`auto-compile:${compileGroup}`, {
      points: Settings.rateLimit.autoCompile[compileGroup] || 25,
      duration: 20,
    })
    autoCompileRateLimiters.set(compileGroup, rateLimiter)
  }
  return rateLimiter
}