[web] download binary files in clsi from filestore via new endpoints (#27505)

* [history-v1] use String.padStart instead of lodash.padStart

* [web] download binary files in clsi from filestore via new endpoints

* [server-ce] tests: Cypress.env() is parsing boolean values

* [server-ce] tests: run history migration as root

GitOrigin-RevId: bdf6c0e542531ccc4b3f13d2ed68ca0d31e580e9

services/filestore/app.js

@@ -119,6 +119,17 @@ app.get(
   fileController.getFile
 )
 
+app.get(
+  '/history/global/hash/:hash',
+  keyBuilder.globalBlobFileKeyMiddleware,
+  fileController.getFile
+)
+app.get(
+  '/history/project/:historyId/hash/:hash',
+  keyBuilder.projectBlobFileKeyMiddleware,
+  fileController.getFile
+)
+
 app.get('/status', function (req, res) {
   if (settings.shuttingDown) {
     res.sendStatus(503) // Service unavailable

services/filestore/app/js/FileController.js

@@ -25,6 +25,7 @@ function getFile(req, res, next) {
     format,
     style,
   }
+  if (req.useSubdirectories) options.useSubdirectories = true
 
   metrics.inc('getFile')
   req.requestLogger.setMessage('getting file')

services/filestore/app/js/KeyBuilder.js

@@ -1,4 +1,5 @@
 const settings = require('@overleaf/settings')
+const projectKey = require('./project_key')
 
 module.exports = {
   getConvertedFolderKey,
@@ -6,6 +7,8 @@ module.exports = {
   userFileKeyMiddleware,
   userProjectKeyMiddleware,
   bucketFileKeyMiddleware,
+  globalBlobFileKeyMiddleware,
+  projectBlobFileKeyMiddleware,
   templateFileKeyMiddleware,
 }
 
@@ -50,6 +53,22 @@ function bucketFileKeyMiddleware(req, res, next) {
   next()
 }
 
+function globalBlobFileKeyMiddleware(req, res, next) {
+  req.bucket = settings.filestore.stores.global_blobs
+  const { hash } = req.params
+  req.key = `${hash.slice(0, 2)}/${hash.slice(2, 4)}/${hash.slice(4)}`
+  req.useSubdirectories = true
+  next()
+}
+
+function projectBlobFileKeyMiddleware(req, res, next) {
+  req.bucket = settings.filestore.stores.project_blobs
+  const { historyId, hash } = req.params
+  req.key = `${projectKey.format(historyId)}/${hash.slice(0, 2)}/${hash.slice(2)}`
+  req.useSubdirectories = true
+  next()
+}
+
 function templateFileKeyMiddleware(req, res, next) {
   const {
     template_id: templateId,

services/filestore/app/js/project_key.js

@@ -0,0 +1,23 @@
+// Keep in sync with services/history-v1/storage/lib/project_key.js
+const path = require('node:path')
+
+//
+// The advice in http://docs.aws.amazon.com/AmazonS3/latest/dev/
+// request-rate-perf-considerations.html is to avoid sequential key prefixes,
+// so we reverse the project ID part of the key as they suggest.
+//
+function format(projectId) {
+  const prefix = naiveReverse(pad(projectId))
+  return path.join(prefix.slice(0, 3), prefix.slice(3, 6), prefix.slice(6))
+}
+
+function pad(number) {
+  return (number || 0).toString().padStart(9, '0')
+}
+
+function naiveReverse(string) {
+  return string.split('').reverse().join('')
+}
+
+exports.format = format
+exports.pad = pad