[filestore] remove user files endpoints (#28125)

* [filestore] remove user files endpoints

* [web] remove user files integration for filestore

GitOrigin-RevId: 565fa68a659c07420ee6141d0f276b4e4d2972e0
This commit is contained in:
Jakob Ackermann
2025-09-01 11:43:37 +02:00
committed by Copybot
parent 21d3879574
commit 319a542e8d
73 changed files with 1481 additions and 3587 deletions

View File

@@ -1,10 +1,10 @@
const Path = require('path')
const DocstoreManager = require('../app/src/Features/Docstore/DocstoreManager')
const DocumentUpdaterHandler = require('../app/src/Features/DocumentUpdater/DocumentUpdaterHandler')
const FileStoreHandler = require('../app/src/Features/FileStore/FileStoreHandler')
const ProjectGetter = require('../app/src/Features/Project/ProjectGetter')
const ProjectEntityMongoUpdateHandler = require('../app/src/Features/Project/ProjectEntityMongoUpdateHandler')
const { waitForDb, db, ObjectId } = require('../app/src/infrastructure/mongodb')
const HistoryManager = require('../app/src/Features/History/HistoryManager')
const logger = require('@overleaf/logger').logger
const args = require('minimist')(process.argv.slice(2), {
@@ -211,10 +211,12 @@ async function checkProject(projectId) {
}
for (const { file, path } of fileEntries) {
try {
const fileSize = await FileStoreHandler.promises.getFileSize(
projectId,
file._id
)
const { contentLength: fileSize } =
await HistoryManager.promises.requestBlobWithProjectId(
projectId,
file.hash,
'HEAD'
)
if (pathCounts.get(path) > 1) {
logFile(projectId, path, { ...file, fileSize }, 'duplicate path')
errors++

View File

@@ -3,7 +3,7 @@ import { ObjectId, db } from '../app/src/infrastructure/mongodb.js'
import ProjectEntityHandler from '../app/src/Features/Project/ProjectEntityHandler.js'
import ProjectGetter from '../app/src/Features/Project/ProjectGetter.js'
import Errors from '../app/src/Features/Errors/Errors.js'
import FileStoreHandler from '../app/src/Features/FileStore/FileStoreHandler.js'
import HistoryManager from '../app/src/Features/History/HistoryManager.js'
// Handles a list of project IDs from stdin, one per line, and outputs the count of files and docs
// in the project, along with the aggregated size in bytes for all files and docs.
@@ -63,24 +63,16 @@ async function countFilesSize(files, projectId) {
return 0
}
const ids = files.map(fileObject => fileObject.file._id)
let totalFileSize = 0
for (const fileId of ids) {
const contentLength = await FileStoreHandler.promises.getFileSize(
projectId,
fileId
)
const size = parseInt(contentLength, 10)
if (isNaN(size)) {
throw new Error(
`Unable to fetch file size for fileId=${fileId} and projectId=${projectId}`
for (const { file } of files) {
const { contentLength } =
await HistoryManager.promises.requestBlobWithProjectId(
projectId,
file.hash,
'HEAD'
)
}
totalFileSize += size
totalFileSize += contentLength
}
return totalFileSize

View File

@@ -7,10 +7,10 @@ import minimist from 'minimist'
import mongodb from 'mongodb-legacy'
import { db } from '../app/src/infrastructure/mongodb.js'
import Errors from '../app/src/Features/Errors/Errors.js'
import FileStoreHandler from '../app/src/Features/FileStore/FileStoreHandler.js'
import ProjectEntityMongoUpdateHandler from '../app/src/Features/Project/ProjectEntityMongoUpdateHandler.js'
import { iterablePaths } from '../app/src/Features/Project/IterablePath.js'
import { scriptRunner } from './lib/ScriptRunner.mjs'
import HistoryManager from '../app/src/Features/History/HistoryManager.js'
const { ObjectId } = mongodb
@@ -57,22 +57,22 @@ async function getProjects() {
async function processProject(project) {
console.log(`Processing project ${project._id}`)
const { docIds, fileIds } = findRefsInFolder(project.rootFolder[0])
const { docIds, fileRefs } = findRefsInFolder(project.rootFolder[0])
for (const docId of docIds) {
if (!(await docExists(docId))) {
await deleteDoc(project._id, docId)
}
}
for (const fileId of fileIds) {
if (!(await fileExists(project._id, fileId))) {
await deleteFile(project._id, fileId)
for (const fileRef of fileRefs) {
if (!(await fileExists(project._id, fileRef.hash))) {
await deleteFile(project._id, fileRef._id)
}
}
}
function findRefsInFolder(folder) {
let docIds = folder.docs.map(doc => doc._id)
let fileIds = folder.fileRefs.map(file => file._id)
let fileIds = folder.fileRefs.slice()
for (const subfolder of iterablePaths(folder, 'folders')) {
const subrefs = findRefsInFolder(subfolder)
docIds = docIds.concat(subrefs.docIds)
@@ -86,10 +86,14 @@ async function docExists(docId) {
return doc != null
}
async function fileExists(projectId, fileId) {
async function fileExists(projectId, hash) {
try {
// Getting the file size to avoid downloading the whole file
await FileStoreHandler.promises.getFileSize(projectId, fileId)
await HistoryManager.promises.requestBlobWithProjectId(
projectId,
hash,
'HEAD'
)
} catch (err) {
if (err instanceof Errors.NotFoundError) {
return false