overleaf-cep/services/web/app/src/Features/Uploads/FileSystemImportManager.js

const fs = require('fs')
const Path = require('path')
const { callbackify } = require('util')
const EditorController = require('../Editor/EditorController')
const Errors = require('../Errors/Errors')
const FileTypeManager = require('./FileTypeManager')
const SafePath = require('../Project/SafePath')
const logger = require('@overleaf/logger')

module.exports = {
  addEntity: callbackify(addEntity),
  importDir: callbackify(importDir),
  importFile: callbackify(importDir),
  promises: {
    addEntity,
    importDir,
    importFile,
  },
}

async function addDoc(userId, projectId, folderId, name, lines, replace) {
  if (replace) {
    const doc = await EditorController.promises.upsertDoc(
      projectId,
      folderId,
      name,
      lines,
      'upload',
      userId
    )
    return doc
  } else {
    const doc = await EditorController.promises.addDoc(
      projectId,
      folderId,
      name,
      lines,
      'upload',
      userId
    )
    return doc
  }
}

async function addFile(userId, projectId, folderId, name, path, replace) {
  if (replace) {
    const file = await EditorController.promises.upsertFile(
      projectId,
      folderId,
      name,
      path,
      null,
      'upload',
      userId
    )
    return file
  } else {
    const file = await EditorController.promises.addFile(
      projectId,
      folderId,
      name,
      path,
      null,
      'upload',
      userId
    )
    return file
  }
}

async function addFolder(userId, projectId, folderId, name, path, replace) {
  const newFolder = await EditorController.promises.addFolder(
    projectId,
    folderId,
    name,
    'upload',
    userId
  )
  await addFolderContents(userId, projectId, newFolder._id, path, replace)
  return newFolder
}

async function addFolderContents(
  userId,
  projectId,
  parentFolderId,
  folderPath,
  replace
) {
  if (!(await _isSafeOnFileSystem(folderPath))) {
    logger.debug(
      { userId, projectId, parentFolderId, folderPath },
      'add folder contents is from symlink, stopping insert'
    )
    throw new Error('path is symlink')
  }
  const entries = (await fs.promises.readdir(folderPath)) || []
  for (const entry of entries) {
    if (FileTypeManager.shouldIgnore(entry)) {
      continue
    }
    await addEntity(
      userId,
      projectId,
      parentFolderId,
      entry,
      `${folderPath}/${entry}`,
      replace
    )
  }
}

async function addEntity(userId, projectId, folderId, name, fsPath, replace) {
  if (!(await _isSafeOnFileSystem(fsPath))) {
    logger.debug(
      { userId, projectId, folderId, fsPath },
      'add entry is from symlink, stopping insert'
    )
    throw new Error('path is symlink')
  }

  if (await FileTypeManager.promises.isDirectory(fsPath)) {
    const newFolder = await addFolder(
      userId,
      projectId,
      folderId,
      name,
      fsPath,
      replace
    )
    return newFolder
  }

  // Here, we cheat a little bit and provide the project path relative to the
  // folder, not the root of the project. This is because we don't know for sure
  // at this point what the final path of the folder will be. The project path
  // is still important for importFile() to be able to figure out if the file is
  // a binary file or an editable document.
  const projectPath = Path.join('/', name)
  const importInfo = await importFile(fsPath, projectPath)
  switch (importInfo.type) {
    case 'file': {
      const entity = await addFile(
        userId,
        projectId,
        folderId,
        name,
        importInfo.fsPath,
        replace
      )
      if (entity != null) {
        entity.type = 'file'
      }
      return entity
    }
    case 'doc': {
      const entity = await addDoc(
        userId,
        projectId,
        folderId,
        name,
        importInfo.lines,
        replace
      )
      if (entity != null) {
        entity.type = 'doc'
      }
      return entity
    }
    default: {
      throw new Error(`unknown import type: ${importInfo.type}`)
    }
  }
}

async function _isSafeOnFileSystem(path) {
  // Use lstat() to ensure we don't follow symlinks. Symlinks from an
  // untrusted source are dangerous.
  const stat = await fs.promises.lstat(path)
  return stat.isFile() || stat.isDirectory()
}

async function importFile(fsPath, projectPath) {
  const stat = await fs.promises.lstat(fsPath)
  if (!stat.isFile()) {
    throw new Error(`can't import ${fsPath}: not a regular file`)
  }
  _validateProjectPath(projectPath)
  const filename = Path.basename(projectPath)

  const { binary, encoding } = await FileTypeManager.promises.getType(
    filename,
    fsPath,
    null
  )
  if (binary) {
    return new FileImport(projectPath, fsPath)
  } else {
    const content = await fs.promises.readFile(fsPath, encoding)
    // Handle Unix, DOS and classic Mac newlines
    const lines = content.split(/\r\n|\n|\r/)
    return new DocImport(projectPath, lines)
  }
}

async function importDir(dirPath) {
  const stat = await fs.promises.lstat(dirPath)
  if (!stat.isDirectory()) {
    throw new Error(`can't import ${dirPath}: not a directory`)
  }
  const entries = []
  for await (const filePath of _walkDir(dirPath)) {
    const projectPath = Path.join('/', Path.relative(dirPath, filePath))
    const importInfo = await importFile(filePath, projectPath)
    entries.push(importInfo)
  }
  return entries
}

function _validateProjectPath(path) {
  if (!SafePath.isAllowedLength(path) || !SafePath.isCleanPath(path)) {
    throw new Errors.InvalidNameError(`Invalid path: ${path}`)
  }
}

async function* _walkDir(dirPath) {
  const entries = await fs.promises.readdir(dirPath)
  for (const entry of entries) {
    const entryPath = Path.join(dirPath, entry)
    if (FileTypeManager.shouldIgnore(entryPath)) {
      continue
    }

    // Use lstat() to ensure we don't follow symlinks. Symlinks from an
    // untrusted source are dangerous.
    const stat = await fs.promises.lstat(entryPath)
    if (stat.isFile()) {
      yield entryPath
    } else if (stat.isDirectory()) {
      yield* _walkDir(entryPath)
    }
  }
}

class FileImport {
  constructor(projectPath, fsPath) {
    this.type = 'file'
    this.projectPath = projectPath
    this.fsPath = fsPath
  }
}

class DocImport {
  constructor(projectPath, lines) {
    this.type = 'doc'
    this.projectPath = projectPath
    this.lines = lines
  }
}