Add Template Gallery support

This commit is contained in:
yu-i-i
2026-07-17 17:56:04 +02:00
parent e0050414d6
commit 2a8b0bef67
68 changed files with 3063 additions and 31 deletions
@@ -0,0 +1,22 @@
import sanitizeHtml from 'sanitize-html'
const sanitizeOptions = {
linksOnly: {
allowedTags: ["a"],
allowedAttributes: { "a": ["href"] }
},
reachText: {
allowedTags: ["h1", "h2", "h3", "h4", "h5", "h6",
"p", "blockquote", "ul", "ol", "li",
"em", "strong", "s", "code", "pre", "hr", "a"],
allowedAttributes: { "a": ["href"] }
},
plainText: {
allowedTags: [],
allowedAttributes: {},
},
}
export function cleanHtml(text, sanitizeType) {
return sanitizeHtml(text, sanitizeOptions[sanitizeType])
}
@@ -0,0 +1,13 @@
import OError from '@overleaf/o-error'
export class TemplateNameConflictError extends OError {
constructor(ownerId, message = 'template_with_this_title_exists_and_owned_by_x') {
super(message, { ownerId })
}
}
export class RecompileRequiredError extends OError {
constructor(message = 'Recompile required') {
super(message, { status: 400 })
}
}
@@ -0,0 +1,162 @@
import logger from '@overleaf/logger'
import ErrorController from '../../../../app/src/Features/Errors/ErrorController.js'
import Errors from '../../../../app/src/Features/Errors/Errors.js'
import SessionManager from '../../../../app/src/Features/Authentication/SessionManager.js'
import TemplateGalleryManager from'./TemplateGalleryManager.mjs'
import { getUserName } from './TemplateGalleryHelper.mjs'
import { TemplateNameConflictError, RecompileRequiredError } from './TemplateErrors.mjs'
import Settings from '@overleaf/settings'
async function createTemplateFromProject(req, res, next) {
const t = req.i18n.translate
try {
const userId = SessionManager.getLoggedInUserId(req.session)
const result = await TemplateGalleryManager.createTemplateFromProject({
projectId: req.params.Project_id,
userId,
templateSource: req.body,
})
if (result.conflict) {
const ownerName = (result.templateOwnerName === 'you') ? t('you') : result.templateOwnerName
const message = `${t('template_with_this_title_exists_and_owned_by_x', { x: ownerName })} `
+ t(result.canOverride ? 'do_you_want_to_overwrite_it' : 'you_cant_overwrite_it')
return res.status(409).json({ canOverride: result.canOverride, message })
}
return res.status(200).json({ template_id: result.templateId })
} catch (error) {
if (error instanceof Errors.InvalidNameError) {
return res.status(error.info?.status || 400).json({ message: error.message })
}
const mainMessage = t('failed_to_publish_as_a_template')
if (error instanceof RecompileRequiredError) {
return res.status(error.info?.status || 400).json({
message: `${mainMessage} ${t('try_recompile_project')}`
})
}
return res.status(400).json({ message: mainMessage })
}
}
async function editTemplate(req, res, next) {
const t = req.i18n.translate
try {
const result = await TemplateGalleryManager.editTemplate({
templateId: req.params.template_id,
updates: req.body
})
res.status(200).json(result)
} catch (error) {
if (error instanceof TemplateNameConflictError) {
const ownerId = error.info?.ownerId
const userId = SessionManager.getLoggedInUserId(req.session)
const ownerName = (ownerId === userId)
? t('you')
: await getUserName(ownerId) || t('unknown')
const message = t(error.message, { x: ownerName })
return res.status(409).json({ message })
}
if (error instanceof Errors.InvalidNameError) {
return res.status(error.info?.status || 400).json({ message: error.message })
}
logger.error({ error }, 'Failure saving template')
return res.status(500).json({ message: t('something_went_wrong_server') })
}
}
async function deleteTemplate(req, res, next) {
const t = req.i18n.translate
try {
await TemplateGalleryManager.deleteTemplate({
templateId: req.params.template_id,
version: req.body.version
})
res.sendStatus(200)
} catch (error) {
logger.error({ error }, 'Failure deleting template')
return res.status(500).json({ message: t('something_went_wrong_server') })
}
}
async function getTemplatePreview(req, res, next) {
try {
const templateId = req.params.template_id
const { version, style } = req.query
const { stream, contentType } = await TemplateGalleryManager.fetchTemplatePreview({ templateId, version, style })
res.setHeader('Content-Type', contentType)
stream.pipe(res)
} catch (error) {
if (error.info?.status == 404) {
return ErrorController.notFound(req, res, next)
}
return res.status(error.info?.status || 400).json(error.info)
}
}
async function templatesCategoryPage(req, res, next) {
const t = req.i18n.translate
try {
let { category } = req.params
const result = await TemplateGalleryManager.getTemplatesPageData(category)
let title
if (result.categoryName) {
title = t('latex_templates') + ' — ' + result.categoryName
} else {
category = null
title = t('templates_page_title')
}
res.render('template_gallery/template-gallery', {
title,
category,
})
} catch (error) {
next(error)
}
}
async function templateDetailsPage(req, res, next) {
const t = req.i18n.translate
try {
const template = await TemplateGalleryManager.getTemplate('_id', req.params.template_id)
res.render('template_gallery/template', {
title: `${t('template')}: ${template.name}`,
template: JSON.stringify(template),
languages: Settings.languages,
})
} catch (error) {
return ErrorController.notFound(req, res, next)
}
}
async function getTemplateJSON(req, res, next) {
try {
const { key, val } = req.query
const template = await TemplateGalleryManager.getTemplate(key, val)
res.json(template)
} catch (error) {
next(error)
}
}
async function getCategoryTemplatesJSON(req, res, next) {
try {
const result = await TemplateGalleryManager.getCategoryTemplates(req.query)
res.json(result)
} catch (error) {
next(error)
}
}
export default {
createTemplateFromProject,
editTemplate,
deleteTemplate,
getTemplatePreview,
templatesCategoryPage,
templateDetailsPage,
getTemplateJSON,
getCategoryTemplatesJSON,
}
@@ -0,0 +1,226 @@
import MarkdownIt from 'markdown-it'
import request from 'request'
import logger from '@overleaf/logger'
import settings from '@overleaf/settings'
import Errors from '../../../../app/src/Features/Errors/Errors.js'
import ProjectGetter from '../../../../app/src/Features/Project/ProjectGetter.js'
import ProjectLocator from '../../../../app/src/Features/Project/ProjectLocator.js'
import ProjectZipStreamManager from '../../../../app/src/Features/Downloads/ProjectZipStreamManager.mjs'
import DocumentUpdaterHandler from '../../../../app/src/Features/DocumentUpdater/DocumentUpdaterHandler.js'
import ClsiManager from '../../../../app/src/Features/Compile/ClsiManager.js'
import CompileManager from '../../../../app/src/Features/Compile/CompileManager.js'
import UserGetter from '../../../../app/src/Features/User/UserGetter.js'
import { fetchStreamWithResponse } from '@overleaf/fetch-utils'
import { Template } from './models/Template.js'
import { RecompileRequiredError } from './TemplateErrors.mjs'
import { cleanHtml } from './CleanHtml.mjs'
const TIMEOUT = 30000
const MAX_PROJECT_NAME_LENGTH = 150
const MAX_FORM_INPUT_LENGTH = 512
const MAX_TEMPLATE_DESCRIPTION_LENGTH = 4096
const markdownIt = new MarkdownIt({ html: false, linkify: true })
function _createZipStreamForProjectAsync(projectId) {
return new Promise((resolve, reject) => {
ProjectZipStreamManager.createZipStreamForProject(projectId, (err, archive) => {
if (err) {
return reject(err)
}
archive.on('error', (err) => reject(err))
resolve(archive)
})
})
}
export function validateTemplateInput({ name, descriptionMD, authorMD, license }) {
if (name?.length > MAX_PROJECT_NAME_LENGTH) {
throw new Errors.InvalidNameError(`Template title exceeds the maximum length of ${MAX_PROJECT_NAME_LENGTH} characters.`)
}
if (descriptionMD?.length > MAX_TEMPLATE_DESCRIPTION_LENGTH) {
throw new Errors.InvalidNameError(`Template description exceeds the maximum length of ${MAX_TEMPLATE_DESCRIPTION_LENGTH} characters.`)
}
if (authorMD?.length > MAX_FORM_INPUT_LENGTH) {
throw new Errors.InvalidNameError('Author name is too long.')
}
if (license?.length > MAX_FORM_INPUT_LENGTH) {
throw new Errors.InvalidNameError('License name is too long.')
}
}
export async function canUserOverrideTemplate(template, userId) {
let templateOwnerId = template.owner
let templateOwnerName = 'you'
let userIsOwner = true
let userIsAdmin
if (templateOwnerId != userId) {
userIsOwner = false
try {
userIsAdmin = (await UserGetter.promises.getUser(userId, { isAdmin: 1 })).isAdmin
} catch {
logger.error({ error, userId }, 'Logged in user does not exist, strange...')
userIsAdmin = false
}
templateOwnerName = await getUserName(templateOwnerId) || 'unknown'
}
const canOverride = userIsOwner || userIsAdmin
return { canOverride, templateOwnerName }
}
export async function getUserName(userId) {
try {
const user = await UserGetter.promises.getUser(userId, {
first_name: 1,
last_name: 1,
})
return ((user?.first_name || "") + " " + (user?.last_name || "")).trim()
} catch (error) {
return 'unknown'
}
}
export async function generateTemplateData(projectId, {
descriptionMD,
authorMD,
category,
license,
name,
}) {
try {
await DocumentUpdaterHandler.promises.flushProjectToMongo(projectId)
const project = await ProjectGetter.promises.getProject(projectId, {
imageName: true,
compiler: true,
spellCheckLanguage: true,
rootDoc_id: true,
rootFolder: true,
})
const { path } = await ProjectLocator.promises.findRootDoc({ project })
const mainFile = path.fileSystem.replace(/^\/+/, '')
const template = {
name,
category,
authorMD,
descriptionMD,
license,
mainFile,
compiler: project.compiler,
imageName: project.imageName,
language: project.spellCheckLanguage,
lastUpdated: new Date(),
}
await renderTemplateHtmlFields(template)
return template
} catch (error) {
logger.error({ error, projectId }, 'Failed to retrieve project data')
throw error
}
}
export async function uploadTemplateAssets(projectId, userId, build, template) {
let zipStream, pdfStream
try {
[zipStream, pdfStream] = await Promise.all([
_createZipStreamForProjectAsync(projectId),
CompileManager.promises
.getProjectCompileLimits(projectId)
.then((limits) =>
ClsiManager.promises.getOutputFileStream(projectId, userId, limits, undefined, build, 'output.pdf')
)
])
} catch (error) {
logger.error({ error, projectId }, 'No output.pdf?')
throw new RecompileRequiredError()
}
try {
const templateUrl = `${settings.apis.filestore.url}/template/${template._id}/v/${template.version}`
const zipUrl = `${templateUrl}/zip`
const pdfUrl = `${templateUrl}/pdf`
const [zipReq, pdfReq] = await Promise.all([
fetchStreamWithResponse(zipUrl, {
method: 'POST',
headers: {
'Content-Type': 'application/octet-stream',
},
body: zipStream,
signal: AbortSignal.timeout(TIMEOUT),
}),
fetchStreamWithResponse(pdfUrl, {
method: 'POST',
headers: {
'Content-Type': 'application/pdf',
},
body: pdfStream,
signal: AbortSignal.timeout(TIMEOUT),
}),
])
const uploadErrors = []
if (zipReq?.response.status !== 200) {
uploadErrors.push({ file: 'zip', uri: zipUrl, statusCode: zipReq.response.status })
}
if (pdfReq?.response.status !== 200) {
uploadErrors.push({ file: 'pdf', uri: pdfUrl, statusCode: pdfReq.response.status })
}
if (uploadErrors.length > 0) {
logger.error({ uploadErrors }, 'Template upload failed')
throw new RecompileRequiredError()
}
} catch (error) {
if (error instanceof RecompileRequiredError) throw error
throw error
}
}
export async function deleteTemplateAssets(templateId, version, deleteFromDb) {
if (deleteFromDb) {
try {
await Template.deleteOne({ _id: templateId }).exec()
} catch (error) {
logger.error({ err, templateId }, 'Failed to delete template from MongoDB')
throw error
}
}
// kick off file deletions, but don't wait
const baseUrl = settings.apis.filestore.url
const urlTemplate = `${baseUrl}/template/${templateId}/v/${version}/zip`
const urlImages = `${baseUrl}/template/${templateId}/v/${version}/pdf`
const optsTemplate = {
method: 'DELETE',
uri: urlTemplate,
timeout: TIMEOUT,
}
const optsImages = {
method: 'DELETE',
uri: urlImages,
timeout: TIMEOUT,
}
request(optsTemplate, (err, response) => {
if (err)
logger.warn({ err, templateId }, 'Failed to delete template zip from filestore')
})
request(optsImages, (err, response) => {
if (err)
logger.warn({ err, templateId }, 'Failed to delete images from filestore')
})
}
export function renderTemplateHtmlFields(updates) {
if (updates.descriptionMD !== undefined) {
const descriptionRawHTML = markdownIt.render(updates.descriptionMD)
updates.description = cleanHtml(descriptionRawHTML, "reachText")
}
if (updates.authorMD !== undefined) {
const authorRawHTML = markdownIt.render(updates.authorMD)
updates.author = cleanHtml(authorRawHTML, "linksOnly")
}
}
@@ -0,0 +1,214 @@
import _ from 'lodash'
import logger from '@overleaf/logger'
import { Readable } from 'stream'
import settings from '@overleaf/settings'
import { OError } from '../../../../app/src/Features/Errors/Errors.js'
import { Template } from './models/Template.js'
import {
validateTemplateInput,
renderTemplateHtmlFields,
uploadTemplateAssets,
deleteTemplateAssets,
canUserOverrideTemplate,
generateTemplateData
} from './TemplateGalleryHelper.mjs'
import { cleanHtml } from './CleanHtml.mjs'
import { TemplateNameConflictError } from './TemplateErrors.mjs'
import { fetchStreamWithResponse } from '@overleaf/fetch-utils'
const TIMEOUT = 30000
async function editTemplate({ templateId, updates }) {
validateTemplateInput(updates)
const template = await Template.findById(templateId)
if (!template) {
throw new OError('Current template not found, strange...', { status: 500, templateId })
}
if (updates.name) {
const conflictingTemplate = await Template.findOne(
{ name: updates.name, _id: { $ne: templateId } },
{ owner: true }
).exec()
if (conflictingTemplate) {
throw new TemplateNameConflictError(String(conflictingTemplate.owner))
}
}
await renderTemplateHtmlFields(updates)
updates.lastUpdated = new Date()
Object.assign(template, updates)
await template.save()
return updates
}
async function deleteTemplate({ templateId, version }) {
await deleteTemplateAssets(templateId, version, true)
}
async function createTemplateFromProject({ projectId, userId, templateSource }) {
validateTemplateInput(templateSource)
let template = await Template.findOne({ name: templateSource.name }).exec()
if (template && !templateSource.override) {
const { canOverride, templateOwnerName } = await canUserOverrideTemplate(template, userId)
return {
conflict: true,
canOverride,
templateOwnerName
}
}
const templateData = await generateTemplateData(projectId, templateSource)
let previousVersionExists
if (!template) {
template = new Template(templateData)
template.owner = userId
previousVersionExists = false
} else {
Object.assign(template, templateData, {
version: template.version + 1,
})
previousVersionExists = true
}
await uploadTemplateAssets(projectId, userId, templateSource.build, template)
await template.save()
if (previousVersionExists) {
deleteTemplateAssets(template._id, template.version - 1, false)
}
return {
conflict: false,
templateId: template._id,
}
}
async function fetchTemplatePreview({ templateId, version, style }) {
if (!templateId || !version) {
throw new OError('Template ID and version are required', { status: 404 })
}
const styleParam = style ? `style=${style}` : ''
const isImage = (style === 'preview' || style === 'thumbnail')
if (style && !isImage) {
throw new OError('Wrong style', { status: 404, style })
}
const pdfUrl = `${settings.apis.filestore.url}/template/${templateId}/v/${version}/pdf?${styleParam}`
const { response } = await fetchStreamWithResponse(pdfUrl, {
method: 'GET',
signal: AbortSignal.timeout(TIMEOUT),
})
if (!response.ok) {
throw new OError(`Failed to fetch file: ${response.statusText}`, { status: 400, templateId, version, styleParam })
}
return {
stream: Readable.from(response.body),
contentType: isImage ? 'application/octet-stream' : 'application/pdf'
}
}
async function getTemplatesPageData(category) {
const categoryName = settings.templateLinks.find(item => item.url.endsWith(`/${category}`))?.name
const templateLinks = categoryName ? undefined : settings.templateLinks.filter(link => link.url !== '/templates/all')
return {
categoryName,
templateLinks
}
}
async function getTemplate(key, val) {
if (!key || !val) {
logger.warn('No key or val provided to getTemplate')
return null
}
const query = { [key]: val }
const template = await Template.findOne(query).exec()
if (!template) return null
return _formatTemplateForPage(template)
}
async function getCategoryTemplates(reqQuery) {
const {
category,
by = 'lastUpdated',
order = 'desc',
} = reqQuery || {}
const query = (category === 'all') ? {} : { category : '/templates/' + category }
const projection = { _id : 1, version : 1, name : 1, author : 1, description : 1, lastUpdated : 1 }
const allTemplates = await Template.find(query, projection).exec()
const formattedTemplates = allTemplates.map(_formatTemplateForList)
const sortedTemplates = _sortTemplates(formattedTemplates, { by, order })
return {
totalSize: sortedTemplates.length,
templates: sortedTemplates,
}
}
function _sortTemplates(templates, sort) {
if (
(sort.by && !['lastUpdated', 'name'].includes(sort.by)) ||
(sort.order && !['asc', 'desc'].includes(sort.order))
) {
throw new OError('Invalid sorting criteria', { status: 400, sort })
}
const sortedTemplates = _.orderBy(
templates,
[sort.by || 'lastUpdated'],
[sort.order || 'desc']
)
return sortedTemplates
}
function _formatTemplateForList(template) {
return {
id: String(template._id),
version: String(template.version),
name: template.name,
author: cleanHtml(template.author, "plainText"),
description: cleanHtml(template.description, "plainText"),
lastUpdated: template.lastUpdated,
}
}
function _formatTemplateForPage(template) {
return {
id: template._id.toString(),
version: template.version.toString(),
category: template.category,
name: template.name,
author: cleanHtml(template.author, "linksOnly"),
authorMD: template.authorMD,
description: cleanHtml(template.description, "reachText"),
descriptionMD: template.descriptionMD,
license: template.license,
lastUpdated: template.lastUpdated,
owner: template.owner,
mainFile: template.mainFile,
compiler: template.compiler,
imageName: template.imageName,
language: template.language,
}
}
export default {
createTemplateFromProject,
editTemplate,
deleteTemplate,
getTemplate,
getCategoryTemplates,
fetchTemplatePreview,
getTemplatesPageData,
}
@@ -0,0 +1,77 @@
import logger from '@overleaf/logger'
import AuthenticationController from '../../../../app/src/Features/Authentication/AuthenticationController.js'
import RateLimiterMiddleware from '../../../../app/src/Features/Security/RateLimiterMiddleware.js'
import { RateLimiter } from '../../../../app/src/infrastructure/RateLimiter.js'
import TemplateGalleryController from './TemplateGalleryController.mjs'
const rateLimiter = new RateLimiter('create-project-from-template', {
points: 20,
duration: 60,
})
const rateLimiterThumbnails = new RateLimiter('gallery-thumbnails', {
points: 360,
duration: 60,
})
export default {
rateLimiter,
apply(webRouter) {
logger.debug({}, 'Init templates router')
webRouter.post(
'/template/new/:Project_id',
AuthenticationController.requireLogin(),
RateLimiterMiddleware.rateLimit(rateLimiter),
TemplateGalleryController.createTemplateFromProject
)
webRouter.get(
'/template/:template_id',
RateLimiterMiddleware.rateLimit(rateLimiter),
TemplateGalleryController.templateDetailsPage
)
webRouter.post(
'/template/:template_id/edit',
AuthenticationController.requireLogin(),
RateLimiterMiddleware.rateLimit(rateLimiter),
TemplateGalleryController.editTemplate
)
webRouter.delete(
'/template/:template_id/delete',
AuthenticationController.requireLogin(),
RateLimiterMiddleware.rateLimit(rateLimiter),
TemplateGalleryController.deleteTemplate
)
webRouter.get(
'/templates/:category?',
RateLimiterMiddleware.rateLimit(rateLimiter),
TemplateGalleryController.templatesCategoryPage
)
webRouter.get(
'/api/template',
RateLimiterMiddleware.rateLimit(rateLimiter),
TemplateGalleryController.getTemplateJSON
)
webRouter.get(
'/api/templates',
RateLimiterMiddleware.rateLimit(rateLimiter),
TemplateGalleryController.getCategoryTemplatesJSON
)
webRouter.get(
'/template/:template_id/preview',
(req, res, next) => {
req.query.style === 'thumbnail'
? RateLimiterMiddleware.rateLimit(rateLimiterThumbnails)(req, res, next)
: RateLimiterMiddleware.rateLimit(rateLimiter)(req, res, next)
},
TemplateGalleryController.getTemplatePreview
)
},
}
@@ -0,0 +1,33 @@
const mongoose = require('../../../../../app/src/infrastructure/Mongoose')
const { Schema } = mongoose
const { ObjectId } = Schema
const TemplateSchema = new Schema(
{
name: { type: String, required: true },
category: { type: String, required: true },
description: { type: String },
descriptionMD: { type: String },
author: { type: String },
authorMD: { type: String },
license: { type: String, required: true },
mainFile: { type: String, required: true },
compiler: { type: String, required: true },
imageName: { type: String },
language: { type: String, required: true },
version: { type: Number, default: 1, required: true },
owner: { type: ObjectId, ref: 'User' },
lastUpdated: {
type: Date,
default() {
return new Date()
},
required: true
},
},
{ minimize: false }
)
exports.Template = mongoose.model('Template', TemplateSchema)
exports.TemplateSchema = TemplateSchema
@@ -0,0 +1,33 @@
import Settings from '@overleaf/settings'
import TemplateGalleryRouter from './app/src/TemplateGalleryRouter.mjs'
const TemplateGalleryModule = {
router: TemplateGalleryRouter,
}
function boolFromEnv(env) {
if (env === undefined || env === null) return undefined
if (typeof env === "string") {
const envLower = env.toLowerCase()
if (envLower === 'true') return true
if (envLower === 'false') return false
}
throw new Error("Invalid value for boolean envirionment variable")
}
Settings.templates = {
nonAdminCanManage: boolFromEnv(process.env.OVERLEAF_NON_ADMIN_CAN_MANAGE_TEMPLATES)
}
Settings.templateLinks = (`${process.env.OVERLEAF_TEMPLATE_KEYS} all`).split(/\s+/).map(key => {
const envKeyBase = key.toUpperCase().replace(/-/g, "_")
const name = process.env[`TEMPLATE_${envKeyBase}_NAME`]
const description = process.env[`TEMPLATE_${envKeyBase}_DESCRIPTION`]
return {
name: name || key,
url: `/templates/${key}`,
description: description || "Templates category"
}
})
export default TemplateGalleryModule