GitLab Synchronization: Initial commit

This commit is contained in:
Florian Porrmann
2026-09-28 00:34:26 +02:00
committed by yu-i-i
parent 738bebb743
commit d458e51918
50 changed files with 4426 additions and 8 deletions
@@ -0,0 +1,24 @@
import mongoose from "../../../../app/src/infrastructure/Mongoose.mjs"
const { Schema } = mongoose
const { ObjectId } = Schema
export const GitLabSyncProjectStatesSchema = new Schema(
{
projectId: { type: ObjectId, ref: 'Project', required: true, unique: true },
repoFullName: { type: String, required: true },
defaultBranchName: { type: String, default: null },
mergeStatus: { type: String, enum: ['clean', 'conflict', 'diverged'], default: 'clean' },
lastSyncCommit: { type: String, default: null },
lastSyncVersion: { type: Number, default: null },
unmergedBranchName: { type: String, default: null },
unmergedBranchHead: { type: String, default: null },
conflictVersion: { type: Number, default: null },
},
{ collection: 'gitlabSyncProjectStates', minimize: false }
)
export const GitLabSyncProjectStates = mongoose.model(
'GitLabSyncProjectStates',
GitLabSyncProjectStatesSchema,
)
@@ -0,0 +1,17 @@
import mongoose from "../../../../app/src/infrastructure/Mongoose.mjs"
const { Schema } = mongoose
const { ObjectId } = Schema
export const GitLabSyncUserCredentialsSchema = new Schema(
{
userId: { type: ObjectId, ref: 'User', required: true, unique: true },
gitlab: { type: String, required: true },
},
{ collection: 'gitlabSyncUserCredentials', minimize: false }
)
export const GitLabSyncUserCredentials = mongoose.model(
'GitLabSyncUserCredentials',
GitLabSyncUserCredentialsSchema,
)
@@ -0,0 +1,82 @@
import logger from '@overleaf/logger'
import Settings from '@overleaf/settings'
import fs from 'node:fs'
import crypto from 'node:crypto'
import Path from 'node:path'
import AccessTokenEncryptorClass from '@overleaf/access-token-encryptor'
const TOKEN_CIPHER_FILE = '/var/lib/overleaf/data/.token-cipher.json'
const TOKEN_CIPHER_LABEL = 'OL_CEP-v3'
let encryptorInstance = null
/**
* Get or create a stable encryptor data that persists across container
* recreations. Priority:
* 1. GITLAB_CIPHER_PASSWORD || TOKEN_CIPHER_PASSWORD env var (explicit user config)
* 2. JSON file with encryptor data in the persistent volume (/var/lib/overleaf/data/)
* — auto-generated on first use, survives container rebuilds
*/
function _getEncryptorData() {
const cipherPassword = process.env.GITLAB_TOKEN_CIPHER_PASSWORD || process.env.TOKEN_CIPHER_PASSWORD
const cipherLabel = process.env.GITLAB_TOKEN_CIPHER_LABEL ||
process.env.TOKEN_CIPHER_LABEL ||
TOKEN_CIPHER_LABEL
if (cipherPassword) {
return {
cipherLabel,
cipherPasswords: {
[cipherLabel]: cipherPassword
}
}
}
// cipherPassword is not set, use file
const cipherFile = process.env.GITLAB_TOKEN_CIPHER_FILE ||
process.env.TOKEN_CIPHER_FILE ||
TOKEN_CIPHER_FILE
try {
return JSON.parse(fs.readFileSync(cipherFile, 'utf8'))
} catch (err) {
if (err.code === 'ENOENT') {
logger.info({ cipherFile }, 'Creating new cipherFile')
} else {
logger.error({ err, cipherFile }, 'Bad cipherFile')
throw err
}
}
// File doesn't exist yet — generate one
try {
const encryptorData = {
cipherLabel,
cipherPasswords: {
[cipherLabel]: crypto.randomBytes(32).toString('base64')
},
}
const dir = Path.dirname(cipherFile)
fs.mkdirSync(dir, { recursive: true })
fs.writeFileSync(cipherFile, JSON.stringify(encryptorData, null, 2), { mode: 0o600 })
return encryptorData
} catch (err) {
logger.error({ err, cipherFile }, 'Cannot create cipherFile')
throw err
}
}
function _getEncryptor() {
if (!encryptorInstance) {
const encryptorData = _getEncryptorData()
encryptorInstance = new AccessTokenEncryptorClass(encryptorData)
}
return encryptorInstance
}
export const AccessTokenEncryptor = {
async encryptJson(json) {
return await _getEncryptor().promises.encryptJson(json)
},
async decryptToJson(encryptedJson) {
return await _getEncryptor().promises.decryptToJson(encryptedJson)
},
}
@@ -0,0 +1,642 @@
import logger from '@overleaf/logger'
import Settings from '@overleaf/settings'
import crypto from 'node:crypto'
import {
fetchJson,
fetchJsonWithResponse,
fetchNothing,
fetchStream,
RequestFailedError,
} from '@overleaf/fetch-utils'
import {
InvalidTokenError,
ExpiredTokenError,
NotFoundError,
GitConflictError,
RateLimitError,
ProviderRequestError,
PermissionDeniedError,
AlreadyExistsError,
} from './GitSyncErrors.mjs'
const GITLAB_URL = process.env.GITLAB_SYNC_URL
const GITLAB_API_BASE = `${GITLAB_URL}/api/v4`
const GITLAB_DEFAULT_BRANCH = process.env.GITLAB_DEFAULT_BRANCH || 'main'
const MAX_PER_PAGE = 100
const REQUEST_TIMEOUT_MS = 60 * 1000
const REQUEST_LONG_TIMEOUT_MS = 600 * 1000
const MERGE_REQUEST_POLL_INTERVAL_MS = process.env.GITLAB_MERGE_REQUEST_POLL_INTERVAL_MS || 1000
const MERGE_REQUEST_TIMEOUT_MS = process.env.GITLAB_MERGE_REQUEST_TIMEOUT_MS || 60_000
const mergeRequestPendingStates = new Set([
'unchecked',
'checking',
'preparing',
'approvals_syncing',
])
const mergeRequestBlockingStates = new Set([
'conflict',
'ci_must_pass',
'ci_still_running',
'commits_status',
'discussions_not_resolved',
'draft_status',
'jira_association_missing',
'merge_request_blocked',
'merge_time',
'need_rebase',
'not_approved',
'not_open',
'requested_changes',
'security_policy_pipeline_check',
'security_policy_violations',
'status_checks_must_pass',
'locked_paths',
'locked_lfs_files',
'title_regex',
])
const maxConcurrency = process.env.GITLAB_API_MAX_CONCURRENCY || 5
function buildHeaders(token) {
return {
Accept: 'application/json',
'Content-Type': 'application/json',
'User-Agent': 'Overleaf-CEP-GitLab-Sync',
...(token ? { Authorization: `Bearer ${token}` } : {}),
}
}
function encodePath(path) {
return encodeURIComponent(path)
}
function projectPath(repoFullName) {
return encodeURIComponent(repoFullName)
}
function isRateLimitError(response) {
return response.headers.get('retry-after') != null
}
function isRepositoryAlreadyExistsError(err) {
if (err.response?.status !== 400) {
return false
}
const json_data = JSON.parse(err.body)
return Object.values(json_data.message).some(value => {
if (Array.isArray(value)) {
return value.includes("has already been taken");
}
return value === "has already been taken";
});
}
function normalizeGitLabError(err, operation) {
const body = err.body || ''
logger.error({ operation, err, body }, 'GitLab API request failed')
if (err.name === 'AbortError') {
throw new ProviderRequestError('GitLab request timed out', { status: 504 }, err)
}
if (!(err instanceof RequestFailedError)) {
throw new ProviderRequestError('Something wrong with GitLab request', { status: 500 }, err)
}
const status = err.response?.status || 500
if (status === 401) {
const json_body = JSON.parse(err.body) || {}
if (json_body && json_body.error && json_body.error.toLowerCase() === 'invalid_token') {
throw new ExpiredTokenError('Token expired', { status }, err)
}
throw new InvalidTokenError('Invalid token', { status }, err)
}
if (status === 404) {
throw new NotFoundError('Not found', { status }, err)
}
if (status === 409) {
throw new GitConflictError()
}
if (status === 403 || status === 429) {
if (isRateLimitError(err.response)) throw new RateLimitError('Rate limit exeeded', { status: 429 }, err)
throw new PermissionDeniedError('Permission denied', { status: 403 }, err)
}
if (status === 400) {
if (isRepositoryAlreadyExistsError(err)) {
throw new AlreadyExistsError('Repository already exists', { status }, err)
}
}
throw new ProviderRequestError('GitLab request failed', { status }, err)
}
// wrappers
function fetchGitLabJson(url, options, operation) {
return fetchJson(url, options).catch(err => {
normalizeGitLabError(err, operation)
})
}
function fetchGitLabJsonWithResponse(url, options, operation) {
return fetchJsonWithResponse(url, options).catch(err => {
normalizeGitLabError(err, operation)
})
}
function fetchGitLabNothing(url, options, operation) {
return fetchNothing(url, options).catch(err => {
normalizeGitLabError(err, operation)
})
}
function fetchGitLabStream(url, options, operation) {
return fetchStream(url, options).catch(err => {
normalizeGitLabError(err, operation)
})
}
async function fetchAllPages(url, options, operation) {
let page = 1
let all = []
while (true) {
const pageUrl = new URL(url.toString())
pageUrl.searchParams.set('page', page.toString())
const { json, response } = await fetchGitLabJsonWithResponse(pageUrl.toString(), options, operation)
all = all.concat(json)
const link = response.headers.get('link') || ''
if (!link.includes('rel="next"')) break
page++
}
return all
}
function getTokenRefreshTimestamp(token, safetyMarginInSec = 300) {
return token.created_at + token.expires_in - safetyMarginInSec;
}
// ---------------------- exports ------------------------------- //
// OAuth
function getOAuth2Url() {
const oAuthUrl = new URL(`${GITLAB_URL}/oauth/authorize`)
oAuthUrl.searchParams.append('client_id', Settings.gitlabSync.clientID)
oAuthUrl.searchParams.append('redirect_uri', Settings.gitlabSync.callbackURL)
oAuthUrl.searchParams.append('response_type', 'code')
oAuthUrl.searchParams.append('scope', 'api read_user read_repository write_repository')
return oAuthUrl
}
function exchangeCodeForToken(code) {
return fetchGitLabJson(`${GITLAB_URL}/oauth/token`, {
method: 'POST',
headers: buildHeaders(),
json: {
client_id: Settings.gitlabSync.clientID,
client_secret: Settings.gitlabSync.clientSecret,
code: code,
grant_type: 'authorization_code',
redirect_uri: Settings.gitlabSync.callbackURL,
},
signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS)
}, 'exchangeCodeForToken').then(r => [r.access_token, r.refresh_token, getTokenRefreshTimestamp(r)] )
}
function refreshToken(refreshToken) {
return fetchGitLabJson(`${GITLAB_URL}/oauth/token`, {
method: 'POST',
headers: buildHeaders(),
json: {
client_id: Settings.gitlabSync.clientID,
client_secret: Settings.gitlabSync.clientSecret,
refresh_token: refreshToken,
grant_type: 'refresh_token',
redirect_uri: Settings.gitlabSync.callbackURL,
},
signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS)
}, 'refreshToken').then(r => [r.access_token, r.refresh_token, getTokenRefreshTimestamp(r)])
}
function revokeToken(token) {
const { clientID, clientSecret } = Settings.gitlabSync
return fetchGitLabNothing(`${GITLAB_URL}/oauth/revoke`, {
method: 'POST',
headers: buildHeaders(),
json: {
client_id: clientID,
client_secret: clientSecret,
token,
},
signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS),
}, 'revokeToken')
}
// user, orgs, permissions
function getUser(token) {
return fetchGitLabJson(`${GITLAB_API_BASE}/user`, {
headers: buildHeaders(token),
signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS),
}, 'getUser').then(({ id, username, name }) => ({ id, login: username, name }))
}
async function getUserAndOrgs(token) {
const user = await fetchGitLabJson(`${GITLAB_API_BASE}/user`, {
headers: buildHeaders(token),
signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS),
}, 'getUserAndOrgs:user')
const groupsUrl = new URL(`${GITLAB_API_BASE}/groups`)
groupsUrl.searchParams.set('all_available', 'true')
groupsUrl.searchParams.set('min_access_level', '30')
groupsUrl.searchParams.set('top_level_only', 'true')
groupsUrl.searchParams.set('per_page', MAX_PER_PAGE.toString())
groupsUrl.searchParams.set('simple', 'true')
const groups = await fetchAllPages(groupsUrl, {
headers: buildHeaders(token),
signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS),
}, 'getUserAndOrgs:groups')
return {
user: user.username,
orgs: groups.map(group => group.full_path),
}
}
function getPushPermission(token, repoFullName) {
return fetchGitLabJson(`${GITLAB_API_BASE}/projects/${projectPath(repoFullName)}`, {
headers: buildHeaders(token),
signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS),
}, 'getPushPermission').then(project => {
const projectAccess = project.permissions?.project_access?.access_level || 0
const groupAccess = project.permissions?.group_access?.access_level || 0
return Math.max(projectAccess, groupAccess) >= 30
})
}
// repos
function resolveNamespaceId(token, org) {
if (org == null) return Promise.resolve(null)
if (/^\d+$/.test(org)) return Promise.resolve(Number(org))
return fetchGitLabJson(`${GITLAB_API_BASE}/groups/${encodePath(org)}`, {
headers: buildHeaders(token),
signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS),
}, 'resolveNamespaceId').then(group => group.id)
}
async function createRepo(token, { name, description, isPublic, org }) {
const namespaceId = await resolveNamespaceId(token, org)
const payload = {
name,
description,
visibility: isPublic ? 'public' : 'private',
initialize_with_readme: false,
default_branch: GITLAB_DEFAULT_BRANCH,
merge_method: 'merge', // Use merge here to allow for conflict resolution to work properly without the need for a rebase
}
if (namespaceId != null) {
payload.namespace_id = namespaceId
}
return fetchGitLabJson(`${GITLAB_API_BASE}/projects`, {
method: 'POST',
headers: buildHeaders(token),
json: payload,
signal: AbortSignal.timeout(REQUEST_LONG_TIMEOUT_MS)
}, 'createRepo')
}
async function listUserRepos(token) {
const url = new URL(`${GITLAB_API_BASE}/projects`)
url.searchParams.set('membership', 'true')
url.searchParams.set('per_page', MAX_PER_PAGE.toString())
url.searchParams.set('simple', 'true')
url.searchParams.set('order_by', 'last_activity_at')
url.searchParams.set('sort', 'desc')
url.searchParams.set('active', 'true') // Only list projects that are not archived and not marked for deletion
const repos = await fetchAllPages(url, {
headers: buildHeaders(token),
signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS),
}, 'listUserRepos')
return repos.map(repo => ({
name: repo.name,
fullName: repo.path_with_namespace,
defaultBranchName: repo.default_branch,
}))
}
// Git (blobs / trees / commits)
function uploadBlob(token, repoFullName, buffer) {
void token
void repoFullName
if (Buffer.isBuffer(buffer)) {
return Promise.resolve(buffer.toString('base64'))
}
if (typeof buffer === 'string') {
return Promise.resolve(buffer)
}
return Promise.resolve(String(buffer))
}
function getBlobStream(token, repoFullName, ref, path) {
const encodedPath = encodePath(path)
return fetchGitLabStream(`${GITLAB_API_BASE}/projects/${projectPath(repoFullName)}/repository/files/${encodedPath}/raw?ref=${encodeURIComponent(ref)}`, {
headers: buildHeaders(token),
signal: AbortSignal.timeout(REQUEST_LONG_TIMEOUT_MS),
}, 'getBlobStream')
}
function createTree(token, repoFullName, entries, baseTree) {
void token
void repoFullName
const existingPaths = new Set((baseTree || []).map(entry => entry.path))
const actions = []
for (const entry of entries) {
if (entry.sha == null) {
actions.push({
action: 'delete',
file_path: entry.path,
})
continue
}
actions.push({
action: existingPaths.has(entry.path) ? 'update' : 'create',
file_path: entry.path,
content: entry.content,
encoding: 'base64',
})
}
return Promise.resolve({
baseTree: baseTree || [],
entries: actions,
})
}
function createCommit(token, repoFullName, { tree, message, branch, start_sha, force = false }) {
const actions = Array.isArray(tree?.entries) ? tree.entries : []
let payload = {
branch: branch || GITLAB_DEFAULT_BRANCH,
commit_message: message,
actions,
}
if (branch && branch != GITLAB_DEFAULT_BRANCH) {
payload.start_sha = start_sha
}
return fetchGitLabJson(`${GITLAB_API_BASE}/projects/${projectPath(repoFullName)}/repository/commits`, {
method: 'POST',
headers: buildHeaders(token),
json: payload,
signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS)
}, 'createCommit').then(r => r.id) // For GitLab id refers to the SHA of the created commit
}
function getCommitTree(token, repoFullName, commit) {
return listBlobsAtCommit(token, repoFullName, commit)
}
async function listBlobsAtCommit(token, repoFullName, commit) {
const url = new URL(`${GITLAB_API_BASE}/projects/${projectPath(repoFullName)}/repository/tree`)
url.searchParams.set('ref', commit)
url.searchParams.set('recursive', 'true')
url.searchParams.set('per_page', MAX_PER_PAGE.toString())
const treeEntries = await fetchAllPages(url, {
headers: buildHeaders(token),
signal: AbortSignal.timeout(REQUEST_LONG_TIMEOUT_MS)
}, 'listBlobsAtCommit')
return treeEntries
.filter(entry => entry.type === 'blob')
.map(entry => ({
sha: entry.id,
path: entry.path,
mode: entry.mode,
type: 'blob',
}))
}
async function getBlobContent(token, repoFullName, sha) {
const url = `${GITLAB_API_BASE}/projects/${projectPath(repoFullName)}/repository/blobs/${sha}`
try {
const json = await fetchGitLabJson(url, {
headers: buildHeaders(token),
signal: AbortSignal.timeout(REQUEST_LONG_TIMEOUT_MS)
}, 'getBlobContent')
return json.content
} catch (err) {
throw OError.tag(err, 'Failed to fetch blob content from GitLab', { repoFullName, sha })
}
}
async function listNewCommitsWithStatus(token, fullName, branchName, fromCommit) {
const url = new URL(`${GITLAB_API_BASE}/projects/${projectPath(fullName)}/repository/compare`)
url.searchParams.set('from', fromCommit)
url.searchParams.set('to', branchName)
url.searchParams.set('straight', 'true')
const data = await fetchGitLabJson(url.toString(), {
headers: buildHeaders(token),
signal: AbortSignal.timeout(REQUEST_LONG_TIMEOUT_MS)
}, 'listNewCommitsWithStatus')
const commits = (data.commits || []).map(c => ({
message: c.message || c.title || '',
author: {
name: c.author_name || '',
email: c.author_email || '',
date: c.created_at || c.committed_date || '',
},
sha: c.id,
}))
return { commits, diverged: commits.length > 0 }
}
// branches
function getBranchHead(token, repoFullName, branchName) {
return fetchGitLabJson(`${GITLAB_API_BASE}/projects/${projectPath(repoFullName)}/repository/branches/${encodePath(branchName)}`, {
headers: buildHeaders(token),
signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS)
}, 'getBranchHead').then(r => r.commit.id)
}
function createBranch(token, repoFullName, branchName, sha) {
return fetchGitLabJson(`${GITLAB_API_BASE}/projects/${projectPath(repoFullName)}/repository/branches`, {
method: 'POST',
headers: buildHeaders(token),
json: {
branch: branchName,
ref: sha,
},
signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS)
}, 'createBranch')
}
function updateBranch(token, repoFullName, branchName, sha, force = false) {
// Update functionality does not exist in GitLab API as it automatically updates the branch when a commit is made to it
}
function deleteBranch(token, repoFullName, branchName) {
return fetchGitLabNothing(`${GITLAB_API_BASE}/projects/${projectPath(repoFullName)}/repository/branches/${encodePath(branchName)}`, {
method: 'DELETE',
headers: buildHeaders(token),
signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS)
}, 'deleteBranch')
}
// merge / compare
async function createMergeRequest(token, repoFullName, sourceBranch, targetBranch, title) {
return fetchGitLabJson(`${GITLAB_API_BASE}/projects/${projectPath(repoFullName)}/merge_requests`, {
method: 'POST',
headers: buildHeaders(token),
json: {
source_branch: sourceBranch,
target_branch: targetBranch,
title,
remove_source_branch: true,
squash: false,
},
signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS),
}, 'createMergeRequest')
}
async function waitForMergeRequestToBeReady(token, repoFullName, mergeRequestIid) {
const startedAt = Date.now()
while (Date.now() - startedAt < MERGE_REQUEST_TIMEOUT_MS) {
const mergeRequest = await fetchGitLabJson(`${GITLAB_API_BASE}/projects/${projectPath(repoFullName)}/merge_requests/${mergeRequestIid}`, {
headers: buildHeaders(token),
signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS),
}, 'getMergeRequest')
const status = mergeRequest.detailed_merge_status
if (status === 'mergeable') {
return {
canMerge: true,
reason: status,
}
}
if (mergeRequestPendingStates.has(status)) {
await new Promise(resolve => setTimeout(resolve, MERGE_REQUEST_POLL_INTERVAL_MS))
continue
}
if (mergeRequestBlockingStates.has(status)) {
return {
canMerge: false,
reason: status,
}
}
// Unknown future GitLab status, treat as unmergeable to be safe
return {
canMerge: false,
reason: `unknown_status:${status}`,
}
}
throw new Error(`Timed out waiting for merge request ${mergeRequestIid} readiness`)
}
async function mergeOpenMergeRequest(token, repoFullName, mergeRequestIid) {
return fetchGitLabJson(`${GITLAB_API_BASE}/projects/${projectPath(repoFullName)}/merge_requests/${mergeRequestIid}/merge`, {
method: 'PUT',
headers: buildHeaders(token),
signal: AbortSignal.timeout(REQUEST_LONG_TIMEOUT_MS),
}, 'mergeOpenMergeRequest')
}
function mergeBranch(token, repoFullName, base, head) {
return createMergeRequest(token, repoFullName, head, base, `Merge ${head} into ${base}`)
.then(async mergeRequest => {
const ret = await waitForMergeRequestToBeReady(token, repoFullName, mergeRequest.iid)
if (!ret.canMerge) {
throw new GitConflictError("Merge request cannot be merged due to conflicts, reason: " + ret.reason)
}
await mergeOpenMergeRequest(token, repoFullName, mergeRequest.iid)
return getBranchHead(token, repoFullName, base)
})
}
function compareCommits(token, repoFullName, from, to) {
const url = new URL(`${GITLAB_API_BASE}/projects/${projectPath(repoFullName)}/repository/compare`)
url.searchParams.set('from', from)
url.searchParams.set('to', to)
url.searchParams.set('straight', 'true')
return fetchGitLabJson(url.toString(), {
headers: buildHeaders(token),
signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS)
}, 'compareCommits').then(r => r.diffs || [])
}
function getRepoZipball(token, repoFullName, sha) {
return fetchGitLabStream(`${GITLAB_API_BASE}/projects/${projectPath(repoFullName)}/repository/archive.zip?sha=${encodeURIComponent(sha)}`, {
headers: buildHeaders(token),
signal: AbortSignal.timeout(REQUEST_LONG_TIMEOUT_MS),
}, 'getRepoZipball')
}
export default {
maxConcurrency,
getOAuth2Url,
exchangeCodeForToken,
refreshToken,
revokeToken,
getUser,
getUserAndOrgs,
getPushPermission,
createRepo,
listUserRepos,
uploadBlob,
getBlobStream,
createTree,
createCommit,
getCommitTree,
listBlobsAtCommit,
getBlobContent,
listNewCommitsWithStatus,
getBranchHead,
createBranch,
updateBranch,
deleteBranch,
mergeBranch,
compareCommits,
getRepoZipball,
}
@@ -0,0 +1,242 @@
import { expressify } from '@overleaf/promise-utils'
import logger from '@overleaf/logger'
import OError from '@overleaf/o-error'
import HttpErrorHandler from '../../../../app/src/Features/Errors/HttpErrorHandler.mjs'
import SessionManager from '../../../../app/src/Features/Authentication/SessionManager.mjs'
import Csrf from '../../../../app/src/infrastructure/Csrf.mjs'
import GitLabSyncHandler from './GitLabSyncHandler.mjs'
import TokenManager from './TokenManager.mjs'
import api from './GitLabApiClient.mjs'
import { doGitMerge } from './GitMerge.mjs'
import { InvalidTokenError, GitNotLinkedError, AlreadyExistsError } from './GitSyncErrors.mjs'
async function getConnectionStatus(req, res) {
const userId = SessionManager.getLoggedInUserId(req.session)
try {
const isConnected = await GitLabSyncHandler.getGitConnState(userId)
res.json(isConnected)
} catch (err) {
const info = OError.getFullInfo(err)
const errStatus = info?.status || 500
logger.error(OError.getFullStack(err))
logger.error({ info }, "failed to check user connection")
return res.status(errStatus).json({ message: err.message })
}
}
async function getProjectState(req, res) {
const { project_id: projectId } = req.params
const userId = SessionManager.getLoggedInUserId(req.session)
try {
const pss = await GitLabSyncHandler.getProjectState(userId, projectId)
return res.json(pss)
} catch (err) {
const info = OError.getFullInfo(err)
const errStatus = info?.status || 500
logger.error(OError.getFullStack(err))
logger.error({ info }, "failed get project sync state")
return res.status(errStatus).json({ message: err.message })
}
}
async function getUserAndOrgs(req, res) {
const userId = SessionManager.getLoggedInUserId(req.session)
const userAndOrgs = await GitLabSyncHandler.getUserAndOrgs(userId)
res.json(userAndOrgs)
}
async function listUserRepos(req, res) {
const userId = SessionManager.getLoggedInUserId(req.session)
try {
const repos = await GitLabSyncHandler.listUserRepos(userId)
res.json({ repos })
} catch (err) {
if (err instanceof InvalidTokenError) {
return res.json(null)
}
const info = OError.getFullInfo(err)
const errStatus = info?.status || 500
logger.error(OError.getFullStack(err))
logger.error({ userId, info }, 'Failed to get repositories list')
return res.status(errStatus).json({ message: err.message })
}
}
async function getMergeOverview(req, res) {
const { project_id: projectId } = req.params
const userId = SessionManager.getLoggedInUserId(req.session)
try {
const commitsAndStatus = await GitLabSyncHandler.getMergeOverview(userId, projectId)
res.json(commitsAndStatus)
} catch (err) {
const info = OError.getFullInfo(err)
const errStatus = info?.status || 500
logger.error(OError.getFullStack(err))
logger.error({ info, projectId }, 'Error listing commits since last sync')
return res.status(errStatus).json({ message: err.message })
}
}
// Import a GitLab repository as a new project
async function importRepo(req, res) {
const userId = SessionManager.getLoggedInUserId(req.session)
const { name, fullName, defaultBranchName } = req.body
try {
const projectId = await GitLabSyncHandler.importRepo(userId, name, fullName, defaultBranchName)
res.json({ projectId })
} catch (error) {
logger.error({ error, userId }, 'Failed to import git repository from server')
res.status(error.status || 500).json({ message: error.message })
}
}
// Redirect user to Git server OAuth2 authorization URL
async function oauth2(req, res) {
const oauth2Url = api.getOAuth2Url()
oauth2Url.searchParams.append('state', req.csrfToken())
res.redirect(oauth2Url.toString())
}
// callback
async function oauth2Callback(req, res) {
const userId = SessionManager.getLoggedInUserId(req.session)
const { code, state } = req.query
try {
await Csrf.promises.validateToken(state, req.session)
} catch {
HttpErrorHandler.forbidden(req, res, 'Invalid CSRF token')
return
}
let token
let refresh_token
let refresh_timestamp
try {
[token, refresh_token, refresh_timestamp] = await api.exchangeCodeForToken(code)
if (!token || !refresh_token) {
HttpErrorHandler.badRequest(req, res, 'Failed to obtain access token from Git server')
return
}
} catch (err) {
const info = OError.getFullInfo(err)
const err_body = err.body
logger.error(OError.getFullStack(err))
logger.error({ info, err_body, userId }, 'Failed to obtain access token from Git server')
HttpErrorHandler.badRequest(req, res, err.message || 'Bad request')
return
}
try {
await TokenManager.saveUserToken(userId, {token, refresh_token, refresh_timestamp})
} catch (err) {
const info = OError.getFullInfo(err)
const errStatus = info?.status || 500
logger.error(OError.getFullStack(err))
logger.error({ info, userId }, 'Error saving user token')
HttpErrorHandler.handleErrorByStatusCode(req, res, err, errStatus)
return
}
// Save success message in session to display on redirect
req.session.projectSyncSuccessMessage =
req.i18n.translate('gitlab_successfully_linked_description')
res.redirect('/user/settings?oauth-complete=gitlab#project-sync')
}
// Unlink user's Git server account
async function unlink(req, res) {
const userId = SessionManager.getLoggedInUserId(req.session)
try {
await TokenManager.removeUserToken(userId)
} catch (err) {
const info = OError.getFullInfo(err)
const errStatus = info?.status || 500
logger.error(OError.getFullStack(err))
logger.error({ info, projectId }, 'Error removing user credentials')
return res.status(errStatus).json({ message: err.message })
}
res.sendStatus(200)
}
// Export project to Git server
// Expected req.body:
// name (string): repository name
// description (string, optional): repository description
// isPublic (boolean, optional): if true, the repository is public
// org (string, optional): if provided, repository will be created under this organization
async function exportProject(req, res) {
const userId = SessionManager.getLoggedInUserId(req.session)
const { project_id: projectId } = req.params
try {
await GitLabSyncHandler.exportProject(userId, projectId, req.body)
} catch (err) {
const info = OError.getFullInfo(err)
const errStatus = info?.status || 500
logger.error(OError.getFullStack(err))
logger.error({ info, projectId }, 'Error exporting project')
let key = 'gitlab_validation_check'
if (err instanceof AlreadyExistsError) key = 'gitlab_validation_name_exists'
else if (errStatus === 401 || errStatus === 403) key = 'gitlab_validation_check_auth'
return res.status(errStatus).json({ key, message: err.message })
}
res.sendStatus(200)
}
async function gitMerge(req, res) {
const { project_id: projectId } = req.params
try {
const userId = SessionManager.getLoggedInUserId(req.session)
const commitMessage = req.body?.message || 'Updates from Overleaf'
const claimConflictIsResolved = req.body.claimConflictIsResolved
const mergeResult = await doGitMerge(userId, projectId, commitMessage, claimConflictIsResolved)
res.json(mergeResult)
} catch (err) {
const info = OError.getFullInfo(err)
const errStatus = info?.status || 500
logger.error(OError.getFullStack(err))
logger.error({ info, projectId }, 'Error syncing project to GitLab')
return res.status(errStatus).json({ message: err.message })
}
}
// Unlink user's Git server account
async function unlinkRepo(req, res) {
const { project_id: projectId } = req.params
const userId = SessionManager.getLoggedInUserId(req.session)
try {
const ownerEmail = await GitLabSyncHandler.unlinkRepo(userId, projectId)
if (ownerEmail) return res.status(403).json({ ownerEmail })
} catch (err) {
const info = OError.getFullInfo(err)
const errStatus = info?.status || 500
logger.error(OError.getFullStack(err))
logger.error({ info, projectId }, 'Error unlinking repo from project')
return res.status(errStatus).json({ message: err.message })
}
res.sendStatus(200)
}
export default {
getConnectionStatus: expressify(getConnectionStatus),
getProjectState: expressify(getProjectState),
oauth2: expressify(oauth2),
unlink: expressify(unlink),
getUserAndOrgs: expressify(getUserAndOrgs),
oauth2Callback: expressify(oauth2Callback),
listUserRepos: expressify(listUserRepos),
importRepo: expressify(importRepo),
exportProject: expressify(exportProject),
getMergeOverview: expressify(getMergeOverview),
gitMerge: expressify(gitMerge),
unlinkRepo: expressify(unlinkRepo),
}
@@ -0,0 +1,255 @@
import Path from 'path'
import fs from 'fs'
import { pipeline } from 'node:stream/promises'
import crypto from 'crypto'
import pLimit from 'p-limit'
import Settings from '@overleaf/settings'
import logger from '@overleaf/logger'
import OError from '@overleaf/o-error'
import ProjectGetter from '../../../../app/src/Features/Project/ProjectGetter.mjs'
import ProjectUploadManager from '../../../../app/src/Features/Uploads/ProjectUploadManager.mjs'
import UserGetter from '../../../../app/src/Features/User/UserGetter.mjs'
import DocumentUpdaterHandler from '../../../../app/src/Features/DocumentUpdater/DocumentUpdaterHandler.mjs'
import api from './GitLabApiClient.mjs'
import SyncStateManager from './SyncStateManager.mjs'
import HistoryManager from './HistoryManager.mjs'
import TokenManager from './TokenManager.mjs'
import { InvalidTokenError, NotFoundError } from './GitSyncErrors.mjs'
async function getGitConnState(userId) {
try {
const tokens = await TokenManager.getUserToken(userId)
if (!tokens) return false;
const { token } = tokens
await api.getUser(token)
return true
} catch (err) {
if (err instanceof InvalidTokenError) {
logger.debug( { err, userId }, 'token invalid, treating as not connected')
return false
}
throw OError.tag(err, 'failed to validate token', { userId })
}
}
async function getProjectState(userId, projectId) {
let pss = null
const projection = { _id: 0, mergeStatus: 1, repoFullName: 1, unmergedBranchName: 1 }
pss = await SyncStateManager.getProjectState(projectId, projection)
if (!pss) {
pss = { mergeStatus: 'need-export' }
try {
const { owner_ref } = await ProjectGetter.promises.getProject(projectId, { owner_ref: 1 })
if (owner_ref.toString() !== userId) {
pss.ownerEmail = await UserGetter.promises.getUserEmail(owner_ref)
}
} catch (err) {
pss.ownerEmail = 'nobody@nowhere'
logger.error({ err }, "failed get user email")
}
return pss
}
let canPush
try {
const tokens = await TokenManager.getUserToken(userId)
if (!tokens) throw new InvalidTokenError('no user token', { userId, status: 400 });
const { token } = tokens
canPush = await api.getPushPermission(token, pss.repoFullName)
} catch (err) {
if ((err instanceof NotFoundError) ||
(err instanceof PermissionDeniedError)
) canPush = false
else throw err
}
if (!canPush) {
pss.mergeStatus = 'need-permission'
// send owner's email to collaborator
// send nothing to the owner
try {
const { owner_ref } = await ProjectGetter.promises.getProject(projectId, { owner_ref: 1 })
if (owner_ref.toString() !== userId) {
pss.ownerEmail = await UserGetter.promises.getUserEmail(owner_ref)
}
} catch (err) {
logger.error({ err, userId }, "failed get user email")
pss.ownerEmail = 'nobody@nowhere'
}
}
return pss
}
async function unlinkRepo(userId, projectId) {
const { owner_ref } = await ProjectGetter.promises.getProject(projectId, { owner_ref: 1 })
if (owner_ref.toString() !== userId) {
let ownerEmail
try {
ownerEmail = await UserGetter.promises.getUserEmail(owner_ref)
} catch (err) {
logger.error({ err, userId }, "failed get user email")
ownerEmail = 'nobody@nowhere'
}
return ownerEmail
}
await SyncStateManager.removeProjectState(projectId)
return null
}
async function listUserRepos(userId) {
const tokens = await TokenManager.getUserToken(userId)
if (!tokens) throw new InvalidTokenError('no user token', { userId, status: 400 });
const { token } = tokens
return await api.listUserRepos(token)
}
async function getUserAndOrgs(userId) {
const tokens = await TokenManager.getUserToken(userId)
if (!tokens) throw new InvalidTokenError('no user token', { userId, status: 400 });
const { token } = tokens
return await api.getUserAndOrgs(token)
}
async function getMergeOverview(userId, projectId) {
const projectSyncState = await SyncStateManager.getProjectState(projectId)
if (!projectSyncState) throw new GitNotLinkedError(projectId)
const { repoFullName, defaultBranchName, lastSyncCommit, lastSyncVersion, mergeStatus } = projectSyncState
if (mergeStatus === 'conflict') return null
const tokens = await TokenManager.getUserToken(userId)
if (!tokens) throw new InvalidTokenError('no user token', { userId, status: 400 });
const { token } = tokens
const currentVersion = await HistoryManager.latestVersion(projectId.toString())
const isProjectUpdated = currentVersion !== lastSyncVersion
try {
const commitsAndStatus = await api.listNewCommitsWithStatus(token, repoFullName, defaultBranchName, lastSyncCommit)
if(commitsAndStatus.diverged) {
await SyncStateManager.updateProjectState(projectId, { mergeStatus: 'diverged' } )
}
return { ...commitsAndStatus, isProjectUpdated }
} catch (err) {
if (err instanceof NotFoundError) {
// what is the reason of NotFoundError? is it a missing commit or not?
let canPush
try {
canPush = await api.getPushPermission(token, repoFullName)
} catch (e) {
logger.error({ error: e }, "failed to check push permission")
canPush = false
}
if (!canPush) throw err
await SyncStateManager.updateProjectState(projectId, { mergeStatus: 'diverged' } )
return { commits: [], diverged: true, isProjectUpdated }
} else {
throw err
}
}
}
async function importRepo(userId, projectName, repoFullName, defaultBranchName) {
const tokens = await TokenManager.getUserToken(userId)
if (!tokens) throw new InvalidTokenError('no user token', { userId, status: 400 });
const { token } = tokens
const defaultBranchHead = await api.getBranchHead(token, repoFullName, defaultBranchName)
const fsPath = Path.join(Settings.path.dumpFolder, `gitlab_import_${crypto.randomUUID()}`)
let project_id
try {
const stream = await api.getRepoZipball(token, repoFullName, defaultBranchHead)
await pipeline(stream, fs.createWriteStream(fsPath))
const { project } = await ProjectUploadManager.promises.createProjectFromZipArchiveWithName(
userId,
projectName,
fsPath
)
project_id = project?._id
} catch (err) {
throw OError.tag(
err,
'failed importing git repo',
{ userId, projectName, repoFullName, defaultBranchName, fsPath }
)
} finally {
fs.promises.rm(fsPath, { force: true }).catch(() => {})
}
try {
const projectVersion = await HistoryManager.latestVersion(project_id.toString())
await SyncStateManager.createProjectState(project_id, {
repoFullName,
mergeStatus: 'clean',
lastSyncCommit: defaultBranchHead,
defaultBranchName,
lastSyncVersion: projectVersion
})
} catch (err) {
// don't throw up
logger.error(
{ err, userId, projectId: project_id.toString(), repoFullName },
'Failed to create state of imported repo'
)
}
return project_id
}
async function exportProject(userId, projectId, repoOptions) {
const isLinked = await SyncStateManager.getProjectState(projectId)
if (isLinked) {
throw new OError('Project is already linked to Git server', { projectId })
}
const tokens = await TokenManager.getUserToken(userId)
if (!tokens) throw new InvalidTokenError('no user token', { userId, status: 400 });
const { token } = tokens
const { path_with_namespace: repoFullName, default_branch: defaultBranchName } =
await api.createRepo(token, repoOptions)
await DocumentUpdaterHandler.promises.flushProjectToMongo(projectId)
const currentVersion = await HistoryManager.latestVersion(projectId)
const currentPaths = await HistoryManager.getPathsAtVersion(projectId, currentVersion)
const limit = pLimit(api.maxConcurrency)
const blobData = await Promise.all(
currentPaths.paths.map((path) =>
limit(async () => {
const buffer = await HistoryManager.getProjectFileBuffer(projectId, currentVersion, path)
const sha = "DON_T_DELETE" // sha needs to be present so that the file is not detected as a deletion
return { path, sha, content: buffer }
})
)
)
const tree = await api.createTree(token, repoFullName, blobData)
const initialCommit = await api.createCommit(token, repoFullName, {
tree,
message: 'Initial Overleaf import',
})
const force = true
return SyncStateManager.createProjectState(projectId, {
mergeStatus: 'clean',
defaultBranchName,
lastSyncCommit: initialCommit,
lastSyncVersion: currentVersion,
repoFullName,
})
}
export default {
exportProject,
getProjectState,
unlinkRepo,
importRepo,
getGitConnState,
listUserRepos,
getUserAndOrgs,
getMergeOverview,
}
@@ -0,0 +1,100 @@
import logger from '@overleaf/logger'
import GitLabSyncController from './GitLabSyncController.mjs'
import AuthenticationController from '../../../../app/src/Features/Authentication/AuthenticationController.mjs'
import AuthorizationMiddleware from '../../../../app/src/Features/Authorization/AuthorizationMiddleware.mjs'
export default {
apply(webRouter) {
logger.debug({}, 'Init gitlab-sync router')
// start the GitLab OAuth flow by redirecting to GitLab authorization page
webRouter.get(
'/user/gitlab-sync/oauth2',
AuthenticationController.requireLogin(),
GitLabSyncController.oauth2
)
// callback for GitLab OAuth flow
webRouter.get(
'/user/gitlab-sync/oauth2/callback',
AuthenticationController.requireLogin(),
GitLabSyncController.oauth2Callback
)
// unlink GitLab account
webRouter.post(
'/user/gitlab-sync/unlink',
AuthenticationController.requireLogin(),
GitLabSyncController.unlink
)
// get user git connection status
webRouter.get(
'/user/gitlab-sync/status',
AuthenticationController.requireLogin(),
GitLabSyncController.getConnectionStatus
)
// get git user name and user's organizations
webRouter.get(
'/user/gitlab-sync/orgs',
AuthenticationController.requireLogin(),
GitLabSyncController.getUserAndOrgs
)
// list user's repos
webRouter.get(
'/user/gitlab-sync/repos',
AuthenticationController.requireLogin(),
GitLabSyncController.listUserRepos
)
// create a new project from git server repo
webRouter.post(
'/project/new/gitlab-sync',
AuthenticationController.requireLogin(),
GitLabSyncController.importRepo
)
// export project to git server
webRouter.post(
'/project/:project_id/gitlab-sync/export',
AuthenticationController.requireLogin(),
AuthorizationMiddleware.ensureUserCanWriteProjectContent,
GitLabSyncController.exportProject
)
// get project sync state
webRouter.get(
'/project/:project_id/gitlab-sync/state',
AuthenticationController.requireLogin(),
AuthorizationMiddleware.ensureUserCanWriteProjectContent,
GitLabSyncController.getProjectState
)
// get overview of the coming merge:
// merge state (clean or diverged), unmerged commits, is OL version changed?
webRouter.get(
'/project/:project_id/gitlab-sync/merge/overview',
AuthenticationController.requireLogin(),
AuthorizationMiddleware.ensureUserCanWriteProjectContent,
GitLabSyncController.getMergeOverview
)
// merge OL project with Git server repo
webRouter.post(
'/project/:project_id/gitlab-sync/merge',
AuthenticationController.requireLogin(),
AuthorizationMiddleware.ensureUserCanWriteProjectContent,
GitLabSyncController.gitMerge
)
// unlink Git repo from Git server
webRouter.delete(
'/project/:project_id/gitlab-sync',
AuthenticationController.requireLogin(),
AuthorizationMiddleware.ensureUserCanWriteProjectContent,
GitLabSyncController.unlinkRepo
)
},
}
@@ -0,0 +1,768 @@
import { normalize } from 'path/posix'
import pLimit from 'p-limit'
import { createHash } from 'crypto'
import logger from '@overleaf/logger'
import SyncStateManager from './SyncStateManager.mjs'
import api from './GitLabApiClient.mjs'
import HistoryManager from './HistoryManager.mjs'
import { GitConflictError } from './GitSyncErrors.mjs'
import TokenManager from './TokenManager.mjs'
import { ObjectId } from '../../../../app/src/infrastructure/mongodb.mjs'
import LockManager from '../../../../app/src/infrastructure/LockManager.mjs'
import ProjectEntityHandler from '../../../../app/src/Features/Project/ProjectEntityHandler.mjs'
import UpdateMerger from '../../../../app/src/Features/ThirdPartyDataStore/UpdateMerger.mjs'
import DocumentUpdaterHandler from '../../../../app/src/Features/DocumentUpdater/DocumentUpdaterHandler.mjs'
const SyncLockManager = LockManager.withTimeout(300) // 5 min
export async function doGitMerge(userId, projectId, message, claimConflictIsResolved) {
return await SyncLockManager.promises.runWithLock(
'gitSync',
projectId,
() => doGitMergeWithoutLock(userId, projectId, message, claimConflictIsResolved)
)
}
async function doGitMergeWithoutLock(userId, projectId, message, claimConflictIsResolved) {
const projectSyncState = await SyncStateManager.getProjectState(projectId)
if (!projectSyncState) throw new GitNotLinkedError(projectId)
const {
repoFullName,
defaultBranchName,
mergeStatus,
unmergedBranchName
} = projectSyncState
// do nothing if conflict is not yet resolved and 'Sync' button is pushed
// do nothing if conflict is already resolved and 'I have manually merged' button is pushed
if (
!claimConflictIsResolved && mergeStatus === 'conflict' ||
claimConflictIsResolved && mergeStatus !== 'conflict'
) return { mergeStatus, repoFullName, unmergedBranchName }
const tokens = await TokenManager.getUserToken(userId)
if (!tokens) throw new InvalidTokenError('no user token', { userId, status: 400 });
const { token } = tokens
await DocumentUpdaterHandler.promises.flushProjectToMongo(projectId)
const currentVersion = await HistoryManager.latestVersion(projectId)
const defaultBranchHead = await api.getBranchHead(token, repoFullName, defaultBranchName)
let newUnmergedBranchName
switch (mergeStatus) {
case 'clean':
newUnmergedBranchName = await resolveCleanSyncState(
token,
userId,
projectSyncState,
currentVersion,
defaultBranchHead,
message,
)
break
case 'conflict':
newUnmergedBranchName = await resolveConflictSyncState(
token,
userId,
projectSyncState,
currentVersion,
defaultBranchHead
)
break
case 'diverged':
newUnmergedBranchName = await resolveDetachedSyncState(
token,
userId,
projectSyncState,
currentVersion,
defaultBranchHead,
message,
)
break
default:
throw new Error('Bad mergeStatus')
}
const newMergeStatus = newUnmergedBranchName ? 'conflict' : 'clean'
return { mergeStatus: newMergeStatus, repoFullName, unmergedBranchName: newUnmergedBranchName }
}
// ----------- clean flow
async function resolveCleanSyncState(
token,
userId,
projectSyncState,
currentVersion,
defaultBranchHead,
message,
) {
const { projectId, repoFullName, defaultBranchName } = projectSyncState
// No changes in OL
if (currentVersion === projectSyncState.lastSyncVersion) {
// No changes in GH too: everything is up to date
if (projectSyncState.lastSyncCommit === defaultBranchHead) return null
// Need to apply only GH changes
const lastSyncCommit = defaultBranchHead
const lastSyncVersion = await applyGitSnapshotToProject({
token,
userId,
projectId,
repoFullName,
lastSyncCommit,
})
await SyncStateManager.updateProjectState(projectId, { lastSyncVersion, lastSyncCommit })
return null
}
try
{
// Possible changes in OL
const olBranchHead =
await exportChangesToGit({
token,
projectId,
repoFullName,
lastSyncVersion: projectSyncState.lastSyncVersion,
currentVersion,
message,
baseCommit: projectSyncState.lastSyncCommit,
})
if (!olBranchHead) {
if ( defaultBranchHead === projectSyncState.lastSyncCommit) {
await SyncStateManager.updateProjectState(projectId, { lastSyncVersion: currentVersion })
return null
}
const lastSyncCommit = defaultBranchHead
const lastSyncVersion = await applyGitSnapshotToProject({
token,
userId,
projectId,
repoFullName,
lastSyncCommit,
})
await SyncStateManager.updateProjectState(projectId, { lastSyncVersion, lastSyncCommit })
return null
}
if (defaultBranchHead === projectSyncState.lastSyncCommit) {
const lastSyncCommit = olBranchHead
const lastSyncVersion = await applyGitSnapshotToProject({
token,
userId,
projectId,
repoFullName,
lastSyncCommit,
})
await SyncStateManager.updateProjectState(projectId, { lastSyncVersion, lastSyncCommit })
}
}
catch (err) {
logger.error({ err, projectId }, 'Error during clean sync state resolution, fallback to merge')
}
// Normal commit failed, fallback to merge, first create a temporary branch with OL changes
const tempBranchName = generateBranchName()
const olBranchHead =
await exportChangesToGit({
token,
projectId,
repoFullName,
lastSyncVersion: projectSyncState.lastSyncVersion,
currentVersion,
message,
baseCommit: projectSyncState.lastSyncCommit,
branch: tempBranchName,
})
// Merge the temporary branch into the default branch
const { mergeCommit, conflict } = await mergeWithTempBranch(
token,
repoFullName,
defaultBranchName,
tempBranchName
)
if (conflict) {
await SyncStateManager.updateProjectState(projectId, {
mergeStatus: 'conflict',
lastSyncCommit: defaultBranchHead,
unmergedBranchName: tempBranchName,
unmergedBranchHead: olBranchHead,
conflictVersion: currentVersion,
})
return tempBranchName
} else {
const lastSyncCommit = mergeCommit
const lastSyncVersion = await applyGitSnapshotToProject({
token,
userId,
projectId,
repoFullName,
lastSyncCommit,
})
await SyncStateManager.updateProjectState(projectId, { lastSyncVersion, lastSyncCommit })
return null
}
}
// ----------- conflict flow
async function resolveConflictSyncState(
token,
userId,
projectSyncState,
currentVersion,
defaultBranchHead
) {
const { projectId, repoFullName, defaultBranchName, conflictVersion } = projectSyncState
// No OL changes after manual conflict resolving: pull GH changes only
if (currentVersion === conflictVersion) {
const lastSyncCommit = defaultBranchHead
const lastSyncVersion = await applyGitSnapshotToProject({
token,
userId,
projectId,
repoFullName,
lastSyncCommit,
})
await SyncStateManager.updateProjectState(projectId, {
mergeStatus: 'clean',
lastSyncVersion,
lastSyncCommit,
conflictVersion: null,
unmergedBranchName: null,
unmergedBranchHead: null
})
return null
}
// OL was changed after manual conflict resolution: need to merge
const prevOlBranchHead = projectSyncState.unmergedBranchHead
// Create a temporary branch with OL changes after manual conflict resolution
const tempBranchName = generateBranchName()
const newOlBranchHead = await exportChangesToGit({
token,
projectId,
repoFullName,
lastSyncVersion: conflictVersion,
currentVersion,
message: '[Updates in Overleaf during conflict resolution]',
baseCommit: prevOlBranchHead,
branch: tempBranchName,
})
// Were there really any changes? (e.g.: file created, then removed: no changes)
if (!newOlBranchHead) {
// nothing to push to GH
const lastSyncCommit = defaultBranchHead
const lastSyncVersion = await applyGitSnapshotToProject({
token,
userId,
projectId,
repoFullName,
lastSyncCommit,
})
await SyncStateManager.updateProjectState(projectId, {
mergeStatus: 'clean',
lastSyncVersion,
lastSyncCommit,
conflictVersion: null,
unmergedBranchName: null,
unmergedBranchHead: null
})
return null
}
// merge
const { mergeCommit, conflict } = await mergeWithTempBranch(
token,
repoFullName,
defaultBranchName,
newOlBranchHead,
tempBranchName,
)
if (conflict) {
// conflict must be resolved manually
await SyncStateManager.updateProjectState(projectId, {
mergeStatus: 'conflict',
lastSyncCommit: defaultBranchHead,
unmergedBranchName: tempBranchName,
unmergedBranchHead: newOlBranchHead,
conflictVersion: currentVersion,
})
return tempBranchName
} else {
// merge succeeded
const lastSyncCommit = mergeCommit
const lastSyncVersion = await applyGitSnapshotToProject({
token,
userId,
projectId,
repoFullName,
lastSyncCommit,
})
await SyncStateManager.updateProjectState(projectId, {
mergeStatus: 'clean',
lastSyncVersion,
lastSyncCommit,
conflictVersion: null,
unmergedBranchName: null,
unmergedBranchHead: null
})
return null
}
}
// ----------- detached flow
// GH repo was force pushed, lastSync commit possibly unavailable
async function resolveDetachedSyncState(
token,
userId,
projectSyncState,
currentVersion,
defaultBranchHead,
message,
) {
const { projectId, repoFullName, defaultBranchName } = projectSyncState
// No changes in OL: re-anchor
if (currentVersion === projectSyncState.lastSyncVersion) {
const lastSyncCommit = defaultBranchHead
const lastSyncVersion = await applyGitSnapshotToProject({
token,
userId,
projectId,
repoFullName,
lastSyncCommit,
})
await SyncStateManager.updateProjectState(projectId, { mergeStatus: 'clean', lastSyncVersion, lastSyncCommit })
return null
}
// Need to compare lastSync OL version (base), current OL version (local) and GH version (remote)
const baseSnapshot = await HistoryManager.getProjectSnapshot(projectId, projectSyncState.lastSyncVersion)
const localSnapshot = await HistoryManager.getProjectSnapshot(projectId, currentVersion)
const remoteBlobMap = await getGitBlobMap(token, repoFullName, defaultBranchHead)
const {
cleanLocalEntries,
conflictRemoteEntries,
conflictLocalEntries,
conflictBaseEntries,
hasConflicts,
hasChanges
} = await buildDetachedSyncPlan({
token,
repoFullName,
projectId,
currentVersion,
baseSnapshot,
localSnapshot,
remoteBlobMap,
})
// were any changes introduced in GH?
if (!hasChanges) {
await SyncStateManager.updateProjectState(projectId, {
mergeStatus: 'clean',
lastSyncVersion: currentVersion,
lastSyncCommit: defaultBranchHead
})
return null
}
if (!hasConflicts) {
let lastSyncCommit = defaultBranchHead
if (cleanLocalEntries.length !== 0) {
lastSyncCommit = await createCommitFromEntries({
token,
repoFullName,
parentCommit: defaultBranchHead,
entries: cleanLocalEntries,
message,
})
}
const lastSyncVersion = await applyGitSnapshotToProject({
token,
userId,
projectId,
repoFullName,
lastSyncCommit,
})
await SyncStateManager.updateProjectState(projectId, { mergeStatus: 'clean', lastSyncVersion, lastSyncCommit })
return null
}
// Resolve potential conflict.
// Build synthetic merge topology:
//
// GH
// /
// H --- B ---
// \
// OL
//
// merge(GH <- OL)
//
// B: conflicted files are restored to lastSyncVersion (merge base)
// GH: in conflicted files remote changes after lastSyncVersion are replayed
// OL: all changes done in Overleaf after lastSyncVersion are applied
// Blobs referenced in conflictBaseEntries in theory could be lost, but for GitHub,
// GitLab and Gitea, blobs from historical commits are normally retained for a
// very long time so the practical probability of failure is likely close to zero.
// TODO: anyway, if error is thrown by api.createTree, upload all blobs
// referenced in conflictBaseEntries and try again
// H --- B
const baseCommit = await createCommitFromEntries({
token,
repoFullName,
parentCommit: defaultBranchHead,
entries: conflictBaseEntries,
message: '[Overleaf GitSync conflict resolution: restore last sync]',
})
// default branch: --- H --- B --- GH
const remoteChangesCommit = await createCommitFromEntries({
token,
repoFullName,
parentCommit: baseCommit,
entries: conflictRemoteEntries,
message: '[Overleaf GitSync conflict resolution: replay repo changes]',
})
const tempBranchName = generateBranchName()
// temp branch: --- H --- B --- OL
const olChangesCommit = await createCommitFromEntries({
token,
repoFullName,
parentCommit: baseCommit,
entries: [...cleanLocalEntries, ...conflictLocalEntries],
message,
branch: tempBranchName,
})
// merge: default <- temp (GH <- OL)
const { mergeCommit, conflict } = await mergeWithTempBranch(
token,
repoFullName,
defaultBranchName,
tempBranchName
)
// auto-merge succeeded, conflicts in editable files were resolved?
if (!conflict) {
const lastSyncCommit = mergeCommit
const lastSyncVersion = await applyGitSnapshotToProject({
token,
userId,
projectId,
repoFullName,
lastSyncCommit,
})
await SyncStateManager.updateProjectState(projectId, { mergeStatus: 'clean', lastSyncVersion, lastSyncCommit })
return null
}
// merge failed: must manually resolve conflicts
await SyncStateManager.updateProjectState(projectId, {
mergeStatus: 'conflict',
lastSyncCommit: remoteChangesCommit,
unmergedBranchName: tempBranchName,
unmergedBranchHead: olChangesCommit,
conflictVersion: currentVersion,
})
return tempBranchName
}
// ----------- Helpers
async function applyGitSnapshotToProject({ token, userId, projectId, repoFullName, lastSyncCommit }) {
const invalidFiles = []
const updates = []
const deletions = []
const currentVersion = await HistoryManager.latestVersion(projectId)
const historySnapshot = await HistoryManager.getProjectSnapshot(projectId, currentVersion)
const blobMap = await getGitBlobMap(token, repoFullName, lastSyncCommit)
const blobPaths = new Set()
for (const [path, sha] of Object.entries(blobMap)) {
const invalid = validateFileName(path)
if (invalid) {
invalidFiles.push(invalid.name)
continue
}
blobPaths.add(path)
const hstSha = historySnapshot[path]?.data?.hash
// File at path is not changed, do not touch it
if (hstSha === sha) continue
updates.push(path)
}
for (const path of Object.keys(historySnapshot)) {
if (blobPaths.has(path)) continue
deletions.push(path)
}
// TODO: do not silently ignore invalidFiles, inform user about them
if (invalidFiles.length > 0) {
logger.warn({ projectId, invalidFiles }, 'Invalid file paths detected at Git server')
}
const limit = pLimit(api.maxConcurrency)
await Promise.all(
updates.map(path => limit(async () => {
const stream = await api.getBlobStream(token, repoFullName, lastSyncCommit, path)
return UpdateMerger.promises.mergeUpdate(userId, projectId, '/' + path, stream, 'gitlab')
}))
)
const localLimit = pLimit(5)
await Promise.all(
deletions.map(path => localLimit(() => {
return UpdateMerger.promises.deleteUpdate(userId, projectId, '/' + path, 'gitlab')
}))
)
return await HistoryManager.latestVersion(projectId)
}
async function buildDetachedSyncPlan({
token,
repoFullName,
projectId,
currentVersion,
baseSnapshot,
localSnapshot,
remoteBlobMap
}) {
const cleanLocalEntries = []
const conflictRemoteEntries = []
const conflictLocalEntries = []
const conflictBaseEntries = []
let hasConflicts = false
let hasChanges = false
const allPaths = new Set([
...Object.keys(baseSnapshot || {}),
...Object.keys(localSnapshot || {}),
...Object.keys(remoteBlobMap || {}),
])
for (const path of allPaths) {
const baseHash = getEntryHash(baseSnapshot[path])
const localHash = getEntryHash(localSnapshot[path])
const remoteHash = remoteBlobMap[path] || null
// OL == GH, do nothing
if (localHash === remoteHash) continue
if (baseHash === remoteHash) {
// changed in OL
hasChanges = true
cleanLocalEntries.push({ path, sha: localHash, content: Buffer.from(localSnapshot[path]?.data?.content || '', 'utf8').toString('base64') })
continue
}
if (baseHash === localHash) {
// changed in GH
hasChanges = true
continue
}
// conflict: changed both in GH and OL
hasChanges = true
hasConflicts = true
conflictRemoteEntries.push({ path, sha: remoteHash, content: await api.getBlobContent(token, repoFullName, remoteHash) })
conflictLocalEntries.push({ path, sha: localHash, content: Buffer.from(localSnapshot[path]?.data?.content || '', 'utf8').toString('base64') })
conflictBaseEntries.push({ path, sha: baseHash, content: Buffer.from(baseSnapshot[path]?.data?.content || '', 'utf8').toString('base64') })
}
return {
cleanLocalEntries,
conflictRemoteEntries,
conflictLocalEntries,
conflictBaseEntries,
hasConflicts,
hasChanges,
}
}
async function createCommitFromEntries({
token,
repoFullName,
parentCommit,
entries,
message,
branch,
}) {
const baseTree = await api.getCommitTree(token, repoFullName, parentCommit)
const newTree = await api.createTree(token, repoFullName, entries, baseTree)
return api.createCommit(token, repoFullName, {
tree: newTree,
start_sha: parentCommit,
message,
branch,
})
}
async function mergeWithTempBranch(
token,
repoFullName,
branchName,
tempBranchName
) {
try {
const mergeCommit = await api.mergeBranch(token, repoFullName, branchName, tempBranchName)
return { conflict: false, mergeCommit }
} catch (err) {
if (err instanceof GitConflictError) {
return { conflict: true }
}
throw err
}
}
async function exportChangesToGit({
token,
projectId,
repoFullName,
lastSyncVersion,
currentVersion,
message,
baseCommit,
branch,
}) {
const diff = await HistoryManager.getProjectFileTreeDiff(
projectId,
lastSyncVersion,
currentVersion
)
const { paths } = await HistoryManager.getPathsAtVersion(projectId, lastSyncVersion)
const existedAtLastSync = new Set(paths)
const deletePaths = new Set()
const upsertPaths = new Set()
for (const item of diff) {
const oldPath = item.pathname
if (item.operation === 'removed') {
// it can happen that removed file was created after last sync
if (existedAtLastSync.has(oldPath)) deletePaths.add(oldPath)
continue
}
if (item.operation === 'renamed') {
if (existedAtLastSync.has(oldPath)) deletePaths.add(oldPath)
upsertPaths.add(item.newPathname)
continue
}
if (item.operation === 'added' || item.operation === 'edited') {
upsertPaths.add(oldPath)
}
}
// if a file is renamed and a new file with the old name is created,
// then the name will be in both sets, need to exclude it from deletePaths
for (const path of upsertPaths) {
if (deletePaths.has(path)) deletePaths.delete(path)
}
// this can happen when a file is created and then removed
if (deletePaths.size === 0 && upsertPaths.size === 0) return null
const limit = pLimit(api.maxConcurrency)
const upsertEntries = await Promise.all(
[...upsertPaths].map(path =>
limit(async () => {
const buffer = await HistoryManager.getProjectFileBuffer(projectId, currentVersion, path)
const sha = "DON_T_DELETE" // sha needs to be present so that the file is not detected as a deletion
return { path, sha, content: buffer }
})
)
)
const deleteEntries = [...deletePaths].map(path => ({ path, sha: null, content: null }))
const entries = [...deleteEntries, ...upsertEntries]
const newCommit = await createCommitFromEntries({
token,
repoFullName,
parentCommit: baseCommit,
entries,
message,
branch,
})
return newCommit
}
function generateBranchName() {
const d = new Date()
const pad = n => `${n}`.padStart(2, '0')
return `overleaf-${d.getFullYear()}-${pad(d.getMonth() + 1)}-${pad(d.getDate())}-${pad(d.getHours())}${pad(d.getMinutes())}`
}
async function getGitBlobMap(token, repoFullName, commit) {
const blobEntries = await api.listBlobsAtCommit(token, repoFullName, commit)
return Object.fromEntries(
blobEntries.map(({ path, sha }) => [path, sha])
)
}
function getEntryHash(entry) {
if (!entry?.data) return null
return (
entry.data.hash ||
gitBlobShaFromString(entry.data.content)
)
}
function gitBlobShaFromString(str) {
if (!str) return null
const content = Buffer.from(str, 'utf8')
const header = Buffer.from(`blob ${content.length}\0`, 'utf8')
const store = Buffer.concat([header, content])
return createHash('sha1').update(store).digest('hex')
}
function validateFileName(name) {
if (
typeof name !== 'string' ||
name.length === 0 ||
name.includes('\0')
) return { name }
const normalized = normalize(name)
if (
normalized !== name ||
normalized.startsWith('/') ||
normalized.startsWith('../')
) return { name }
return null
}
@@ -0,0 +1,28 @@
import OError from '@overleaf/o-error'
class GitNotLinkedError extends OError {
constructor(projectId) {
super('git server is not linked to project', { projectId })
}
}
class AlreadyExistsError extends OError {}
class GitConflictError extends OError {}
class InvalidTokenError extends OError {}
class ExpiredTokenError extends OError {}
class NotFoundError extends OError {}
class PermissionDeniedError extends OError {}
class RateLimitError extends OError {}
class ProviderRequestError extends OError {}
export {
AlreadyExistsError,
GitNotLinkedError,
GitConflictError,
InvalidTokenError,
ExpiredTokenError,
NotFoundError,
PermissionDeniedError,
RateLimitError,
ProviderRequestError,
}
@@ -0,0 +1,97 @@
import Settings from '@overleaf/settings'
import logger from '@overleaf/logger'
import OError from '@overleaf/o-error'
import { fetchJson, fetchStream } from '@overleaf/fetch-utils'
import HistoryManager from '../../../../app/src/Features/History/HistoryManager.mjs'
const urlBase = `${Settings.apis.project_history.url}/project`
async function latestVersion(projectId) {
const url = `${urlBase}/${projectId}/version`
try {
await HistoryManager.promises.flushProject(projectId)
const json = await fetchJson(url)
return json.version
} catch (err) {
throw OError.tag(err, 'Failed to get the latest project version from history', { projectId })
}
}
async function getProjectSnapshot(projectId, version) {
const url = `${urlBase}/${projectId}/version/${version}`
try {
const snapshot = await fetchJson(url)
return snapshot.files || {}
} catch (err) {
throw OError.tag(err, 'Failed to get the project snapshot from history', { projectId })
}
}
async function getPathsAtVersion(projectId, version) {
const url = `${urlBase}/${projectId}/paths/version/${version}`
try {
const json = await fetchJson(url)
return json
} catch (err) {
throw OError.tag(err, 'Failed to fetch file paths from history', { projectId, version })
}
}
// response field 'diff' is an array of objects:
// {
// pathname: string,
// operation: 'added' | 'removed' | 'edited'| 'renamed' | undefined,
// ...
// }
async function getProjectFileTreeDiff(projectId, fromV, toV) {
const url = `${urlBase}/${projectId}/filetree/diff?from=${fromV}&to=${toV}`
try {
const json = await fetchJson(url)
return json.diff || []
} catch (err) {
throw OError.tag(err, 'Failed to fetch filetree diff from history', { projectId, fromV, toV })
}
}
// return file from history as a base64 encoded string
async function getProjectFileBuffer(projectId, version, filePath) {
const url = `${urlBase}/${projectId}/version/${version}/${encodeURIComponent(filePath)}`
try {
const stream = await fetchStream(url)
return await _streamToBase64(stream)
} catch (err) {
throw OError.tag(err, 'Failed to fetch file from history', { projectId, version, filePath })
}
}
async function _streamToBase64(stream) {
const parts = []
let leftover = null
for await (const chunk of stream) {
const data = leftover ? Buffer.concat([leftover, chunk]) : chunk
const remainder = data.length % 3
const completeLength = data.length - remainder
if (completeLength > 0) {
parts.push(data.subarray(0, completeLength).toString('base64'))
}
leftover = remainder ? data.subarray(completeLength) : null
}
if (leftover) {
parts.push(leftover.toString('base64'))
}
return parts.join('')
}
export default {
latestVersion,
getPathsAtVersion,
getProjectSnapshot,
getProjectFileBuffer,
getProjectFileTreeDiff,
}
@@ -0,0 +1,32 @@
import { GitLabSyncProjectStates } from '../models/gitlabSyncProjectStates.mjs'
import Mongo from '../../../../app/src/Features/Helpers/Mongo.mjs'
const { normalizeQuery } = Mongo
function getProjectState(projectId, projection = {}) {
return GitLabSyncProjectStates.findOne(normalizeQuery({ projectId }), projection).lean()
}
function createProjectState(projectId, data) {
return GitLabSyncProjectStates.create({
projectId: normalizeQuery(projectId),
...data
})
}
function updateProjectState(projectId, data) {
return GitLabSyncProjectStates.updateOne(
normalizeQuery({ projectId }),
{ $set: data },
)
}
function removeProjectState(projectId) {
return GitLabSyncProjectStates.deleteMany(normalizeQuery({ projectId }))
}
export default {
getProjectState,
createProjectState,
updateProjectState,
removeProjectState,
}
@@ -0,0 +1,115 @@
import logger from '@overleaf/logger'
import OError from '@overleaf/o-error'
import Mongo from '../../../../app/src/Features/Helpers/Mongo.mjs'
import { GitLabSyncUserCredentials } from '../models/gitlabSyncUserCredentials.mjs'
import { AccessTokenEncryptor } from './AccessTokenEncryptorHelper.mjs'
import { InvalidTokenError } from './GitSyncErrors.mjs'
import api from './GitLabApiClient.mjs'
const { normalizeQuery } = Mongo
async function encryptAccessToken(accessToken) {
try {
return await AccessTokenEncryptor.encryptJson(accessToken)
} catch (err) {
throw OError.tag('failed to encrypt token', err)
}
}
async function decryptAccessToken(tokenEncrypted) {
try {
return await AccessTokenEncryptor.decryptToJson(tokenEncrypted)
} catch (err) {
throw new InvalidTokenError('failed to decrypt token', { status: 401 }, err)
}
}
async function refreshToken(userId, tokens) {
logger.info({ userId }, "refreshing GitLab token")
if (!tokens.refresh_token) {
logger.error("refresh_token invalid")
return false
}
let token
let refresh_token
let refresh_timestamp
try {
[token, refresh_token, refresh_timestamp] = await api.refreshToken(tokens.refresh_token)
if (!token || !refresh_token) {
HttpErrorHandler.badRequest(req, res, 'Failed to refresh access token')
return false
}
} catch (err) {
const info = OError.getFullInfo(err)
logger.error(OError.getFullStack(err))
logger.error({ info, userId }, 'Failed to refresh access token')
HttpErrorHandler.badRequest(req, res, err.message || 'Bad request')
return false
}
try {
await saveUserToken(userId, { token, refresh_token, refresh_timestamp })
} catch (err) {
const info = OError.getFullInfo(err)
const errStatus = info?.status || 500
logger.error(OError.getFullStack(err))
logger.error({ info, userId }, 'Error saving user token')
HttpErrorHandler.handleErrorByStatusCode(req, res, err, errStatus)
return false
}
return true
}
// ------------------------- exports -------------------------- //
async function getUserToken(userId) {
const credentials = await GitLabSyncUserCredentials.findOne(normalizeQuery({ userId }))
if (!credentials) throw new InvalidTokenError('no user token', { userId, status: 400 })
let tokens = await decryptAccessToken(credentials.gitlab)
const now = Math.floor(Date.now() / 1000);
if (!tokens.refresh_timestamp || tokens.refresh_timestamp < now) {
const refreshed = await refreshToken(userId, tokens)
if (refreshed) {
return await getUserToken(userId)
}
}
return tokens
}
async function saveUserToken(userId, accessToken) {
const tokenEncrypted = await encryptAccessToken(accessToken)
await GitLabSyncUserCredentials.findOneAndUpdate(
normalizeQuery({ userId }),
{ $set: { gitlab: tokenEncrypted } },
{ upsert: true }
)
}
// Try to revoke user's token, then remove it
async function removeUserToken(userId) {
let token
try {
token = await getUserToken(userId)
} catch (err) {
logger.warn({ err, userId }, 'failed to get user token')
}
// fire-and-forget, but still handle errors
if (token) {
api.revokeToken(token).catch(err => {
logger.warn({ err, userId }, 'failed to revoke user token')
})
}
await GitLabSyncUserCredentials.deleteOne(normalizeQuery({ userId }))
return
}
export default {
saveUserToken,
getUserToken,
removeUserToken
}