diff --git a/services/web/app/src/infrastructure/ExpressLocals.mjs b/services/web/app/src/infrastructure/ExpressLocals.mjs index de214c9b3e..83a7fb4e0b 100644 --- a/services/web/app/src/infrastructure/ExpressLocals.mjs +++ b/services/web/app/src/infrastructure/ExpressLocals.mjs @@ -418,6 +418,7 @@ export default async function (webRouter, privateApiRouter, publicApiRouter) { cioSiteId: Settings.analytics?.cio?.siteId, linkedInInsightsPartnerId: Settings.analytics?.linkedIn?.partnerId, githubSyncEnabled: !!Settings.githubSync?.clientID && !!Settings.githubSync?.clientSecret, + gitlabSyncEnabled: !!Settings.gitlabSync?.clientID && !!Settings.gitlabSync?.clientSecret, zoteroEnabled: !!Settings.zotero?.clientKey && !!Settings.zotero?.clientSecret, enablePandocConversions: Settings.enablePandocConversions, mixpanelLabsToken: diff --git a/services/web/app/src/infrastructure/Features.mjs b/services/web/app/src/infrastructure/Features.mjs index 2a1a7a38e2..7085b9ca9e 100644 --- a/services/web/app/src/infrastructure/Features.mjs +++ b/services/web/app/src/infrastructure/Features.mjs @@ -16,6 +16,7 @@ const trackChangesModuleAvailable = * @property {Object | undefined} apis.linkedUrlProxy * @property {string | undefined} apis.linkedUrlProxy.url * @property {boolean | undefined} enableGithubSync + * @property {boolean | undefined} enableGitLabSync * @property {boolean | undefined} enableGitBridge * @property {boolean | undefined} enableHomepage * @property {boolean | undefined} enableSaml @@ -61,6 +62,8 @@ const Features = { return Boolean(Settings.disableLinkSharing) === false case 'github-sync': return Boolean(Settings.enableGithubSync) + case 'gitlab-sync': + return Boolean(Settings.enableGitLabSync) case 'git-bridge': return Boolean(Settings.enableGitBridge) case 'oauth': diff --git a/services/web/app/src/util/third-party-icons.mjs b/services/web/app/src/util/third-party-icons.mjs index 5970a8e97c..7d19ef4790 100644 --- a/services/web/app/src/util/third-party-icons.mjs +++ b/services/web/app/src/util/third-party-icons.mjs @@ -6,6 +6,7 @@ const thirdPartyIcons = [ { file: 'git.svg', label: 'Git' }, { file: 'github.svg', label: 'GitHub' }, + { file: 'gitlab.svg', label: 'GitLab' }, { file: 'dropbox.svg', label: 'Dropbox' }, { file: 'zotero_64x64x32.png', label: 'Zotero' }, { file: 'mendeley.svg', label: 'Mendeley' }, diff --git a/services/web/config/settings.defaults.js b/services/web/config/settings.defaults.js index 673070b461..4dbb7b1bf8 100644 --- a/services/web/config/settings.defaults.js +++ b/services/web/config/settings.defaults.js @@ -1098,6 +1098,10 @@ module.exports = { __dirname, '../modules/github-sync/frontend/js/components/github-sync-widget.tsx' ), + Path.resolve( + __dirname, + '../modules/gitlab-sync/frontend/js/components/gitlab-sync-widget.tsx' + ), ], referenceLinkingWidgets: [ Path.resolve( @@ -1117,6 +1121,18 @@ module.exports = { '../modules/github-sync/frontend/js/components/import-from-github-menu.tsx' ), ], + importProjectFromGitLabModalWrapper: [ + Path.resolve( + __dirname, + '../modules/gitlab-sync/frontend/js/components/import-from-gitlab-modal-wrapper.tsx' + ), + ], + importProjectFromGitLabMenu: [ + Path.resolve( + __dirname, + '../modules/gitlab-sync/frontend/js/components/import-from-gitlab-menu.tsx' + ), + ], editorLeftMenuSync: [], editorLeftMenuManageTemplate: [], menubarExtraComponents: [ @@ -1178,6 +1194,10 @@ module.exports = { __dirname, '../modules/github-sync/frontend/js/components/github-integration-card.tsx' ), + Path.resolve( + __dirname, + '../modules/gitlab-sync/frontend/js/components/gitlab-integration-card.tsx' + ), Path.resolve( __dirname, '../modules/git-bridge/frontend/js/card/components/git-integration-card.tsx' @@ -1220,6 +1240,7 @@ module.exports = { 'template-gallery', 'git-bridge', 'github-sync', + 'gitlab-sync', 'zotero', ], viewIncludes: {}, diff --git a/services/web/frontend/js/features/history/components/change-list/origin.tsx b/services/web/frontend/js/features/history/components/change-list/origin.tsx index 98d8767019..2311328456 100644 --- a/services/web/frontend/js/features/history/components/change-list/origin.tsx +++ b/services/web/frontend/js/features/history/components/change-list/origin.tsx @@ -9,6 +9,7 @@ function Origin({ origin }: Pick) { if (origin?.kind === 'upload') result = t('history_entry_origin_upload') if (origin?.kind === 'git-bridge') result = t('history_entry_origin_git') if (origin?.kind === 'github') result = t('history_entry_origin_github') + if (origin?.kind === 'gitlab') result = t('history_entry_origin_gitlab') if (result) { return ({result}) diff --git a/services/web/frontend/js/features/history/services/types/shared.ts b/services/web/frontend/js/features/history/services/types/shared.ts index 1e66f64afe..a67075cd85 100644 --- a/services/web/frontend/js/features/history/services/types/shared.ts +++ b/services/web/frontend/js/features/history/services/types/shared.ts @@ -20,6 +20,7 @@ export interface Meta { | 'upload' | 'git-bridge' | 'github' + | 'gitlab' | 'history-resync' | 'history-migration' } diff --git a/services/web/frontend/js/features/ide-react/components/rail/rail.tsx b/services/web/frontend/js/features/ide-react/components/rail/rail.tsx index 796fe5d369..ea0f08e507 100644 --- a/services/web/frontend/js/features/ide-react/components/rail/rail.tsx +++ b/services/web/frontend/js/features/ide-react/components/rail/rail.tsx @@ -71,7 +71,7 @@ export const RailLayout = () => { const { features } = useProjectContext() const { isRestrictedTokenMember } = useEditorContext() const gitBridgeEnabled = getMeta('ol-gitBridgeEnabled') - const { isOverleaf, githubSyncEnabled, zoteroEnabled } = getMeta('ol-ExposedSettings') + const { isOverleaf, githubSyncEnabled, gitlabSyncEnabled, zoteroEnabled } = getMeta('ol-ExposedSettings') const { view, setSettingsShown, focusMode } = useLayoutContext() @@ -119,7 +119,7 @@ export const RailLayout = () => { icon: 'integration_instructions', title: t('integrations'), component: , - hide: !isOverleaf && !gitBridgeEnabled && !githubSyncEnabled && !zoteroEnabled, + hide: !isOverleaf && !gitBridgeEnabled && !githubSyncEnabled && !gitlabSyncEnabled && !zoteroEnabled, }, { key: 'review-panel', @@ -149,6 +149,7 @@ export const RailLayout = () => { isOverleaf, gitBridgeEnabled, githubSyncEnabled, + gitlabSyncEnabled, zoteroEnabled, ] ) diff --git a/services/web/frontend/js/features/ide-react/context/editor-manager-context.tsx b/services/web/frontend/js/features/ide-react/context/editor-manager-context.tsx index 5fb07e50e1..f5cdf6f79d 100644 --- a/services/web/frontend/js/features/ide-react/context/editor-manager-context.tsx +++ b/services/web/frontend/js/features/ide-react/context/editor-manager-context.tsx @@ -235,7 +235,7 @@ export const EditorManagerProvider: FC = ({ } if ( update.meta.type === 'external' && - (update.meta.source === 'git-bridge' || update.meta.source === 'github') + (update.meta.source === 'git-bridge' || update.meta.source === 'github' || update.meta.source === 'gitlab') ) { return } diff --git a/services/web/frontend/js/features/integrations-panel/integrations-panel.tsx b/services/web/frontend/js/features/integrations-panel/integrations-panel.tsx index c931dd79b8..f7f54bebb2 100644 --- a/services/web/frontend/js/features/integrations-panel/integrations-panel.tsx +++ b/services/web/frontend/js/features/integrations-panel/integrations-panel.tsx @@ -12,6 +12,7 @@ const integrationPanelComponents = allIntegrationPanelComponents.filter( ({ path }) => (getMeta('ol-gitBridgeEnabled') || !path.includes('git-bridge')) && (getMeta('ol-ExposedSettings').githubSyncEnabled || !path.includes('github-sync')) && + (getMeta('ol-ExposedSettings').gitlabSyncEnabled || !path.includes('gitlab-sync')) && (getMeta('ol-ExposedSettings').zoteroEnabled || !path.includes('zotero')) ) diff --git a/services/web/frontend/js/features/project-list/components/new-project-button.tsx b/services/web/frontend/js/features/project-list/components/new-project-button.tsx index 1ee6786e24..66f548400d 100644 --- a/services/web/frontend/js/features/project-list/components/new-project-button.tsx +++ b/services/web/frontend/js/features/project-list/components/new-project-button.tsx @@ -58,7 +58,7 @@ function NewProjectButton({ align, }: NewProjectButtonProps) { const { t } = useTranslation() - const { templateLinks, githubSyncEnabled } = getMeta('ol-ExposedSettings') + const { templateLinks, githubSyncEnabled, gitlabSyncEnabled } = getMeta('ol-ExposedSettings') const [modal, setModal] = useState>(null) const portalTemplates = getMeta('ol-portalTemplates') || [] @@ -169,11 +169,19 @@ function NewProjectButton({ const [importProjectFromGithubMenu] = !githubSyncEnabled ? [] : importOverleafModules( 'importProjectFromGithubMenu' ) + + const [importProjectFromGitLabMenu] = !gitlabSyncEnabled ? [] : importOverleafModules( + 'importProjectFromGitLabMenu' + ) const ImportProjectFromGithubMenu: JSXElementConstructor<{ onClick: (e: React.MouseEvent) => void }> = importProjectFromGithubMenu?.import.default + const ImportProjectFromGitLabMenu: JSXElementConstructor<{ + onClick: (e: React.MouseEvent) => void + }> = importProjectFromGitLabMenu?.import.default + return ( <> )} -
  • - {ImportProjectFromGithubMenu && ( + {ImportProjectFromGithubMenu && ( +
  • handleModalMenuClick(e, { @@ -268,8 +276,20 @@ function NewProjectButton({ }) } /> - )} -
    • + + )} + {ImportProjectFromGitLabMenu && ( +
  • + + handleModalMenuClick(e, { + modalVariant: 'import_from_gitlab', + dropdownMenuEvent: 'import-from-gitlab', + }) + } + /> +
    • + )} {portalTemplates.length > 0 ? ( <> diff --git a/services/web/frontend/js/features/project-list/components/new-project-button/new-project-button-modal.tsx b/services/web/frontend/js/features/project-list/components/new-project-button/new-project-button-modal.tsx index 6f9f273a3a..ae1c90326e 100644 --- a/services/web/frontend/js/features/project-list/components/new-project-button/new-project-button-modal.tsx +++ b/services/web/frontend/js/features/project-list/components/new-project-button/new-project-button-modal.tsx @@ -15,6 +15,7 @@ export type NewProjectButtonModalVariant = | 'example_project' | 'upload_project' | 'import_from_github' + | 'import_from_gitlab' | 'import_docx' | 'import_markdown' @@ -36,6 +37,14 @@ function NewProjectButtonModal({ onHide: () => void }> = importProjectFromGithubModalWrapper?.import.default + const [importProjectFromGitLabModalWrapper] = importOverleafModules( + 'importProjectFromGitLabModalWrapper' + ) + const ImportProjectFromGitLabModalWrapper: JSXElementConstructor<{ + onHide: () => void + }> = importProjectFromGitLabModalWrapper?.import.default + + const location = useLocation() const openProject = useCallback( @@ -82,6 +91,8 @@ function NewProjectButtonModal({ ) case 'import_from_github': return + case 'import_from_gitlab': + return default: return null } diff --git a/services/web/frontend/js/features/project-list/components/welcome-message-new/welcome-message-create-new-project-dropdown.tsx b/services/web/frontend/js/features/project-list/components/welcome-message-new/welcome-message-create-new-project-dropdown.tsx index 08ad1f630e..980d85939c 100644 --- a/services/web/frontend/js/features/project-list/components/welcome-message-new/welcome-message-create-new-project-dropdown.tsx +++ b/services/web/frontend/js/features/project-list/components/welcome-message-new/welcome-message-create-new-project-dropdown.tsx @@ -68,7 +68,7 @@ function WelcomeMessageCreateNewProjectDropdown({ useFeatureFlag('import-markdown') && getMeta('ol-ExposedSettings').enablePandocConversions - const { isOverleaf, githubSyncEnabled } = getMeta('ol-ExposedSettings') + const { isOverleaf, githubSyncEnabled, gitlabSyncEnabled } = getMeta('ol-ExposedSettings') const handleDropdownItemClick = useCallback( ( @@ -187,6 +187,23 @@ function WelcomeMessageCreateNewProjectDropdown({ )} + {(isOverleaf || gitlabSyncEnabled) && ( +
  • + + handleDropdownItemClick( + e, + 'import_from_gitlab', + 'import-from-gitlab' + ) + } + tabIndex={-1} + > + {t('import_from_gitlab')} + +
    • + )} {(portalTemplates?.length ?? 0) > 0 ? ( <> diff --git a/services/web/frontend/js/features/settings/components/linking-section.tsx b/services/web/frontend/js/features/settings/components/linking-section.tsx index b54e6b2280..17d6986fd9 100644 --- a/services/web/frontend/js/features/settings/components/linking-section.tsx +++ b/services/web/frontend/js/features/settings/components/linking-section.tsx @@ -19,11 +19,12 @@ const availableLangFeedbackLinkingWidgets = importOverleafModules( const gitBridgeEnabled = getMeta('ol-gitBridgeEnabled') const githubSyncEnabled = getMeta('ol-ExposedSettings').githubSyncEnabled +const gitlabSyncEnabled = getMeta('ol-ExposedSettings').gitlabSyncEnabled const zoteroEnabled = getMeta('ol-ExposedSettings').zoteroEnabled const availableIntegrationLinkingWidgets = allAvailableIntegrationLinkingWidgets.filter( ({ path }) => - (githubSyncEnabled || !path.includes('github-sync')) + (githubSyncEnabled || !path.includes('github-sync')) && (gitlabSyncEnabled || !path.includes('gitlab-sync')) ) function LinkingSection() { @@ -51,7 +52,7 @@ function LinkingSection() { }[] const renderSyncSection = - getMeta('ol-isSaas') || gitBridgeEnabled || githubSyncEnabled + getMeta('ol-isSaas') || gitBridgeEnabled || githubSyncEnabled || gitlabSyncEnabled const allIntegrationLinkingWidgets = integrationLinkingWidgets.concat( oauth2ServerComponents diff --git a/services/web/frontend/js/shared/svgs/gitlab-logo-black.tsx b/services/web/frontend/js/shared/svgs/gitlab-logo-black.tsx new file mode 100644 index 0000000000..3ad82fe6c6 --- /dev/null +++ b/services/web/frontend/js/shared/svgs/gitlab-logo-black.tsx @@ -0,0 +1,18 @@ +function GitLabLogoBlack({ size = 20 }: { size?: number }) { + return ( + + + + ) +} + +export default GitLabLogoBlack \ No newline at end of file diff --git a/services/web/frontend/js/shared/svgs/gitlab-logo.tsx b/services/web/frontend/js/shared/svgs/gitlab-logo.tsx new file mode 100644 index 0000000000..47a59e8903 --- /dev/null +++ b/services/web/frontend/js/shared/svgs/gitlab-logo.tsx @@ -0,0 +1,32 @@ +function GitLabLogo({ size = 40 }: { size?: number }) { + return ( + + + + + + + + + ) +} + +export default GitLabLogo \ No newline at end of file diff --git a/services/web/frontend/stylesheets/modules/all.scss b/services/web/frontend/stylesheets/modules/all.scss index 346258022e..56b415e2d5 100644 --- a/services/web/frontend/stylesheets/modules/all.scss +++ b/services/web/frontend/stylesheets/modules/all.scss @@ -16,4 +16,5 @@ @import '../../../modules/admin-tools/frontend/styles/manage-users-page'; @import '../../../modules/admin-tools/frontend/styles/pagination-dark'; @import '../../../modules/github-sync/frontend/styles/github-sync'; +@import '../../../modules/gitlab-sync/frontend/styles/gitlab-sync'; @import '../../../modules/reference-picker/frontend/styles/reference-picker'; diff --git a/services/web/locales/en.json b/services/web/locales/en.json index 1c2accb783..7dc9c42c4e 100644 --- a/services/web/locales/en.json +++ b/services/web/locales/en.json @@ -246,6 +246,11 @@ "ask_proj_owner_to_upgrade_for_full_history": "Please ask the project owner to upgrade to access this project’s full history.", "ask_repo_owner_to_reconnect": "Ask the GitHub repository owner (<0>__repoOwnerEmail__) to reconnect the project.", "ask_repo_owner_to_renew_overleaf_subscription": "Ask the GitHub repository owner (<0>__repoOwnerEmail__) to renew their __appName__ subscription and reconnect the project.", + "ask_proj_owner_to_add_you_as_gitlab_collaborator": "You don’t have permission to push to the linked repository <0>__repoFullName__. Ask the owner of the project (<1>__projectOwnerEmail__) to add you as a collaborator to this repository.", + "ask_proj_owner_to_export_to_gitlab": "Ask the owner of the project (<0>__projectOwnerEmail__) to export the project to a Git repository and add you as a collaborator.", + "ask_proj_owner_to_unlink_from_current_gitlab": "Ask the owner of the project (<0>__projectOwnerEmail__) to unlink the project from the current git repository and create a connection to a different repository.", + "ask_gitlab_repo_owner_to_reconnect": "Ask the GitLab repository owner (<0>__repoOwnerEmail__) to reconnect the project.", + "ask_gitlab_repo_owner_to_renew_overleaf_subscription": "Ask the GitLab repository owner (<0>__repoOwnerEmail__) to renew their __appName__ subscription and reconnect the project.", "at_most_x_libraries_can_be_selected": "At most __maxCount__ libraries can be selected", "attach_image_or_pdf": "Attach image or PDF", "audit_logging": "Audit logging", @@ -386,6 +391,7 @@ "checking": "Checking", "checking_dropbox_status": "Checking Dropbox status", "checking_project_github_status": "Checking project status in GitHub", + "checking_project_gitlab_status": "Checking project status in GitLab", "choose_a_custom_color": "Choose a custom color", "choose_a_new_password": "Choose a new password", "choose_from_group_members": "Choose from group members", @@ -510,6 +516,7 @@ "contacting_the_sales_team": "Contacting the Sales team", "continue": "Continue", "continue_github_merge": "I have manually merged. Continue", + "continue_gitlab_merge": "I have manually merged. Continue", "continue_to": "Continue to __appName__", "continue_using_free_features": "Continue using our free features", "continue_with_free_plan": "Continue with free plan", @@ -544,6 +551,7 @@ "create_new_tag": "Create new tag", "create_password": "Create password", "create_project_in_github": "Create a GitHub repository", + "create_project_in_gitlab": "Create a GitLab repository", "create_your_account": "Create your account", "created": "Created", "created_at": "Created at", @@ -876,6 +884,7 @@ "export_as_markdown": "Export as Markdown (.md)", "export_csv": "Export CSV", "export_project_to_github": "Export Project to GitHub", + "export_project_to_gitlab": "Export Project to GitLab", "failed": "Failed", "failed_to_consent_to_workbench_terms": "Failed to consent to workbench terms. Please try again later.", "failed_to_publish_as_a_template": "Failed to publish as a template.", @@ -977,6 +986,7 @@ "from_external_url": "From external URL", "from_filename": "From <0>__filename__", "from_github": "From GitHub", + "from_gitlab": "From GitLab", "from_project_files": "From project files", "from_provider": "From __provider__", "from_url": "From URL", @@ -1072,6 +1082,41 @@ "github_workflow_files_delete_github_repo": "The repository has been created on GitHub but linking was unsuccessful. You will have to delete GitHub repository or choose a new name.", "github_workflow_files_error": "The __appName__ GitHub sync service couldn’t sync GitHub Workflow files (in .github/workflows/). Please authorize __appName__ to edit your GitHub workflow files and try again.", "github_workflow_files_error_non_owner": "The __appName__ GitHub sync service couldn’t sync GitHub Workflow files (in .github/workflows/). Please ensure the project owner has authorized __appName__ to edit this project’s GitHub workflow files and try again.", + "gitlab": "GitLab", + "gitlab_commit_message_placeholder": "Commit message for changes made in __appName__...", + "gitlab_credentials_expired": "Your GitLab authorization credentials have expired", + "gitlab_empty_repository_error": "It looks like your GitLab repository is empty or not yet available. Create a new file on GitLab.com then try again.", + "gitlab_file_name_error": "This repository cannot be imported, because it contains file(s) with an invalid filename:", + "gitlab_file_sync_error": "We are unable to sync the following files:", + "gitlab_git_and_dropbox_integrations": "<0>GitLab, <0>Git and <0>Dropbox integrations", + "gitlab_git_and_reference_manager_integrations": "GitLab, Git and reference manager integrations", + "gitlab_git_folder_error": "This project contains a .git folder at the top level, indicating that it is already a git repository. The Overleaf GitLab sync service cannot sync git histories. Please remove the .git folder and try again.", + "gitlab_integration": "GitLab integration", + "gitlab_integration_lowercase": "Git and GitLab integration", + "gitlab_is_no_longer_connected": "GitLab is no longer connected to this project.", + "gitlab_is_premium": "GitLab Sync is a premium feature", + "gitlab_large_files_error": "Merge failed: your GitLab repository contains files over the 50mb file size limit ", + "gitlab_manual_merge_user_prompt": "Please manually resolve the merge request associated with the branch <0>__sharelatex_branch__. Click below to continue, after you have manually merged.", + "gitlab_merge_failed_error": "Your changes in __appName__ and GitLab could not be automatically merged.", + "gitlab_no_master_branch_error": "This repository cannot be imported as it is missing a default branch. Please make sure the project has a default branch", + "gitlab_private_description": "You choose who can see and commit to this repository.", + "gitlab_public_description": "Anyone can see this repository. You choose who can commit.", + "gitlab_repository_diverged": "The default branch of the linked repository has been force-pushed. The list of recent commits below may be incomplete due to rewritten history since the last sync.", + "gitlab_successfully_linked_description": "Thanks, we’ve successfully linked your GitLab account to __appName__. You can now export your __appName__ projects to GitLab, or import projects from your GitLab repositories.", + "gitlab_symlink_error": "Your GitLab repository contains symbolic link files, which are not currently supported by Overleaf. Please remove these and try again.", + "gitlab_sync_description": "With GitLab Sync you can link your __appName__ projects to GitLab repositories, create new commits from __appName__, and merge commits from GitLab.", + "gitlab_sync_error": "Sorry, there was a problem checking our GitLab service. Please try again in a few moments.", + "gitlab_sync_repository_not_found_description": "The linked repository has either been removed, or you no longer have access to it. You can set up sync with a new repository by cloning the project and using the ‘GitLab’ menu item. You can also unlink the repository from this project.", + "gitlab_timeout_error": "Syncing your Overleaf project with GitLab has timed out. This may be due to the overall size of your project, or the number of files/changes to sync, being too large.", + "gitlab_too_many_files_error": "This repository cannot be imported as it exceeds the maximum number of files allowed", + "gitlab_validation_check": "Please check that the repository name is valid, and that you have permission to create the repository.", + "gitlab_validation_check_auth": "Please check that you have permission to create the repository.", + "gitlab_validation_name_exists": "Repository with this name already exists.", + "gitlab_validation_name_invalid": "Repository name is invalid.", + "gitlab_workflow_authorize": "Authorize GitLab Workflow files", + "gitlab_workflow_files_delete_gitlab_repo": "The repository has been created on GitLab but linking was unsuccessful. You will have to delete GitLab repository or choose a new name.", + "gitlab_workflow_files_error": "The __appName__ GitLab sync service couldn’t sync GitLab Workflow files (in .gitlab/workflows/). Please authorize __appName__ to edit your GitLab workflow files and try again.", + "gitlab_workflow_files_error_non_owner": "The __appName__ GitLab sync service couldn’t sync GitLab Workflow files (in .gitlab/workflows/). Please ensure the project owner has authorized __appName__ to edit this project’s GitLab workflow files and try again.", "give-ai-consent": "Give AI consent", "give_feedback": "Give feedback", "global": "global", @@ -1153,6 +1198,7 @@ "history_entry_origin_dropbox": "via Dropbox", "history_entry_origin_git": "via Git", "history_entry_origin_github": "via GitHub", + "history_entry_origin_gitlab": "via GitLab", "history_entry_origin_upload": "upload", "history_label_created_by": "Created by", "history_label_project_current_state": "Current state", @@ -1217,7 +1263,9 @@ "import_a_bibtex_file_from_your_provider_account": "Import a BibTeX file from your __provider__ account", "import_document_description": "Content will be imported from the selected document. Formatting may not be reproduced exactly.", "import_existing_projects_from_github": "Import existing projects from GitHub", + "import_existing_projects_from_gitlab": "Import existing projects from GitLab", "import_from_github": "Import from GitHub", + "import_from_gitlab": "Import from GitLab", "import_idp_metadata": "Import IdP metadata", "import_markdown_file": "Import Markdown file", "import_to_sharelatex": "Import to __appName__", @@ -1232,6 +1280,7 @@ "imported_from_zotero_at_date_by": "Imported from Zotero at __formattedDate__ __relativeDate__ by __importedByName__", "importing": "Importing", "importing_and_merging_changes_in_github": "Importing and merging changes in GitHub", + "importing_and_merging_changes_in_gitlab": "Importing and merging changes in GitLab", "in_order_to_have_a_secure_account_make_sure_your_password": "To help keep your account secure, make sure your new password:", "in_order_to_match_institutional_metadata_2": "In order to match your institutional metadata, we’ve linked your account using <0>__email__.", "in_order_to_match_institutional_metadata_associated": "In order to match your institutional metadata, your account is associated with the email __email__.", @@ -1298,6 +1347,7 @@ "integrations": "Integrations", "integrations_include": "Integrations include:", "integrations_like_github": "Integrations like GitHub Sync", + "integrations_like_gitlab": "Integrations like GitLab Sync", "interested_in_cheaper_personal_plan": "Would you be interested in the cheaper <0>__price__ Personal plan?", "invalid_certificate": "Invalid certificate. Please check the certificate and try again.", "invalid_confirmation_code": "That didn’t work. Please check the code and try again.", @@ -1480,6 +1530,8 @@ "link_sharing_is_on": "Link sharing is on", "link_to_github": "Link to your GitHub account", "link_to_github_description": "You need to authorise __appName__ to access your GitHub account to allow us to sync your projects.", + "link_to_gitlab": "Link to your GitLab account", + "link_to_gitlab_description": "You need to authorise __appName__ to access your GitLab account to allow us to sync your projects.", "link_to_mendeley": "Link to Mendeley", "link_to_papers": "Link to Papers", "link_to_zotero": "Link to Zotero", @@ -1493,9 +1545,11 @@ "loading": "Loading", "loading_content": "Creating Project", "loading_github_repositories": "Loading your GitHub repositories", + "loading_gitlab_repositories": "Loading your GitLab repositories", "loading_prices": "loading prices", "loading_python_runtime": "Loading Python runtime...", "loading_recent_github_commits": "Loading recent commits", + "loading_recent_gitlab_commits": "Loading recent commits", "loading_references": "Loading references", "local_account": "Local account", "log_entry_description": "Log entry with level: __level__", @@ -1716,6 +1770,7 @@ "no_members": "No members", "no_messages_yet": "No messages yet", "no_new_commits_in_github": "No new commits in GitHub since last merge.", + "no_new_commits_in_gitlab": "No new commits in GitLab since last merge.", "no_one_has_commented_or_left_any_suggestions_yet": "No one has commented or left any suggestions yet.", "no_one_invited_yet": "No one invited yet", "no_one_sharing_link_is_disabled": "No one—sharing link is disabled", @@ -1792,6 +1847,7 @@ "only_importer_can_refresh": "Only the person who originally imported this __provider__ file can refresh it.", "only_invited_people": "Only invited people", "only_project_owner_can_link_github": "Only the project owner can link this project to a GitHub repository. Please contact the project owner if you would like to set this up.", + "only_project_owner_can_link_gitlab": "Only the project owner can link this project to a GitLab repository. Please contact the project owner if you would like to set this up.", "open_action_menu": "Open __name__ action menu", "open_advanced_reference_search": "Open advanced reference search", "open_as_template": "Open as Template", @@ -2046,6 +2102,7 @@ "project_linked_to": "This project is linked to", "project_name": "Project name", "project_not_linked_to_github": "This project is not linked to a GitHub repository. You can create a repository for it in GitHub:", + "project_not_linked_to_gitlab": "This project is not linked to a GitLab repository. You can create a repository for it in GitLab:", "project_notifications": "Project notifications", "project_notifications_muted_description": "You are not receiving any notifications, as you have disabled all project notifications.", "project_ownership_transfer_confirmation_1": "Are you sure you want to make <0>__user__ the owner of <1>__project__?", @@ -2079,9 +2136,12 @@ "publisher_account": "Publisher Account", "publishing": "Publishing", "pull_github_changes_into_sharelatex": "Pull GitHub changes into __appName__", + "pull_gitlab_changes_into_sharelatex": "Pull GitLab changes into __appName__", "purge": "Purge", "push_sharelatex_changes_to_github": "Push __appName__ changes to GitHub", + "push_sharelatex_changes_to_gitlab": "Push __appName__ changes to GitLab", "push_to_github_pull_to_overleaf": "Push to GitHub, pull to __appName__", + "push_to_gitlab_pull_to_overleaf": "Push to GitLab, pull to __appName__", "quoted_text": "Quoted text", "raw_logs": "Raw logs", "raw_logs_description": "Raw logs from the LaTeX compiler", @@ -2102,9 +2162,11 @@ "real_time_track_changes": "Real-time track-changes", "realtime_track_changes": "Real-time track changes", "reauthorize_github_account": "Reauthorize your GitHub Account", + "reauthorize_gitlab_account": "Reauthorize your GitLab Account", "recaptcha_conditions": "The site is protected by reCAPTCHA.", "recent": "Recent", "recent_commits_in_github": "Recent commits in GitHub", + "recent_commits_in_gitlab": "Recent commits in GitLab", "recommended": "Recommended", "recompile": "Recompile", "recompile_from_scratch": "Recompile from scratch", @@ -2371,6 +2433,7 @@ "select_from_source_files": "select from source files", "select_from_your_computer": "select from your computer", "select_github_repository": "Select a GitHub repository to import into __appName__.", + "select_gitlab_repository": "Select a GitLab repository to import into __appName__.", "select_image_from_project_files": "Select image from project files", "select_project": "Select __project__", "select_projects": "Select Projects", @@ -2636,11 +2699,15 @@ "sync_all_projects_with_dropbox": "Sync all projects with Dropbox.", "sync_dropbox_github": "Sync with Dropbox and GitHub", "sync_project_to_github_explanation": "Any changes you have made in __appName__ will be committed and merged with any updates in GitHub.", + "sync_project_to_gitlab_explanation": "Any changes you have made in __appName__ will be committed and merged with any updates in GitLab.", "sync_to_dropbox": "Sync to Dropbox", "sync_to_github": "Sync to GitHub", + "sync_to_gitlab": "Sync to GitLab", "sync_with_a_github_repository": "Sync with a GitHub repository.", + "sync_with_a_gitlab_repository": "Sync with a GitLab repository.", "sync_with_dropbox": "Sync with Dropbox", "sync_with_github": "Sync with GitHub", + "sync_with_gitlab": "Sync with GitLab", "synctex_error_recompile_and_try_again": "That didn’t work. Recompile and try again.", "synctex_failed": "Couldn’t find the corresponding source file", "syntax_checks": "Syntax checks", @@ -2790,6 +2857,7 @@ "to_continue_using_upgrade_or_change_your_browser": "To continue using __appName__ without problems you need to upgrade or change to a <0>supported browser.", "to_fix_this_you_can": "To fix this, you can:", "to_fix_this_you_can_ask_the_github_repository_owner": "To fix this, you can ask the GitHub repository owner (<0>__repoOwnerEmail__) to renew their __appName__ subscription and reconnect the project.", + "to_fix_this_you_can_ask_the_gitlab_repository_owner": "To fix this, you can ask the GitLab repository owner (<0>__repoOwnerEmail__) to renew their __appName__ subscription and reconnect the project.", "to_insert_or_move_a_caption_make_sure_tabular_is_directly_within_table": "To insert or move a caption, make sure \\begin{tabular} is directly within a table environment", "to_keep_edit_access": "To keep edit access, ask the project owner to upgrade their plan or reduce the number of people with edit access.", "to_modify_your_subscription_go_to": "To modify your subscription go to", @@ -2942,6 +3010,8 @@ "unlink_from_sso": "Unlink from SSO", "unlink_github_repository": "Unlink GitHub repository", "unlink_github_warning": "Any projects that you have synced with GitHub will be disconnected and no longer kept in sync with GitHub. Are you sure you want to unlink your GitHub account?", + "unlink_gitlab_repository": "Unlink GitLab repository", + "unlink_gitlab_warning": "Any projects that you have synced with GitLab will be disconnected and no longer kept in sync with GitLab. Are you sure you want to unlink your GitLab account?", "unlink_linked_accounts": "Unlink any linked accounts (such as ORCID ID, IEEE). <0>Remove them in Account settings (under Linked Accounts).", "unlink_linked_google_account": "Unlink your Google account. <0>Remove it in Account settings (under Linked Accounts).", "unlink_provider_account_title": "Unlink __provider__ Account", @@ -2949,6 +3019,7 @@ "unlink_reference": "Unlink __provider__ References Provider", "unlink_sso": "Unlink SSO", "unlink_the_project_from_the_current_github_repo": "Unlink the project from the current GitHub repository and create a connection to a repository you own.", + "unlink_the_project_from_the_current_gitlab_repo": "Unlink the project from the current GitLab repository and create a connection to a repository you own.", "unlink_user_explanation": "You’re about to remove the SSO login option for <0>__email__. This will force them to reauthenticate their Overleaf account with your IdP. They’ll receive an email asking them to do this.", "unlink_users": "Unlink users", "unlink_warning_reference": "Warning: When you unlink your account from __provider__ you will not be able to import references into your projects.", @@ -3153,6 +3224,7 @@ "work_or_university_sso": "Work/university single sign-on", "work_with_non_overleaf_users": "Work with non-Overleaf users", "work_with_other_github_users": "Work with other GitHub users", + "work_with_other_gitlab_users": "Work with other GitLab users", "writefull": "Writefull", "x_changes_in": "__count__ change in", "x_changes_in_plural": "__count__ changes in", @@ -3221,6 +3293,7 @@ "you_will_be_able_to_reassign_subscription": "You will be able to reassign their subscription membership to another person in your organization", "youll_get_best_results_in_visual_but_can_be_used_in_source": "You’ll get the best results from using this tool in the <0>Visual Editor, although you can still use it to insert tables in the <1>Code Editor. Once you’ve selected the number of rows and columns you need, the table will appear in your document and you can double click in a cell to add contents to it.", "youll_need_to_ask_the_github_repository_owner": "You’ll need to ask the GitHub repository owner (<0>__repoOwnerEmail__) to reconnect the project.", + "youll_need_to_ask_the_gitlab_repository_owner": "You’ll need to ask the GitLab repository owner (<0>__repoOwnerEmail__) to reconnect the project.", "youll_no_longer_need_to_remember_credentials": "You’ll no longer need to remember a separate email address and password. Instead, you will use single-sign on to login to Overleaf. <0>Read more about SSO.", "your_account_is_managed_by_admin_cant_join_additional_group": "Your __appName__ account is managed by your current group admin (__admin__). This means you can’t join additional group subscriptions. <0>Read more about Managed Users.", "your_account_is_managed_by_your_group_admin": "Your account is managed by your group admin. You can’t change or delete your email address.", diff --git a/services/web/modules/gitlab-sync/app/models/gitlabSyncProjectStates.mjs b/services/web/modules/gitlab-sync/app/models/gitlabSyncProjectStates.mjs new file mode 100644 index 0000000000..8a5f7421f6 --- /dev/null +++ b/services/web/modules/gitlab-sync/app/models/gitlabSyncProjectStates.mjs @@ -0,0 +1,24 @@ +import mongoose from "../../../../app/src/infrastructure/Mongoose.mjs" + +const { Schema } = mongoose +const { ObjectId } = Schema + +export const GitLabSyncProjectStatesSchema = new Schema( + { + projectId: { type: ObjectId, ref: 'Project', required: true, unique: true }, + repoFullName: { type: String, required: true }, + defaultBranchName: { type: String, default: null }, + mergeStatus: { type: String, enum: ['clean', 'conflict', 'diverged'], default: 'clean' }, + lastSyncCommit: { type: String, default: null }, + lastSyncVersion: { type: Number, default: null }, + unmergedBranchName: { type: String, default: null }, + unmergedBranchHead: { type: String, default: null }, + conflictVersion: { type: Number, default: null }, + }, + { collection: 'gitlabSyncProjectStates', minimize: false } +) + +export const GitLabSyncProjectStates = mongoose.model( + 'GitLabSyncProjectStates', + GitLabSyncProjectStatesSchema, +) diff --git a/services/web/modules/gitlab-sync/app/models/gitlabSyncUserCredentials.mjs b/services/web/modules/gitlab-sync/app/models/gitlabSyncUserCredentials.mjs new file mode 100644 index 0000000000..063b6d67a9 --- /dev/null +++ b/services/web/modules/gitlab-sync/app/models/gitlabSyncUserCredentials.mjs @@ -0,0 +1,17 @@ +import mongoose from "../../../../app/src/infrastructure/Mongoose.mjs" + +const { Schema } = mongoose +const { ObjectId } = Schema + +export const GitLabSyncUserCredentialsSchema = new Schema( + { + userId: { type: ObjectId, ref: 'User', required: true, unique: true }, + gitlab: { type: String, required: true }, + }, + { collection: 'gitlabSyncUserCredentials', minimize: false } +) + +export const GitLabSyncUserCredentials = mongoose.model( + 'GitLabSyncUserCredentials', + GitLabSyncUserCredentialsSchema, +) diff --git a/services/web/modules/gitlab-sync/app/src/AccessTokenEncryptorHelper.mjs b/services/web/modules/gitlab-sync/app/src/AccessTokenEncryptorHelper.mjs new file mode 100644 index 0000000000..dd72d34bc5 --- /dev/null +++ b/services/web/modules/gitlab-sync/app/src/AccessTokenEncryptorHelper.mjs @@ -0,0 +1,82 @@ +import logger from '@overleaf/logger' +import Settings from '@overleaf/settings' +import fs from 'node:fs' +import crypto from 'node:crypto' +import Path from 'node:path' +import AccessTokenEncryptorClass from '@overleaf/access-token-encryptor' + +const TOKEN_CIPHER_FILE = '/var/lib/overleaf/data/.token-cipher.json' +const TOKEN_CIPHER_LABEL = 'OL_CEP-v3' + +let encryptorInstance = null + +/** + * Get or create a stable encryptor data that persists across container + * recreations. Priority: + * 1. GITLAB_CIPHER_PASSWORD || TOKEN_CIPHER_PASSWORD env var (explicit user config) + * 2. JSON file with encryptor data in the persistent volume (/var/lib/overleaf/data/) + * — auto-generated on first use, survives container rebuilds + */ +function _getEncryptorData() { + const cipherPassword = process.env.GITLAB_TOKEN_CIPHER_PASSWORD || process.env.TOKEN_CIPHER_PASSWORD + const cipherLabel = process.env.GITLAB_TOKEN_CIPHER_LABEL || + process.env.TOKEN_CIPHER_LABEL || + TOKEN_CIPHER_LABEL + if (cipherPassword) { + return { + cipherLabel, + cipherPasswords: { + [cipherLabel]: cipherPassword + } + } + } + // cipherPassword is not set, use file + const cipherFile = process.env.GITLAB_TOKEN_CIPHER_FILE || + process.env.TOKEN_CIPHER_FILE || + TOKEN_CIPHER_FILE + try { + return JSON.parse(fs.readFileSync(cipherFile, 'utf8')) + } catch (err) { + if (err.code === 'ENOENT') { + logger.info({ cipherFile }, 'Creating new cipherFile') + } else { + logger.error({ err, cipherFile }, 'Bad cipherFile') + throw err + } + } + // File doesn't exist yet — generate one + try { + const encryptorData = { + cipherLabel, + cipherPasswords: { + [cipherLabel]: crypto.randomBytes(32).toString('base64') + }, + } + + const dir = Path.dirname(cipherFile) + fs.mkdirSync(dir, { recursive: true }) + fs.writeFileSync(cipherFile, JSON.stringify(encryptorData, null, 2), { mode: 0o600 }) + return encryptorData + + } catch (err) { + logger.error({ err, cipherFile }, 'Cannot create cipherFile') + throw err + } +} + +function _getEncryptor() { + if (!encryptorInstance) { + const encryptorData = _getEncryptorData() + encryptorInstance = new AccessTokenEncryptorClass(encryptorData) + } + return encryptorInstance +} + +export const AccessTokenEncryptor = { + async encryptJson(json) { + return await _getEncryptor().promises.encryptJson(json) + }, + async decryptToJson(encryptedJson) { + return await _getEncryptor().promises.decryptToJson(encryptedJson) + }, +} diff --git a/services/web/modules/gitlab-sync/app/src/GitLabApiClient.mjs b/services/web/modules/gitlab-sync/app/src/GitLabApiClient.mjs new file mode 100644 index 0000000000..7a17bd31cd --- /dev/null +++ b/services/web/modules/gitlab-sync/app/src/GitLabApiClient.mjs @@ -0,0 +1,638 @@ +import logger from '@overleaf/logger' +import Settings from '@overleaf/settings' +import crypto from 'node:crypto' + +import { + fetchJson, + fetchJsonWithResponse, + fetchNothing, + fetchStream, + RequestFailedError, +} from '@overleaf/fetch-utils' + +import { + InvalidTokenError, + ExpiredTokenError, + NotFoundError, + GitConflictError, + RateLimitError, + ProviderRequestError, + PermissionDeniedError, + AlreadyExistsError, +} from './GitSyncErrors.mjs' + +const GITLAB_URL = process.env.GITLAB_SYNC_URL +const GITLAB_API_BASE = `${GITLAB_URL}/api/v4` +const GITLAB_DEFAULT_BRANCH = process.env.GITLAB_DEFAULT_BRANCH || 'main' +const MAX_PER_PAGE = 100 + +const REQUEST_TIMEOUT_MS = 60 * 1000 +const REQUEST_LONG_TIMEOUT_MS = 600 * 1000 + +const MERGE_REQUEST_POLL_INTERVAL_MS = process.env.GITLAB_MERGE_REQUEST_POLL_INTERVAL_MS || 1000 +const MERGE_REQUEST_TIMEOUT_MS = process.env.GITLAB_MERGE_REQUEST_TIMEOUT_MS || 60_000 + +const mergeRequestPendingStates = new Set([ + 'unchecked', + 'checking', + 'preparing', + 'approvals_syncing', +]) + +const mergeRequestBlockingStates = new Set([ + 'conflict', + 'ci_must_pass', + 'ci_still_running', + 'commits_status', + 'discussions_not_resolved', + 'draft_status', + 'jira_association_missing', + 'merge_request_blocked', + 'merge_time', + 'need_rebase', + 'not_approved', + 'not_open', + 'requested_changes', + 'security_policy_pipeline_check', + 'security_policy_violations', + 'status_checks_must_pass', + 'locked_paths', + 'locked_lfs_files', + 'title_regex', +]) + + + +const maxConcurrency = process.env.GITLAB_API_MAX_CONCURRENCY || 5 + +function buildHeaders(token) { + return { + Accept: 'application/json', + 'Content-Type': 'application/json', + 'User-Agent': 'Overleaf-CEP-GitLab-Sync', + ...(token ? { Authorization: `Bearer ${token}` } : {}), + } +} + +function encodePath(path) { + return encodeURIComponent(path) +} + +function projectPath(repoFullName) { + return encodeURIComponent(repoFullName) +} + +function isRateLimitError(response) { + return response.headers.get('retry-after') != null +} + +function isRepositoryAlreadyExistsError(err) { + if (err.response?.status !== 400) { + return false + } + + const json_data = JSON.parse(err.body) + return Object.values(json_data.message).some(value => { + if (Array.isArray(value)) { + return value.includes("has already been taken"); + } + return value === "has already been taken"; + }); +} + +function normalizeGitLabError(err, operation) { + const body = err.body || '' + logger.error({ operation, err, body }, 'GitLab API request failed') + + if (err.name === 'AbortError') { + throw new ProviderRequestError('GitLab request timed out', { status: 504 }, err) + } + + if (!(err instanceof RequestFailedError)) { + throw new ProviderRequestError('Something wrong with GitLab request', { status: 500 }, err) + } + + const status = err.response?.status || 500 + + if (status === 401) { + const json_body = JSON.parse(err.body) || {} + if (json_body && json_body.error && json_body.error.toLowerCase() === 'invalid_token') { + throw new ExpiredTokenError('Token expired', { status }, err) + } + + throw new InvalidTokenError('Invalid token', { status }, err) + } + + if (status === 404) { + throw new NotFoundError('Not found', { status }, err) + } + + if (status === 409) { + throw new GitConflictError() + } + + if (status === 403 || status === 429) { + if (isRateLimitError(err.response)) throw new RateLimitError('Rate limit exeeded', { status: 429 }, err) + throw new PermissionDeniedError('Permission denied', { status: 403 }, err) + } + + if (status === 400) { + if (isRepositoryAlreadyExistsError(err)) { + throw new AlreadyExistsError('Repository already exists', { status }, err) + } + } + + throw new ProviderRequestError('GitLab request failed', { status }, err) +} + +// wrappers +function fetchGitLabJson(url, options, operation) { + return fetchJson(url, options).catch(err => { + normalizeGitLabError(err, operation) + }) +} + +function fetchGitLabJsonWithResponse(url, options, operation) { + return fetchJsonWithResponse(url, options).catch(err => { + normalizeGitLabError(err, operation) + }) +} + +function fetchGitLabNothing(url, options, operation) { + return fetchNothing(url, options).catch(err => { + normalizeGitLabError(err, operation) + }) +} + +function fetchGitLabStream(url, options, operation) { + return fetchStream(url, options).catch(err => { + normalizeGitLabError(err, operation) + }) +} + +async function fetchAllPages(url, options, operation) { + let page = 1 + let all = [] + + while (true) { + const pageUrl = new URL(url.toString()) + pageUrl.searchParams.set('page', page.toString()) + + const { json, response } = await fetchGitLabJsonWithResponse(pageUrl.toString(), options, operation) + all = all.concat(json) + + const link = response.headers.get('link') || '' + if (!link.includes('rel="next"')) break + page++ + } + + return all +} + +// ---------------------- exports ------------------------------- // + +// OAuth +function getOAuth2Url() { + const oAuthUrl = new URL(`${GITLAB_URL}/oauth/authorize`) + oAuthUrl.searchParams.append('client_id', Settings.gitlabSync.clientID) + oAuthUrl.searchParams.append('redirect_uri', Settings.gitlabSync.callbackURL) + oAuthUrl.searchParams.append('response_type', 'code') + oAuthUrl.searchParams.append('scope', 'api read_user read_repository write_repository') + return oAuthUrl +} + +function exchangeCodeForToken(code) { + return fetchGitLabJson(`${GITLAB_URL}/oauth/token`, { + method: 'POST', + headers: buildHeaders(), + json: { + client_id: Settings.gitlabSync.clientID, + client_secret: Settings.gitlabSync.clientSecret, + code: code, + grant_type: 'authorization_code', + redirect_uri: Settings.gitlabSync.callbackURL, + }, + signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS) + }, 'exchangeCodeForToken').then(r => [r.access_token, r.refresh_token] ) +} + +function refreshToken(refreshToken) { + return fetchGitLabJson(`${GITLAB_URL}/oauth/token`, { + method: 'POST', + headers: buildHeaders(), + json: { + client_id: Settings.gitlabSync.clientID, + client_secret: Settings.gitlabSync.clientSecret, + refresh_token: refreshToken, + grant_type: 'refresh_token', + redirect_uri: Settings.gitlabSync.callbackURL, + }, + signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS) + }, 'refreshToken').then(r => [r.access_token, r.refresh_token]) +} + +function revokeToken(token) { + const { clientID, clientSecret } = Settings.gitlabSync + return fetchGitLabNothing(`${GITLAB_URL}/oauth/revoke`, { + method: 'POST', + headers: buildHeaders(), + json: { + client_id: clientID, + client_secret: clientSecret, + token, + }, + signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS), + }, 'revokeToken') +} + +// user, orgs, permissions +function getUser(token) { + return fetchGitLabJson(`${GITLAB_API_BASE}/user`, { + headers: buildHeaders(token), + signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS), + }, 'getUser').then(({ id, username, name }) => ({ id, login: username, name })) +} + +async function getUserAndOrgs(token) { + const user = await fetchGitLabJson(`${GITLAB_API_BASE}/user`, { + headers: buildHeaders(token), + signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS), + }, 'getUserAndOrgs:user') + + const groupsUrl = new URL(`${GITLAB_API_BASE}/groups`) + groupsUrl.searchParams.set('all_available', 'true') + groupsUrl.searchParams.set('min_access_level', '30') + groupsUrl.searchParams.set('top_level_only', 'true') + groupsUrl.searchParams.set('per_page', MAX_PER_PAGE.toString()) + groupsUrl.searchParams.set('simple', 'true') + + const groups = await fetchAllPages(groupsUrl, { + headers: buildHeaders(token), + signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS), + }, 'getUserAndOrgs:groups') + + return { + user: user.username, + orgs: groups.map(group => group.full_path), + } +} + +function getPushPermission(token, repoFullName) { + return fetchGitLabJson(`${GITLAB_API_BASE}/projects/${projectPath(repoFullName)}`, { + headers: buildHeaders(token), + signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS), + }, 'getPushPermission').then(project => { + const projectAccess = project.permissions?.project_access?.access_level || 0 + const groupAccess = project.permissions?.group_access?.access_level || 0 + return Math.max(projectAccess, groupAccess) >= 30 + }) +} + +// repos +function resolveNamespaceId(token, org) { + if (org == null) return Promise.resolve(null) + if (/^\d+$/.test(org)) return Promise.resolve(Number(org)) + + return fetchGitLabJson(`${GITLAB_API_BASE}/groups/${encodePath(org)}`, { + headers: buildHeaders(token), + signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS), + }, 'resolveNamespaceId').then(group => group.id) +} + +async function createRepo(token, { name, description, isPublic, org }) { + const namespaceId = await resolveNamespaceId(token, org) + const payload = { + name, + description, + visibility: isPublic ? 'public' : 'private', + initialize_with_readme: false, + default_branch: GITLAB_DEFAULT_BRANCH, + merge_method: 'merge', // Use merge here to allow for conflict resolution to work properly without the need for a rebase + } + + if (namespaceId != null) { + payload.namespace_id = namespaceId + } + + return fetchGitLabJson(`${GITLAB_API_BASE}/projects`, { + method: 'POST', + headers: buildHeaders(token), + json: payload, + signal: AbortSignal.timeout(REQUEST_LONG_TIMEOUT_MS) + }, 'createRepo') +} + +async function listUserRepos(token) { + const url = new URL(`${GITLAB_API_BASE}/projects`) + url.searchParams.set('membership', 'true') + url.searchParams.set('per_page', MAX_PER_PAGE.toString()) + url.searchParams.set('simple', 'true') + url.searchParams.set('order_by', 'last_activity_at') + url.searchParams.set('sort', 'desc') + url.searchParams.set('active', 'true') // Only list projects that are not archived and not marked for deletion + + const repos = await fetchAllPages(url, { + headers: buildHeaders(token), + signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS), + }, 'listUserRepos') + + return repos.map(repo => ({ + name: repo.name, + fullName: repo.path_with_namespace, + defaultBranchName: repo.default_branch, + })) +} + +// Git (blobs / trees / commits) +function uploadBlob(token, repoFullName, buffer) { + void token + void repoFullName + + if (Buffer.isBuffer(buffer)) { + return Promise.resolve(buffer.toString('base64')) + } + + if (typeof buffer === 'string') { + return Promise.resolve(buffer) + } + + return Promise.resolve(String(buffer)) +} + +function getBlobStream(token, repoFullName, ref, path) { + const encodedPath = encodePath(path) + return fetchGitLabStream(`${GITLAB_API_BASE}/projects/${projectPath(repoFullName)}/repository/files/${encodedPath}/raw?ref=${encodeURIComponent(ref)}`, { + headers: buildHeaders(token), + signal: AbortSignal.timeout(REQUEST_LONG_TIMEOUT_MS), + }, 'getBlobStream') +} + +function createTree(token, repoFullName, entries, baseTree) { + void token + void repoFullName + + const existingPaths = new Set((baseTree || []).map(entry => entry.path)) + const actions = [] + + for (const entry of entries) { + if (entry.sha == null) { + actions.push({ + action: 'delete', + file_path: entry.path, + }) + continue + } + + actions.push({ + action: existingPaths.has(entry.path) ? 'update' : 'create', + file_path: entry.path, + content: entry.content, + encoding: 'base64', + }) + } + + return Promise.resolve({ + baseTree: baseTree || [], + entries: actions, + }) +} + +function createCommit(token, repoFullName, { tree, message, branch, start_sha, force = false }) { + const actions = Array.isArray(tree?.entries) ? tree.entries : [] + + let payload = { + branch: branch || GITLAB_DEFAULT_BRANCH, + commit_message: message, + actions, + } + + if (branch && branch != GITLAB_DEFAULT_BRANCH) { + payload.start_sha = start_sha + } + + return fetchGitLabJson(`${GITLAB_API_BASE}/projects/${projectPath(repoFullName)}/repository/commits`, { + method: 'POST', + headers: buildHeaders(token), + json: payload, + signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS) + }, 'createCommit').then(r => r.id) // For GitLab id refers to the SHA of the created commit +} + +function getCommitTree(token, repoFullName, commit) { + return listBlobsAtCommit(token, repoFullName, commit) +} + +async function listBlobsAtCommit(token, repoFullName, commit) { + const url = new URL(`${GITLAB_API_BASE}/projects/${projectPath(repoFullName)}/repository/tree`) + url.searchParams.set('ref', commit) + url.searchParams.set('recursive', 'true') + url.searchParams.set('per_page', MAX_PER_PAGE.toString()) + + const treeEntries = await fetchAllPages(url, { + headers: buildHeaders(token), + signal: AbortSignal.timeout(REQUEST_LONG_TIMEOUT_MS) + }, 'listBlobsAtCommit') + + return treeEntries + .filter(entry => entry.type === 'blob') + .map(entry => ({ + sha: entry.id, + path: entry.path, + mode: entry.mode, + type: 'blob', + })) +} + +async function getBlobContent(token, repoFullName, sha) { + const url = `${GITLAB_API_BASE}/projects/${projectPath(repoFullName)}/repository/blobs/${sha}` + + try { + const json = await fetchGitLabJson(url, { + headers: buildHeaders(token), + signal: AbortSignal.timeout(REQUEST_LONG_TIMEOUT_MS) + }, 'getBlobContent') + return json.content + } catch (err) { + throw OError.tag(err, 'Failed to fetch blob content from GitLab', { repoFullName, sha }) + } +} + +async function listNewCommitsWithStatus(token, fullName, branchName, fromCommit) { + const url = new URL(`${GITLAB_API_BASE}/projects/${projectPath(fullName)}/repository/compare`) + url.searchParams.set('from', fromCommit) + url.searchParams.set('to', branchName) + url.searchParams.set('straight', 'true') + + const data = await fetchGitLabJson(url.toString(), { + headers: buildHeaders(token), + signal: AbortSignal.timeout(REQUEST_LONG_TIMEOUT_MS) + }, 'listNewCommitsWithStatus') + + const commits = (data.commits || []).map(c => ({ + message: c.message || c.title || '', + author: { + name: c.author_name || '', + email: c.author_email || '', + date: c.created_at || c.committed_date || '', + }, + sha: c.id, + })) + + return { commits, diverged: commits.length > 0 } +} + +// branches +function getBranchHead(token, repoFullName, branchName) { + return fetchGitLabJson(`${GITLAB_API_BASE}/projects/${projectPath(repoFullName)}/repository/branches/${encodePath(branchName)}`, { + headers: buildHeaders(token), + signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS) + }, 'getBranchHead').then(r => r.commit.id) +} + +function createBranch(token, repoFullName, branchName, sha) { + return fetchGitLabJson(`${GITLAB_API_BASE}/projects/${projectPath(repoFullName)}/repository/branches`, { + method: 'POST', + headers: buildHeaders(token), + json: { + branch: branchName, + ref: sha, + }, + signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS) + }, 'createBranch') +} + +function updateBranch(token, repoFullName, branchName, sha, force = false) { +// Update functionality does not exist in GitLab API as it automatically updates the branch when a commit is made to it +} + +function deleteBranch(token, repoFullName, branchName) { + return fetchGitLabNothing(`${GITLAB_API_BASE}/projects/${projectPath(repoFullName)}/repository/branches/${encodePath(branchName)}`, { + method: 'DELETE', + headers: buildHeaders(token), + signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS) + }, 'deleteBranch') +} + +// merge / compare +async function createMergeRequest(token, repoFullName, sourceBranch, targetBranch, title) { + return fetchGitLabJson(`${GITLAB_API_BASE}/projects/${projectPath(repoFullName)}/merge_requests`, { + method: 'POST', + headers: buildHeaders(token), + json: { + source_branch: sourceBranch, + target_branch: targetBranch, + title, + remove_source_branch: true, + squash: false, + }, + signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS), + }, 'createMergeRequest') +} + +async function waitForMergeRequestToBeReady(token, repoFullName, mergeRequestIid) { + const startedAt = Date.now() + + while (Date.now() - startedAt < MERGE_REQUEST_TIMEOUT_MS) { + const mergeRequest = await fetchGitLabJson(`${GITLAB_API_BASE}/projects/${projectPath(repoFullName)}/merge_requests/${mergeRequestIid}`, { + headers: buildHeaders(token), + signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS), + }, 'getMergeRequest') + + const status = mergeRequest.detailed_merge_status + + if (status === 'mergeable') { + return { + canMerge: true, + reason: status, + } + } + + if (mergeRequestPendingStates.has(status)) { + await new Promise(resolve => setTimeout(resolve, MERGE_REQUEST_POLL_INTERVAL_MS)) + continue + } + + if (mergeRequestBlockingStates.has(status)) { + return { + canMerge: false, + reason: status, + } + } + + // Unknown future GitLab status, treat as unmergeable to be safe + return { + canMerge: false, + reason: `unknown_status:${status}`, + } + } + + throw new Error(`Timed out waiting for merge request ${mergeRequestIid} readiness`) +} + +async function mergeOpenMergeRequest(token, repoFullName, mergeRequestIid) { + return fetchGitLabJson(`${GITLAB_API_BASE}/projects/${projectPath(repoFullName)}/merge_requests/${mergeRequestIid}/merge`, { + method: 'PUT', + headers: buildHeaders(token), + signal: AbortSignal.timeout(REQUEST_LONG_TIMEOUT_MS), + }, 'mergeOpenMergeRequest') +} + +function mergeBranch(token, repoFullName, base, head) { + return createMergeRequest(token, repoFullName, head, base, `Merge ${head} into ${base}`) + .then(async mergeRequest => { + const ret = await waitForMergeRequestToBeReady(token, repoFullName, mergeRequest.iid) + if (!ret.canMerge) { + throw new GitConflictError("Merge request cannot be merged due to conflicts, reason: " + ret.reason) + } + + await mergeOpenMergeRequest(token, repoFullName, mergeRequest.iid) + return getBranchHead(token, repoFullName, base) + }) +} + +function compareCommits(token, repoFullName, from, to) { + const url = new URL(`${GITLAB_API_BASE}/projects/${projectPath(repoFullName)}/repository/compare`) + url.searchParams.set('from', from) + url.searchParams.set('to', to) + url.searchParams.set('straight', 'true') + + return fetchGitLabJson(url.toString(), { + headers: buildHeaders(token), + signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS) + }, 'compareCommits').then(r => r.diffs || []) +} + +function getRepoZipball(token, repoFullName, sha) { + return fetchGitLabStream(`${GITLAB_API_BASE}/projects/${projectPath(repoFullName)}/repository/archive.zip?sha=${encodeURIComponent(sha)}`, { + headers: buildHeaders(token), + signal: AbortSignal.timeout(REQUEST_LONG_TIMEOUT_MS), + }, 'getRepoZipball') +} + +export default { + maxConcurrency, + getOAuth2Url, + exchangeCodeForToken, + refreshToken, + revokeToken, + getUser, + getUserAndOrgs, + getPushPermission, + createRepo, + listUserRepos, + uploadBlob, + getBlobStream, + createTree, + createCommit, + getCommitTree, + listBlobsAtCommit, + getBlobContent, + listNewCommitsWithStatus, + getBranchHead, + createBranch, + updateBranch, + deleteBranch, + mergeBranch, + compareCommits, + getRepoZipball, +} \ No newline at end of file diff --git a/services/web/modules/gitlab-sync/app/src/GitLabSyncController.mjs b/services/web/modules/gitlab-sync/app/src/GitLabSyncController.mjs new file mode 100644 index 0000000000..12ad582196 --- /dev/null +++ b/services/web/modules/gitlab-sync/app/src/GitLabSyncController.mjs @@ -0,0 +1,241 @@ +import { expressify } from '@overleaf/promise-utils' +import logger from '@overleaf/logger' +import OError from '@overleaf/o-error' +import HttpErrorHandler from '../../../../app/src/Features/Errors/HttpErrorHandler.mjs' +import SessionManager from '../../../../app/src/Features/Authentication/SessionManager.mjs' +import Csrf from '../../../../app/src/infrastructure/Csrf.mjs' +import GitLabSyncHandler from './GitLabSyncHandler.mjs' +import TokenManager from './TokenManager.mjs' +import api from './GitLabApiClient.mjs' +import { doGitMerge } from './GitMerge.mjs' +import { InvalidTokenError, GitNotLinkedError, AlreadyExistsError } from './GitSyncErrors.mjs' + +async function getConnectionStatus(req, res) { + const userId = SessionManager.getLoggedInUserId(req.session) + + try { + const isConnected = await GitLabSyncHandler.getGitConnState(userId) + res.json(isConnected) + } catch (err) { + const info = OError.getFullInfo(err) + const errStatus = info?.status || 500 + logger.error(OError.getFullStack(err)) + logger.error({ info }, "failed to check user connection") + return res.status(errStatus).json({ message: err.message }) + } +} + +async function getProjectState(req, res) { + const { project_id: projectId } = req.params + const userId = SessionManager.getLoggedInUserId(req.session) + + try { + const pss = await GitLabSyncHandler.getProjectState(userId, projectId) + return res.json(pss) + } catch (err) { + const info = OError.getFullInfo(err) + const errStatus = info?.status || 500 + logger.error(OError.getFullStack(err)) + logger.error({ info }, "failed get project sync state") + return res.status(errStatus).json({ message: err.message }) + } +} + +async function getUserAndOrgs(req, res) { + const userId = SessionManager.getLoggedInUserId(req.session) + const userAndOrgs = await GitLabSyncHandler.getUserAndOrgs(userId) + res.json(userAndOrgs) +} +async function listUserRepos(req, res) { + const userId = SessionManager.getLoggedInUserId(req.session) + + try { + const repos = await GitLabSyncHandler.listUserRepos(userId) + res.json({ repos }) + + } catch (err) { + if (err instanceof InvalidTokenError) { + return res.json(null) + } + const info = OError.getFullInfo(err) + const errStatus = info?.status || 500 + logger.error(OError.getFullStack(err)) + logger.error({ userId, info }, 'Failed to get repositories list') + return res.status(errStatus).json({ message: err.message }) + } +} + +async function getMergeOverview(req, res) { + const { project_id: projectId } = req.params + const userId = SessionManager.getLoggedInUserId(req.session) + try { + const commitsAndStatus = await GitLabSyncHandler.getMergeOverview(userId, projectId) + res.json(commitsAndStatus) + + } catch (err) { + const info = OError.getFullInfo(err) + const errStatus = info?.status || 500 + logger.error(OError.getFullStack(err)) + logger.error({ info, projectId }, 'Error listing commits since last sync') + return res.status(errStatus).json({ message: err.message }) + } +} + +// Import a GitLab repository as a new project +async function importRepo(req, res) { + const userId = SessionManager.getLoggedInUserId(req.session) + const { name, fullName, defaultBranchName } = req.body + + try { + const projectId = await GitLabSyncHandler.importRepo(userId, name, fullName, defaultBranchName) + res.json({ projectId }) + } catch (error) { + logger.error({ error, userId }, 'Failed to import git repository from server') + res.status(error.status || 500).json({ message: error.message }) + } +} + +// Redirect user to Git server OAuth2 authorization URL +async function oauth2(req, res) { + const oauth2Url = api.getOAuth2Url() + oauth2Url.searchParams.append('state', req.csrfToken()) + res.redirect(oauth2Url.toString()) +} + +// callback +async function oauth2Callback(req, res) { + const userId = SessionManager.getLoggedInUserId(req.session) + const { code, state } = req.query + + try { + await Csrf.promises.validateToken(state, req.session) + } catch { + HttpErrorHandler.forbidden(req, res, 'Invalid CSRF token') + return + } + + let token + let refresh_token + try { + [token, refresh_token] = await api.exchangeCodeForToken(code) + if (!token || !refresh_token) { + HttpErrorHandler.badRequest(req, res, 'Failed to obtain access token from Git server') + return + } + } catch (err) { + const info = OError.getFullInfo(err) + const err_body = err.body + logger.error(OError.getFullStack(err)) + logger.error({ info, err_body, userId }, 'Failed to obtain access token from Git server') + HttpErrorHandler.badRequest(req, res, err.message || 'Bad request') + return + } + + try { + await TokenManager.saveUserToken(userId, {token, refresh_token}) + } catch (err) { + const info = OError.getFullInfo(err) + const errStatus = info?.status || 500 + logger.error(OError.getFullStack(err)) + logger.error({ info, userId }, 'Error saving user token') + HttpErrorHandler.handleErrorByStatusCode(req, res, err, errStatus) + return + } + + // Save success message in session to display on redirect + req.session.projectSyncSuccessMessage = + req.i18n.translate('gitlab_successfully_linked_description') + res.redirect('/user/settings?oauth-complete=gitlab#project-sync') +} + +// Unlink user's Git server account +async function unlink(req, res) { + const userId = SessionManager.getLoggedInUserId(req.session) + try { + await TokenManager.removeUserToken(userId) + } catch (err) { + const info = OError.getFullInfo(err) + const errStatus = info?.status || 500 + logger.error(OError.getFullStack(err)) + logger.error({ info, projectId }, 'Error removing user credentials') + return res.status(errStatus).json({ message: err.message }) + } + res.sendStatus(200) +} + +// Export project to Git server +// Expected req.body: +// name (string): repository name +// description (string, optional): repository description +// isPublic (boolean, optional): if true, the repository is public +// org (string, optional): if provided, repository will be created under this organization + +async function exportProject(req, res) { + const userId = SessionManager.getLoggedInUserId(req.session) + const { project_id: projectId } = req.params + + try { + await GitLabSyncHandler.exportProject(userId, projectId, req.body) + + } catch (err) { + const info = OError.getFullInfo(err) + const errStatus = info?.status || 500 + logger.error(OError.getFullStack(err)) + logger.error({ info, projectId }, 'Error exporting project') + let key = 'gitlab_validation_check' + if (err instanceof AlreadyExistsError) key = 'gitlab_validation_name_exists' + else if (errStatus === 401 || errStatus === 403) key = 'gitlab_validation_check_auth' + return res.status(errStatus).json({ key, message: err.message }) + } + res.sendStatus(200) +} + +async function gitMerge(req, res) { + const { project_id: projectId } = req.params + try { + const userId = SessionManager.getLoggedInUserId(req.session) + const commitMessage = req.body?.message || 'Updates from Overleaf' + const claimConflictIsResolved = req.body.claimConflictIsResolved + + const mergeResult = await doGitMerge(userId, projectId, commitMessage, claimConflictIsResolved) + + res.json(mergeResult) + } catch (err) { + const info = OError.getFullInfo(err) + const errStatus = info?.status || 500 + logger.error(OError.getFullStack(err)) + logger.error({ info, projectId }, 'Error syncing project to GitLab') + return res.status(errStatus).json({ message: err.message }) + } +} + +// Unlink user's Git server account +async function unlinkRepo(req, res) { + const { project_id: projectId } = req.params + const userId = SessionManager.getLoggedInUserId(req.session) + try { + const ownerEmail = await GitLabSyncHandler.unlinkRepo(userId, projectId) + if (ownerEmail) return res.status(403).json({ ownerEmail }) + } catch (err) { + const info = OError.getFullInfo(err) + const errStatus = info?.status || 500 + logger.error(OError.getFullStack(err)) + logger.error({ info, projectId }, 'Error unlinking repo from project') + return res.status(errStatus).json({ message: err.message }) + } + res.sendStatus(200) +} +export default { + getConnectionStatus: expressify(getConnectionStatus), + getProjectState: expressify(getProjectState), + oauth2: expressify(oauth2), + unlink: expressify(unlink), + getUserAndOrgs: expressify(getUserAndOrgs), + oauth2Callback: expressify(oauth2Callback), + listUserRepos: expressify(listUserRepos), + importRepo: expressify(importRepo), + exportProject: expressify(exportProject), + getMergeOverview: expressify(getMergeOverview), + gitMerge: expressify(gitMerge), + unlinkRepo: expressify(unlinkRepo), +} diff --git a/services/web/modules/gitlab-sync/app/src/GitLabSyncHandler.mjs b/services/web/modules/gitlab-sync/app/src/GitLabSyncHandler.mjs new file mode 100644 index 0000000000..2c1f75c3d2 --- /dev/null +++ b/services/web/modules/gitlab-sync/app/src/GitLabSyncHandler.mjs @@ -0,0 +1,302 @@ +import Path from 'path' +import fs from 'fs' +import { pipeline } from 'node:stream/promises' +import crypto from 'crypto' +import pLimit from 'p-limit' +import Settings from '@overleaf/settings' +import logger from '@overleaf/logger' +import OError from '@overleaf/o-error' +import ProjectGetter from '../../../../app/src/Features/Project/ProjectGetter.mjs' +import ProjectUploadManager from '../../../../app/src/Features/Uploads/ProjectUploadManager.mjs' +import UserGetter from '../../../../app/src/Features/User/UserGetter.mjs' +import DocumentUpdaterHandler from '../../../../app/src/Features/DocumentUpdater/DocumentUpdaterHandler.mjs' +import api from './GitLabApiClient.mjs' +import SyncStateManager from './SyncStateManager.mjs' +import HistoryManager from './HistoryManager.mjs' +import TokenManager from './TokenManager.mjs' +import { InvalidTokenError, ExpiredTokenError, NotFoundError } from './GitSyncErrors.mjs' + +async function refreshGitToken(userId) { + logger.info({ userId }, 'token expired, trying to refresh') + + const tokens = await TokenManager.getUserToken(userId) + let { token, refresh_token } = tokens + + if (!tokens) { + logger.error("tokens invalid") + return false; + } + + if (!refresh_token) { + logger.error("refresh_token invalid") + return false + } + + try { + [token, refresh_token] = await api.refreshToken(refresh_token) + if (!token || !refresh_token) { + HttpErrorHandler.badRequest(req, res, 'Failed to refresh access token') + return false + } + } catch (err) { + const info = OError.getFullInfo(err) + logger.error(OError.getFullStack(err)) + logger.error({ info, userId }, 'Failed to refresh access token') + HttpErrorHandler.badRequest(req, res, err.message || 'Bad request') + return false + } + + try { + await TokenManager.saveUserToken(userId, { token, refresh_token }) + } catch (err) { + const info = OError.getFullInfo(err) + const errStatus = info?.status || 500 + logger.error(OError.getFullStack(err)) + logger.error({ info, userId }, 'Error saving user token') + HttpErrorHandler.handleErrorByStatusCode(req, res, err, errStatus) + return false + } + + return true +} + +async function getGitConnState(userId) { + try { + const tokens = await TokenManager.getUserToken(userId) + if (!tokens) return false; + const { token } = tokens + + await api.getUser(token) + return true + } catch (err) { + if (err instanceof InvalidTokenError) { + logger.debug( { err, userId }, 'token invalid, treating as not connected') + return false + } else if (err instanceof ExpiredTokenError) { + logger.info("Token has expired") + return await refreshGitToken(userId) + } + throw OError.tag(err, 'failed to validate token', { userId }) + } +} + +async function getProjectState(userId, projectId) { + let pss = null + const projection = { _id: 0, mergeStatus: 1, repoFullName: 1, unmergedBranchName: 1 } + pss = await SyncStateManager.getProjectState(projectId, projection) + if (!pss) { + pss = { mergeStatus: 'need-export' } + try { + const { owner_ref } = await ProjectGetter.promises.getProject(projectId, { owner_ref: 1 }) + if (owner_ref.toString() !== userId) { + pss.ownerEmail = await UserGetter.promises.getUserEmail(owner_ref) + } + } catch (err) { + pss.ownerEmail = 'nobody@nowhere' + logger.error({ err }, "failed get user email") + } + return pss + } + let canPush + try { + const tokens = await TokenManager.getUserToken(userId) + if (!tokens) throw new InvalidTokenError('no user token', { userId, status: 400 }); + const { token } = tokens + canPush = await api.getPushPermission(token, pss.repoFullName) + } catch (err) { + if ((err instanceof NotFoundError) || + (err instanceof PermissionDeniedError) + ) canPush = false + else throw err + } + if (!canPush) { + pss.mergeStatus = 'need-permission' + // send owner's email to collaborator + // send nothing to the owner + try { + const { owner_ref } = await ProjectGetter.promises.getProject(projectId, { owner_ref: 1 }) + if (owner_ref.toString() !== userId) { + pss.ownerEmail = await UserGetter.promises.getUserEmail(owner_ref) + } + } catch (err) { + logger.error({ err, userId }, "failed get user email") + pss.ownerEmail = 'nobody@nowhere' + } + } + return pss +} + +async function unlinkRepo(userId, projectId) { + const { owner_ref } = await ProjectGetter.promises.getProject(projectId, { owner_ref: 1 }) + if (owner_ref.toString() !== userId) { + let ownerEmail + try { + ownerEmail = await UserGetter.promises.getUserEmail(owner_ref) + } catch (err) { + logger.error({ err, userId }, "failed get user email") + ownerEmail = 'nobody@nowhere' + } + return ownerEmail + } + await SyncStateManager.removeProjectState(projectId) + return null +} + + +async function listUserRepos(userId) { + const tokens = await TokenManager.getUserToken(userId) + if (!tokens) throw new InvalidTokenError('no user token', { userId, status: 400 }); + const { token } = tokens + return await api.listUserRepos(token) +} + +async function getUserAndOrgs(userId) { + const tokens = await TokenManager.getUserToken(userId) + if (!tokens) throw new InvalidTokenError('no user token', { userId, status: 400 }); + const { token } = tokens + return await api.getUserAndOrgs(token) +} + +async function getMergeOverview(userId, projectId) { + const projectSyncState = await SyncStateManager.getProjectState(projectId) + if (!projectSyncState) throw new GitNotLinkedError(projectId) + const { repoFullName, defaultBranchName, lastSyncCommit, lastSyncVersion, mergeStatus } = projectSyncState + + if (mergeStatus === 'conflict') return null + + const tokens = await TokenManager.getUserToken(userId) + if (!tokens) throw new InvalidTokenError('no user token', { userId, status: 400 }); + const { token } = tokens + const currentVersion = await HistoryManager.latestVersion(projectId.toString()) + const isProjectUpdated = currentVersion !== lastSyncVersion + try { + const commitsAndStatus = await api.listNewCommitsWithStatus(token, repoFullName, defaultBranchName, lastSyncCommit) + if(commitsAndStatus.diverged) { + await SyncStateManager.updateProjectState(projectId, { mergeStatus: 'diverged' } ) + } + return { ...commitsAndStatus, isProjectUpdated } + } catch (err) { + if (err instanceof NotFoundError) { + // what is the reason of NotFoundError? is it a missing commit or not? + let canPush + try { + canPush = await api.getPushPermission(token, repoFullName) + } catch (e) { + logger.error({ error: e }, "failed to check push permission") + canPush = false + } + if (!canPush) throw err + await SyncStateManager.updateProjectState(projectId, { mergeStatus: 'diverged' } ) + return { commits: [], diverged: true, isProjectUpdated } + } else { + throw err + } + } +} + +async function importRepo(userId, projectName, repoFullName, defaultBranchName) { + const tokens = await TokenManager.getUserToken(userId) + if (!tokens) throw new InvalidTokenError('no user token', { userId, status: 400 }); + const { token } = tokens + const defaultBranchHead = await api.getBranchHead(token, repoFullName, defaultBranchName) + const fsPath = Path.join(Settings.path.dumpFolder, `gitlab_import_${crypto.randomUUID()}`) + + let project_id + + try { + const stream = await api.getRepoZipball(token, repoFullName, defaultBranchHead) + await pipeline(stream, fs.createWriteStream(fsPath)) + + const { project } = await ProjectUploadManager.promises.createProjectFromZipArchiveWithName( + userId, + projectName, + fsPath + ) + + project_id = project?._id + } catch (err) { + throw OError.tag( + err, + 'failed importing git repo', + { userId, projectName, repoFullName, defaultBranchName, fsPath } + ) + } finally { + fs.promises.rm(fsPath, { force: true }).catch(() => {}) + } + + try { + const projectVersion = await HistoryManager.latestVersion(project_id.toString()) + await SyncStateManager.createProjectState(project_id, { + repoFullName, + mergeStatus: 'clean', + lastSyncCommit: defaultBranchHead, + defaultBranchName, + lastSyncVersion: projectVersion + }) + } catch (err) { +// don't throw up + logger.error( + { err, userId, projectId: project_id.toString(), repoFullName }, + 'Failed to create state of imported repo' + ) + } + + return project_id +} + +async function exportProject(userId, projectId, repoOptions) { + const isLinked = await SyncStateManager.getProjectState(projectId) + if (isLinked) { + throw new OError('Project is already linked to Git server', { projectId }) + } + + const tokens = await TokenManager.getUserToken(userId) + if (!tokens) throw new InvalidTokenError('no user token', { userId, status: 400 }); + const { token } = tokens + + const { path_with_namespace: repoFullName, default_branch: defaultBranchName } = + await api.createRepo(token, repoOptions) + + await DocumentUpdaterHandler.promises.flushProjectToMongo(projectId) + + const currentVersion = await HistoryManager.latestVersion(projectId) + const currentPaths = await HistoryManager.getPathsAtVersion(projectId, currentVersion) + + const limit = pLimit(api.maxConcurrency) + const blobData = await Promise.all( + currentPaths.paths.map((path) => + limit(async () => { + const buffer = await HistoryManager.getProjectFileBuffer(projectId, currentVersion, path) + const sha = "DON_T_DELETE" // sha needs to be present so that the file is not detected as a deletion + return { path, sha, content: buffer } + }) + ) + ) + + const tree = await api.createTree(token, repoFullName, blobData) + const initialCommit = await api.createCommit(token, repoFullName, { + tree, + message: 'Initial Overleaf import', + }) + + const force = true + + return SyncStateManager.createProjectState(projectId, { + mergeStatus: 'clean', + defaultBranchName, + lastSyncCommit: initialCommit, + lastSyncVersion: currentVersion, + repoFullName, + }) +} + +export default { + exportProject, + getProjectState, + unlinkRepo, + importRepo, + getGitConnState, + listUserRepos, + getUserAndOrgs, + getMergeOverview, +} diff --git a/services/web/modules/gitlab-sync/app/src/GitLabSyncRouter.mjs b/services/web/modules/gitlab-sync/app/src/GitLabSyncRouter.mjs new file mode 100644 index 0000000000..971d5380f2 --- /dev/null +++ b/services/web/modules/gitlab-sync/app/src/GitLabSyncRouter.mjs @@ -0,0 +1,100 @@ +import logger from '@overleaf/logger' + +import GitLabSyncController from './GitLabSyncController.mjs' +import AuthenticationController from '../../../../app/src/Features/Authentication/AuthenticationController.mjs' +import AuthorizationMiddleware from '../../../../app/src/Features/Authorization/AuthorizationMiddleware.mjs' + +export default { + apply(webRouter) { + logger.debug({}, 'Init gitlab-sync router') + // start the GitLab OAuth flow by redirecting to GitLab authorization page + webRouter.get( + '/user/gitlab-sync/oauth2', + AuthenticationController.requireLogin(), + GitLabSyncController.oauth2 + ) + + // callback for GitLab OAuth flow + webRouter.get( + '/user/gitlab-sync/oauth2/callback', + AuthenticationController.requireLogin(), + GitLabSyncController.oauth2Callback + ) + + // unlink GitLab account + webRouter.post( + '/user/gitlab-sync/unlink', + AuthenticationController.requireLogin(), + GitLabSyncController.unlink + ) + + // get user git connection status + webRouter.get( + '/user/gitlab-sync/status', + AuthenticationController.requireLogin(), + GitLabSyncController.getConnectionStatus + ) + + // get git user name and user's organizations + webRouter.get( + '/user/gitlab-sync/orgs', + AuthenticationController.requireLogin(), + GitLabSyncController.getUserAndOrgs + ) + + // list user's repos + webRouter.get( + '/user/gitlab-sync/repos', + AuthenticationController.requireLogin(), + GitLabSyncController.listUserRepos + ) + + // create a new project from git server repo + webRouter.post( + '/project/new/gitlab-sync', + AuthenticationController.requireLogin(), + GitLabSyncController.importRepo + ) + + // export project to git server + webRouter.post( + '/project/:project_id/gitlab-sync/export', + AuthenticationController.requireLogin(), + AuthorizationMiddleware.ensureUserCanWriteProjectContent, + GitLabSyncController.exportProject + ) + + // get project sync state + webRouter.get( + '/project/:project_id/gitlab-sync/state', + AuthenticationController.requireLogin(), + AuthorizationMiddleware.ensureUserCanWriteProjectContent, + GitLabSyncController.getProjectState + ) + + // get overview of the coming merge: + // merge state (clean or diverged), unmerged commits, is OL version changed? + webRouter.get( + '/project/:project_id/gitlab-sync/merge/overview', + AuthenticationController.requireLogin(), + AuthorizationMiddleware.ensureUserCanWriteProjectContent, + GitLabSyncController.getMergeOverview + ) + + // merge OL project with Git server repo + webRouter.post( + '/project/:project_id/gitlab-sync/merge', + AuthenticationController.requireLogin(), + AuthorizationMiddleware.ensureUserCanWriteProjectContent, + GitLabSyncController.gitMerge + ) + + // unlink Git repo from Git server + webRouter.delete( + '/project/:project_id/gitlab-sync', + AuthenticationController.requireLogin(), + AuthorizationMiddleware.ensureUserCanWriteProjectContent, + GitLabSyncController.unlinkRepo + ) + }, +} diff --git a/services/web/modules/gitlab-sync/app/src/GitMerge.mjs b/services/web/modules/gitlab-sync/app/src/GitMerge.mjs new file mode 100644 index 0000000000..7a2456e711 --- /dev/null +++ b/services/web/modules/gitlab-sync/app/src/GitMerge.mjs @@ -0,0 +1,768 @@ +import { normalize } from 'path/posix' +import pLimit from 'p-limit' +import { createHash } from 'crypto' +import logger from '@overleaf/logger' +import SyncStateManager from './SyncStateManager.mjs' +import api from './GitLabApiClient.mjs' +import HistoryManager from './HistoryManager.mjs' +import { GitConflictError } from './GitSyncErrors.mjs' +import TokenManager from './TokenManager.mjs' +import { ObjectId } from '../../../../app/src/infrastructure/mongodb.mjs' +import LockManager from '../../../../app/src/infrastructure/LockManager.mjs' +import ProjectEntityHandler from '../../../../app/src/Features/Project/ProjectEntityHandler.mjs' +import UpdateMerger from '../../../../app/src/Features/ThirdPartyDataStore/UpdateMerger.mjs' +import DocumentUpdaterHandler from '../../../../app/src/Features/DocumentUpdater/DocumentUpdaterHandler.mjs' + +const SyncLockManager = LockManager.withTimeout(300) // 5 min + +export async function doGitMerge(userId, projectId, message, claimConflictIsResolved) { + return await SyncLockManager.promises.runWithLock( + 'gitSync', + projectId, + () => doGitMergeWithoutLock(userId, projectId, message, claimConflictIsResolved) + ) +} + +async function doGitMergeWithoutLock(userId, projectId, message, claimConflictIsResolved) { + const projectSyncState = await SyncStateManager.getProjectState(projectId) + if (!projectSyncState) throw new GitNotLinkedError(projectId) + const { + repoFullName, + defaultBranchName, + mergeStatus, + unmergedBranchName + } = projectSyncState + + // do nothing if conflict is not yet resolved and 'Sync' button is pushed + // do nothing if conflict is already resolved and 'I have manually merged' button is pushed + if ( + !claimConflictIsResolved && mergeStatus === 'conflict' || + claimConflictIsResolved && mergeStatus !== 'conflict' + ) return { mergeStatus, repoFullName, unmergedBranchName } + + const tokens = await TokenManager.getUserToken(userId) + if (!tokens) throw new InvalidTokenError('no user token', { userId, status: 400 }); + const { token } = tokens + + await DocumentUpdaterHandler.promises.flushProjectToMongo(projectId) + + const currentVersion = await HistoryManager.latestVersion(projectId) + const defaultBranchHead = await api.getBranchHead(token, repoFullName, defaultBranchName) + + let newUnmergedBranchName + + switch (mergeStatus) { + + case 'clean': + newUnmergedBranchName = await resolveCleanSyncState( + token, + userId, + projectSyncState, + currentVersion, + defaultBranchHead, + message, + ) + break + + case 'conflict': + newUnmergedBranchName = await resolveConflictSyncState( + token, + userId, + projectSyncState, + currentVersion, + defaultBranchHead + ) + break + + case 'diverged': + newUnmergedBranchName = await resolveDetachedSyncState( + token, + userId, + projectSyncState, + currentVersion, + defaultBranchHead, + message, + ) + break + + default: + throw new Error('Bad mergeStatus') + } + + const newMergeStatus = newUnmergedBranchName ? 'conflict' : 'clean' + return { mergeStatus: newMergeStatus, repoFullName, unmergedBranchName: newUnmergedBranchName } +} + +// ----------- clean flow +async function resolveCleanSyncState( + token, + userId, + projectSyncState, + currentVersion, + defaultBranchHead, + message, +) { + const { projectId, repoFullName, defaultBranchName } = projectSyncState + + // No changes in OL + if (currentVersion === projectSyncState.lastSyncVersion) { + // No changes in GH too: everything is up to date + if (projectSyncState.lastSyncCommit === defaultBranchHead) return null + + // Need to apply only GH changes + const lastSyncCommit = defaultBranchHead + const lastSyncVersion = await applyGitSnapshotToProject({ + token, + userId, + projectId, + repoFullName, + lastSyncCommit, + }) + + await SyncStateManager.updateProjectState(projectId, { lastSyncVersion, lastSyncCommit }) + return null + } + + try + { + // Possible changes in OL + const olBranchHead = + await exportChangesToGit({ + token, + projectId, + repoFullName, + lastSyncVersion: projectSyncState.lastSyncVersion, + currentVersion, + message, + baseCommit: projectSyncState.lastSyncCommit, + }) + + if (!olBranchHead) { + if ( defaultBranchHead === projectSyncState.lastSyncCommit) { + await SyncStateManager.updateProjectState(projectId, { lastSyncVersion: currentVersion }) + return null + } + + const lastSyncCommit = defaultBranchHead + const lastSyncVersion = await applyGitSnapshotToProject({ + token, + userId, + projectId, + repoFullName, + lastSyncCommit, + }) + await SyncStateManager.updateProjectState(projectId, { lastSyncVersion, lastSyncCommit }) + return null + } + + if (defaultBranchHead === projectSyncState.lastSyncCommit) { + const lastSyncCommit = olBranchHead + const lastSyncVersion = await applyGitSnappshotToProject({ + token, + userId, + projectId, + repoFullName, + lastSyncCommit, + }) + await SyncStateManager.updateProjectState(projectId, { lastSyncVersion, lastSyncCommit }) + } + } + catch (err) { + logger.error({ err, projectId }, 'Error during clean sync state resolution, fallback to merge') + } + + // Normal commit failed, fallback to merge, first create a temporary branch with OL changes + const tempBranchName = generateBranchName() + const olBranchHead = + await exportChangesToGit({ + token, + projectId, + repoFullName, + lastSyncVersion: projectSyncState.lastSyncVersion, + currentVersion, + message, + baseCommit: projectSyncState.lastSyncCommit, + branch: tempBranchName, + }) + + // Merge the temporary branch into the default branch + const { mergeCommit, conflict } = await mergeWithTempBranch( + token, + repoFullName, + defaultBranchName, + tempBranchName + ) + + if (conflict) { + await SyncStateManager.updateProjectState(projectId, { + mergeStatus: 'conflict', + lastSyncCommit: defaultBranchHead, + unmergedBranchName: tempBranchName, + unmergedBranchHead: olBranchHead, + conflictVersion: currentVersion, + }) + return tempBranchName + } else { + const lastSyncCommit = mergeCommit + const lastSyncVersion = await applyGitSnapshotToProject({ + token, + userId, + projectId, + repoFullName, + lastSyncCommit, + }) + await SyncStateManager.updateProjectState(projectId, { lastSyncVersion, lastSyncCommit }) + return null + } +} + +// ----------- conflict flow +async function resolveConflictSyncState( + token, + userId, + projectSyncState, + currentVersion, + defaultBranchHead +) { + const { projectId, repoFullName, defaultBranchName, conflictVersion } = projectSyncState + + // No OL changes after manual conflict resolving: pull GH changes only + if (currentVersion === conflictVersion) { + const lastSyncCommit = defaultBranchHead + const lastSyncVersion = await applyGitSnapshotToProject({ + token, + userId, + projectId, + repoFullName, + lastSyncCommit, + }) + + await SyncStateManager.updateProjectState(projectId, { + mergeStatus: 'clean', + lastSyncVersion, + lastSyncCommit, + conflictVersion: null, + unmergedBranchName: null, + unmergedBranchHead: null + }) + return null + } + + // OL was changed after manual conflict resolution: need to merge + const prevOlBranchHead = projectSyncState.unmergedBranchHead + + // Create a temporary branch with OL changes after manual conflict resolution + const tempBranchName = generateBranchName() + const newOlBranchHead = await exportChangesToGit({ + token, + projectId, + repoFullName, + lastSyncVersion: conflictVersion, + currentVersion, + message: '[Updates in Overleaf during conflict resolution]', + baseCommit: prevOlBranchHead, + branch: tempBranchName, + }) + + // Were there really any changes? (e.g.: file created, then removed: no changes) + if (!newOlBranchHead) { + // nothing to push to GH + const lastSyncCommit = defaultBranchHead + const lastSyncVersion = await applyGitSnapshotToProject({ + token, + userId, + projectId, + repoFullName, + lastSyncCommit, + }) + + await SyncStateManager.updateProjectState(projectId, { + mergeStatus: 'clean', + lastSyncVersion, + lastSyncCommit, + conflictVersion: null, + unmergedBranchName: null, + unmergedBranchHead: null + }) + return null + } + + // merge + const { mergeCommit, conflict } = await mergeWithTempBranch( + token, + repoFullName, + defaultBranchName, + newOlBranchHead, + tempBranchName, + ) + + if (conflict) { + // conflict must be resolved manually + await SyncStateManager.updateProjectState(projectId, { + mergeStatus: 'conflict', + lastSyncCommit: defaultBranchHead, + unmergedBranchName: tempBranchName, + unmergedBranchHead: newOlBranchHead, + conflictVersion: currentVersion, + }) + return tempBranchName + } else { + // merge succeeded + const lastSyncCommit = mergeCommit + const lastSyncVersion = await applyGitSnapshotToProject({ + token, + userId, + projectId, + repoFullName, + lastSyncCommit, + }) + await SyncStateManager.updateProjectState(projectId, { + mergeStatus: 'clean', + lastSyncVersion, + lastSyncCommit, + conflictVersion: null, + unmergedBranchName: null, + unmergedBranchHead: null + }) + return null + } +} + +// ----------- detached flow +// GH repo was force pushed, lastSync commit possibly unavailable +async function resolveDetachedSyncState( + token, + userId, + projectSyncState, + currentVersion, + defaultBranchHead, + message, +) { + const { projectId, repoFullName, defaultBranchName } = projectSyncState + + // No changes in OL: re-anchor + if (currentVersion === projectSyncState.lastSyncVersion) { + const lastSyncCommit = defaultBranchHead + const lastSyncVersion = await applyGitSnapshotToProject({ + token, + userId, + projectId, + repoFullName, + lastSyncCommit, + }) + await SyncStateManager.updateProjectState(projectId, { mergeStatus: 'clean', lastSyncVersion, lastSyncCommit }) + return null + } + + // Need to compare lastSync OL version (base), current OL version (local) and GH version (remote) + const baseSnapshot = await HistoryManager.getProjectSnapshot(projectId, projectSyncState.lastSyncVersion) + const localSnapshot = await HistoryManager.getProjectSnapshot(projectId, currentVersion) + const remoteBlobMap = await getGitBlobMap(token, repoFullName, defaultBranchHead) + + const { + cleanLocalEntries, + conflictRemoteEntries, + conflictLocalEntries, + conflictBaseEntries, + hasConflicts, + hasChanges + } = await buildDetachedSyncPlan({ + token, + repoFullName, + projectId, + currentVersion, + baseSnapshot, + localSnapshot, + remoteBlobMap, + }) + + // were any changes introduced in GH? + if (!hasChanges) { + await SyncStateManager.updateProjectState(projectId, { + mergeStatus: 'clean', + lastSyncVersion: currentVersion, + lastSyncCommit: defaultBranchHead + }) + return null + } + + if (!hasConflicts) { + let lastSyncCommit = defaultBranchHead + if (cleanLocalEntries.length !== 0) { + lastSyncCommit = await createCommitFromEntries({ + token, + repoFullName, + parentCommit: defaultBranchHead, + entries: cleanLocalEntries, + message, + }) + } + const lastSyncVersion = await applyGitSnapshotToProject({ + token, + userId, + projectId, + repoFullName, + lastSyncCommit, + }) + await SyncStateManager.updateProjectState(projectId, { mergeStatus: 'clean', lastSyncVersion, lastSyncCommit }) + return null + } + + // Resolve potential conflict. + // Build synthetic merge topology: + // + // GH + // / + // H --- B --- + // \ + // OL + // + // merge(GH <- OL) + // + // B: conflicted files are restored to lastSyncVersion (merge base) + // GH: in conflicted files remote changes after lastSyncVersion are replayed + // OL: all changes done in Overleaf after lastSyncVersion are applied + + // Blobs referenced in conflictBaseEntries in theory could be lost, but for GitHub, + // GitLab and Gitea, blobs from historical commits are normally retained for a + // very long time so the practical probability of failure is likely close to zero. + // TODO: anyway, if error is thrown by api.createTree, upload all blobs + // referenced in conflictBaseEntries and try again + + // H --- B + const baseCommit = await createCommitFromEntries({ + token, + repoFullName, + parentCommit: defaultBranchHead, + entries: conflictBaseEntries, + message: '[Overleaf GitSync conflict resolution: restore last sync]', + }) + + // default branch: --- H --- B --- GH + const remoteChangesCommit = await createCommitFromEntries({ + token, + repoFullName, + parentCommit: baseCommit, + entries: conflictRemoteEntries, + message: '[Overleaf GitSync conflict resolution: replay repo changes]', + }) + + const tempBranchName = generateBranchName() + // temp branch: --- H --- B --- OL + const olChangesCommit = await createCommitFromEntries({ + token, + repoFullName, + parentCommit: baseCommit, + entries: [...cleanLocalEntries, ...conflictLocalEntries], + message, + branch: tempBranchName, + }) + + // merge: default <- temp (GH <- OL) + const { mergeCommit, conflict } = await mergeWithTempBranch( + token, + repoFullName, + defaultBranchName, + tempBranchName + ) + + // auto-merge succeeded, conflicts in editable files were resolved? + if (!conflict) { + const lastSyncCommit = mergeCommit + const lastSyncVersion = await applyGitSnapshotToProject({ + token, + userId, + projectId, + repoFullName, + lastSyncCommit, + }) + await SyncStateManager.updateProjectState(projectId, { mergeStatus: 'clean', lastSyncVersion, lastSyncCommit }) + return null + } + + // merge failed: must manually resolve conflicts + await SyncStateManager.updateProjectState(projectId, { + mergeStatus: 'conflict', + lastSyncCommit: remoteChangesCommit, + unmergedBranchName: tempBranchName, + unmergedBranchHead: olChangesCommit, + conflictVersion: currentVersion, + }) + return tempBranchName +} + +// ----------- Helpers +async function applyGitSnapshotToProject({ token, userId, projectId, repoFullName, lastSyncCommit }) { + + const invalidFiles = [] + const updates = [] + const deletions = [] + + const currentVersion = await HistoryManager.latestVersion(projectId) + const historySnapshot = await HistoryManager.getProjectSnapshot(projectId, currentVersion) + const blobMap = await getGitBlobMap(token, repoFullName, lastSyncCommit) + const blobPaths = new Set() + + for (const [path, sha] of Object.entries(blobMap)) { + const invalid = validateFileName(path) + + if (invalid) { + invalidFiles.push(invalid.name) + continue + } + + blobPaths.add(path) + + const hstSha = historySnapshot[path]?.data?.hash + // File at path is not changed, do not touch it + if (hstSha === sha) continue + + updates.push(path) + } + + for (const path of Object.keys(historySnapshot)) { + if (blobPaths.has(path)) continue + deletions.push(path) + } + + // TODO: do not silently ignore invalidFiles, inform user about them + if (invalidFiles.length > 0) { + logger.warn({ projectId, invalidFiles }, 'Invalid file paths detected at Git server') + } + const limit = pLimit(api.maxConcurrency) + await Promise.all( + updates.map(path => limit(async () => { + const stream = await api.getBlobStream(token, repoFullName, lastSyncCommit, path) + return UpdateMerger.promises.mergeUpdate(userId, projectId, '/' + path, stream, 'gitlab') + })) + ) + const localLimit = pLimit(5) + await Promise.all( + deletions.map(path => localLimit(() => { + return UpdateMerger.promises.deleteUpdate(userId, projectId, '/' + path, 'gitlab') + })) + ) + + return await HistoryManager.latestVersion(projectId) +} + +async function buildDetachedSyncPlan({ + token, + repoFullName, + projectId, + currentVersion, + baseSnapshot, + localSnapshot, + remoteBlobMap +}) { + const cleanLocalEntries = [] + const conflictRemoteEntries = [] + const conflictLocalEntries = [] + const conflictBaseEntries = [] + + let hasConflicts = false + let hasChanges = false + + const allPaths = new Set([ + ...Object.keys(baseSnapshot || {}), + ...Object.keys(localSnapshot || {}), + ...Object.keys(remoteBlobMap || {}), + ]) + + for (const path of allPaths) { + const baseHash = getEntryHash(baseSnapshot[path]) + const localHash = getEntryHash(localSnapshot[path]) + const remoteHash = remoteBlobMap[path] || null + + // OL == GH, do nothing + if (localHash === remoteHash) continue + + if (baseHash === remoteHash) { + // changed in OL + hasChanges = true + cleanLocalEntries.push({ path, sha: localHash, content: Buffer.from(localSnapshot[path]?.data?.content || '', 'utf8').toString('base64') }) + continue + } + + if (baseHash === localHash) { + // changed in GH + hasChanges = true + continue + } + // conflict: changed both in GH and OL + hasChanges = true + hasConflicts = true + + conflictRemoteEntries.push({ path, sha: remoteHash, content: await api.getBlobContent(token, repoFullName, remoteHash) }) + conflictLocalEntries.push({ path, sha: localHash, content: Buffer.from(localSnapshot[path]?.data?.content || '', 'utf8').toString('base64') }) + conflictBaseEntries.push({ path, sha: baseHash, content: Buffer.from(baseSnapshot[path]?.data?.content || '', 'utf8').toString('base64') }) + } + return { + cleanLocalEntries, + conflictRemoteEntries, + conflictLocalEntries, + conflictBaseEntries, + hasConflicts, + hasChanges, + } +} + +async function createCommitFromEntries({ + token, + repoFullName, + parentCommit, + entries, + message, + branch, +}) { + const baseTree = await api.getCommitTree(token, repoFullName, parentCommit) + const newTree = await api.createTree(token, repoFullName, entries, baseTree) + return api.createCommit(token, repoFullName, { + tree: newTree, + start_sha: parentCommit, + message, + branch, + }) +} + +async function mergeWithTempBranch( + token, + repoFullName, + branchName, + tempBranchName +) { + try { + const mergeCommit = await api.mergeBranch(token, repoFullName, branchName, tempBranchName) + + return { conflict: false, mergeCommit } + + } catch (err) { + if (err instanceof GitConflictError) { + return { conflict: true } + } + throw err + } +} + +async function exportChangesToGit({ + token, + projectId, + repoFullName, + lastSyncVersion, + currentVersion, + message, + baseCommit, + branch, +}) { + const diff = await HistoryManager.getProjectFileTreeDiff( + projectId, + lastSyncVersion, + currentVersion + ) + + const { paths } = await HistoryManager.getPathsAtVersion(projectId, lastSyncVersion) + const existedAtLastSync = new Set(paths) + + const deletePaths = new Set() + const upsertPaths = new Set() + + for (const item of diff) { + const oldPath = item.pathname + + if (item.operation === 'removed') { + // it can happen that removed file was created after last sync + if (existedAtLastSync.has(oldPath)) deletePaths.add(oldPath) + continue + } + + if (item.operation === 'renamed') { + if (existedAtLastSync.has(oldPath)) deletePaths.add(oldPath) + upsertPaths.add(item.newPathname) + continue + } + if (item.operation === 'added' || item.operation === 'edited') { + upsertPaths.add(oldPath) + } + } + // if a file is renamed and a new file with the old name is created, + // then the name will be in both sets, need to exclude it from deletePaths + for (const path of upsertPaths) { + if (deletePaths.has(path)) deletePaths.delete(path) + } + + // this can happen when a file is created and then removed + if (deletePaths.size === 0 && upsertPaths.size === 0) return null + + const limit = pLimit(api.maxConcurrency) + const upsertEntries = await Promise.all( + [...upsertPaths].map(path => + limit(async () => { + const buffer = await HistoryManager.getProjectFileBuffer(projectId, currentVersion, path) + const sha = "DON_T_DELETE" // sha needs to be present so that the file is not detected as a deletion + return { path, sha, content: buffer } + }) + ) + ) + + const deleteEntries = [...deletePaths].map(path => ({ path, sha: null, content: null })) + const entries = [...deleteEntries, ...upsertEntries] + + const newCommit = await createCommitFromEntries({ + token, + repoFullName, + parentCommit: baseCommit, + entries, + message, + branch, + }) + + return newCommit +} + +function generateBranchName() { + const d = new Date() + const pad = n => `${n}`.padStart(2, '0') + return `overleaf-${d.getFullYear()}-${pad(d.getMonth() + 1)}-${pad(d.getDate())}-${pad(d.getHours())}${pad(d.getMinutes())}` +} + +async function getGitBlobMap(token, repoFullName, commit) { + const blobEntries = await api.listBlobsAtCommit(token, repoFullName, commit) + return Object.fromEntries( + blobEntries.map(({ path, sha }) => [path, sha]) + ) +} + +function getEntryHash(entry) { + if (!entry?.data) return null + return ( + entry.data.hash || + gitBlobShaFromString(entry.data.content) + ) +} + +function gitBlobShaFromString(str) { + if (!str) return null + const content = Buffer.from(str, 'utf8') + const header = Buffer.from(`blob ${content.length}\0`, 'utf8') + const store = Buffer.concat([header, content]) + + return createHash('sha1').update(store).digest('hex') +} + +function validateFileName(name) { + if ( + typeof name !== 'string' || + name.length === 0 || + name.includes('\0') + ) return { name } + + const normalized = normalize(name) + + if ( + normalized !== name || + normalized.startsWith('/') || + normalized.startsWith('../') + ) return { name } + + return null +} diff --git a/services/web/modules/gitlab-sync/app/src/GitSyncErrors.mjs b/services/web/modules/gitlab-sync/app/src/GitSyncErrors.mjs new file mode 100644 index 0000000000..730558432c --- /dev/null +++ b/services/web/modules/gitlab-sync/app/src/GitSyncErrors.mjs @@ -0,0 +1,28 @@ +import OError from '@overleaf/o-error' + +class GitNotLinkedError extends OError { + constructor(projectId) { + super('git server is not linked to project', { projectId }) + } +} + +class AlreadyExistsError extends OError {} +class GitConflictError extends OError {} +class InvalidTokenError extends OError {} +class ExpiredTokenError extends OError {} +class NotFoundError extends OError {} +class PermissionDeniedError extends OError {} +class RateLimitError extends OError {} +class ProviderRequestError extends OError {} + +export { + AlreadyExistsError, + GitNotLinkedError, + GitConflictError, + InvalidTokenError, + ExpiredTokenError, + NotFoundError, + PermissionDeniedError, + RateLimitError, + ProviderRequestError, +} diff --git a/services/web/modules/gitlab-sync/app/src/HistoryManager.mjs b/services/web/modules/gitlab-sync/app/src/HistoryManager.mjs new file mode 100644 index 0000000000..b0ff41e692 --- /dev/null +++ b/services/web/modules/gitlab-sync/app/src/HistoryManager.mjs @@ -0,0 +1,97 @@ +import Settings from '@overleaf/settings' +import logger from '@overleaf/logger' +import OError from '@overleaf/o-error' +import { fetchJson, fetchStream } from '@overleaf/fetch-utils' +import HistoryManager from '../../../../app/src/Features/History/HistoryManager.mjs' + +const urlBase = `${Settings.apis.project_history.url}/project` + +async function latestVersion(projectId) { + const url = `${urlBase}/${projectId}/version` + try { + await HistoryManager.promises.flushProject(projectId) + const json = await fetchJson(url) + return json.version + } catch (err) { + throw OError.tag(err, 'Failed to get the latest project version from history', { projectId }) + } +} + +async function getProjectSnapshot(projectId, version) { + const url = `${urlBase}/${projectId}/version/${version}` + try { + const snapshot = await fetchJson(url) + return snapshot.files || {} + } catch (err) { + throw OError.tag(err, 'Failed to get the project snapshot from history', { projectId }) + } +} + +async function getPathsAtVersion(projectId, version) { + const url = `${urlBase}/${projectId}/paths/version/${version}` + try { + const json = await fetchJson(url) + return json + } catch (err) { + throw OError.tag(err, 'Failed to fetch file paths from history', { projectId, version }) + } +} + +// response field 'diff' is an array of objects: +// { +// pathname: string, +// operation: 'added' | 'removed' | 'edited'| 'renamed' | undefined, +// ... +// } +async function getProjectFileTreeDiff(projectId, fromV, toV) { + const url = `${urlBase}/${projectId}/filetree/diff?from=${fromV}&to=${toV}` + try { + const json = await fetchJson(url) + return json.diff || [] + } catch (err) { + throw OError.tag(err, 'Failed to fetch filetree diff from history', { projectId, fromV, toV }) + } +} + +// return file from history as a base64 encoded string +async function getProjectFileBuffer(projectId, version, filePath) { + const url = `${urlBase}/${projectId}/version/${version}/${encodeURIComponent(filePath)}` + try { + const stream = await fetchStream(url) + return await _streamToBase64(stream) + } catch (err) { + throw OError.tag(err, 'Failed to fetch file from history', { projectId, version, filePath }) + } +} + +async function _streamToBase64(stream) { + const parts = [] + let leftover = null + + for await (const chunk of stream) { + const data = leftover ? Buffer.concat([leftover, chunk]) : chunk + + const remainder = data.length % 3 + const completeLength = data.length - remainder + + if (completeLength > 0) { + parts.push(data.subarray(0, completeLength).toString('base64')) + } + + leftover = remainder ? data.subarray(completeLength) : null + } + + if (leftover) { + parts.push(leftover.toString('base64')) + } + + return parts.join('') +} + +export default { + latestVersion, + getPathsAtVersion, + getProjectSnapshot, + getProjectFileBuffer, + getProjectFileTreeDiff, +} diff --git a/services/web/modules/gitlab-sync/app/src/SyncStateManager.mjs b/services/web/modules/gitlab-sync/app/src/SyncStateManager.mjs new file mode 100644 index 0000000000..7be2869144 --- /dev/null +++ b/services/web/modules/gitlab-sync/app/src/SyncStateManager.mjs @@ -0,0 +1,32 @@ +import { GitLabSyncProjectStates } from '../models/gitlabSyncProjectStates.mjs' +import Mongo from '../../../../app/src/Features/Helpers/Mongo.mjs' +const { normalizeQuery } = Mongo + +function getProjectState(projectId, projection = {}) { + return GitLabSyncProjectStates.findOne(normalizeQuery({ projectId }), projection).lean() +} + +function createProjectState(projectId, data) { + return GitLabSyncProjectStates.create({ + projectId: normalizeQuery(projectId), + ...data + }) +} + +function updateProjectState(projectId, data) { + return GitLabSyncProjectStates.updateOne( + normalizeQuery({ projectId }), + { $set: data }, + ) +} + +function removeProjectState(projectId) { + return GitLabSyncProjectStates.deleteMany(normalizeQuery({ projectId })) +} + +export default { + getProjectState, + createProjectState, + updateProjectState, + removeProjectState, +} diff --git a/services/web/modules/gitlab-sync/app/src/TokenManager.mjs b/services/web/modules/gitlab-sync/app/src/TokenManager.mjs new file mode 100644 index 0000000000..1f7f11a33c --- /dev/null +++ b/services/web/modules/gitlab-sync/app/src/TokenManager.mjs @@ -0,0 +1,64 @@ +import logger from '@overleaf/logger' +import OError from '@overleaf/o-error' +import Mongo from '../../../../app/src/Features/Helpers/Mongo.mjs' +import { GitLabSyncUserCredentials } from '../models/gitlabSyncUserCredentials.mjs' +import { AccessTokenEncryptor } from './AccessTokenEncryptorHelper.mjs' +import { InvalidTokenError } from './GitSyncErrors.mjs' + +const { normalizeQuery } = Mongo + +async function encryptAccessToken(accessToken) { + try { + return await AccessTokenEncryptor.encryptJson(accessToken) + } catch (err) { + throw OError.tag('failed to encrypt token', err) + } +} + +async function decryptAccessToken(tokenEncrypted) { + try { + return await AccessTokenEncryptor.decryptToJson(tokenEncrypted) + } catch (err) { + throw new InvalidTokenError('failed to decrypt token', { status: 401 }, err) + } +} + +// ------------------------- exports -------------------------- // +async function getUserToken(userId) { + const credentials = await GitLabSyncUserCredentials.findOne(normalizeQuery({ userId })) + if (!credentials) throw new InvalidTokenError('no user token', { userId, status: 400 }) + return await decryptAccessToken(credentials.gitlab) +} + +async function saveUserToken(userId, accessToken) { + const tokenEncrypted = await encryptAccessToken(accessToken) + await GitLabSyncUserCredentials.findOneAndUpdate( + normalizeQuery({ userId }), + { $set: { gitlab: tokenEncrypted } }, + { upsert: true } + ) +} + +// Try to revoke user's token, then remove it +async function removeUserToken(userId) { + let token + try { + token = await getUserToken(userId) + } catch (err) { + logger.warn({ err, userId }, 'failed to get user token') + } + // fire-and-forget, but still handle errors + if (token) { + GitLabApiClient.revokeToken(token).catch(err => { + logger.warn({ err, userId }, 'failed to revoke user token') + }) + } + await GitLabSyncUserCredentials.deleteOne(normalizeQuery({ userId })) + return +} + +export default { + saveUserToken, + getUserToken, + removeUserToken +} diff --git a/services/web/modules/gitlab-sync/frontend/js/components/gitlab-integration-card.tsx b/services/web/modules/gitlab-sync/frontend/js/components/gitlab-integration-card.tsx new file mode 100644 index 0000000000..217e9ed84f --- /dev/null +++ b/services/web/modules/gitlab-sync/frontend/js/components/gitlab-integration-card.tsx @@ -0,0 +1,41 @@ +import { useState } from 'react' +import { useTranslation } from 'react-i18next' +import { useProjectContext } from '@/shared/context/project-context' +import GitLabLogo from '@/shared/svgs/gitlab-logo' +import IntegrationCard from '@/features/integrations-panel/integration-card' +import GitSyncModal from './modals/git-sync-modal' +import { GitSyncModalStatus } from '../types/git-sync-types' + +const GitLabSyncCard = () => { + const { t } = useTranslation() + + const [showModal, setShowModal] = useState(false) + const { projectId, name } = useProjectContext() + const [modalStatus, setModalStatus] = useState('loading') + + return ( + <> + } + showPaywallBadge={false} + onClick={() => setShowModal(true)} + > + + { + setShowModal(false) + setModalStatus('loading') + }} + projectId={projectId} + projectName={name} + /> + + ) +} + +export default GitLabSyncCard diff --git a/services/web/modules/gitlab-sync/frontend/js/components/gitlab-sync-widget.tsx b/services/web/modules/gitlab-sync/frontend/js/components/gitlab-sync-widget.tsx new file mode 100644 index 0000000000..a609dd4de0 --- /dev/null +++ b/services/web/modules/gitlab-sync/frontend/js/components/gitlab-sync-widget.tsx @@ -0,0 +1,174 @@ +import { useCallback, useEffect, useState } from 'react' +import { useTranslation } from 'react-i18next' +import getMeta from '@/utils/meta' +import { getJSON, postJSON } from '@/infrastructure/fetch-json' +import useAsync from '@/shared/hooks/use-async' +import { debugConsole } from '@/utils/debugging' +import OLButton from '@/shared/components/ol/ol-button' +import { + OLModal, + OLModalBody, + OLModalFooter, + OLModalHeader, + OLModalTitle, +} from '@/shared/components/ol/ol-modal' +import OLNotification from '@/shared/components/ol/ol-notification' +import GitLabLogo from '@/shared/svgs/gitlab-logo' + +export const GitLabSyncWidget = function GitLabSyncWidget() { + const { t } = useTranslation() + const { appName } = getMeta('ol-ExposedSettings') + + const { + isLoading: isCheckingConn, + isError: isErrorConnCheck, + runAsync: runAsyncConnCheck, + data: isConnected, + setData: setConnState, + } = useAsync() + + const { + isLoading: isUnlinking, + isError: isErrorUnlink, + runAsync: runAsyncUnlink, + } = useAsync() + + const [showUnlinkModal, setShowUnlinkModal] = useState(false) + + const handleConnCheck = useCallback(() => { + runAsyncConnCheck(getJSON('/user/gitlab-sync/status')).catch(err => + debugConsole.error(err?.data?.message || err?.message || err), + ) + }, [runAsyncConnCheck]) + + useEffect(() => { + handleConnCheck() + }, [handleConnCheck]) + + const handleUnlink = useCallback(() => { + runAsyncUnlink(postJSON('/user/gitlab-sync/unlink')) + .then(() => setConnState(false)) + .catch(err => debugConsole.error(err?.data?.message || err?.message || err)) + .finally(() => setShowUnlinkModal(false)) + }, [runAsyncUnlink]) + + if (isCheckingConn) { + return ( +
    +
    + +
    + +
    +
    +

    GitLab

    +
    + +

    + {t('loading')}… +

    +
    +
    + ) + } + + return ( + <> +
    +
    + +
    + +
    +
    +

    GitLab

    +
    + +

    + {t('gitlab_sync_description', { appName })} +

    + + {isErrorConnCheck && ( + + )} + + {isErrorUnlink && ( + + )} +
    + +
    + {isConnected ? ( + setShowUnlinkModal(true)} + disabled={isUnlinking} + > + {isUnlinking ? t('unlinking') : t('unlink')} + + ) : isErrorConnCheck ? ( + + {t('reconnect')} + + ) : ( + + {t('link')} + + )} +
    +
    + + setShowUnlinkModal(false)} + backdrop="static" + > + + + {t('unlink_provider_account_title', { + provider: 'GitLab', + })} + + + + +

    + {t('unlink_gitlab_warning', { + provider: 'GitLab', + })} +

    +     + + + setShowUnlinkModal(false)} + > + {t('cancel')} + + + + {isUnlinking ? t('unlinking') : t('unlink')} + + +     + + ) +} diff --git a/services/web/modules/gitlab-sync/frontend/js/components/import-from-gitlab-menu.tsx b/services/web/modules/gitlab-sync/frontend/js/components/import-from-gitlab-menu.tsx new file mode 100644 index 0000000000..72b1ef0409 --- /dev/null +++ b/services/web/modules/gitlab-sync/frontend/js/components/import-from-gitlab-menu.tsx @@ -0,0 +1,12 @@ +import React from 'react' +import { useTranslation } from 'react-i18next' +import { DropdownItem } from '@/shared/components/dropdown/dropdown-menu' + +export default function ImportFromGitLabMenu({ onClick }: { onClick: (e: React.MouseEvent) => void }) { + const { t } = useTranslation() + return ( + + {t('import_from_gitlab')} + + ) +} \ No newline at end of file diff --git a/services/web/modules/gitlab-sync/frontend/js/components/import-from-gitlab-modal-wrapper.tsx b/services/web/modules/gitlab-sync/frontend/js/components/import-from-gitlab-modal-wrapper.tsx new file mode 100644 index 0000000000..f1879c1a61 --- /dev/null +++ b/services/web/modules/gitlab-sync/frontend/js/components/import-from-gitlab-modal-wrapper.tsx @@ -0,0 +1,218 @@ +import React, { useEffect } from 'react' +import { useTranslation } from 'react-i18next' +import useAsync from '@/shared/hooks/use-async' +import { debugConsole } from '@/utils/debugging' +import { + postJSON, + getJSON, +} from '@/infrastructure/fetch-json' +import getMeta from '@/utils/meta' +import { + OLModal, + OLModalBody, + OLModalFooter, + OLModalHeader, + OLModalTitle, +} from '@/shared/components/ol/ol-modal' +import OLButton from '@/shared/components/ol/ol-button' +import OLSpinner from '@/shared/components/ol/ol-spinner' +import OLNotification from '@/shared/components/ol/ol-notification' + +type GitSyncRepo = { + name: string + fullName: string + defaultBranchName: string +} + +type GitSyncReposResponse = { + repos?: GitSyncRepo[] +} + +function ImportFromGitLabModalContent({ handleHide }: { handleHide: () => void }) { + const { t } = useTranslation() + const { appName } = getMeta('ol-ExposedSettings') + + const { + isLoading, + isSuccess, + isError, + data, + runAsync, + } = useAsync() + + useEffect(() => { + runAsync(getJSON('/user/gitlab-sync/repos')) + .catch(err => debugConsole.error(err?.data?.message || err?.message || err)) + }, [runAsync]) + + const reposExist = data?.repos != null + const repos = reposExist ? data.repos : [] + + const { + isLoading: isImporting, + isSuccess: isImported, + isError: isErrorImport, + runAsync: runAsyncImport, + } = useAsync<{ projectId: string }>() + + useEffect(() => { + if (isImported) { + handleHide() + } + }, [handleHide, isImported]) + + const showLinkToGitLab = !isImporting && isSuccess && !reposExist + const showRepos = !isImporting && isSuccess && reposExist + + const handleImport = (repo: GitSyncRepo) => { + + runAsyncImport( + postJSON('/project/new/gitlab-sync', { + body: repo + }) + ) + .then(data => { + window.location.href = `/project/${data.projectId}` + }) + .catch(err => debugConsole.error(err?.data?.message || err?.message || err)) + } + + return ( + <> + + {t('import_from_gitlab')} + + + + {isLoading && ( + + + {t('loading_gitlab_repositories')} + + )} + + {isImporting && ( + + + {t('importing')} + + )} + + {isError && ( +
    + +
    + )} + + {showLinkToGitLab && ( +
    +

    + {t('link_to_gitlab_description', { appName })} +

    + + {t('link_to_gitlab')} + +
    + )} + + {showRepos && ( +
    + {repos.length === 0 ? ( +

    + {t('you_dont_have_any_repositories')} +

    + ) : ( + <> +

    + {t('select_gitlab_repository', { appName })} +

    +
    + + + {repos.map(repo => ( + + + + + ))} + +
    + {repo.name} + + + handleImport(repo)} + > + {t('import_to_sharelatex', { appName: 'Overleaf' })} + +
    +
    + + )} +
    + )} + + {isErrorImport && ( +
    + +
    + )} + +     + + + {!isImporting && ( + + + {t('cant_see_what_youre_looking_for_question')} + + + )} + + {t('cancel')} + + + + ) +} + +export default function ImportFromGitLabModal({ onHide }: { onHide: () => void }) { + return ( + + + + ) +} diff --git a/services/web/modules/gitlab-sync/frontend/js/components/modals/git-sync-cannot-export-modal.tsx b/services/web/modules/gitlab-sync/frontend/js/components/modals/git-sync-cannot-export-modal.tsx new file mode 100644 index 0000000000..8e09df8401 --- /dev/null +++ b/services/web/modules/gitlab-sync/frontend/js/components/modals/git-sync-cannot-export-modal.tsx @@ -0,0 +1,47 @@ +import { useTranslation, Trans } from 'react-i18next' +import OLNotification from '@/shared/components/ol/ol-notification' +import { + OLModalBody, + OLModalFooter, +} from '@/shared/components/ol/ol-modal' +import OLButton from '@/shared/components/ol/ol-button' +import { ProjectSyncState } from '../../types/git-sync-types' + +type GitSyncNeedPermissionModalProps = { + projectSyncState: ProjectSyncState + handleHide: () => void +} + +const GitSyncCannotExportModal = ({ projectSyncState, handleHide }: GitSyncCannotExportModalProps) => { + const { t } = useTranslation() + return ( + <> + + : <> + ]} + /> + )} + /> + + + + {t('close')} + + + + ) +} + +export default GitSyncCannotExportModal diff --git a/services/web/modules/gitlab-sync/frontend/js/components/modals/git-sync-confirm-unlink-modal.tsx b/services/web/modules/gitlab-sync/frontend/js/components/modals/git-sync-confirm-unlink-modal.tsx new file mode 100644 index 0000000000..f82b7ff0aa --- /dev/null +++ b/services/web/modules/gitlab-sync/frontend/js/components/modals/git-sync-confirm-unlink-modal.tsx @@ -0,0 +1,101 @@ +import { useTranslation, Trans } from 'react-i18next' +import useAsync from '@/shared/hooks/use-async' +import { deleteJSON } from '@/infrastructure/fetch-json' +import { OLModalBody, OLModalFooter } from '@/shared/components/ol/ol-modal' +import OLButton from '@/shared/components/ol/ol-button' +import OLNotification from '@/shared/components/ol/ol-notification' +import { debugConsole } from '@/utils/debugging' +import { ProjectSyncState, GitSyncModalStatus } from '../../types/git-sync-types' + +type GitSyncUnlinkModalProps = { + handleHide: () => void + setModalStatus: (modalStatus: GitSyncModalStatus) => void + projectSyncState: ProjectSyncState + projectId: string +} + +const GitSyncUnlinkModal = ({ + handleHide, + setModalStatus, + projectSyncState, + projectId, +}: GitSyncInitModalProps) => { + const { t } = useTranslation() + + const { + error, + isLoading, + runAsync, + } = useAsync() + + const handleUnlink = () => { + runAsync(deleteJSON(`/project/${projectId}/gitlab-sync`)) + .then(() => setModalStatus('need-export')) + .catch(err => debugConsole.error(err?.data?.message || err?.message || err)) + } + + return ( + <> + +

    + {t('project_linked_to')}:  + + {projectSyncState.repoFullName} + +

    + +
    + + {!error ? ( + + ) : ( + : <> + ]} + /> + ) : ( + t('something_went_wrong_server') + ) + } + /> + )} +     + + + +
    + + {t('confirm')} + + + + {t('cancel')} + +
    +     + + ) +} + +export default GitSyncUnlinkModal diff --git a/services/web/modules/gitlab-sync/frontend/js/components/modals/git-sync-conflict-modal.tsx b/services/web/modules/gitlab-sync/frontend/js/components/modals/git-sync-conflict-modal.tsx new file mode 100644 index 0000000000..1d53ea77d2 --- /dev/null +++ b/services/web/modules/gitlab-sync/frontend/js/components/modals/git-sync-conflict-modal.tsx @@ -0,0 +1,57 @@ +import { useTranslation, Trans } from 'react-i18next' +import getMeta from '@/utils/meta' +import { + OLModalBody, + OLModalFooter, +} from '@/shared/components/ol/ol-modal' +import OLButton from '@/shared/components/ol/ol-button' +import OLNotification from '@/shared/components/ol/ol-notification' + +import { ProjectSyncState, GitSyncModalStatus } from '../../types/git-sync-types' + +type GitSyncConflictModalProps = { + projectSyncState: ProjectSyncState + handleHide: () => void + setModalStatus: (modalStatus: GitSyncModalStatus) => void +} + +const GitSyncConflictModal = ({ projectSyncState, handleHide, setModalStatus }: GitSyncConflictModalProps) => { + const { t } = useTranslation() + const { appName } = getMeta('ol-ExposedSettings') + + return ( + <> + + +

    + ]} + /> +

    +     + + + {t('close')} + + setModalStatus('run-merge-resolved') + } + > + {t('continue_gitlab_merge')} + + + + ) +} + +export default GitSyncConflictModal diff --git a/services/web/modules/gitlab-sync/frontend/js/components/modals/git-sync-export-modal.tsx b/services/web/modules/gitlab-sync/frontend/js/components/modals/git-sync-export-modal.tsx new file mode 100644 index 0000000000..0861500a27 --- /dev/null +++ b/services/web/modules/gitlab-sync/frontend/js/components/modals/git-sync-export-modal.tsx @@ -0,0 +1,229 @@ +import { useTranslation } from 'react-i18next' +import { useState, useEffect } from 'react' +import useAsync from '@/shared/hooks/use-async' +import { debugConsole } from '@/utils/debugging' +import { + getJSON, + postJSON +} from '@/infrastructure/fetch-json' +import { + OLModalBody, + OLModalFooter, +} from '@/shared/components/ol/ol-modal' +import OLNotification from '@/shared/components/ol/ol-notification' +import OLButton from '@/shared/components/ol/ol-button' +import OLForm from '@/shared/components/ol/ol-form' +import OLFormGroup from '@/shared/components/ol/ol-form-group' +import OLFormControl from '@/shared/components/ol/ol-form-control' +import OLFormLabel from '@/shared/components/ol/ol-form-label' +import OLFormCheckbox from '@/shared/components/ol/ol-form-checkbox' +import OLFormSelect from '@/shared/components/ol/ol-form-select' +import OLRow from '@/shared/components/ol/ol-row' +import OLCol from '@/shared/components/ol/ol-col' +import { GitSyncModalStatus } from '../../types/git-sync-types' + +type OrgsResponse = { + user: string + orgs: string[] +} + +type GitSyncExportModalProps = { + projectId: string + projectName: string + handleHide: () => void + setModalStatus: (modalStatus: GitSyncModalStatus) => void +} + +const GitSyncExportModal = ({ + projectId, + projectName, + handleHide, + setModalStatus +}: GitSyncExportModalProps) => { + const { t } = useTranslation() + + const [selectedOwner, setSelectedOwner] = useState('') + const [errorMessage, setErrorMessage] = useState('') + const [repoName, setRepoName] = useState(projectName) + const [description, setDescription] = useState('') + const [visibility, setVisibility] = useState<'public' | 'private'>('private') + + const { + runAsync: runAsyncUserAndOrgs, + data: userAndOrgs, + error: errorUserAndOrgs + } = useAsync() + + useEffect(() => { + runAsyncUserAndOrgs(getJSON('/user/gitlab-sync/orgs')) + .then(userAndOrgs => setSelectedOwner(userAndOrgs?.user)) + .catch(err => debugConsole.error(err?.data?.message || err?.message || err)) + }, []) + + const { isLoading, error, setError, runAsync } = useAsync() + + const createRepo = () => { + const isPublic = visibility === 'public' + const org = selectedOwner === userAndOrgs?.user ? undefined : selectedOwner + + runAsync(postJSON(`/project/${projectId}/gitlab-sync/export`, { + body: { + name: repoName, + description, + isPublic, + org, + }, + })) + .then(() => setModalStatus('loading')) + .catch(err => { + debugConsole.error(err?.data?.message || err?.message || err) + if (!err?.data?.key) setError(t('something_went_wrong_server')) + else setError(t(err.data.key)) + }) + } + + return ( + <> + +

    {t('export_project_to_gitlab')}

    +

    {t('project_not_linked_to_gitlab')}

    + + {error && ( + + )} + + {errorUserAndOrgs && ( + + )} + + + + + + + {selectedOwner === userAndOrgs?.user + ? t('owner') + : t('organization')} + + setSelectedOwner(e.target.value)} + > + + {userAndOrgs?.orgs.map(org => ( + + ))} + + + + + + + + {t('repository_name')} + + setRepoName(e.target.value)} + /> + + + + + + + + + {t('description')} ({t('optional')}) + + setDescription(e.target.value)} + /> + + + + +
    + +
    + + {t('repository_visibility')} + + + + + + setVisibility('public')} + label={t('public', { defaultValue: 'Public' })} + description={t('gitlab_public_description')} + /> + + + + + + setVisibility('private')} + label={t('private', { defaultValue: 'Private' })} + description={t('gitlab_private_description')} + /> + + + +
    +     +     + + + + {t('cancel')} + + + + {t('create_project_in_gitlab')} + + + + ) +} + +export default GitSyncExportModal diff --git a/services/web/modules/gitlab-sync/frontend/js/components/modals/git-sync-loading-modal.tsx b/services/web/modules/gitlab-sync/frontend/js/components/modals/git-sync-loading-modal.tsx new file mode 100644 index 0000000000..6153864486 --- /dev/null +++ b/services/web/modules/gitlab-sync/frontend/js/components/modals/git-sync-loading-modal.tsx @@ -0,0 +1,54 @@ +import { useTranslation } from 'react-i18next' +import { + OLModalBody, + OLModalFooter, +} from '@/shared/components/ol/ol-modal' +import OLButton from '@/shared/components/ol/ol-button' +import OLNotification from '@/shared/components/ol/ol-notification' +import { GitSyncModalStatus } from '../../types/git-sync-types' + +type GitSyncLoadingModalProps = { + handleHide: () => void + setModalStatus: (modalStatus: GitSyncModalStatus) => void + errorMessage: string | null +} +const GitSyncLoadingModal = ({ handleHide, setModalStatus, errorMessage }: GitSyncLoadingModalProps) => { + const { t } = useTranslation() + + return ( + <> + +
    + + {t('checking_project_gitlab_status')} +
    +     + + {errorMessage && ( + + )} + + + {errorMessage && ( + setModalStatus('confirm-unlink')} + > + {t('unlink')} + + )} + + {t('cancel')} + + + + ) +} + +export default GitSyncLoadingModal diff --git a/services/web/modules/gitlab-sync/frontend/js/components/modals/git-sync-merge-modal.tsx b/services/web/modules/gitlab-sync/frontend/js/components/modals/git-sync-merge-modal.tsx new file mode 100644 index 0000000000..b54dcf01c0 --- /dev/null +++ b/services/web/modules/gitlab-sync/frontend/js/components/modals/git-sync-merge-modal.tsx @@ -0,0 +1,129 @@ +import { useEffect } from 'react' +import { useTranslation } from 'react-i18next' +import { debugConsole } from '@/utils/debugging' +import { postJSON } from '@/infrastructure/fetch-json' +import useAsync from '@/shared/hooks/use-async' +import clientId from '@/utils/client-id' +import { + OLModalBody, + OLModalFooter, +} from '@/shared/components/ol/ol-modal' +import OLButton from '@/shared/components/ol/ol-button' +import OLNotification from '@/shared/components/ol/ol-notification' +import { GitSyncModalStatus, ProjectSyncState } from '../../types/git-sync-types' + +import { useReferencesContext } from '@/features/ide-react/context/references-context' + +type GitSyncMergeModalProps = { + handleHide: () => void + modalStatus: GitSyncModalStatus + setModalStatus: (modalStatus: GitSyncModalStatus) => void + setProjectSyncState: (projectSyncState: ProjectSyncState) => void + commitMessage: string + setCommitMessage: (message: string) => void + projectId: string +} + +const GitSyncMergeModal = ({ + handleHide, + modalStatus, + setModalStatus, + setProjectSyncState, + commitMessage, + setCommitMessage, + projectId +}: GitSyncMergeModalProps) => { + + const { t } = useTranslation() + const { error, isError, isLoading, setError, runAsync } = useAsync() + const { indexAllReferences } = useReferencesContext() + + useEffect(() => { + if (modalStatus !== 'run-merge' && + modalStatus !== 'run-merge-resolved' + ) return + + const claimConflictIsResolved = (modalStatus === 'run-merge-resolved') + + runAsync( + postJSON(`/project/${projectId}/gitlab-sync/merge`, { + body: { message: commitMessage, claimConflictIsResolved }, + }) + ) + .then(async data => { + switch (data.mergeStatus) { + + case 'clean': + setModalStatus('merge-overview') + + await postJSON(`/project/${projectId}/flush`) + .catch(e => debugConsole.error(e)) + + indexAllReferences(true) + + break + + case 'conflict': + setModalStatus('show-conflict') + break + + default: + const message = `Unexpected merge status received: ${data.mergeStatus}` + const msError = new Error(message) + setError(msError) + throw msError + } + setProjectSyncState(data) + setCommitMessage('') + }) + .catch(err => { + debugConsole.error(err?.data?.message || err?.message || err) + if (err?.info?.statusCode === 403 || err?.info?.statusCode === 404) setModalStatus('loading') + }) + + }, [ + modalStatus, + projectId, + commitMessage, + runAsync, + ]) + + if (!isLoading && !isError) return + + return ( + <> + +
    + + + {t('importing_and_merging_changes_in_gitlab')} +
    +     + + {error && ( + + )} + + + + {t('close')} + + + + ) +} + +export default GitSyncMergeModal diff --git a/services/web/modules/gitlab-sync/frontend/js/components/modals/git-sync-merge-overview-modal.tsx b/services/web/modules/gitlab-sync/frontend/js/components/modals/git-sync-merge-overview-modal.tsx new file mode 100644 index 0000000000..cca142d3b8 --- /dev/null +++ b/services/web/modules/gitlab-sync/frontend/js/components/modals/git-sync-merge-overview-modal.tsx @@ -0,0 +1,238 @@ +import { useEffect } from 'react' +import { useTranslation } from 'react-i18next' +import useAsync from '@/shared/hooks/use-async' +import { getJSON } from '@/infrastructure/fetch-json' +import { OLModalBody, OLModalFooter } from '@/shared/components/ol/ol-modal' +import OLForm from '@/shared/components/ol/ol-form' +import OLFormGroup from '@/shared/components/ol/ol-form-group' +import OLFormControl from '@/shared/components/ol/ol-form-control' +import OLRow from '@/shared/components/ol/ol-row' +import OLCol from '@/shared/components/ol/ol-col' +import OLButton from '@/shared/components/ol/ol-button' +import OLIconButton from '@/shared/components/ol/ol-icon-button' +import OLNotification from '@/shared/components/ol/ol-notification' +import { debugConsole } from '@/utils/debugging' +import { ProjectSyncState, GitSyncModalStatus } from '../../types/git-sync-types' + +type GitSyncMergeOverviewModalProps = { + handleHide: () => void + setModalStatus: (modalStatus: GitSyncModalStatus) => void + projectSyncState: ProjectSyncState + projectId: string + commitMessage: string + setCommitMessage: (message: string) => void + +} + +type GitCommit = { + message: string + author: { + name: string + email: string + date: string + } + sha: string +} + +type UnmergedCommitsResponse = { + commits: GitCommit[] + diverged: boolean + isProjectUpdated: boolean +} + +const GitSyncMergeOverviewModal = ({ + handleHide, + setModalStatus, + projectSyncState, + projectId, + commitMessage, + setCommitMessage +}: GitSyncMergeOverviewModalProps) => { + const { t } = useTranslation() + const appName = 'Overleaf' + + const { + data, + setData, + error, + isSuccess, + isLoading, + runAsync, + } = useAsync() + + const loadUnmergedCommits = () => { + setData({}) + + runAsync(getJSON(`/project/${projectId}/gitlab-sync/merge/overview`)) + .then(data => { if (!data) setModalStatus('loading') }) + .catch(err => { + debugConsole.error(err?.data?.message || err?.message || err) + if (err?.info?.statusCode === 403 || + err?.info?.statusCode === 404 || + err?.info?.statusCode === 401 + ) setModalStatus('loading') + }) + } + + useEffect(() => { + loadUnmergedCommits() + }, [projectId, runAsync]) + + return ( + <> + +

    + {t('project_linked_to')}:  + + {projectSyncState.repoFullName} + +

    + + {isLoading && ( + <> +
    + + {t('loading')} +
    + + )} + + {isSuccess && ( + <> +
    + + {data.diverged && ( + + )} + +

    + {t('recent_commits_in_gitlab')} + + +

    + + {data.commits?.length === 0 && ( +
    +

    {t('no_new_commits_in_gitlab')}

    +
    + )} + + {data.commits?.length > 0 && ( +
    + {data.commits.map((commit: GitCommit) => ( +
    + + + {commit.sha.substring(0, 7)} + + + + + {commit.message} + + +
    + by {commit.author.name} <{commit.author.email}> +
    +
    + ))} +
    + )} + + {data.isProjectUpdated && ( + <> +
    + +

    {t('sync_project_to_gitlab_explanation', { appName })}

    + + + + setCommitMessage(e.target.value)} + /> + + + +     + + )} + + )} + + {error && ( + + )} + +     + + +
    + +
    + setModalStatus('confirm-unlink')} + disabled={!isSuccess && error?.info?.statusCode !== 404} + > + {t('unlink')} + +
    + +
    + setModalStatus('run-merge')} + disabled={!isSuccess} + > + {t('sync')} + + + + {t('close')} + +
    + +
    +     + + ) +} + +export default GitSyncMergeOverviewModal diff --git a/services/web/modules/gitlab-sync/frontend/js/components/modals/git-sync-modal.tsx b/services/web/modules/gitlab-sync/frontend/js/components/modals/git-sync-modal.tsx new file mode 100644 index 0000000000..7aa392ef56 --- /dev/null +++ b/services/web/modules/gitlab-sync/frontend/js/components/modals/git-sync-modal.tsx @@ -0,0 +1,226 @@ +import { useTranslation } from 'react-i18next' +import { useState, useEffect } from 'react' +import { + OLModalHeader, + OLModalTitle, + OLModal, +} from '@/shared/components/ol/ol-modal' + +import { getJSON } from '@/infrastructure/fetch-json' +import useAsync from '@/shared/hooks/use-async' +import { debugConsole } from '@/utils/debugging' + +import GitSyncLoadingModal from './git-sync-loading-modal' +import GitSyncMergeOverviewModal from './git-sync-merge-overview-modal' +import GitSyncMergeModal from './git-sync-merge-modal' +import GitSyncConflictModal from './git-sync-conflict-modal' +import GitSyncExportModal from './git-sync-export-modal' +import GitSyncNeedAuthModal from './git-sync-need-auth-modal' +import GitSyncNeedPermissionModal from './git-sync-need-permission-modal' +import GitSyncConfirmUnlinkModal from './git-sync-confirm-unlink-modal' +import GitSyncUnlinkUnavailableModal from './git-sync-unlink-unavailable-modal' +import GitSyncCannotExportModal from './git-sync-cannot-export-modal' + +import { GitSyncModalStatus, ProjectSyncState } from '../../types/git-sync-types' + +type GitSyncModalProps = { + show: boolean + handleHide: () => void + projectId: string + projectName: string + modalStatus: GitSyncModalStatus + setModalStatus: (modalStatus: GitSyncModalStatus) => void +} + +function GitSyncModal({ + show, + handleHide, + projectId, + projectName, + modalStatus, + setModalStatus, +}: GitSyncModalProps) { + + const { t } = useTranslation() + const [commitMessage, setCommitMessage] = useState('') + const [errorMessage, setErrorMessage] = useState('') + +const { + runAsync: runAsyncConn, + error: errorConn, + setError: setErrorConn, + data: dataConn, +} = useAsync() + +const { + runAsync, + data: projectSyncState, + setData: setProjectSyncState, + error, + setError, +} = useAsync() + + useEffect(() => { + if (!show) return + if (modalStatus !== 'loading') return + + const fetchProjectState = () => { + setError(null) + setErrorMessage('') + + runAsync(getJSON(`/project/${projectId}/gitlab-sync/state`)) + .then(pss => { + switch (pss.mergeStatus) { + // pss.ownerEmail is set to the email address of the project owner + // or null if the user is the project owner + // only owner can export project to git server + case 'need-export': + if (!pss.ownerEmail) setModalStatus('need-export') + else setModalStatus('cannot-export') + break + + case 'clean': + case 'diverged': + setModalStatus('merge-overview') + break + + case 'conflict': + setModalStatus('show-conflict') + break + + // user does not have push permission to repo + // pss.ownerEmail is set to the email address of the owner + // or null if the user is the project owner + // if the user is owner, it seems that repo is not accessible + case 'need-permission': { + if (!pss.ownerEmail) setModalStatus('unlink-unavailable') + else setModalStatus('need-permission') + break + } + } + }) + .catch(err => { + debugConsole.error(err?.data?.message || err?.message || err) + setErrorMessage(t('generic_something_went_wrong')) + }) + } + + const fetchConnectionStatus = () => { + setErrorConn(null) + setErrorMessage('') + + runAsyncConn(getJSON(`/user/gitlab-sync/status`)) + .then(isConnected => { + if (!isConnected) { + setModalStatus('need-auth') + return + } + fetchProjectState() + }) + .catch(err => { + debugConsole.error(err?.data?.message || err?.message || err) + setErrorMessage(t('generic_something_went_wrong')) + }) + } + + fetchConnectionStatus() + + }, [show, modalStatus, projectId, runAsyncConn, runAsync]) + + return ( + + + {t('sync_with_gitlab')} + + + {modalStatus === 'loading' && ( + + )} + + {modalStatus === 'need-export' && ( + + )} + + {modalStatus === 'cannot-export' && ( + + )} + + {modalStatus === 'need-auth' && ( + + )} + + {modalStatus === 'need-permission' && ( + + )} + + {modalStatus === 'merge-overview' && ( + + )} + + {modalStatus === 'confirm-unlink' && ( + + )} + + {modalStatus === 'unlink-unavailable' && ( + + )} + + {(modalStatus === 'run-merge' || + modalStatus === 'run-merge-resolved') && ( + + )} + + {modalStatus === 'show-conflict' && ( + + )} + + + ) +} + +export default GitSyncModal diff --git a/services/web/modules/gitlab-sync/frontend/js/components/modals/git-sync-need-auth-modal.tsx b/services/web/modules/gitlab-sync/frontend/js/components/modals/git-sync-need-auth-modal.tsx new file mode 100644 index 0000000000..d405ead2ef --- /dev/null +++ b/services/web/modules/gitlab-sync/frontend/js/components/modals/git-sync-need-auth-modal.tsx @@ -0,0 +1,43 @@ +import { useTranslation } from 'react-i18next' +import getMeta from '@/utils/meta' +import { + OLModalBody, + OLModalFooter, +} from '@/shared/components/ol/ol-modal' +import OLButton from '@/shared/components/ol/ol-button' + +const GitSyncNeedAuthModal = ({ handleHide }: { handleHide: () => void }) => { + const { t } = useTranslation() + const { appName } = getMeta('ol-ExposedSettings') + return ( + <> + +

    {t('link_to_gitlab_description', { appName })}

    +     + + + {t('close')} + + + { + handleHide() + window.open( + '/user/gitlab-sync/oauth2', + 'gitlabAuth', + 'width=600,height=700' + ) + }} + > + {t('link_to_gitlab')} + + + + ) +} + +export default GitSyncNeedAuthModal diff --git a/services/web/modules/gitlab-sync/frontend/js/components/modals/git-sync-need-permission-modal.tsx b/services/web/modules/gitlab-sync/frontend/js/components/modals/git-sync-need-permission-modal.tsx new file mode 100644 index 0000000000..406ef729d8 --- /dev/null +++ b/services/web/modules/gitlab-sync/frontend/js/components/modals/git-sync-need-permission-modal.tsx @@ -0,0 +1,57 @@ +import { useTranslation, Trans } from 'react-i18next' +import OLNotification from '@/shared/components/ol/ol-notification' +import { + OLModalBody, + OLModalFooter, +} from '@/shared/components/ol/ol-modal' +import OLButton from '@/shared/components/ol/ol-button' +import { ProjectSyncState } from '../../types/git-sync-types' + +type GitSyncNeedPermissionModalProps = { + projectSyncState: ProjectSyncState + handleHide: () => void +} + +const GitSyncNeedPermissionModal = ({ projectSyncState, handleHide }: GitSyncConflictModalProps) => { + const { t } = useTranslation() + return ( + <> + + + ) : ( + <> + ), + projectSyncState.ownerEmail ? : <> + ]} + /> + )} + /> + + + + {t('close')} + + + + ) +} + +export default GitSyncNeedPermissionModal diff --git a/services/web/modules/gitlab-sync/frontend/js/components/modals/git-sync-unlink-unavailable-modal.tsx b/services/web/modules/gitlab-sync/frontend/js/components/modals/git-sync-unlink-unavailable-modal.tsx new file mode 100644 index 0000000000..4650bf4d6a --- /dev/null +++ b/services/web/modules/gitlab-sync/frontend/js/components/modals/git-sync-unlink-unavailable-modal.tsx @@ -0,0 +1,85 @@ +import { useTranslation, Trans } from 'react-i18next' +import useAsync from '@/shared/hooks/use-async' +import { deleteJSON } from '@/infrastructure/fetch-json' +import { OLModalBody, OLModalFooter } from '@/shared/components/ol/ol-modal' +import OLButton from '@/shared/components/ol/ol-button' +import OLNotification from '@/shared/components/ol/ol-notification' +import { debugConsole } from '@/utils/debugging' +import { GitSyncModalStatus } from '../../types/git-sync-types' + +type GitSyncUnlinkUnavailableModalProps = { + handleHide: () => void + setModalStatus: (modalStatus: GitSyncModalStatus) => void + projectId: string +} + +const GitSyncUnlinkUnavailableModal = ({ + handleHide, + setModalStatus, + projectId, +}: GitSyncInitModalProps) => { + const { t } = useTranslation() + + const { + error, + isLoading, + runAsync, + } = useAsync() + + const handleUnlink = () => { + runAsync(deleteJSON(`/project/${projectId}/gitlab-sync`)) + .then(() => setModalStatus('need-export')) + .catch(err => debugConsole.error(err?.data?.message || err?.message || err)) + } + + return ( + <> + + + + {error && ( + : <> + ]} + /> + ) : ( + t('something_went_wrong_server') + ) + } + /> + )} + + + +
    + + {t('unlink_gitlab_repository')} + + + + {t('cancel')} + +
    +     + + ) +} + +export default GitSyncUnlinkUnavailableModal diff --git a/services/web/modules/gitlab-sync/frontend/js/types/git-sync-types.ts b/services/web/modules/gitlab-sync/frontend/js/types/git-sync-types.ts new file mode 100644 index 0000000000..325cee8424 --- /dev/null +++ b/services/web/modules/gitlab-sync/frontend/js/types/git-sync-types.ts @@ -0,0 +1,19 @@ +export type GitSyncModalStatus = + 'loading' + | 'need-export' + | 'cannot-export' + | 'need-auth' + | 'need-permission' + | 'merge-overview' + | 'confirm-unlink' + | 'unlink-unavailable' + | 'run-merge' + | 'run-merge-resolved' + | 'show-conflict' + +export type ProjectSyncState = { + mergeStatus: 'clean' | 'conflict' | 'diverged' | 'need-export' | 'need-permission' + repoFullName: string + unmergedBranchName: string | null + ownerEmail?: string +} diff --git a/services/web/modules/gitlab-sync/frontend/styles/gitlab-sync.scss b/services/web/modules/gitlab-sync/frontend/styles/gitlab-sync.scss new file mode 100644 index 0000000000..a992b6369f --- /dev/null +++ b/services/web/modules/gitlab-sync/frontend/styles/gitlab-sync.scss @@ -0,0 +1,8 @@ +.gitlab-sync-commits-heading { + align-items: center; + display: flex; + font-size: var(--font-size-03); + font-weight: 600; + justify-content: space-between; + line-height: var(--line-height-03); +} diff --git a/services/web/modules/gitlab-sync/index.mjs b/services/web/modules/gitlab-sync/index.mjs new file mode 100644 index 0000000000..cc7316d6e1 --- /dev/null +++ b/services/web/modules/gitlab-sync/index.mjs @@ -0,0 +1,55 @@ +import Settings from '@overleaf/settings' +import Modules from '../../app/src/infrastructure/Modules.mjs' +import logger from '@overleaf/logger' + +let GitLabSyncModule = {} +if (process.env.GITLAB_SYNC_ENABLED?.toLowerCase() === 'true') { + logger.debug({}, 'Enabling GitLab Sync module') + + // The GITLAB_URL variable has to be set for this module to work, otherwise the module will not be enabled + if (process.env.GITLAB_SYNC_URL === undefined || process.env.GITLAB_SYNC_URL === '') { + logger.warn({}, 'GitLab Sync module is enabled but GITLAB_SYNC_URL is not set, stopping module initialization') + } else { + const [{ default: GitLabSyncRouter }, + { default: SyncStateManager }, + { default: TokenManager } + ] = + await Promise.all([ + import('./app/src/GitLabSyncRouter.mjs'), + import('./app/src/SyncStateManager.mjs'), + import('./app/src/TokenManager.mjs'), + ]) + + const siteUrl = Settings.siteUrl.replace(/\/+$/, '') || 'http://localhost' + Settings.gitlabSync = { + clientID: process.env.GITLAB_SYNC_CLIENT_ID, + clientSecret: process.env.GITLAB_SYNC_CLIENT_SECRET, + callbackURL: `${siteUrl}/user/gitlab-sync/oauth2/callback`, + }, + + // Delete project sync state from mongo (hook 'projectExpired') + Modules.hooks.attach('projectExpired', async projectId => { + try { + await SyncStateManager.removeProjectState(projectId) + logger.debug({ projectId }, 'on project expire: removed Git sync state') + } catch (err) { + logger.warn({ projectId, err }, 'on project expire: failed to remove Git sync state') + } + }) + + // Delete user gitlab token from mongo (hook 'expireDeletedUser') + Modules.hooks.attach('expireDeletedUser', async userId => { + try { + await TokenManager.removeUserToken(userId) + } catch (err) { + logger.warn({ userId, err }, 'on user expire: failed removing user token') + } + }) + + GitLabSyncModule = { + router: GitLabSyncRouter, + } + } +} + +export default GitLabSyncModule diff --git a/services/web/test/unit/src/infrastructure/Features.test.mjs b/services/web/test/unit/src/infrastructure/Features.test.mjs index 8f0ee593ff..978b9683b7 100644 --- a/services/web/test/unit/src/infrastructure/Features.test.mjs +++ b/services/web/test/unit/src/infrastructure/Features.test.mjs @@ -55,6 +55,7 @@ describe('Features', function () { expect(ctx.Features.hasFeature('analytics')).to.be.false expect(ctx.Features.hasFeature('git-bridge')).to.be.false expect(ctx.Features.hasFeature('github-sync')).to.be.false + expect(ctx.Features.hasFeature('gitlab-sync')).to.be.false expect(ctx.Features.hasFeature('homepage')).to.be.false expect(ctx.Features.hasFeature('link-url')).to.be.false expect(ctx.Features.hasFeature('oauth')).to.be.false @@ -78,6 +79,7 @@ describe('Features', function () { expect(ctx.Features.hasFeature('analytics')).to.be.false expect(ctx.Features.hasFeature('git-bridge')).to.be.false expect(ctx.Features.hasFeature('github-sync')).to.be.false + expect(ctx.Features.hasFeature('gitlab-sync')).to.be.false expect(ctx.Features.hasFeature('homepage')).to.be.false expect(ctx.Features.hasFeature('link-url')).to.be.false expect(ctx.Features.hasFeature('oauth')).to.be.false @@ -108,6 +110,7 @@ describe('Features', function () { expect(ctx.Features.hasFeature('link-url')).to.be.false expect(ctx.Features.hasFeature('git-bridge')).to.be.false expect(ctx.Features.hasFeature('github-sync')).to.be.false + expect(ctx.Features.hasFeature('gitlab-sync')).to.be.false expect(ctx.Features.hasFeature('homepage')).to.be.false expect(ctx.Features.hasFeature('oauth')).to.be.false expect(ctx.Features.hasFeature('saml')).to.be.false @@ -118,6 +121,7 @@ describe('Features', function () { ctx.settings.enableHomepage = true ctx.settings.enableGitBridge = true ctx.settings.enableGithubSync = true + ctx.settings.enableGitLabSync = true ctx.settings.enableSaml = true ctx.settings.oauth = true ctx.settings.enabledLinkedFileTypes = ['url', 'project_file'] @@ -127,6 +131,7 @@ describe('Features', function () { expect(ctx.Features.hasFeature('affiliations')).to.be.true expect(ctx.Features.hasFeature('analytics')).to.be.true expect(ctx.Features.hasFeature('github-sync')).to.be.true + expect(ctx.Features.hasFeature('gitlab-sync')).to.be.true expect(ctx.Features.hasFeature('git-bridge')).to.be.true expect(ctx.Features.hasFeature('homepage')).to.be.true expect(ctx.Features.hasFeature('link-url')).to.be.true diff --git a/services/web/types/exposed-settings.ts b/services/web/types/exposed-settings.ts index 0bea6e641a..4ae4312389 100644 --- a/services/web/types/exposed-settings.ts +++ b/services/web/types/exposed-settings.ts @@ -53,6 +53,7 @@ export type ExposedSettings = { templatesEnabled?: boolean linkedInInsightsPartnerId?: string githubSyncEnabled: boolean + gitlabSyncEnabled: boolean zoteroEnabled: boolean enablePandocConversions: boolean }