From c60ceaf932f17041e27eef1e8a806d01f628f268 Mon Sep 17 00:00:00 2001 From: Liangjun Song <146005915+adai26@users.noreply.github.com> Date: Fri, 11 Apr 2025 09:06:46 +0100 Subject: [PATCH] Merge pull request #24466 from overleaf/ls-script-runner Script runner GitOrigin-RevId: 4cc7004f05177dba2a2151aa6db7e75fb679d11d --- .../web/app/src/infrastructure/mongodb.js | 1 + services/web/app/src/models/ScriptLog.mjs | 27 +++++++ services/web/app/views/layout-react.pug | 2 + .../layout/navbar-marketing-bootstrap-5.pug | 3 + .../web/app/views/layout/navbar-marketing.pug | 4 + .../views/layout/navbar-website-redesign.pug | 4 + .../bootstrap-5/navbar/admin-menu.tsx | 7 ++ .../bootstrap-5/navbar/default-navbar.tsx | 2 + .../types/default-navbar-metadata.ts | 1 + services/web/frontend/js/utils/meta.ts | 4 + ...403133427_create_index_for_script_logs.mjs | 32 ++++++++ .../web/scripts/example/track_progress.mjs | 31 ++++++++ services/web/scripts/lib/README.md | 55 ++++++++++++++ services/web/scripts/lib/ScriptRunner.mjs | 75 +++++++++++++++++++ 14 files changed, 248 insertions(+) create mode 100644 services/web/app/src/models/ScriptLog.mjs create mode 100644 services/web/migrations/20250403133427_create_index_for_script_logs.mjs create mode 100644 services/web/scripts/example/track_progress.mjs create mode 100644 services/web/scripts/lib/README.md create mode 100644 services/web/scripts/lib/ScriptRunner.mjs diff --git a/services/web/app/src/infrastructure/mongodb.js b/services/web/app/src/infrastructure/mongodb.js index 70753dd38d..aa7aa4ac44 100644 --- a/services/web/app/src/infrastructure/mongodb.js +++ b/services/web/app/src/infrastructure/mongodb.js @@ -81,6 +81,7 @@ const db = { userAuditLogEntries: internalDb.collection('userAuditLogEntries'), users: internalDb.collection('users'), onboardingDataCollection: internalDb.collection('onboardingDataCollection'), + scriptLogs: internalDb.collection('scriptLogs'), } const connectionPromise = mongoClient.connect() diff --git a/services/web/app/src/models/ScriptLog.mjs b/services/web/app/src/models/ScriptLog.mjs new file mode 100644 index 0000000000..9cc6b8655f --- /dev/null +++ b/services/web/app/src/models/ScriptLog.mjs @@ -0,0 +1,27 @@ +import Mongoose from '../infrastructure/Mongoose.js' + +export const ScriptLogSchema = new Mongoose.Schema( + { + canonicalName: { type: String, required: true }, + filePathAtVersion: { type: String, required: true }, + imageVersion: { type: String, required: true }, + podName: { type: String, required: true }, + startTime: { type: Date, default: Date.now }, + endTime: { type: Date, default: null }, + username: { type: String, required: true }, + status: { + type: String, + enum: ['pending', 'success', 'error'], + default: 'pending', + required: true, + }, + vars: { type: Object, required: true }, + progressLogs: { + type: [{ timestamp: Date, message: String }], + required: true, + }, + }, + { minimize: false, collection: 'scriptLogs' } +) + +export const ScriptLog = Mongoose.model('ScriptLog', ScriptLogSchema) diff --git a/services/web/app/views/layout-react.pug b/services/web/app/views/layout-react.pug index 4fba24c226..f3dc8e6a06 100644 --- a/services/web/app/views/layout-react.pug +++ b/services/web/app/views/layout-react.pug @@ -19,6 +19,7 @@ block append meta - const staffAccess = sessionUser?.staffAccess - const canDisplaySplitTestMenu = hasFeature('saas') && (canDisplayAdminMenu || staffAccess?.splitTestMetrics || staffAccess?.splitTestManagement) - const canDisplaySurveyMenu = hasFeature('saas') && canDisplayAdminMenu + - const canDisplayScriptLogMenu = hasFeature('saas') && canDisplayAdminMenu - const enableUpgradeButton = projectDashboardReact && usersBestSubscription && (usersBestSubscription.type === 'free' || usersBestSubscription.type === 'standalone-ai-add-on') - const showSignUpLink = hasFeature('registration-page') @@ -29,6 +30,7 @@ block append meta canDisplayAdminRedirect, canDisplaySplitTestMenu, canDisplaySurveyMenu, + canDisplayScriptLogMenu, enableUpgradeButton, suppressNavbarRight: !!suppressNavbarRight, suppressNavContentLinks: !!suppressNavContentLinks, diff --git a/services/web/app/views/layout/navbar-marketing-bootstrap-5.pug b/services/web/app/views/layout/navbar-marketing-bootstrap-5.pug index 1c527c93d2..ee94394bc4 100644 --- a/services/web/app/views/layout/navbar-marketing-bootstrap-5.pug +++ b/services/web/app/views/layout/navbar-marketing-bootstrap-5.pug @@ -27,6 +27,7 @@ nav.navbar.navbar-default.navbar-main.navbar-expand-lg(class={ - var canDisplayAdminRedirect = canRedirectToAdminDomain() - var canDisplaySplitTestMenu = hasFeature('saas') && (canDisplayAdminMenu || (getSessionUser() && getSessionUser().staffAccess && (getSessionUser().staffAccess.splitTestMetrics || getSessionUser().staffAccess.splitTestManagement))) - var canDisplaySurveyMenu = hasFeature('saas') && canDisplayAdminMenu + - var canDisplayScriptLogMenu = hasFeature('saas') && canDisplayAdminMenu if (typeof suppressNavbarRight === "undefined") button.navbar-toggler.collapsed( @@ -66,6 +67,8 @@ nav.navbar.navbar-default.navbar-main.navbar-expand-lg(class={ +dropdown-menu-link-item()(href="/admin/split-test") Manage Feature Flags if canDisplaySurveyMenu +dropdown-menu-link-item()(href="/admin/survey") Manage Surveys + if canDisplayScriptLogMenu + +dropdown-menu-link-item()(href="/admin/script-logs") View Script Logs // loop over header_extras each item in nav.header_extras diff --git a/services/web/app/views/layout/navbar-marketing.pug b/services/web/app/views/layout/navbar-marketing.pug index ebf698f726..c07c959543 100644 --- a/services/web/app/views/layout/navbar-marketing.pug +++ b/services/web/app/views/layout/navbar-marketing.pug @@ -30,6 +30,7 @@ nav.navbar.navbar-default.navbar-main - var canDisplayAdminRedirect = canRedirectToAdminDomain() - var canDisplaySplitTestMenu = hasFeature('saas') && (canDisplayAdminMenu || (getSessionUser() && getSessionUser().staffAccess && (getSessionUser().staffAccess.splitTestMetrics || getSessionUser().staffAccess.splitTestManagement))) - var canDisplaySurveyMenu = hasFeature('saas') && canDisplayAdminMenu + - var canDisplayScriptLogMenu = hasFeature('saas') && canDisplayAdminMenu if (typeof(suppressNavbarRight) == "undefined") .navbar-collapse.collapse#navbar-main-collapse @@ -66,6 +67,9 @@ nav.navbar.navbar-default.navbar-main if canDisplaySurveyMenu li a(href="/admin/survey") Manage Surveys + if canDisplayScriptLogMenu + li + a(href="/admin/script-logs") View Script Logs // loop over header_extras each item in nav.header_extras diff --git a/services/web/app/views/layout/navbar-website-redesign.pug b/services/web/app/views/layout/navbar-website-redesign.pug index 3e18ca94bc..c4b712e955 100644 --- a/services/web/app/views/layout/navbar-website-redesign.pug +++ b/services/web/app/views/layout/navbar-website-redesign.pug @@ -30,6 +30,7 @@ nav.navbar.navbar-default.navbar-main.website-redesign-navbar - var canDisplayAdminRedirect = canRedirectToAdminDomain() - var canDisplaySplitTestMenu = hasFeature('saas') && (canDisplayAdminMenu || (getSessionUser() && getSessionUser().staffAccess && (getSessionUser().staffAccess.splitTestMetrics || getSessionUser().staffAccess.splitTestManagement))) - var canDisplaySurveyMenu = hasFeature('saas') && canDisplayAdminMenu + - var canDisplayScriptLogMenu = hasFeature('saas') && canDisplayAdminMenu if (typeof(suppressNavbarRight) == "undefined") .navbar-collapse.collapse#navbar-main-collapse @@ -66,6 +67,9 @@ nav.navbar.navbar-default.navbar-main.website-redesign-navbar if canDisplaySurveyMenu li a(href="/admin/survey") Manage Surveys + if canDisplayScriptLogMenu + li + a(href="/admin/script-logs") View Script Logs // loop over header_extras each item in nav.header_extras diff --git a/services/web/frontend/js/features/ui/components/bootstrap-5/navbar/admin-menu.tsx b/services/web/frontend/js/features/ui/components/bootstrap-5/navbar/admin-menu.tsx index c50fa00911..fdc670423a 100644 --- a/services/web/frontend/js/features/ui/components/bootstrap-5/navbar/admin-menu.tsx +++ b/services/web/frontend/js/features/ui/components/bootstrap-5/navbar/admin-menu.tsx @@ -8,6 +8,7 @@ export default function AdminMenu({ canDisplayAdminRedirect, canDisplaySplitTestMenu, canDisplaySurveyMenu, + canDisplayScriptLogMenu, adminUrl, }: Pick< DefaultNavbarMetadata, @@ -15,6 +16,7 @@ export default function AdminMenu({ | 'canDisplayAdminRedirect' | 'canDisplaySplitTestMenu' | 'canDisplaySurveyMenu' + | 'canDisplayScriptLogMenu' | 'adminUrl' >) { const sendProjectListMB = useSendProjectListMB() @@ -57,6 +59,11 @@ export default function AdminMenu({ Manage Surveys ) : null} + {canDisplayScriptLogMenu ? ( + + View Script Logs + + ) : null} ) } diff --git a/services/web/frontend/js/features/ui/components/bootstrap-5/navbar/default-navbar.tsx b/services/web/frontend/js/features/ui/components/bootstrap-5/navbar/default-navbar.tsx index b7a843d99f..9066f5bbe7 100644 --- a/services/web/frontend/js/features/ui/components/bootstrap-5/navbar/default-navbar.tsx +++ b/services/web/frontend/js/features/ui/components/bootstrap-5/navbar/default-navbar.tsx @@ -22,6 +22,7 @@ function DefaultNavbar(props: DefaultNavbarMetadata) { canDisplayAdminRedirect, canDisplaySplitTestMenu, canDisplaySurveyMenu, + canDisplayScriptLogMenu, enableUpgradeButton, suppressNavbarRight, suppressNavContentLinks, @@ -101,6 +102,7 @@ function DefaultNavbar(props: DefaultNavbarMetadata) { canDisplayAdminRedirect={canDisplayAdminRedirect} canDisplaySplitTestMenu={canDisplaySplitTestMenu} canDisplaySurveyMenu={canDisplaySurveyMenu} + canDisplayScriptLogMenu={canDisplayScriptLogMenu} adminUrl={adminUrl} /> ) : null} diff --git a/services/web/frontend/js/features/ui/components/types/default-navbar-metadata.ts b/services/web/frontend/js/features/ui/components/types/default-navbar-metadata.ts index e2970d85ec..b9d6b0c506 100644 --- a/services/web/frontend/js/features/ui/components/types/default-navbar-metadata.ts +++ b/services/web/frontend/js/features/ui/components/types/default-navbar-metadata.ts @@ -10,6 +10,7 @@ export type DefaultNavbarMetadata = { canDisplayAdminRedirect: boolean canDisplaySplitTestMenu: boolean canDisplaySurveyMenu: boolean + canDisplayScriptLogMenu: boolean enableUpgradeButton: boolean suppressNavbarRight: boolean suppressNavContentLinks: boolean diff --git a/services/web/frontend/js/utils/meta.ts b/services/web/frontend/js/utils/meta.ts index 0f188c0976..12236949ad 100644 --- a/services/web/frontend/js/utils/meta.ts +++ b/services/web/frontend/js/utils/meta.ts @@ -51,6 +51,7 @@ import { Publisher } from '../../../types/subscription/dashboard/publisher' import { SubscriptionChangePreview } from '../../../types/subscription/subscription-change-preview' import { DefaultNavbarMetadata } from '@/features/ui/components/types/default-navbar-metadata' import { FooterMetadata } from '@/features/ui/components/types/footer-metadata' +import type { ScriptLogType } from '../../../modules/admin-panel/frontend/js/features/script-logs/script-log' export interface Meta { 'ol-ExposedSettings': ExposedSettings 'ol-addonPrices': Record @@ -162,6 +163,7 @@ export interface Meta { 'ol-oauthProviders': OAuthProviders 'ol-odcRole': string 'ol-overallThemes': OverallThemeMeta[] + 'ol-pages': number 'ol-passwordStrengthOptions': PasswordStrengthOptions 'ol-paywallPlans': { [key: string]: string } 'ol-personalAccessTokens': AccessToken[] | undefined @@ -188,6 +190,8 @@ export interface Meta { 'ol-recurlySubdomain': string 'ol-ro-mirror-on-client-no-local-storage': boolean 'ol-samlError': SAMLError | undefined + 'ol-script-log': ScriptLogType + 'ol-script-logs': ScriptLogType[] 'ol-settingsGroupSSO': { enabled: boolean } | undefined 'ol-settingsPlans': Plan[] 'ol-shouldAllowEditingDetails': boolean diff --git a/services/web/migrations/20250403133427_create_index_for_script_logs.mjs b/services/web/migrations/20250403133427_create_index_for_script_logs.mjs new file mode 100644 index 0000000000..0ca677a82a --- /dev/null +++ b/services/web/migrations/20250403133427_create_index_for_script_logs.mjs @@ -0,0 +1,32 @@ +/* eslint-disable no-unused-vars */ + +import Helpers from './lib/helpers.mjs' + +const tags = ['saas'] + +const indexes = [ + { + key: { canonicalName: 1 }, + name: 'canonicalName_1', + }, + { + key: { username: 1 }, + name: 'username_1', + }, +] + +const migrate = async client => { + const { db } = client + await Helpers.addIndexesToCollection(db.scriptLogs, indexes) +} + +const rollback = async client => { + const { db } = client + await Helpers.dropIndexesFromCollection(db.scriptLogs, indexes) +} + +export default { + tags, + migrate, + rollback, +} diff --git a/services/web/scripts/example/track_progress.mjs b/services/web/scripts/example/track_progress.mjs new file mode 100644 index 0000000000..531eb877a6 --- /dev/null +++ b/services/web/scripts/example/track_progress.mjs @@ -0,0 +1,31 @@ +// Import the script runner utility (adjust the path as needed) +import { scriptRunner } from '../lib/ScriptRunner.mjs' + +const subJobs = 30 + +/** + * Your script's main work goes here. + * It must be an async function and accept `trackProgress`. + * @param {(message: string) => Promise} trackProgress - Call this to log progress. + */ +async function main(trackProgress) { + for (let i = 0; i < subJobs; i++) { + await new Promise(resolve => setTimeout(() => resolve(), 1000)) + await trackProgress(`Job in progress ${i + 1}/${subJobs}`) + } + await trackProgress('Job finished') +} + +// Define any variables your script needs (optional) +const scriptVariables = { + subJobs, +} + +// --- Execute the script using the runner with async/await --- +try { + await scriptRunner(main, scriptVariables) + process.exit() +} catch (error) { + console.error(error) + process.exit(1) +} diff --git a/services/web/scripts/lib/README.md b/services/web/scripts/lib/README.md new file mode 100644 index 0000000000..8dae2db07f --- /dev/null +++ b/services/web/scripts/lib/README.md @@ -0,0 +1,55 @@ +# Script Runner + +## Overview + +The Script Runner wraps your script's main logic to automatically handle logging, status tracking (success/error), and progress updates. Script execution status can be viewed from "Script Logs" portal page. + +## Features + +- Automatically logs the start and end of your script. +- Records the final status ('success' or 'error'). +- Provides a simple function (`trackProgress`) to your script for logging custom progress steps. +- Captures script parameters and basic environment details. + +## Usage + +1. **Import `scriptRunner`**. +2. **Define your script's main logic** as an `async` function that accepts `trackProgress` as its argument (can ignore `trackProgress` if you don't need to track progress). +3. **Call `scriptRunner`**, passing your function and any variables it needs. +4. **Check script execution status** by visiting the "Script Logs" portal page using the URL printed in the console output. + +**Example:** + +```javascript +// Import the script runner utility (adjust the path as needed) +import { scriptRunner } from './lib/ScriptRunner.mjs' + +const subJobs = 30 + +/** + * Your script's main work goes here. + * It must be an async function and accept `trackProgress`. + * @param {(message: string) => void} trackProgress - Call this to log progress. + */ +async function main(trackProgress) { + for (let i = 0; i < subJobs; i++) { + await new Promise(resolve => setTimeout(() => resolve(), 1000)) + await trackProgress(`Job in progress ${i + 1}/${subJobs}`) + } + await trackProgress('Job finished') +} + +// Define any variables your script needs (optional) +const scriptVariables = { + subJobs, +} + +// --- Execute the script using the runner with async/await --- +try { + await scriptRunner(main, scriptVariables) + process.exit() +} catch (error) { + console.error(error) + process.exit(1) +} +``` diff --git a/services/web/scripts/lib/ScriptRunner.mjs b/services/web/scripts/lib/ScriptRunner.mjs new file mode 100644 index 0000000000..1708fa9310 --- /dev/null +++ b/services/web/scripts/lib/ScriptRunner.mjs @@ -0,0 +1,75 @@ +import { ScriptLog } from '../../app/src/models/ScriptLog.mjs' +import Settings from '@overleaf/settings' + +async function beforeScriptExecution(canonicalName, vars, scriptPath) { + let log = new ScriptLog({ + canonicalName, + filePathAtVersion: scriptPath, + podName: process.env.OL_POD_NAME, + username: process.env.OL_USERNAME, + imageVersion: process.env.OL_IMAGE_VERSION, + vars, + }) + log = await log.save() + console.log( + '\n==================================' + + '\n✨ Your script is running!' + + '\n📊 Track progress at:' + + `\n${Settings.adminUrl}/admin/script-log/${log._id}` + + '\n==================================\n' + ) + return log._id +} + +async function afterScriptExecution(logId, status) { + await ScriptLog.findByIdAndUpdate(logId, { status, endTime: new Date() }) +} + +/** + * @param {(trackProgress: (progress: string) => Promise) => Promise} main - Main function for the script + * @param {Object} vars - Variables to be used in the script + * @param {string} canonicalName - The canonical name of the script, default to filename + * @param {string} scriptPath - The file path of the script, default to process.argv[1] + * @returns {Promise} + * @async + */ +export async function scriptRunner( + main, + vars = {}, + canonicalName = process.argv[1].split('/').pop().split('.')[0], + scriptPath = process.argv[1] +) { + const isSaaS = Boolean(Settings.overleaf) + if (!isSaaS) { + await main(async message => { + console.warn(message) + }) + return + } + const logId = await beforeScriptExecution(canonicalName, vars, scriptPath) + + async function trackProgress(message) { + try { + console.warn(message) + await ScriptLog.findByIdAndUpdate(logId, { + $push: { + progressLogs: { + timestamp: new Date(), + message, + }, + }, + }) + } catch (error) { + console.error('Error tracking progress:', error) + } + } + + try { + await main(trackProgress) + } catch (error) { + await afterScriptExecution(logId, 'error') + throw error + } + + await afterScriptExecution(logId, 'success') +}