diff --git a/package-lock.json b/package-lock.json index 6d0d50ba77..51456fb0b2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -55085,11 +55085,11 @@ "mocha-multi-reporters": "^1.5.1", "mock-fs": "^5.1.2", "node-fetch": "^2.7.0", - "sandboxed-module": "^2.0.4", "sinon": "~9.0.1", "sinon-chai": "^3.7.0", "timekeeper": "2.2.0", - "typescript": "^5.0.4" + "typescript": "^5.0.4", + "vitest": "^4.0.0" } }, "services/clsi-cache": { @@ -55264,6 +55264,585 @@ "url": "https://github.com/sponsors/isaacs" } }, + "services/clsi/node_modules/@esbuild/aix-ppc64": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.27.2.tgz", + "integrity": "sha512-GZMB+a0mOMZs4MpDbj8RJp4cw+w1WV5NYD6xzgvzUJ5Ek2jerwfO2eADyI6ExDSUED+1X8aMbegahsJi+8mgpw==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "services/clsi/node_modules/@esbuild/android-arm": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.27.2.tgz", + "integrity": "sha512-DVNI8jlPa7Ujbr1yjU2PfUSRtAUZPG9I1RwW4F4xFB1Imiu2on0ADiI/c3td+KmDtVKNbi+nffGDQMfcIMkwIA==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "services/clsi/node_modules/@esbuild/android-arm64": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.27.2.tgz", + "integrity": "sha512-pvz8ZZ7ot/RBphf8fv60ljmaoydPU12VuXHImtAs0XhLLw+EXBi2BLe3OYSBslR4rryHvweW5gmkKFwTiFy6KA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "services/clsi/node_modules/@esbuild/android-x64": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.27.2.tgz", + "integrity": "sha512-z8Ank4Byh4TJJOh4wpz8g2vDy75zFL0TlZlkUkEwYXuPSgX8yzep596n6mT7905kA9uHZsf/o2OJZubl2l3M7A==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "services/clsi/node_modules/@esbuild/darwin-arm64": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.27.2.tgz", + "integrity": "sha512-davCD2Zc80nzDVRwXTcQP/28fiJbcOwvdolL0sOiOsbwBa72kegmVU0Wrh1MYrbuCL98Omp5dVhQFWRKR2ZAlg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "services/clsi/node_modules/@esbuild/darwin-x64": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.27.2.tgz", + "integrity": "sha512-ZxtijOmlQCBWGwbVmwOF/UCzuGIbUkqB1faQRf5akQmxRJ1ujusWsb3CVfk/9iZKr2L5SMU5wPBi1UWbvL+VQA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "services/clsi/node_modules/@esbuild/freebsd-arm64": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.27.2.tgz", + "integrity": "sha512-lS/9CN+rgqQ9czogxlMcBMGd+l8Q3Nj1MFQwBZJyoEKI50XGxwuzznYdwcav6lpOGv5BqaZXqvBSiB/kJ5op+g==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "services/clsi/node_modules/@esbuild/freebsd-x64": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.27.2.tgz", + "integrity": "sha512-tAfqtNYb4YgPnJlEFu4c212HYjQWSO/w/h/lQaBK7RbwGIkBOuNKQI9tqWzx7Wtp7bTPaGC6MJvWI608P3wXYA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "services/clsi/node_modules/@esbuild/linux-arm": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.27.2.tgz", + "integrity": "sha512-vWfq4GaIMP9AIe4yj1ZUW18RDhx6EPQKjwe7n8BbIecFtCQG4CfHGaHuh7fdfq+y3LIA2vGS/o9ZBGVxIDi9hw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "services/clsi/node_modules/@esbuild/linux-arm64": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.27.2.tgz", + "integrity": "sha512-hYxN8pr66NsCCiRFkHUAsxylNOcAQaxSSkHMMjcpx0si13t1LHFphxJZUiGwojB1a/Hd5OiPIqDdXONia6bhTw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "services/clsi/node_modules/@esbuild/linux-ia32": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.27.2.tgz", + "integrity": "sha512-MJt5BRRSScPDwG2hLelYhAAKh9imjHK5+NE/tvnRLbIqUWa+0E9N4WNMjmp/kXXPHZGqPLxggwVhz7QP8CTR8w==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "services/clsi/node_modules/@esbuild/linux-loong64": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.27.2.tgz", + "integrity": "sha512-lugyF1atnAT463aO6KPshVCJK5NgRnU4yb3FUumyVz+cGvZbontBgzeGFO1nF+dPueHD367a2ZXe1NtUkAjOtg==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "services/clsi/node_modules/@esbuild/linux-mips64el": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.27.2.tgz", + "integrity": "sha512-nlP2I6ArEBewvJ2gjrrkESEZkB5mIoaTswuqNFRv/WYd+ATtUpe9Y09RnJvgvdag7he0OWgEZWhviS1OTOKixw==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "services/clsi/node_modules/@esbuild/linux-ppc64": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.27.2.tgz", + "integrity": "sha512-C92gnpey7tUQONqg1n6dKVbx3vphKtTHJaNG2Ok9lGwbZil6DrfyecMsp9CrmXGQJmZ7iiVXvvZH6Ml5hL6XdQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "services/clsi/node_modules/@esbuild/linux-riscv64": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.27.2.tgz", + "integrity": "sha512-B5BOmojNtUyN8AXlK0QJyvjEZkWwy/FKvakkTDCziX95AowLZKR6aCDhG7LeF7uMCXEJqwa8Bejz5LTPYm8AvA==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "services/clsi/node_modules/@esbuild/linux-s390x": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.27.2.tgz", + "integrity": "sha512-p4bm9+wsPwup5Z8f4EpfN63qNagQ47Ua2znaqGH6bqLlmJ4bx97Y9JdqxgGZ6Y8xVTixUnEkoKSHcpRlDnNr5w==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "services/clsi/node_modules/@esbuild/linux-x64": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.27.2.tgz", + "integrity": "sha512-uwp2Tip5aPmH+NRUwTcfLb+W32WXjpFejTIOWZFw/v7/KnpCDKG66u4DLcurQpiYTiYwQ9B7KOeMJvLCu/OvbA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "services/clsi/node_modules/@esbuild/netbsd-arm64": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.27.2.tgz", + "integrity": "sha512-Kj6DiBlwXrPsCRDeRvGAUb/LNrBASrfqAIok+xB0LxK8CHqxZ037viF13ugfsIpePH93mX7xfJp97cyDuTZ3cw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "services/clsi/node_modules/@esbuild/netbsd-x64": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.27.2.tgz", + "integrity": "sha512-HwGDZ0VLVBY3Y+Nw0JexZy9o/nUAWq9MlV7cahpaXKW6TOzfVno3y3/M8Ga8u8Yr7GldLOov27xiCnqRZf0tCA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "services/clsi/node_modules/@esbuild/openbsd-arm64": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.27.2.tgz", + "integrity": "sha512-DNIHH2BPQ5551A7oSHD0CKbwIA/Ox7+78/AWkbS5QoRzaqlev2uFayfSxq68EkonB+IKjiuxBFoV8ESJy8bOHA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "services/clsi/node_modules/@esbuild/openbsd-x64": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.27.2.tgz", + "integrity": "sha512-/it7w9Nb7+0KFIzjalNJVR5bOzA9Vay+yIPLVHfIQYG/j+j9VTH84aNB8ExGKPU4AzfaEvN9/V4HV+F+vo8OEg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "services/clsi/node_modules/@esbuild/sunos-x64": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.27.2.tgz", + "integrity": "sha512-kMtx1yqJHTmqaqHPAzKCAkDaKsffmXkPHThSfRwZGyuqyIeBvf08KSsYXl+abf5HDAPMJIPnbBfXvP2ZC2TfHg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "services/clsi/node_modules/@esbuild/win32-arm64": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.27.2.tgz", + "integrity": "sha512-Yaf78O/B3Kkh+nKABUF++bvJv5Ijoy9AN1ww904rOXZFLWVc5OLOfL56W+C8F9xn5JQZa3UX6m+IktJnIb1Jjg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "services/clsi/node_modules/@esbuild/win32-ia32": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.27.2.tgz", + "integrity": "sha512-Iuws0kxo4yusk7sw70Xa2E2imZU5HoixzxfGCdxwBdhiDgt9vX9VUCBhqcwY7/uh//78A1hMkkROMJq9l27oLQ==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "services/clsi/node_modules/@esbuild/win32-x64": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.27.2.tgz", + "integrity": "sha512-sRdU18mcKf7F+YgheI/zGf5alZatMUTKj/jNS6l744f9u3WFu4v7twcUI9vu4mknF4Y9aDlblIie0IM+5xxaqQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "services/clsi/node_modules/@opentelemetry/api": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.0.tgz", + "integrity": "sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==", + "dev": true, + "license": "Apache-2.0", + "optional": true, + "peer": true, + "engines": { + "node": ">=8.0.0" + } + }, + "services/clsi/node_modules/@types/chai": { + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/@types/chai/-/chai-5.2.3.tgz", + "integrity": "sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/deep-eql": "*", + "assertion-error": "^2.0.1" + } + }, + "services/clsi/node_modules/@vitest/expect": { + "version": "4.0.16", + "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-4.0.16.tgz", + "integrity": "sha512-eshqULT2It7McaJkQGLkPjPjNph+uevROGuIMJdG3V+0BSR2w9u6J9Lwu+E8cK5TETlfou8GRijhafIMhXsimA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@standard-schema/spec": "^1.0.0", + "@types/chai": "^5.2.2", + "@vitest/spy": "4.0.16", + "@vitest/utils": "4.0.16", + "chai": "^6.2.1", + "tinyrainbow": "^3.0.3" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "services/clsi/node_modules/@vitest/expect/node_modules/chai": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/chai/-/chai-6.2.2.tgz", + "integrity": "sha512-NUPRluOfOiTKBKvWPtSD4PhFvWCqOi0BGStNWs57X9js7XGTprSmFoz5F0tWhR4WPjNeR9jXqdC7/UpSJTnlRg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "services/clsi/node_modules/@vitest/mocker": { + "version": "4.0.16", + "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-4.0.16.tgz", + "integrity": "sha512-yb6k4AZxJTB+q9ycAvsoxGn+j/po0UaPgajllBgt1PzoMAAmJGYFdDk0uCcRcxb3BrME34I6u8gHZTQlkqSZpg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/spy": "4.0.16", + "estree-walker": "^3.0.3", + "magic-string": "^0.30.21" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "msw": "^2.4.9", + "vite": "^6.0.0 || ^7.0.0-0" + }, + "peerDependenciesMeta": { + "msw": { + "optional": true + }, + "vite": { + "optional": true + } + } + }, + "services/clsi/node_modules/@vitest/pretty-format": { + "version": "4.0.16", + "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-4.0.16.tgz", + "integrity": "sha512-eNCYNsSty9xJKi/UdVD8Ou16alu7AYiS2fCPRs0b1OdhJiV89buAXQLpTbe+X8V9L6qrs9CqyvU7OaAopJYPsA==", + "dev": true, + "license": "MIT", + "dependencies": { + "tinyrainbow": "^3.0.3" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "services/clsi/node_modules/@vitest/runner": { + "version": "4.0.16", + "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-4.0.16.tgz", + "integrity": "sha512-VWEDm5Wv9xEo80ctjORcTQRJ539EGPB3Pb9ApvVRAY1U/WkHXmmYISqU5E79uCwcW7xYUV38gwZD+RV755fu3Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/utils": "4.0.16", + "pathe": "^2.0.3" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "services/clsi/node_modules/@vitest/snapshot": { + "version": "4.0.16", + "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-4.0.16.tgz", + "integrity": "sha512-sf6NcrYhYBsSYefxnry+DR8n3UV4xWZwWxYbCJUt2YdvtqzSPR7VfGrY0zsv090DAbjFZsi7ZaMi1KnSRyK1XA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/pretty-format": "4.0.16", + "magic-string": "^0.30.21", + "pathe": "^2.0.3" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "services/clsi/node_modules/@vitest/spy": { + "version": "4.0.16", + "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-4.0.16.tgz", + "integrity": "sha512-4jIOWjKP0ZUaEmJm00E0cOBLU+5WE0BpeNr3XN6TEF05ltro6NJqHWxXD0kA8/Zc8Nh23AT8WQxwNG+WeROupw==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "services/clsi/node_modules/@vitest/utils": { + "version": "4.0.16", + "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-4.0.16.tgz", + "integrity": "sha512-h8z9yYhV3e1LEfaQ3zdypIrnAg/9hguReGZoS7Gl0aBG5xgA410zBqECqmaF/+RkTggRsfnzc1XaAHA6bmUufA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/pretty-format": "4.0.16", + "tinyrainbow": "^3.0.3" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "services/clsi/node_modules/assertion-error": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-2.0.1.tgz", + "integrity": "sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + } + }, "services/clsi/node_modules/diff": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz", @@ -55273,6 +55852,89 @@ "node": ">=0.3.1" } }, + "services/clsi/node_modules/esbuild": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.27.2.tgz", + "integrity": "sha512-HyNQImnsOC7X9PMNaCIeAm4ISCQXs5a5YasTXVliKv4uuBo1dKrG0A+uQS8M5eXjVMnLg3WgXaKvprHlFJQffw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.27.2", + "@esbuild/android-arm": "0.27.2", + "@esbuild/android-arm64": "0.27.2", + "@esbuild/android-x64": "0.27.2", + "@esbuild/darwin-arm64": "0.27.2", + "@esbuild/darwin-x64": "0.27.2", + "@esbuild/freebsd-arm64": "0.27.2", + "@esbuild/freebsd-x64": "0.27.2", + "@esbuild/linux-arm": "0.27.2", + "@esbuild/linux-arm64": "0.27.2", + "@esbuild/linux-ia32": "0.27.2", + "@esbuild/linux-loong64": "0.27.2", + "@esbuild/linux-mips64el": "0.27.2", + "@esbuild/linux-ppc64": "0.27.2", + "@esbuild/linux-riscv64": "0.27.2", + "@esbuild/linux-s390x": "0.27.2", + "@esbuild/linux-x64": "0.27.2", + "@esbuild/netbsd-arm64": "0.27.2", + "@esbuild/netbsd-x64": "0.27.2", + "@esbuild/openbsd-arm64": "0.27.2", + "@esbuild/openbsd-x64": "0.27.2", + "@esbuild/openharmony-arm64": "0.27.2", + "@esbuild/sunos-x64": "0.27.2", + "@esbuild/win32-arm64": "0.27.2", + "@esbuild/win32-ia32": "0.27.2", + "@esbuild/win32-x64": "0.27.2" + } + }, + "services/clsi/node_modules/fdir": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", + "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12.0.0" + }, + "peerDependencies": { + "picomatch": "^3 || ^4" + }, + "peerDependenciesMeta": { + "picomatch": { + "optional": true + } + } + }, + "services/clsi/node_modules/magic-string": { + "version": "0.30.21", + "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.21.tgz", + "integrity": "sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.5.5" + } + }, + "services/clsi/node_modules/picomatch": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.3.tgz", + "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, "services/clsi/node_modules/sinon": { "version": "9.0.3", "resolved": "https://registry.npmjs.org/sinon/-/sinon-9.0.3.tgz", @@ -55304,6 +55966,197 @@ "node": ">=8" } }, + "services/clsi/node_modules/tinyexec": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-1.0.2.tgz", + "integrity": "sha512-W/KYk+NFhkmsYpuHq5JykngiOCnxeVL8v8dFnqxSD8qEEdRfXk1SDM6JzNqcERbcGYj9tMrDQBYV9cjgnunFIg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "services/clsi/node_modules/tinyrainbow": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/tinyrainbow/-/tinyrainbow-3.0.3.tgz", + "integrity": "sha512-PSkbLUoxOFRzJYjjxHJt9xro7D+iilgMX/C9lawzVuYiIdcihh9DXmVibBe8lmcFrRi/VzlPjBxbN7rH24q8/Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, + "services/clsi/node_modules/vite": { + "version": "7.3.1", + "resolved": "https://registry.npmjs.org/vite/-/vite-7.3.1.tgz", + "integrity": "sha512-w+N7Hifpc3gRjZ63vYBXA56dvvRlNWRczTdmCBBa+CotUzAPf5b7YMdMR/8CQoeYE5LX3W4wj6RYTgonm1b9DA==", + "dev": true, + "license": "MIT", + "dependencies": { + "esbuild": "^0.27.0", + "fdir": "^6.5.0", + "picomatch": "^4.0.3", + "postcss": "^8.5.6", + "rollup": "^4.43.0", + "tinyglobby": "^0.2.15" + }, + "bin": { + "vite": "bin/vite.js" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "funding": { + "url": "https://github.com/vitejs/vite?sponsor=1" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + }, + "peerDependencies": { + "@types/node": "^20.19.0 || >=22.12.0", + "jiti": ">=1.21.0", + "less": "^4.0.0", + "lightningcss": "^1.21.0", + "sass": "^1.70.0", + "sass-embedded": "^1.70.0", + "stylus": ">=0.54.8", + "sugarss": "^5.0.0", + "terser": "^5.16.0", + "tsx": "^4.8.1", + "yaml": "^2.4.2" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + }, + "jiti": { + "optional": true + }, + "less": { + "optional": true + }, + "lightningcss": { + "optional": true + }, + "sass": { + "optional": true + }, + "sass-embedded": { + "optional": true + }, + "stylus": { + "optional": true + }, + "sugarss": { + "optional": true + }, + "terser": { + "optional": true + }, + "tsx": { + "optional": true + }, + "yaml": { + "optional": true + } + } + }, + "services/clsi/node_modules/vitest": { + "version": "4.0.16", + "resolved": "https://registry.npmjs.org/vitest/-/vitest-4.0.16.tgz", + "integrity": "sha512-E4t7DJ9pESL6E3I8nFjPa4xGUd3PmiWDLsDztS2qXSJWfHtbQnwAWylaBvSNY48I3vr8PTqIZlyK8TE3V3CA4Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/expect": "4.0.16", + "@vitest/mocker": "4.0.16", + "@vitest/pretty-format": "4.0.16", + "@vitest/runner": "4.0.16", + "@vitest/snapshot": "4.0.16", + "@vitest/spy": "4.0.16", + "@vitest/utils": "4.0.16", + "es-module-lexer": "^1.7.0", + "expect-type": "^1.2.2", + "magic-string": "^0.30.21", + "obug": "^2.1.1", + "pathe": "^2.0.3", + "picomatch": "^4.0.3", + "std-env": "^3.10.0", + "tinybench": "^2.9.0", + "tinyexec": "^1.0.2", + "tinyglobby": "^0.2.15", + "tinyrainbow": "^3.0.3", + "vite": "^6.0.0 || ^7.0.0", + "why-is-node-running": "^2.3.0" + }, + "bin": { + "vitest": "vitest.mjs" + }, + "engines": { + "node": "^20.0.0 || ^22.0.0 || >=24.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "@edge-runtime/vm": "*", + "@opentelemetry/api": "^1.9.0", + "@types/node": "^20.0.0 || ^22.0.0 || >=24.0.0", + "@vitest/browser-playwright": "4.0.16", + "@vitest/browser-preview": "4.0.16", + "@vitest/browser-webdriverio": "4.0.16", + "@vitest/ui": "4.0.16", + "happy-dom": "*", + "jsdom": "*" + }, + "peerDependenciesMeta": { + "@edge-runtime/vm": { + "optional": true + }, + "@opentelemetry/api": { + "optional": true + }, + "@types/node": { + "optional": true + }, + "@vitest/browser-playwright": { + "optional": true + }, + "@vitest/browser-preview": { + "optional": true + }, + "@vitest/browser-webdriverio": { + "optional": true + }, + "@vitest/ui": { + "optional": true + }, + "happy-dom": { + "optional": true + }, + "jsdom": { + "optional": true + } + } + }, + "services/clsi/node_modules/yaml": { + "version": "2.8.2", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.8.2.tgz", + "integrity": "sha512-mplynKqc1C2hTVYxd0PU2xQAc22TI1vShAYGksCCfxbn/dFwnHTNi1bvYsBTkhdUNtGIf5xNOg938rrSSYvS9A==", + "dev": true, + "license": "ISC", + "optional": true, + "peer": true, + "bin": { + "yaml": "bin.mjs" + }, + "engines": { + "node": ">= 14.6" + }, + "funding": { + "url": "https://github.com/sponsors/eemeli" + } + }, "services/contacts": { "name": "@overleaf/contacts", "dependencies": { diff --git a/services/clsi/app.js b/services/clsi/app.js index 080c533de9..1b1f4e3208 100644 --- a/services/clsi/app.js +++ b/services/clsi/app.js @@ -1,36 +1,39 @@ // Metrics must be initialized before importing anything else -require('@overleaf/metrics/initialize') +import '@overleaf/metrics/initialize.js' -const CompileController = require('./app/js/CompileController') -const ContentController = require('./app/js/ContentController') -const Settings = require('@overleaf/settings') -const logger = require('@overleaf/logger') +import CompileController from './app/js/CompileController.js' +import ContentController from './app/js/ContentController.js' +import Settings from '@overleaf/settings' +import logger from '@overleaf/logger' +import LoggerSerializers from './app/js/LoggerSerializers.js' + +import Metrics from '@overleaf/metrics' +import smokeTest from './test/smoke/js/SmokeTests.js' +import ContentTypeMapper from './app/js/ContentTypeMapper.js' +import Errors from './app/js/Errors.js' +import OutputController from './app/js/OutputController.js' +import Path from 'node:path' + +import ProjectPersistenceManager from './app/js/ProjectPersistenceManager.js' +import OutputCacheManager from './app/js/OutputCacheManager.js' +import ContentCacheManager from './app/js/ContentCacheManager.js' + +import express from 'express' +import bodyParser from 'body-parser' + +import ForbidSymlinks from './app/js/StaticServerForbidSymlinks.js' + +import net from 'node:net' +import os from 'node:os' logger.initialize('clsi') -const LoggerSerializers = require('./app/js/LoggerSerializers') logger.logger.serializers.clsiRequest = LoggerSerializers.clsiRequest -const Metrics = require('@overleaf/metrics') - -const smokeTest = require('./test/smoke/js/SmokeTests') -const ContentTypeMapper = require('./app/js/ContentTypeMapper') -const Errors = require('./app/js/Errors') -const { createOutputZip } = require('./app/js/OutputController') - -const Path = require('node:path') - Metrics.open_sockets.monitor(true) Metrics.memory.monitor(logger) Metrics.leaked_sockets.monitor(logger) -const ProjectPersistenceManager = require('./app/js/ProjectPersistenceManager') -const OutputCacheManager = require('./app/js/OutputCacheManager') -const ContentCacheManager = require('./app/js/ContentCacheManager') - ProjectPersistenceManager.init() OutputCacheManager.init() - -const express = require('express') -const bodyParser = require('body-parser') const app = express() Metrics.injectMetricsRoute(app) @@ -126,8 +129,6 @@ app.get( CompileController.wordcount ) -const ForbidSymlinks = require('./app/js/StaticServerForbidSymlinks') - // create a static server which does not allow access to any symlinks // avoids possible mismatch of root directory between middleware check // and serving the files @@ -155,14 +156,14 @@ const staticOutputServer = ForbidSymlinks( app.get( '/project/:project_id/build/:build_id/output/output.zip', bodyParser.json(), - createOutputZip + OutputController.createOutputZip ) // This needs to be before GET /project/:project_id/user/:user_id/build/:build_id/output/* app.get( '/project/:project_id/user/:user_id/build/:build_id/output/output.zip', bodyParser.json(), - createOutputZip + OutputController.createOutputZip ) app.get( @@ -275,9 +276,6 @@ app.use(function (error, req, res, next) { } }) -const net = require('node:net') -const os = require('node:os') - let STATE = 'up' const loadTcpServer = net.createServer(function (socket) { @@ -360,7 +358,7 @@ const host = Settings.internal.clsi.host const loadTcpPort = Settings.internal.load_balancer_agent.load_port const loadHttpPort = Settings.internal.load_balancer_agent.local_port -if (!module.parent) { +if (import.meta.main) { // Called directly // handle uncaught exceptions when running in production @@ -394,4 +392,4 @@ if (!module.parent) { }) } -module.exports = app +export default app diff --git a/services/clsi/app/js/CLSICacheHandler.js b/services/clsi/app/js/CLSICacheHandler.js index 792866941b..5c49c9528b 100644 --- a/services/clsi/app/js/CLSICacheHandler.js +++ b/services/clsi/app/js/CLSICacheHandler.js @@ -1,23 +1,25 @@ // @ts-check -const crypto = require('node:crypto') -const fs = require('node:fs') -const Path = require('node:path') -const { pipeline } = require('node:stream/promises') -const { createGzip, createGunzip } = require('node:zlib') -const { crc32 } = require('node:zlib') -const tarFs = require('tar-fs') -const _ = require('lodash') -const { +import crypto from 'node:crypto' +import fs from 'node:fs' +import Path from 'node:path' +import { pipeline } from 'node:stream/promises' +import { crc32, createGzip, createGunzip } from 'node:zlib' +import tarFs from 'tar-fs' +import _ from 'lodash' +import { fetchNothing, fetchStream, RequestFailedError, -} = require('@overleaf/fetch-utils') -const logger = require('@overleaf/logger') -const Metrics = require('@overleaf/metrics') -const Settings = require('@overleaf/settings') -const { MeteredStream } = require('@overleaf/stream-utils') -const { CACHE_SUBDIR } = require('./OutputCacheManager') -const { isExtraneousFile } = require('./ResourceWriter') +} from '@overleaf/fetch-utils' +import logger from '@overleaf/logger' +import Metrics from '@overleaf/metrics' +import Settings from '@overleaf/settings' +import { MeteredStream } from '@overleaf/stream-utils' +import OutputCacheManager from './OutputCacheManager.js' +import ResourceWriter from './ResourceWriter.js' + +const { CACHE_SUBDIR } = OutputCacheManager +const { isExtraneousFile } = ResourceWriter const TIMEOUT = 5_000 /** @@ -426,7 +428,7 @@ async function downloadLatestCompileCache(projectId, userId, compileDir) { return !abort } -module.exports = { +export default { notifyCLSICacheAboutBuild, downloadLatestCompileCache, downloadOutputDotSynctexFromCompileCache, diff --git a/services/clsi/app/js/CommandRunner.js b/services/clsi/app/js/CommandRunner.js index 61d58ddd14..d98cfcc4d3 100644 --- a/services/clsi/app/js/CommandRunner.js +++ b/services/clsi/app/js/CommandRunner.js @@ -5,16 +5,16 @@ * DS207: Consider shorter variations of null checks * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ +import Settings from '@overleaf/settings' +import logger from '@overleaf/logger' let commandRunnerPath -const Settings = require('@overleaf/settings') -const logger = require('@overleaf/logger') if ((Settings.clsi != null ? Settings.clsi.dockerRunner : undefined) === true) { - commandRunnerPath = './DockerRunner' + commandRunnerPath = './DockerRunner.js' } else { - commandRunnerPath = './LocalCommandRunner' + commandRunnerPath = './LocalCommandRunner.js' } logger.debug({ commandRunnerPath }, 'selecting command runner for clsi') -const CommandRunner = require(commandRunnerPath) +const CommandRunner = (await import(commandRunnerPath)).default -module.exports = CommandRunner +export default CommandRunner diff --git a/services/clsi/app/js/CompileController.js b/services/clsi/app/js/CompileController.js index c5300c6ec3..17b031d661 100644 --- a/services/clsi/app/js/CompileController.js +++ b/services/clsi/app/js/CompileController.js @@ -1,12 +1,14 @@ -const Path = require('node:path') -const RequestParser = require('./RequestParser') -const CompileManager = require('./CompileManager') -const Settings = require('@overleaf/settings') -const Metrics = require('@overleaf/metrics') -const ProjectPersistenceManager = require('./ProjectPersistenceManager') -const logger = require('@overleaf/logger') -const Errors = require('./Errors') -const { notifyCLSICacheAboutBuild } = require('./CLSICacheHandler') +import Path from 'node:path' +import RequestParser from './RequestParser.js' +import CompileManager from './CompileManager.js' +import Settings from '@overleaf/settings' +import Metrics from '@overleaf/metrics' +import ProjectPersistenceManager from './ProjectPersistenceManager.js' +import logger from '@overleaf/logger' +import Errors from './Errors.js' +import CLSICacheHandler from './CLSICacheHandler.js' + +const { notifyCLSICacheAboutBuild } = CLSICacheHandler let lastSuccessfulCompileTimestamp = 0 @@ -272,7 +274,7 @@ function status(req, res, next) { res.send('OK') } -module.exports = { +export default { compile, stopCompile, clearCache, diff --git a/services/clsi/app/js/CompileManager.js b/services/clsi/app/js/CompileManager.js index 131c2a5cc6..e40f02b501 100644 --- a/services/clsi/app/js/CompileManager.js +++ b/services/clsi/app/js/CompileManager.js @@ -1,33 +1,33 @@ -const fsPromises = require('node:fs/promises') -const os = require('node:os') -const Path = require('node:path') -const { callbackify } = require('node:util') +import fsPromises from 'node:fs/promises' +import os from 'node:os' +import Path from 'node:path' +import { callbackify } from 'node:util' +import Settings from '@overleaf/settings' +import logger from '@overleaf/logger' +import OError from '@overleaf/o-error' +import ResourceWriter from './ResourceWriter.js' +import LatexRunner from './LatexRunner.js' +import OutputFileFinder from './OutputFileFinder.js' +import OutputCacheManager from './OutputCacheManager.js' +import ClsiMetrics from './Metrics.js' +import DraftModeManager from './DraftModeManager.js' +import TikzManager from './TikzManager.js' +import LockManager from './LockManager.js' +import Errors from './Errors.js' +import CommandRunner from './CommandRunner.js' +import ContentCacheMetrics from './ContentCacheMetrics.js' +import SynctexOutputParser from './SynctexOutputParser.js' +import CLSICacheHandler from './CLSICacheHandler.js' +import StatsManager from './StatsManager.js' +import SafeReader from './SafeReader.js' +import LatexMetrics from './LatexMetrics.js' +import { callbackifyMultiResult } from '@overleaf/promise-utils' -const Settings = require('@overleaf/settings') -const logger = require('@overleaf/logger') -const OError = require('@overleaf/o-error') - -const ResourceWriter = require('./ResourceWriter') -const LatexRunner = require('./LatexRunner') -const OutputFileFinder = require('./OutputFileFinder') -const OutputCacheManager = require('./OutputCacheManager') -const ClsiMetrics = require('./Metrics') -const DraftModeManager = require('./DraftModeManager') -const TikzManager = require('./TikzManager') -const LockManager = require('./LockManager') -const Errors = require('./Errors') -const CommandRunner = require('./CommandRunner') -const { emitPdfStats } = require('./ContentCacheMetrics') -const SynctexOutputParser = require('./SynctexOutputParser') -const { - downloadLatestCompileCache, - downloadOutputDotSynctexFromCompileCache, -} = require('./CLSICacheHandler') -const StatsManager = require('./StatsManager') -const SafeReader = require('./SafeReader') -const { enableLatexMkMetrics, addLatexFdbMetrics } = require('./LatexMetrics') -const { callbackifyMultiResult } = require('@overleaf/promise-utils') -const { shouldSkipMetrics } = require('./Metrics') +const { downloadLatestCompileCache, downloadOutputDotSynctexFromCompileCache } = + CLSICacheHandler +const { emitPdfStats } = ContentCacheMetrics +const { enableLatexMkMetrics, addLatexFdbMetrics } = LatexMetrics +const { shouldSkipMetrics } = ClsiMetrics const KNOWN_LATEXMK_RULES = new Set([ 'biber', @@ -848,7 +848,7 @@ function _emitMetrics(request, status, stats, timings) { } } -module.exports = { +export default { doCompileWithLock: callbackify(doCompileWithLock), stopCompile: callbackify(stopCompile), clearProject: callbackify(clearProject), diff --git a/services/clsi/app/js/ContentCacheManager.js b/services/clsi/app/js/ContentCacheManager.js index 5457c0dce0..dc69ce1208 100644 --- a/services/clsi/app/js/ContentCacheManager.js +++ b/services/clsi/app/js/ContentCacheManager.js @@ -2,21 +2,24 @@ * ContentCacheManager - maintains a cache of stream hashes from a PDF file */ -const { callbackify } = require('node:util') -const fs = require('node:fs') -const crypto = require('node:crypto') -const Path = require('node:path') -const Settings = require('@overleaf/settings') -const OError = require('@overleaf/o-error') -const pLimit = require('p-limit') -const { parseXrefTable } = require('./XrefParser') -const { +import { callbackify } from 'node:util' + +import fs from 'node:fs' +import crypto from 'node:crypto' +import Path from 'node:path' +import Settings from '@overleaf/settings' +import OError from '@overleaf/o-error' +import pLimit from 'p-limit' +import XrefParser from './XrefParser.js' +import { QueueLimitReachedError, TimedOutError, NoXrefTableError, -} = require('./Errors') -const workerpool = require('workerpool') -const Metrics = require('@overleaf/metrics') +} from './Errors.js' +import workerpool from 'workerpool' +import Metrics from '@overleaf/metrics' + +const { parseXrefTable } = XrefParser /** * @type {import('workerpool').WorkerPool} @@ -24,14 +27,17 @@ const Metrics = require('@overleaf/metrics') let WORKER_POOL // NOTE: Check for main thread to avoid recursive start of pool. if (Settings.pdfCachingEnableWorkerPool && workerpool.isMainThread) { - WORKER_POOL = workerpool.pool(Path.join(__dirname, 'ContentCacheWorker.js'), { - // Cap number of worker threads. - maxWorkers: Settings.pdfCachingWorkerPoolSize, - // Warmup workers. - minWorkers: Settings.pdfCachingWorkerPoolSize, - // Limit queue back-log - maxQueueSize: Settings.pdfCachingWorkerPoolBackLogLimit, - }) + WORKER_POOL = workerpool.pool( + Path.join(import.meta.dirname, 'ContentCacheWorker.js'), + { + // Cap number of worker threads. + maxWorkers: Settings.pdfCachingWorkerPoolSize, + // Warmup workers. + minWorkers: Settings.pdfCachingWorkerPoolSize, + // Limit queue back-log + maxQueueSize: Settings.pdfCachingWorkerPoolBackLogLimit, + } + ) setInterval(() => { const { totalWorkers, @@ -431,7 +437,7 @@ function promiseMapWithLimit(concurrency, array, fn) { return Promise.all(array.map(x => limit(() => fn(x)))) } -module.exports = { +export default { HASH_REGEX: /^[0-9a-f]{64}$/, update: callbackify(update), promises: { diff --git a/services/clsi/app/js/ContentCacheMetrics.js b/services/clsi/app/js/ContentCacheMetrics.js index cb617ff856..10697f138a 100644 --- a/services/clsi/app/js/ContentCacheMetrics.js +++ b/services/clsi/app/js/ContentCacheMetrics.js @@ -1,6 +1,6 @@ -const logger = require('@overleaf/logger') -const Metrics = require('@overleaf/metrics') -const os = require('node:os') +import logger from '@overleaf/logger' +import Metrics from '@overleaf/metrics' +import os from 'node:os' let CACHED_LOAD = { expires: -1, @@ -141,6 +141,6 @@ function emitPdfCachingStats(stats, timings, request) { ) } -module.exports = { +export default { emitPdfStats, } diff --git a/services/clsi/app/js/ContentCacheWorker.js b/services/clsi/app/js/ContentCacheWorker.js index 9ecb5b16dc..dab28c3526 100644 --- a/services/clsi/app/js/ContentCacheWorker.js +++ b/services/clsi/app/js/ContentCacheWorker.js @@ -1,4 +1,4 @@ -const workerpool = require('workerpool') -const ContentCacheManager = require('./ContentCacheManager') +import workerpool from 'workerpool' +import ContentCacheManager from './ContentCacheManager.js' workerpool.worker(ContentCacheManager.promises) diff --git a/services/clsi/app/js/ContentController.js b/services/clsi/app/js/ContentController.js index 96eba613ed..95a402fa69 100644 --- a/services/clsi/app/js/ContentController.js +++ b/services/clsi/app/js/ContentController.js @@ -1,7 +1,7 @@ -const Path = require('node:path') -const send = require('send') -const Settings = require('@overleaf/settings') -const OutputCacheManager = require('./OutputCacheManager') +import Path from 'node:path' +import send from 'send' +import Settings from '@overleaf/settings' +import OutputCacheManager from './OutputCacheManager.js' const ONE_DAY_S = 24 * 60 * 60 const ONE_DAY_MS = ONE_DAY_S * 1000 @@ -21,4 +21,4 @@ function getPdfRange(req, res, next) { send(req, path).pipe(res) } -module.exports = { getPdfRange } +export default { getPdfRange } diff --git a/services/clsi/app/js/ContentTypeMapper.js b/services/clsi/app/js/ContentTypeMapper.js index 5bf0c31423..5d0c5c113c 100644 --- a/services/clsi/app/js/ContentTypeMapper.js +++ b/services/clsi/app/js/ContentTypeMapper.js @@ -3,13 +3,13 @@ */ // TODO: This file was created by bulk-decaffeinate. // Fix any style issues and re-enable lint. +import Path from 'node:path' let ContentTypeMapper -const Path = require('node:path') // here we coerce html, css and js to text/plain, // otherwise choose correct mime type based on file extension, // falling back to octet-stream -module.exports = ContentTypeMapper = { +export default ContentTypeMapper = { map(path) { switch (Path.extname(path)) { case '.txt': diff --git a/services/clsi/app/js/DockerLockManager.js b/services/clsi/app/js/DockerLockManager.js index 97804b7499..700a365583 100644 --- a/services/clsi/app/js/DockerLockManager.js +++ b/services/clsi/app/js/DockerLockManager.js @@ -7,12 +7,12 @@ * DS207: Consider shorter variations of null checks * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ +import logger from '@overleaf/logger' let LockManager -const logger = require('@overleaf/logger') const LockState = {} // locks for docker container operations, by container name -module.exports = LockManager = { +export default LockManager = { MAX_LOCK_HOLD_TIME: 15000, // how long we can keep a lock MAX_LOCK_WAIT_TIME: 10000, // how long we wait for a lock LOCK_TEST_INTERVAL: 1000, // retry time diff --git a/services/clsi/app/js/DraftModeManager.js b/services/clsi/app/js/DraftModeManager.js index cf8ababc47..9c87de8095 100644 --- a/services/clsi/app/js/DraftModeManager.js +++ b/services/clsi/app/js/DraftModeManager.js @@ -1,6 +1,6 @@ -const fsPromises = require('node:fs/promises') -const { callbackify } = require('node:util') -const logger = require('@overleaf/logger') +import fsPromises from 'node:fs/promises' +import { callbackify } from 'node:util' +import logger from '@overleaf/logger' async function injectDraftMode(filename) { const content = await fsPromises.readFile(filename, { encoding: 'utf8' }) @@ -18,7 +18,7 @@ async function injectDraftMode(filename) { await fsPromises.writeFile(filename, modifiedContent, { encoding: 'utf8' }) } -module.exports = { +export default { injectDraftMode: callbackify(injectDraftMode), promises: { injectDraftMode }, } diff --git a/services/clsi/app/js/Errors.js b/services/clsi/app/js/Errors.js index 64c3c7b59a..83f3f04299 100644 --- a/services/clsi/app/js/Errors.js +++ b/services/clsi/app/js/Errors.js @@ -1,13 +1,10 @@ -/* eslint-disable - no-proto, - no-unused-vars, -*/ +/* eslint-disable no-proto + */ // TODO: This file was created by bulk-decaffeinate. // Fix any style issues and re-enable lint. -const OError = require('@overleaf/o-error') +import OError from '@overleaf/o-error' -let Errors -function NotFoundError(message) { +export function NotFoundError(message) { const error = new Error(message) error.name = 'NotFoundError' error.__proto__ = NotFoundError.prototype @@ -15,7 +12,7 @@ function NotFoundError(message) { } NotFoundError.prototype.__proto__ = Error.prototype -function FilesOutOfSyncError(message) { +export function FilesOutOfSyncError(message) { const error = new Error(message) error.name = 'FilesOutOfSyncError' error.__proto__ = FilesOutOfSyncError.prototype @@ -23,7 +20,7 @@ function FilesOutOfSyncError(message) { } FilesOutOfSyncError.prototype.__proto__ = Error.prototype -function AlreadyCompilingError(message) { +export function AlreadyCompilingError(message) { const error = new Error(message) error.name = 'AlreadyCompilingError' error.__proto__ = AlreadyCompilingError.prototype @@ -31,13 +28,13 @@ function AlreadyCompilingError(message) { } AlreadyCompilingError.prototype.__proto__ = Error.prototype -class QueueLimitReachedError extends OError {} -class TimedOutError extends OError {} -class NoXrefTableError extends OError {} -class TooManyCompileRequestsError extends OError {} -class InvalidParameter extends OError {} +export class QueueLimitReachedError extends OError {} +export class TimedOutError extends OError {} +export class NoXrefTableError extends OError {} +export class TooManyCompileRequestsError extends OError {} +export class InvalidParameter extends OError {} -module.exports = Errors = { +export default { QueueLimitReachedError, TimedOutError, NotFoundError, diff --git a/services/clsi/app/js/LatexMetrics.js b/services/clsi/app/js/LatexMetrics.js index 1a99813841..1783064324 100644 --- a/services/clsi/app/js/LatexMetrics.js +++ b/services/clsi/app/js/LatexMetrics.js @@ -339,4 +339,4 @@ function convertToArray(object) { .sort((a, b) => b.size - a.size) // sort by size descending } -module.exports = { enableLatexMkMetrics, addLatexMkMetrics, addLatexFdbMetrics } +export default { enableLatexMkMetrics, addLatexMkMetrics, addLatexFdbMetrics } diff --git a/services/clsi/app/js/LatexRunner.js b/services/clsi/app/js/LatexRunner.js index 118c78ea5f..323bd01456 100644 --- a/services/clsi/app/js/LatexRunner.js +++ b/services/clsi/app/js/LatexRunner.js @@ -1,10 +1,12 @@ -const Path = require('node:path') -const { promisify } = require('node:util') -const Settings = require('@overleaf/settings') -const logger = require('@overleaf/logger') -const CommandRunner = require('./CommandRunner') -const { addLatexMkMetrics } = require('./LatexMetrics') -const fs = require('node:fs') +import Path from 'node:path' +import { promisify } from 'node:util' +import Settings from '@overleaf/settings' +import logger from '@overleaf/logger' +import CommandRunner from './CommandRunner.js' +import LatexMetrics from './LatexMetrics.js' +import fs from 'node:fs' + +const { addLatexMkMetrics } = LatexMetrics const ProcessTable = {} // table of currently running jobs (pids or docker container names) @@ -203,7 +205,7 @@ function _buildLatexCommand(mainFile, opts = {}) { return command } -module.exports = { +export default { runLatex, killLatex, promises: { diff --git a/services/clsi/app/js/LocalCommandRunner.js b/services/clsi/app/js/LocalCommandRunner.js index aa62825443..ea9b85526b 100644 --- a/services/clsi/app/js/LocalCommandRunner.js +++ b/services/clsi/app/js/LocalCommandRunner.js @@ -11,15 +11,15 @@ * DS207: Consider shorter variations of null checks * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ +import { spawn } from 'node:child_process' +import { promisify } from 'node:util' +import _ from 'lodash' +import logger from '@overleaf/logger' let CommandRunner -const { spawn } = require('node:child_process') -const { promisify } = require('node:util') -const _ = require('lodash') -const logger = require('@overleaf/logger') logger.debug('using standard command runner') -module.exports = CommandRunner = { +export default CommandRunner = { run( projectId, command, @@ -106,7 +106,7 @@ module.exports = CommandRunner = { }, } -module.exports.promises = { +CommandRunner.promises = { run: promisify(CommandRunner.run), kill: promisify(CommandRunner.kill), } diff --git a/services/clsi/app/js/LockManager.js b/services/clsi/app/js/LockManager.js index a44810fe2e..f7dc4bcc19 100644 --- a/services/clsi/app/js/LockManager.js +++ b/services/clsi/app/js/LockManager.js @@ -1,8 +1,8 @@ -const logger = require('@overleaf/logger') -const Errors = require('./Errors') -const RequestParser = require('./RequestParser') -const Metrics = require('@overleaf/metrics') -const Settings = require('@overleaf/settings') +import logger from '@overleaf/logger' +import Errors from './Errors.js' +import RequestParser from './RequestParser.js' +import Metrics from '@overleaf/metrics' +import Settings from '@overleaf/settings' // The lock timeout should be higher than the maximum end-to-end compile time. // Here, we use the maximum compile timeout plus 2 minutes. @@ -63,4 +63,4 @@ class Lock { } } -module.exports = { acquire } +export default { acquire } diff --git a/services/clsi/app/js/LoggerSerializers.js b/services/clsi/app/js/LoggerSerializers.js index 20b4330b52..82d445f20e 100644 --- a/services/clsi/app/js/LoggerSerializers.js +++ b/services/clsi/app/js/LoggerSerializers.js @@ -1,4 +1,4 @@ -const Path = require('node:path') +import Path from 'node:path' const CLSI_REQUEST_SERIALIZED_PROPERTIES = [ 'compiler', @@ -16,7 +16,7 @@ const CLSI_REQUEST_SERIALIZED_PROPERTIES = [ 'syncType', ] -module.exports = { +export default { /** * Serializer for a CLSI request object. * Only includes properties useful for logging. diff --git a/services/clsi/app/js/Metrics.js b/services/clsi/app/js/Metrics.js index 98c4e48cd2..05a9518c69 100644 --- a/services/clsi/app/js/Metrics.js +++ b/services/clsi/app/js/Metrics.js @@ -1,4 +1,4 @@ -const { prom } = require('@overleaf/metrics') +import { prom } from '@overleaf/metrics' const COMPILE_TIME_BUCKETS = [ 0.5, 1, 1.5, 2, 3, 4, 5, 6, 8, 10, 15, 20, 25, 30, 45, 60, 75, 90, 120, 150, @@ -74,7 +74,7 @@ function shouldSkipMetrics(request) { ) } -module.exports = { +export default { compilesTotal, compileDurationSeconds, e2eCompileDurationSeconds, diff --git a/services/clsi/app/js/OutputCacheManager.js b/services/clsi/app/js/OutputCacheManager.js index 8336d0075a..1abdc7a6fc 100644 --- a/services/clsi/app/js/OutputCacheManager.js +++ b/services/clsi/app/js/OutputCacheManager.js @@ -1,21 +1,20 @@ -let OutputCacheManager -const { callbackify, promisify } = require('node:util') -const async = require('async') -const fs = require('node:fs') -const Path = require('node:path') -const logger = require('@overleaf/logger') -const _ = require('lodash') -const Settings = require('@overleaf/settings') -const crypto = require('node:crypto') -const Metrics = require('@overleaf/metrics') - -const OutputFileOptimiser = require('./OutputFileOptimiser') -const ContentCacheManager = require('./ContentCacheManager') -const { +import { callbackify, promisify } from 'node:util' +import async from 'async' +import fs from 'node:fs' +import Path from 'node:path' +import logger from '@overleaf/logger' +import _ from 'lodash' +import Settings from '@overleaf/settings' +import crypto from 'node:crypto' +import Metrics from '@overleaf/metrics' +import OutputFileOptimiser from './OutputFileOptimiser.js' +import ContentCacheManager from './ContentCacheManager.js' +import { QueueLimitReachedError, TimedOutError, NoXrefTableError, -} = require('./Errors') +} from './Errors.js' +let OutputCacheManager const OLDEST_BUILD_DIR = new Map() const PENDING_PROJECT_ACTIONS = new Map() @@ -101,7 +100,7 @@ async function queueDirOperation(dir, fn) { return p } -module.exports = OutputCacheManager = { +export default OutputCacheManager = { CONTENT_SUBDIR: 'content', CACHE_SUBDIR: 'generated-files', ARCHIVE_SUBDIR: 'archived-logs', diff --git a/services/clsi/app/js/OutputController.js b/services/clsi/app/js/OutputController.js index e5048c4c25..4a95d2ae73 100644 --- a/services/clsi/app/js/OutputController.js +++ b/services/clsi/app/js/OutputController.js @@ -1,6 +1,6 @@ -const OutputFileArchiveManager = require('./OutputFileArchiveManager') -const { expressify } = require('@overleaf/promise-utils') -const { pipeline } = require('node:stream/promises') +import OutputFileArchiveManager from './OutputFileArchiveManager.js' +import { expressify } from '@overleaf/promise-utils' +import { pipeline } from 'node:stream/promises' async function createOutputZip(req, res) { const { @@ -20,4 +20,4 @@ async function createOutputZip(req, res) { await pipeline(archive, res) } -module.exports = { createOutputZip: expressify(createOutputZip) } +export default { createOutputZip: expressify(createOutputZip) } diff --git a/services/clsi/app/js/OutputFileArchiveManager.js b/services/clsi/app/js/OutputFileArchiveManager.js index 64c5198392..dd8e52e2cc 100644 --- a/services/clsi/app/js/OutputFileArchiveManager.js +++ b/services/clsi/app/js/OutputFileArchiveManager.js @@ -1,10 +1,10 @@ -const archiver = require('archiver') -const OutputCacheManager = require('./OutputCacheManager') -const OutputFileFinder = require('./OutputFileFinder') -const Settings = require('@overleaf/settings') -const { open } = require('node:fs/promises') -const { NotFoundError } = require('./Errors') -const logger = require('@overleaf/logger') +import archiver from 'archiver' +import OutputCacheManager from './OutputCacheManager.js' +import OutputFileFinder from './OutputFileFinder.js' +import Settings from '@overleaf/settings' +import { open } from 'node:fs/promises' +import { NotFoundError } from './Errors.js' +import logger from '@overleaf/logger' // NOTE: Updating this list requires a corresponding change in // * services/web/frontend/js/features/pdf-preview/util/file-list.ts @@ -20,7 +20,7 @@ function getContentDir(projectId, userId) { return `${Settings.path.outputDir}/${subDir}/` } -module.exports = { +export default { async archiveFilesForBuild(projectId, userId, build) { logger.debug({ projectId, userId, build }, 'Will create zip file') diff --git a/services/clsi/app/js/OutputFileFinder.js b/services/clsi/app/js/OutputFileFinder.js index e62038c614..5b581a9a7d 100644 --- a/services/clsi/app/js/OutputFileFinder.js +++ b/services/clsi/app/js/OutputFileFinder.js @@ -1,6 +1,6 @@ -const Path = require('node:path') -const fs = require('node:fs') -const { callbackifyMultiResult } = require('@overleaf/promise-utils') +import Path from 'node:path' +import fs from 'node:fs' +import { callbackifyMultiResult } from '@overleaf/promise-utils' async function walkFolder(compileDir, d, files, allEntries) { const dirents = await fs.promises.readdir(Path.join(compileDir, d), { @@ -42,7 +42,7 @@ async function findOutputFiles(resources, directory) { } } -module.exports = { +export default { findOutputFiles: callbackifyMultiResult(findOutputFiles, [ 'outputFiles', 'allEntries', diff --git a/services/clsi/app/js/OutputFileOptimiser.js b/services/clsi/app/js/OutputFileOptimiser.js index 5859c006da..57b20afcec 100644 --- a/services/clsi/app/js/OutputFileOptimiser.js +++ b/services/clsi/app/js/OutputFileOptimiser.js @@ -12,15 +12,15 @@ * DS207: Consider shorter variations of null checks * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ +import fs from 'node:fs' +import Path from 'node:path' +import { spawn } from 'node:child_process' +import logger from '@overleaf/logger' +import Metrics from '@overleaf/metrics' +import _ from 'lodash' let OutputFileOptimiser -const fs = require('node:fs') -const Path = require('node:path') -const { spawn } = require('node:child_process') -const logger = require('@overleaf/logger') -const Metrics = require('@overleaf/metrics') -const _ = require('lodash') -module.exports = OutputFileOptimiser = { +export default OutputFileOptimiser = { optimiseFile(src, dst, callback) { // check output file (src) and see if we can optimise it, storing // the result in the build directory (dst) diff --git a/services/clsi/app/js/ProjectPersistenceManager.js b/services/clsi/app/js/ProjectPersistenceManager.js index 41cdd07f4d..05fb1c3616 100644 --- a/services/clsi/app/js/ProjectPersistenceManager.js +++ b/services/clsi/app/js/ProjectPersistenceManager.js @@ -7,17 +7,17 @@ * DS207: Consider shorter variations of null checks * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ +import UrlCache from './UrlCache.js' +import CompileManager from './CompileManager.js' +import async from 'async' +import logger from '@overleaf/logger' +import Metrics from '@overleaf/metrics' +import Settings from '@overleaf/settings' +import { callbackify } from 'node:util' +import Path from 'node:path' +import fs from 'node:fs' let ProjectPersistenceManager -const UrlCache = require('./UrlCache') -const CompileManager = require('./CompileManager') -const async = require('async') -const logger = require('@overleaf/logger') const oneDay = 24 * 60 * 60 * 1000 -const Metrics = require('@overleaf/metrics') -const Settings = require('@overleaf/settings') -const { callbackify } = require('node:util') -const Path = require('node:path') -const fs = require('node:fs') // projectId -> timestamp mapping. const LAST_ACCESS = new Map() @@ -87,7 +87,7 @@ async function refreshExpiryTimeout() { ) } -module.exports = ProjectPersistenceManager = { +export default ProjectPersistenceManager = { EXPIRY_TIMEOUT: Settings.project_cache_length_ms || oneDay * 2.5, isAnyDiskLow() { diff --git a/services/clsi/app/js/RequestParser.js b/services/clsi/app/js/RequestParser.js index 4e9d722921..cd37b8b40e 100644 --- a/services/clsi/app/js/RequestParser.js +++ b/services/clsi/app/js/RequestParser.js @@ -1,5 +1,5 @@ -const settings = require('@overleaf/settings') -const OutputCacheManager = require('./OutputCacheManager') +import settings from '@overleaf/settings' +import OutputCacheManager from './OutputCacheManager.js' const VALID_COMPILERS = ['pdflatex', 'latex', 'xelatex', 'lualatex'] const MAX_TIMEOUT = 600 @@ -247,4 +247,4 @@ function _checkPath(path) { return path } -module.exports = { parse, MAX_TIMEOUT } +export default { parse, MAX_TIMEOUT } diff --git a/services/clsi/app/js/ResourceStateManager.js b/services/clsi/app/js/ResourceStateManager.js index a5f747e1cd..c0e0662744 100644 --- a/services/clsi/app/js/ResourceStateManager.js +++ b/services/clsi/app/js/ResourceStateManager.js @@ -1,10 +1,10 @@ -const Path = require('node:path') -const fs = require('node:fs') -const logger = require('@overleaf/logger') -const Errors = require('./Errors') -const SafeReader = require('./SafeReader') +import Path from 'node:path' +import fs from 'node:fs' +import logger from '@overleaf/logger' +import Errors from './Errors.js' +import SafeReader from './SafeReader.js' -module.exports = { +export default { // The sync state is an identifier which must match for an // incremental update to be allowed. // diff --git a/services/clsi/app/js/ResourceWriter.js b/services/clsi/app/js/ResourceWriter.js index 7348ac2107..dc74e9f024 100644 --- a/services/clsi/app/js/ResourceWriter.js +++ b/services/clsi/app/js/ResourceWriter.js @@ -12,22 +12,25 @@ * DS207: Consider shorter variations of null checks * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ +import { promisify } from 'node:util' +import UrlCache from './UrlCache.js' +import Path from 'node:path' +import fs from 'node:fs' +import async from 'async' +import OutputFileFinder from './OutputFileFinder.js' +import ResourceStateManager from './ResourceStateManager.js' +import Metrics from '@overleaf/metrics' +import logger from '@overleaf/logger' +import settings from '@overleaf/settings' +import ClsiMetrics from './Metrics.js' + +const { shouldSkipMetrics } = ClsiMetrics + let ResourceWriter -const { promisify } = require('node:util') -const UrlCache = require('./UrlCache') -const Path = require('node:path') -const fs = require('node:fs') -const async = require('async') -const OutputFileFinder = require('./OutputFileFinder') -const ResourceStateManager = require('./ResourceStateManager') -const Metrics = require('@overleaf/metrics') -const logger = require('@overleaf/logger') -const settings = require('@overleaf/settings') -const { shouldSkipMetrics } = require('./Metrics') const parallelFileDownloads = settings.parallelFileDownloads || 1 -module.exports = ResourceWriter = { +export default ResourceWriter = { syncResourcesToDisk(request, basePath, callback) { if (callback == null) { callback = function () {} @@ -375,7 +378,7 @@ module.exports = ResourceWriter = { }, } -module.exports.promises = { +ResourceWriter.promises = { syncResourcesToDisk: promisify(ResourceWriter.syncResourcesToDisk), saveIncrementalResourcesToDisk: promisify( ResourceWriter.saveIncrementalResourcesToDisk diff --git a/services/clsi/app/js/SafeReader.js b/services/clsi/app/js/SafeReader.js index c59a0d2e87..7bc95cb920 100644 --- a/services/clsi/app/js/SafeReader.js +++ b/services/clsi/app/js/SafeReader.js @@ -11,12 +11,12 @@ * DS207: Consider shorter variations of null checks * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ +import fs from 'node:fs' +import logger from '@overleaf/logger' +import { promisifyMultiResult } from '@overleaf/promise-utils' let SafeReader -const fs = require('node:fs') -const logger = require('@overleaf/logger') -const { promisifyMultiResult } = require('@overleaf/promise-utils') -module.exports = SafeReader = { +export default SafeReader = { // safely read up to size bytes from a file and return result as a // string @@ -62,6 +62,6 @@ module.exports = SafeReader = { }, } -module.exports.promises = { +SafeReader.promises = { readFile: promisifyMultiResult(SafeReader.readFile, ['result', 'bytesRead']), } diff --git a/services/clsi/app/js/StaticServerForbidSymlinks.js b/services/clsi/app/js/StaticServerForbidSymlinks.js index a5ec774396..5ba783d259 100644 --- a/services/clsi/app/js/StaticServerForbidSymlinks.js +++ b/services/clsi/app/js/StaticServerForbidSymlinks.js @@ -12,13 +12,13 @@ * DS207: Consider shorter variations of null checks * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ +import Path from 'node:path' +import fs from 'node:fs' +import Settings from '@overleaf/settings' +import logger from '@overleaf/logger' let ForbidSymlinks -const Path = require('node:path') -const fs = require('node:fs') -const Settings = require('@overleaf/settings') -const logger = require('@overleaf/logger') -module.exports = ForbidSymlinks = function (staticFn, root, options) { +export default ForbidSymlinks = function (staticFn, root, options) { const expressStatic = staticFn(root, options) const basePath = Path.resolve(root) return function (req, res, next) { diff --git a/services/clsi/app/js/StatsManager.js b/services/clsi/app/js/StatsManager.js index 888bd2ec8a..2bcc00e552 100644 --- a/services/clsi/app/js/StatsManager.js +++ b/services/clsi/app/js/StatsManager.js @@ -1,5 +1,7 @@ -const crypto = require('node:crypto') -const { shouldSkipMetrics } = require('./Metrics') +import crypto from 'node:crypto' +import Metrics from './Metrics.js' + +const { shouldSkipMetrics } = Metrics /** * Consistently sample a keyspace with a given sample percentage. @@ -46,4 +48,4 @@ function sampleRequest(request, samplingPercentage) { } } -module.exports = { sampleByHash, sampleRequest } +export default { sampleByHash, sampleRequest } diff --git a/services/clsi/app/js/SynctexOutputParser.js b/services/clsi/app/js/SynctexOutputParser.js index 5b2d237825..ea720a7cb2 100644 --- a/services/clsi/app/js/SynctexOutputParser.js +++ b/services/clsi/app/js/SynctexOutputParser.js @@ -1,4 +1,4 @@ -const Path = require('node:path') +import Path from 'node:path' /** * Parse output from the `synctex view` command @@ -110,4 +110,4 @@ function _setFloatProp(record, prop, value) { } } -module.exports = { parseViewOutput, parseEditOutput } +export default { parseViewOutput, parseEditOutput } diff --git a/services/clsi/app/js/TikzManager.js b/services/clsi/app/js/TikzManager.js index ca9db6b005..bb74fc0146 100644 --- a/services/clsi/app/js/TikzManager.js +++ b/services/clsi/app/js/TikzManager.js @@ -10,19 +10,19 @@ * DS207: Consider shorter variations of null checks * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ +import fs from 'node:fs' +import Path from 'node:path' +import { promisify } from 'node:util' +import ResourceWriter from './ResourceWriter.js' +import SafeReader from './SafeReader.js' +import logger from '@overleaf/logger' let TikzManager -const fs = require('node:fs') -const Path = require('node:path') -const { promisify } = require('node:util') -const ResourceWriter = require('./ResourceWriter') -const SafeReader = require('./SafeReader') -const logger = require('@overleaf/logger') // for \tikzexternalize or pstool to work the main file needs to match the // jobname. Since we set the -jobname to output, we have to create a // copy of the main file as 'output.tex'. -module.exports = TikzManager = { +export default TikzManager = { checkMainFile(compileDir, mainFile, resources, callback) { // if there's already an output.tex file, we don't want to touch it if (callback == null) { @@ -103,7 +103,7 @@ module.exports = TikzManager = { }, } -module.exports.promises = { +TikzManager.promises = { checkMainFile: promisify(TikzManager.checkMainFile), injectOutputFile: promisify(TikzManager.injectOutputFile), } diff --git a/services/clsi/app/js/UrlCache.js b/services/clsi/app/js/UrlCache.js index 304474bbed..83abc95104 100644 --- a/services/clsi/app/js/UrlCache.js +++ b/services/clsi/app/js/UrlCache.js @@ -10,12 +10,13 @@ * DS207: Consider shorter variations of null checks * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -const UrlFetcher = require('./UrlFetcher') -const Settings = require('@overleaf/settings') -const fs = require('node:fs') -const Path = require('node:path') -const { callbackify } = require('node:util') -const Metrics = require('@overleaf/metrics') +import UrlFetcher from './UrlFetcher.js' + +import Settings from '@overleaf/settings' +import fs from 'node:fs' +import Path from 'node:path' +import { callbackify } from 'node:util' +import Metrics from '@overleaf/metrics' const PENDING_DOWNLOADS = new Map() @@ -120,7 +121,7 @@ async function download(url, fallbackURL, cachePath) { } } -module.exports = { +export default { clearProject: callbackify(clearProject), createProjectDir: callbackify(createProjectDir), downloadUrlToFile: callbackify(downloadUrlToFile), diff --git a/services/clsi/app/js/UrlFetcher.js b/services/clsi/app/js/UrlFetcher.js index 7bcef0d606..70faed00dd 100644 --- a/services/clsi/app/js/UrlFetcher.js +++ b/services/clsi/app/js/UrlFetcher.js @@ -1,15 +1,15 @@ -const fs = require('node:fs') -const logger = require('@overleaf/logger') -const Settings = require('@overleaf/settings') -const { +import fs from 'node:fs' +import logger from '@overleaf/logger' +import Settings from '@overleaf/settings' +import { CustomHttpAgent, CustomHttpsAgent, fetchStream, RequestFailedError, -} = require('@overleaf/fetch-utils') -const { URL } = require('node:url') -const { pipeline } = require('node:stream/promises') -const Metrics = require('@overleaf/metrics') +} from '@overleaf/fetch-utils' +import { URL } from 'node:url' +import { pipeline } from 'node:stream/promises' +import Metrics from '@overleaf/metrics' const MAX_CONNECT_TIME = 1000 const httpAgent = new CustomHttpAgent({ connectTimeout: MAX_CONNECT_TIME }) @@ -121,6 +121,8 @@ function inferSource(url) { return 'unknown' } -module.exports.promises = { - pipeUrlToFileWithRetry, +export default { + promises: { + pipeUrlToFileWithRetry, + }, } diff --git a/services/clsi/app/js/XrefParser.js b/services/clsi/app/js/XrefParser.js index 5f2d154679..2c885577da 100644 --- a/services/clsi/app/js/XrefParser.js +++ b/services/clsi/app/js/XrefParser.js @@ -1,5 +1,5 @@ -const { NoXrefTableError } = require('./Errors') -const fs = require('node:fs') +import { NoXrefTableError } from './Errors.js' +import fs from 'node:fs' const { O_RDONLY, O_NOFOLLOW } = fs.constants const MAX_XREF_FILE_SIZE = 1024 * 1024 @@ -62,6 +62,6 @@ async function parseXrefTable(filePath, pdfFileSize) { } } -module.exports = { +export default { parseXrefTable, } diff --git a/services/clsi/buildscript.txt b/services/clsi/buildscript.txt index 95531da126..2cf30ef479 100644 --- a/services/clsi/buildscript.txt +++ b/services/clsi/buildscript.txt @@ -7,4 +7,6 @@ clsi --node-version=22.18.0 --pipeline-owner=🚉 Platform --public-repo=True +--test-unit-vitest=True +--tsconfig-extra-includes=vitest.config.unit.cjs --use-large-ci-runner=True diff --git a/services/clsi/config/settings.defaults.js b/services/clsi/config/settings.defaults.cjs similarity index 94% rename from services/clsi/config/settings.defaults.js rename to services/clsi/config/settings.defaults.cjs index 3f0b77cbd0..d9baedc40f 100644 --- a/services/clsi/config/settings.defaults.js +++ b/services/clsi/config/settings.defaults.cjs @@ -1,5 +1,6 @@ const Path = require('node:path') const os = require('node:os') +const fs = require('node:fs') const isPreEmptible = process.env.PREEMPTIBLE === 'TRUE' const CLSI_SERVER_ID = os.hostname().replace('-ctr', '') @@ -96,6 +97,15 @@ if (process.env.ALLOWED_COMPILE_GROUPS) { } if ((process.env.DOCKER_RUNNER || process.env.SANDBOXED_COMPILES) === 'true') { + if ( + !fs.existsSync(Path.join(__dirname, '..', 'app', 'js', 'DockerRunner.js')) + ) { + console.error( + 'Sandboxed compiles are only available with Overleaf Server Pro. Compare Server Pro with Community Edition here: https://docs.overleaf.com/on-premises/welcome/server-pro-vs.-community-edition' + ) + process.exit(1) + } + module.exports.clsi = { dockerRunner: true, docker: { diff --git a/services/clsi/docker-compose.ci.yml b/services/clsi/docker-compose.ci.yml index aaea30076a..63d612315b 100644 --- a/services/clsi/docker-compose.ci.yml +++ b/services/clsi/docker-compose.ci.yml @@ -7,12 +7,14 @@ services: image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER volumes: - ./reports:/overleaf/services/clsi/reports + - ../../tsconfig.backend.json:/overleaf/tsconfig.backend.json command: npm run test:unit:_run environment: CI: MONGO_CONNECTION_STRING: mongodb://mongo/test-overleaf NODE_ENV: test NODE_OPTIONS: "--unhandled-rejections=strict" + VITEST_NO_CACHE: true test_acceptance: diff --git a/services/clsi/docker-compose.yml b/services/clsi/docker-compose.yml index e1eb0ff322..2688f8e702 100644 --- a/services/clsi/docker-compose.yml +++ b/services/clsi/docker-compose.yml @@ -12,6 +12,7 @@ services: - .:/overleaf/services/clsi - ../../node_modules:/overleaf/node_modules - ../../libraries:/overleaf/libraries + - ../../tsconfig.backend.json:/overleaf/tsconfig.backend.json working_dir: /overleaf/services/clsi environment: MOCHA_GREP: ${MOCHA_GREP} diff --git a/services/clsi/package.json b/services/clsi/package.json index b16dc7d157..3052d74ccf 100644 --- a/services/clsi/package.json +++ b/services/clsi/package.json @@ -3,12 +3,13 @@ "description": "A Node.js implementation of the CLSI LaTeX web-API", "private": true, "main": "app.js", + "type": "module", "scripts": { "start": "node app.js", "test:acceptance:_run": "mocha --recursive --timeout 15000 --exit $@ test/acceptance/js", "test:acceptance": "npm run test:acceptance:_run -- --grep=$MOCHA_GREP", - "test:unit:_run": "mocha --recursive --exit $@ test/unit/js", - "test:unit": "npm run test:unit:_run -- --grep=$MOCHA_GREP", + "test:unit:_run": "vitest --config ./vitest.config.unit.cjs", + "test:unit": "npm run test:unit:_run", "nodemon": "node --watch app.js", "lint": "eslint --max-warnings 0 --format unix .", "format": "prettier --list-different $PWD/'**/{*.*js,*.ts}'", @@ -45,10 +46,10 @@ "mocha-multi-reporters": "^1.5.1", "mock-fs": "^5.1.2", "node-fetch": "^2.7.0", - "sandboxed-module": "^2.0.4", "sinon": "~9.0.1", "sinon-chai": "^3.7.0", "timekeeper": "2.2.0", - "typescript": "^5.0.4" + "typescript": "^5.0.4", + "vitest": "^4.0.0" } } diff --git a/services/clsi/scripts/demo-pdfjs-Xref.js b/services/clsi/scripts/demo-pdfjs-Xref.js index 1f55c571a7..aacb9dae62 100644 --- a/services/clsi/scripts/demo-pdfjs-Xref.js +++ b/services/clsi/scripts/demo-pdfjs-Xref.js @@ -1,11 +1,11 @@ -const fs = require('node:fs') -const { parseXrefTable } = require('../app/lib/pdfjs/parseXrefTable') +import fs from 'node:fs' +import XrefParser from '../app/js/XrefParser.js' const pdfPath = process.argv[2] async function main() { const size = (await fs.promises.stat(pdfPath)).size - const { xRefEntries } = await parseXrefTable(pdfPath, size) + const { xRefEntries } = await XrefParser.parseXrefTable(pdfPath, size) console.log('Xref entries', xRefEntries) } diff --git a/services/clsi/test/acceptance/js/AllowedImageNamesTests.js b/services/clsi/test/acceptance/js/AllowedImageNamesTests.js index 62d4dc8222..03d4731cda 100644 --- a/services/clsi/test/acceptance/js/AllowedImageNamesTests.js +++ b/services/clsi/test/acceptance/js/AllowedImageNamesTests.js @@ -1,6 +1,6 @@ -const Client = require('./helpers/Client') -const ClsiApp = require('./helpers/ClsiApp') -const { expect } = require('chai') +import Client from './helpers/Client.js' +import ClsiApp from './helpers/ClsiApp.js' +import { expect } from 'chai' describe('AllowedImageNames', function () { beforeEach(async function () { diff --git a/services/clsi/test/acceptance/js/BrokenLatexFileTests.js b/services/clsi/test/acceptance/js/BrokenLatexFileTests.js index 738a13a81f..7229b7b88d 100644 --- a/services/clsi/test/acceptance/js/BrokenLatexFileTests.js +++ b/services/clsi/test/acceptance/js/BrokenLatexFileTests.js @@ -1,6 +1,6 @@ -const Client = require('./helpers/Client') -const ClsiApp = require('./helpers/ClsiApp') -const { expect } = require('chai') +import Client from './helpers/Client.js' +import ClsiApp from './helpers/ClsiApp.js' +import { expect } from 'chai' describe('Broken LaTeX file', function () { before(async function () { diff --git a/services/clsi/test/acceptance/js/DeleteOldFilesTest.js b/services/clsi/test/acceptance/js/DeleteOldFilesTest.js index e5e230e49f..7435afef77 100644 --- a/services/clsi/test/acceptance/js/DeleteOldFilesTest.js +++ b/services/clsi/test/acceptance/js/DeleteOldFilesTest.js @@ -1,5 +1,5 @@ -const Client = require('./helpers/Client') -const ClsiApp = require('./helpers/ClsiApp') +import Client from './helpers/Client.js' +import ClsiApp from './helpers/ClsiApp.js' describe('Deleting Old Files', function () { before(async function () { diff --git a/services/clsi/test/acceptance/js/ExampleDocumentTests.js b/services/clsi/test/acceptance/js/ExampleDocumentTests.js index dc2f0fe669..192f8fa903 100644 --- a/services/clsi/test/acceptance/js/ExampleDocumentTests.js +++ b/services/clsi/test/acceptance/js/ExampleDocumentTests.js @@ -1,19 +1,19 @@ -const Client = require('./helpers/Client') -const fetch = require('node-fetch') -const Stream = require('node:stream') -const fs = require('node:fs') -const fsPromises = require('node:fs/promises') -const ChildProcess = require('node:child_process') -const { promisify } = require('node:util') -const ClsiApp = require('./helpers/ClsiApp') -const Path = require('node:path') +import Client from './helpers/Client.js' +import fetch from 'node-fetch' +import Stream from 'node:stream' +import fs from 'node:fs' +import fsPromises from 'node:fs/promises' +import ChildProcess from 'node:child_process' +import { promisify } from 'node:util' +import ClsiApp from './helpers/ClsiApp.js' +import Path from 'node:path' +import process from 'node:process' const fixturePath = path => { if (path.slice(0, 3) === 'tmp') { return '/tmp/clsi_acceptance_tests' + path.slice(3) } - return Path.join(__dirname, '../fixtures/', path) + return Path.join(import.meta.dirname, '../fixtures/', path) } -const process = require('node:process') const pipeline = promisify(Stream.pipeline) console.log( process.pid, diff --git a/services/clsi/test/acceptance/js/SimpleLatexFileTests.js b/services/clsi/test/acceptance/js/SimpleLatexFileTests.js index 3042f0a4e9..1a0632e132 100644 --- a/services/clsi/test/acceptance/js/SimpleLatexFileTests.js +++ b/services/clsi/test/acceptance/js/SimpleLatexFileTests.js @@ -1,8 +1,8 @@ -const Client = require('./helpers/Client') -const { fetchNothing, fetchString } = require('@overleaf/fetch-utils') -const ClsiApp = require('./helpers/ClsiApp') -const { expect } = require('chai') -const Settings = require('@overleaf/settings') +import Client from './helpers/Client.js' +import { fetchNothing, fetchString } from '@overleaf/fetch-utils' +import ClsiApp from './helpers/ClsiApp.js' +import { expect } from 'chai' +import Settings from '@overleaf/settings' describe('Simple LaTeX file', function () { const content = `\ diff --git a/services/clsi/test/acceptance/js/SmokeTest.js b/services/clsi/test/acceptance/js/SmokeTest.js index a5cb31da12..293f762e29 100644 --- a/services/clsi/test/acceptance/js/SmokeTest.js +++ b/services/clsi/test/acceptance/js/SmokeTest.js @@ -1,6 +1,6 @@ -const Client = require('./helpers/Client') -const ClsiApp = require('./helpers/ClsiApp') -const { expect } = require('chai') +import Client from './helpers/Client.js' +import ClsiApp from './helpers/ClsiApp.js' +import { expect } from 'chai' describe('Smoke Test', function () { before(async function () { diff --git a/services/clsi/test/acceptance/js/Stats.js b/services/clsi/test/acceptance/js/Stats.js index 98e6a4cacd..a276f8c639 100644 --- a/services/clsi/test/acceptance/js/Stats.js +++ b/services/clsi/test/acceptance/js/Stats.js @@ -1,5 +1,5 @@ -const { fetchString } = require('@overleaf/fetch-utils') -const Settings = require('@overleaf/settings') +import { fetchString } from '@overleaf/fetch-utils' +import Settings from '@overleaf/settings' after(async function () { const metrics = await fetchString(`${Settings.apis.clsi.url}/metrics`) console.error('-- metrics --') diff --git a/services/clsi/test/acceptance/js/StopCompile.js b/services/clsi/test/acceptance/js/StopCompile.js index 33a5160bff..f7b49e3150 100644 --- a/services/clsi/test/acceptance/js/StopCompile.js +++ b/services/clsi/test/acceptance/js/StopCompile.js @@ -1,7 +1,7 @@ -const { promisify } = require('node:util') -const Client = require('./helpers/Client') -const ClsiApp = require('./helpers/ClsiApp') -const { expect } = require('chai') +import { promisify } from 'node:util' +import Client from './helpers/Client.js' +import ClsiApp from './helpers/ClsiApp.js' +import { expect } from 'chai' const sleep = promisify(setTimeout) diff --git a/services/clsi/test/acceptance/js/SynctexTests.js b/services/clsi/test/acceptance/js/SynctexTests.js index 13fc213c02..7c3c0569cd 100644 --- a/services/clsi/test/acceptance/js/SynctexTests.js +++ b/services/clsi/test/acceptance/js/SynctexTests.js @@ -1,6 +1,6 @@ -const Client = require('./helpers/Client') -const { expect } = require('chai') -const ClsiApp = require('./helpers/ClsiApp') +import Client from './helpers/Client.js' +import { expect } from 'chai' +import ClsiApp from './helpers/ClsiApp.js' describe('Syncing', function () { before(async function () { diff --git a/services/clsi/test/acceptance/js/TimeoutTests.js b/services/clsi/test/acceptance/js/TimeoutTests.js index 70926e3aee..aaf7db85b9 100644 --- a/services/clsi/test/acceptance/js/TimeoutTests.js +++ b/services/clsi/test/acceptance/js/TimeoutTests.js @@ -1,6 +1,6 @@ -const Client = require('./helpers/Client') -const ClsiApp = require('./helpers/ClsiApp') -const { expect } = require('chai') +import Client from './helpers/Client.js' +import ClsiApp from './helpers/ClsiApp.js' +import { expect } from 'chai' describe('Timed out compile', function () { before(async function () { diff --git a/services/clsi/test/acceptance/js/UrlCachingTests.js b/services/clsi/test/acceptance/js/UrlCachingTests.js index b698af5488..7b11249a59 100644 --- a/services/clsi/test/acceptance/js/UrlCachingTests.js +++ b/services/clsi/test/acceptance/js/UrlCachingTests.js @@ -1,16 +1,18 @@ -const express = require('express') -const Path = require('node:path') -const Client = require('./helpers/Client') -const sinon = require('sinon') -const ClsiApp = require('./helpers/ClsiApp') -const { fetchString } = require('@overleaf/fetch-utils') -const Settings = require('@overleaf/settings') +import express from 'express' +import Path from 'node:path' +import Client from './helpers/Client.js' +import sinon from 'sinon' +import ClsiApp from './helpers/ClsiApp.js' +import { fetchString } from '@overleaf/fetch-utils' +import Settings from '@overleaf/settings' const Server = { run() { const app = express() - const staticServer = express.static(Path.join(__dirname, '../fixtures/')) + const staticServer = express.static( + Path.join(import.meta.dirname, '../fixtures/') + ) const alreadyFailed = new Map() app.get('/fail/:times/:id', (req, res) => { diff --git a/services/clsi/test/acceptance/js/WordcountTests.js b/services/clsi/test/acceptance/js/WordcountTests.js index 50e2506d6a..eabf6bd972 100644 --- a/services/clsi/test/acceptance/js/WordcountTests.js +++ b/services/clsi/test/acceptance/js/WordcountTests.js @@ -1,8 +1,8 @@ -const Client = require('./helpers/Client') -const { expect } = require('chai') -const path = require('node:path') -const fs = require('node:fs') -const ClsiApp = require('./helpers/ClsiApp') +import Client from './helpers/Client.js' +import { expect } from 'chai' +import path from 'node:path' +import fs from 'node:fs' +import ClsiApp from './helpers/ClsiApp.js' describe('Syncing', function () { before(async function () { @@ -11,7 +11,7 @@ describe('Syncing', function () { { path: 'main.tex', content: fs.readFileSync( - path.join(__dirname, '../fixtures/naugty_strings.txt'), + path.join(import.meta.dirname, '../fixtures/naugty_strings.txt'), 'utf-8' ), }, diff --git a/services/clsi/test/acceptance/js/helpers/Client.js b/services/clsi/test/acceptance/js/helpers/Client.js index 1b5b1f9b08..6448d0b4fb 100644 --- a/services/clsi/test/acceptance/js/helpers/Client.js +++ b/services/clsi/test/acceptance/js/helpers/Client.js @@ -1,12 +1,8 @@ -const express = require('express') -const { - fetchJson, - fetchNothing, - fetchString, -} = require('@overleaf/fetch-utils') -const fs = require('node:fs') -const fsPromises = require('node:fs/promises') -const Settings = require('@overleaf/settings') +import express from 'express' +import { fetchJson, fetchNothing, fetchString } from '@overleaf/fetch-utils' +import fs from 'node:fs' +import fsPromises from 'node:fs/promises' +import Settings from '@overleaf/settings' const host = Settings.apis.clsi.url @@ -187,7 +183,7 @@ function smokeTest() { }) } -module.exports = { +export default { randomId, compile, stopCompile, diff --git a/services/clsi/test/acceptance/js/helpers/ClsiApp.js b/services/clsi/test/acceptance/js/helpers/ClsiApp.js index 5d5bc57233..1f0725c0c6 100644 --- a/services/clsi/test/acceptance/js/helpers/ClsiApp.js +++ b/services/clsi/test/acceptance/js/helpers/ClsiApp.js @@ -1,5 +1,5 @@ -const app = require('../../../../app') -const Settings = require('@overleaf/settings') +import app from '../../../../app.js' +import Settings from '@overleaf/settings' function startApp() { return new Promise((resolve, reject) => { @@ -26,6 +26,6 @@ async function ensureRunning() { await appStartedPromise } -module.exports = { +export default { ensureRunning, } diff --git a/services/clsi/test/bench/hashbench.js b/services/clsi/test/bench/hashbench.js index 1e19af6a35..49ccf45feb 100644 --- a/services/clsi/test/bench/hashbench.js +++ b/services/clsi/test/bench/hashbench.js @@ -1,9 +1,9 @@ -const ContentCacheManager = require('../../app/js/ContentCacheManager') -const fs = require('node:fs') -const crypto = require('node:crypto') -const path = require('node:path') -const os = require('node:os') -const async = require('async') +import ContentCacheManager from '../../app/js/ContentCacheManager.js' +import fs from 'node:fs' +import crypto from 'node:crypto' +import path from 'node:path' +import os from 'node:os' +import async from 'async' const _createHash = crypto.createHash const files = process.argv.slice(2) diff --git a/services/clsi/test/load/js/loadTest.js b/services/clsi/test/load/js/loadTest.js index 189ab29fb7..450feac9c1 100644 --- a/services/clsi/test/load/js/loadTest.js +++ b/services/clsi/test/load/js/loadTest.js @@ -1,8 +1,8 @@ -const { fetchNothing } = require('@overleaf/fetch-utils') -const Settings = require('@overleaf/settings') -const async = require('async') -const fs = require('node:fs') -const _ = require('lodash') +import { fetchNothing } from '@overleaf/fetch-utils' +import Settings from '@overleaf/settings' +import async from 'async' +import fs from 'node:fs' +import _ from 'lodash' const concurentCompiles = 5 const totalCompiles = 50 diff --git a/services/clsi/test/setup.js b/services/clsi/test/setup.js index b17507bf92..39bb0b9f12 100644 --- a/services/clsi/test/setup.js +++ b/services/clsi/test/setup.js @@ -1,29 +1,8 @@ -const chai = require('chai') -const sinonChai = require('sinon-chai') -const chaiAsPromised = require('chai-as-promised') -const SandboxedModule = require('sandboxed-module') +import chai from 'chai' +import sinonChai from 'sinon-chai' +import chaiAsPromised from 'chai-as-promised' // Setup chai chai.should() chai.use(sinonChai) chai.use(chaiAsPromised) - -// Global SandboxedModule settings -SandboxedModule.configure({ - requires: { - '@overleaf/logger': { - debug() {}, - log() {}, - info() {}, - warn() {}, - error() {}, - err() {}, - }, - }, - globals: { Buffer, console, process, URL, Math }, - sourceTransformers: { - removeNodePrefix: function (source) { - return source.replace(/require\(['"]node:/g, "require('") - }, - }, -}) diff --git a/services/clsi/test/smoke/js/SmokeTests.js b/services/clsi/test/smoke/js/SmokeTests.js index 4d280690ca..97618e7d21 100644 --- a/services/clsi/test/smoke/js/SmokeTests.js +++ b/services/clsi/test/smoke/js/SmokeTests.js @@ -1,12 +1,12 @@ -const request = require('request') -const Settings = require('@overleaf/settings') +import request from 'request' +import Settings from '@overleaf/settings' const buildUrl = path => `http://${Settings.internal.clsi.host}:${Settings.internal.clsi.port}/${path}` const url = buildUrl(`project/smoketest-${process.pid}/compile`) -module.exports = { +export default { sendNewResult(res) { this._run(error => this._sendResponse(res, error)) }, diff --git a/services/clsi/test/unit/js/CompileController.test.js b/services/clsi/test/unit/js/CompileController.test.js new file mode 100644 index 0000000000..ebdf4cb2c4 --- /dev/null +++ b/services/clsi/test/unit/js/CompileController.test.js @@ -0,0 +1,519 @@ +import { vi, describe, beforeEach, it } from 'vitest' +import sinon from 'sinon' +import Errors from '../../../app/js/Errors.js' +import path from 'node:path' +const modulePath = path.join( + import.meta.dirname, + '../../../app/js/CompileController' +) + +describe('CompileController', () => { + beforeEach(async ctx => { + ctx.buildId = 'build-id-123' + + vi.doMock('../../../app/js/CompileManager', () => ({ + default: (ctx.CompileManager = {}), + })) + + vi.doMock('../../../app/js/RequestParser', () => ({ + default: (ctx.RequestParser = {}), + })) + + vi.doMock('@overleaf/settings', () => ({ + default: (ctx.Settings = { + apis: { + clsi: { + url: 'http://clsi.example.com', + outputUrlPrefix: '/zone/b', + downloadHost: 'http://localhost:3013', + }, + clsiCache: { + enabled: false, + url: 'http://localhost:3044', + }, + }, + }), + })) + + vi.doMock('@overleaf/metrics', () => ({ + default: { + Timer: sinon.stub().returns({ done: sinon.stub() }), + }, + })) + + vi.doMock('../../../app/js/ProjectPersistenceManager', () => ({ + default: (ctx.ProjectPersistenceManager = {}), + })) + + vi.doMock('../../../app/js/CLSICacheHandler', () => ({ + default: { + notifyCLSICacheAboutBuild: sinon.stub(), + downloadLatestCompileCache: sinon.stub().resolves(), + downloadOutputDotSynctexFromCompileCache: sinon.stub().resolves(), + }, + })) + + vi.doMock('../../../app/js/Errors', () => ({ + default: (ctx.Errors = Errors), + })) + + ctx.CompileController = (await import(modulePath)).default + ctx.Settings.externalUrl = 'http://www.example.com' + ctx.req = {} + ctx.res = {} + ctx.next = sinon.stub() + }) + + describe('compile', () => { + beforeEach(ctx => { + ctx.req.body = { + compile: 'mock-body', + } + ctx.req.params = { project_id: (ctx.project_id = 'project-id-123') } + ctx.request = { + compile: 'mock-parsed-request', + } + ctx.request_with_project_id = { + compile: ctx.request.compile, + project_id: ctx.project_id, + } + ctx.output_files = [ + { + path: 'output.pdf', + type: 'pdf', + size: 1337, + build: 1234, + }, + { + path: 'output.log', + type: 'log', + build: 1234, + }, + ] + ctx.RequestParser.parse = sinon.stub().callsArgWith(1, null, ctx.request) + ctx.ProjectPersistenceManager.markProjectAsJustAccessed = sinon + .stub() + .callsArg(1) + ctx.stats = { foo: 1 } + ctx.timings = { bar: 2 } + ctx.res.status = sinon.stub().returnsThis() + ctx.res.send = sinon.stub() + + ctx.CompileManager.doCompileWithLock = sinon + .stub() + .callsFake((_req, stats, timings, cb) => { + Object.assign(stats, ctx.stats) + Object.assign(timings, ctx.timings) + cb(null, { + outputFiles: ctx.output_files, + buildId: ctx.buildId, + }) + }) + }) + + describe('successfully', () => { + beforeEach(ctx => { + ctx.CompileController.compile(ctx.req, ctx.res) + }) + + it('should parse the request', ctx => { + ctx.RequestParser.parse.calledWith(ctx.req.body).should.equal(true) + }) + + it('should run the compile for the specified project', ctx => { + ctx.CompileManager.doCompileWithLock + .calledWith(ctx.request_with_project_id) + .should.equal(true) + }) + + it('should mark the project as accessed', ctx => { + ctx.ProjectPersistenceManager.markProjectAsJustAccessed + .calledWith(ctx.project_id) + .should.equal(true) + }) + + it('should return the JSON response', ctx => { + ctx.res.status.calledWith(200).should.equal(true) + ctx.res.send + .calledWith({ + compile: { + status: 'success', + error: null, + stats: ctx.stats, + timings: ctx.timings, + buildId: ctx.buildId, + outputUrlPrefix: '/zone/b', + outputFiles: ctx.output_files.map(file => ({ + url: `${ctx.Settings.apis.clsi.url}/project/${ctx.project_id}/build/${file.build}/output/${file.path}`, + ...file, + })), + clsiCacheShard: undefined, + }, + }) + .should.equal(true) + }) + }) + + describe('without a outputUrlPrefix', () => { + beforeEach(ctx => { + ctx.Settings.apis.clsi.outputUrlPrefix = '' + ctx.CompileController.compile(ctx.req, ctx.res) + }) + + it('should return the JSON response with empty outputUrlPrefix', ctx => { + ctx.res.status.calledWith(200).should.equal(true) + ctx.res.send + .calledWith({ + compile: { + status: 'success', + error: null, + stats: ctx.stats, + timings: ctx.timings, + buildId: ctx.buildId, + outputUrlPrefix: '', + outputFiles: ctx.output_files.map(file => ({ + url: `${ctx.Settings.apis.clsi.url}/project/${ctx.project_id}/build/${file.build}/output/${file.path}`, + ...file, + })), + clsiCacheShard: undefined, + }, + }) + .should.equal(true) + }) + }) + + describe('with user provided fake_output.pdf', () => { + beforeEach(ctx => { + ctx.output_files = [ + { + path: 'fake_output.pdf', + type: 'pdf', + build: 1234, + }, + { + path: 'output.log', + type: 'log', + build: 1234, + }, + ] + ctx.CompileManager.doCompileWithLock = sinon + .stub() + .callsFake((_req, stats, timings, cb) => { + Object.assign(stats, ctx.stats) + Object.assign(timings, ctx.timings) + cb(null, { + outputFiles: ctx.output_files, + buildId: ctx.buildId, + }) + }) + ctx.CompileController.compile(ctx.req, ctx.res) + }) + + it('should return the JSON response with status failure', ctx => { + ctx.res.status.calledWith(200).should.equal(true) + ctx.res.send.should.have.been.calledWith({ + compile: { + status: 'failure', + error: null, + stats: ctx.stats, + timings: ctx.timings, + outputUrlPrefix: '/zone/b', + buildId: ctx.buildId, + outputFiles: ctx.output_files.map(file => ({ + url: `${ctx.Settings.apis.clsi.url}/project/${ctx.project_id}/build/${file.build}/output/${file.path}`, + ...file, + })), + clsiCacheShard: undefined, + }, + }) + }) + }) + + describe('with an empty output.pdf', () => { + beforeEach(ctx => { + ctx.output_files = [ + { + path: 'output.pdf', + type: 'pdf', + size: 0, + build: 1234, + }, + { + path: 'output.log', + type: 'log', + build: 1234, + }, + ] + ctx.CompileManager.doCompileWithLock = sinon + .stub() + .callsFake((_req, stats, timings, cb) => { + Object.assign(stats, ctx.stats) + Object.assign(timings, ctx.timings) + cb(null, { + outputFiles: ctx.output_files, + buildId: ctx.buildId, + }) + }) + ctx.CompileController.compile(ctx.req, ctx.res) + }) + + it('should return the JSON response with status failure', ctx => { + ctx.res.status.calledWith(200).should.equal(true) + ctx.res.send.should.have.been.calledWith({ + compile: { + status: 'failure', + error: null, + stats: ctx.stats, + buildId: ctx.buildId, + timings: ctx.timings, + outputUrlPrefix: '/zone/b', + outputFiles: ctx.output_files.map(file => ({ + url: `${ctx.Settings.apis.clsi.url}/project/${ctx.project_id}/build/${file.build}/output/${file.path}`, + ...file, + })), + clsiCacheShard: undefined, + }, + }) + }) + }) + + describe('with an error', () => { + beforeEach(ctx => { + const error = new Error((ctx.message = 'error message')) + error.buildId = ctx.buildId + ctx.CompileManager.doCompileWithLock = sinon + .stub() + .callsFake((_req, stats, timings, cb) => { + Object.assign(stats, ctx.stats) + Object.assign(timings, ctx.timings) + cb(error) + }) + ctx.CompileController.compile(ctx.req, ctx.res) + }) + + it('should return the JSON response with the error', ctx => { + ctx.res.status.calledWith(500).should.equal(true) + ctx.res.send + .calledWith({ + compile: { + status: 'error', + error: ctx.message, + outputUrlPrefix: '/zone/b', + outputFiles: [], + buildId: ctx.buildId, + stats: ctx.stats, + timings: ctx.timings, + clsiCacheShard: undefined, + }, + }) + .should.equal(true) + }) + }) + + describe('with too many compile requests error', () => { + beforeEach(ctx => { + const error = new Errors.TooManyCompileRequestsError( + 'too many concurrent compile requests' + ) + ctx.CompileManager.doCompileWithLock = sinon + .stub() + .callsFake((_req, stats, timings, cb) => { + Object.assign(stats, ctx.stats) + Object.assign(timings, ctx.timings) + cb(error) + }) + ctx.CompileController.compile(ctx.req, ctx.res) + }) + + it('should return the JSON response with the error', ctx => { + ctx.res.status.calledWith(503).should.equal(true) + ctx.res.send + .calledWith({ + compile: { + status: 'unavailable', + error: 'too many concurrent compile requests', + outputUrlPrefix: '/zone/b', + outputFiles: [], + stats: ctx.stats, + timings: ctx.timings, + // JSON.stringify will omit these undefined values + buildId: undefined, + clsiCacheShard: undefined, + }, + }) + .should.equal(true) + }) + }) + + describe('when the request times out', () => { + beforeEach(ctx => { + ctx.error = new Error((ctx.message = 'container timed out')) + ctx.error.timedout = true + ctx.CompileManager.doCompileWithLock = sinon + .stub() + .callsFake((_req, stats, timings, cb) => { + Object.assign(stats, ctx.stats) + Object.assign(timings, ctx.timings) + cb(ctx.error) + }) + ctx.CompileController.compile(ctx.req, ctx.res) + }) + + it('should return the JSON response with the timeout status', ctx => { + ctx.res.status.calledWith(200).should.equal(true) + ctx.res.send + .calledWith({ + compile: { + status: 'timedout', + error: ctx.message, + outputUrlPrefix: '/zone/b', + outputFiles: [], + stats: ctx.stats, + timings: ctx.timings, + // JSON.stringify will omit these undefined values + buildId: undefined, + clsiCacheShard: undefined, + }, + }) + .should.equal(true) + }) + }) + + describe('when the request returns no output files', () => { + beforeEach(ctx => { + ctx.CompileManager.doCompileWithLock = sinon + .stub() + .callsFake((_req, stats, timings, cb) => { + Object.assign(stats, ctx.stats) + Object.assign(timings, ctx.timings) + cb(null, {}) + }) + ctx.CompileController.compile(ctx.req, ctx.res) + }) + + it('should return the JSON response with the failure status', ctx => { + ctx.res.status.calledWith(200).should.equal(true) + ctx.res.send + .calledWith({ + compile: { + error: null, + status: 'failure', + outputUrlPrefix: '/zone/b', + outputFiles: [], + stats: ctx.stats, + timings: ctx.timings, + // JSON.stringify will omit these undefined values + buildId: undefined, + clsiCacheShard: undefined, + }, + }) + .should.equal(true) + }) + }) + }) + + describe('syncFromCode', () => { + beforeEach(ctx => { + ctx.file = 'main.tex' + ctx.line = 42 + ctx.column = 5 + ctx.project_id = 'mock-project-id' + ctx.req.params = { project_id: ctx.project_id } + ctx.req.query = { + file: ctx.file, + line: ctx.line.toString(), + column: ctx.column.toString(), + } + ctx.res.json = sinon.stub() + + ctx.CompileManager.syncFromCode = sinon + .stub() + .yields(null, (ctx.pdfPositions = ['mock-positions']), true) + ctx.CompileController.syncFromCode(ctx.req, ctx.res, ctx.next) + }) + + it('should find the corresponding location in the PDF', ctx => { + ctx.CompileManager.syncFromCode + .calledWith(ctx.project_id, undefined, ctx.file, ctx.line, ctx.column) + .should.equal(true) + }) + + it('should return the positions', ctx => { + ctx.res.json + .calledWith({ + pdf: ctx.pdfPositions, + downloadedFromCache: true, + }) + .should.equal(true) + }) + }) + + describe('syncFromPdf', () => { + beforeEach(ctx => { + ctx.page = 5 + ctx.h = 100.23 + ctx.v = 45.67 + ctx.project_id = 'mock-project-id' + ctx.req.params = { project_id: ctx.project_id } + ctx.req.query = { + page: ctx.page.toString(), + h: ctx.h.toString(), + v: ctx.v.toString(), + } + ctx.res.json = sinon.stub() + + ctx.CompileManager.syncFromPdf = sinon + .stub() + .yields(null, (ctx.codePositions = ['mock-positions']), true) + ctx.CompileController.syncFromPdf(ctx.req, ctx.res, ctx.next) + }) + + it('should find the corresponding location in the code', ctx => { + ctx.CompileManager.syncFromPdf + .calledWith(ctx.project_id, undefined, ctx.page, ctx.h, ctx.v) + .should.equal(true) + }) + + it('should return the positions', ctx => { + ctx.res.json + .calledWith({ + code: ctx.codePositions, + downloadedFromCache: true, + }) + .should.equal(true) + }) + }) + + describe('wordcount', () => { + beforeEach(ctx => { + ctx.file = 'main.tex' + ctx.project_id = 'mock-project-id' + ctx.req.params = { project_id: ctx.project_id } + ctx.req.query = { + file: ctx.file, + image: (ctx.image = 'example.com/image'), + } + ctx.res.json = sinon.stub() + + ctx.CompileManager.wordcount = sinon + .stub() + .callsArgWith(4, null, (ctx.texcount = ['mock-texcount'])) + }) + + it('should return the word count of a file', ctx => { + ctx.CompileController.wordcount(ctx.req, ctx.res, ctx.next) + ctx.CompileManager.wordcount + .calledWith(ctx.project_id, undefined, ctx.file, ctx.image) + .should.equal(true) + }) + + it('should return the texcount info', ctx => { + ctx.CompileController.wordcount(ctx.req, ctx.res, ctx.next) + ctx.res.json + .calledWith({ + texcount: ctx.texcount, + }) + .should.equal(true) + }) + }) +}) diff --git a/services/clsi/test/unit/js/CompileControllerTests.js b/services/clsi/test/unit/js/CompileControllerTests.js deleted file mode 100644 index 2ac8d9c2d7..0000000000 --- a/services/clsi/test/unit/js/CompileControllerTests.js +++ /dev/null @@ -1,507 +0,0 @@ -const SandboxedModule = require('sandboxed-module') -const sinon = require('sinon') -const modulePath = require('node:path').join( - __dirname, - '../../../app/js/CompileController' -) -const Errors = require('../../../app/js/Errors') - -describe('CompileController', function () { - beforeEach(function () { - this.buildId = 'build-id-123' - this.CompileController = SandboxedModule.require(modulePath, { - requires: { - './CompileManager': (this.CompileManager = {}), - './RequestParser': (this.RequestParser = {}), - '@overleaf/settings': (this.Settings = { - apis: { - clsi: { - url: 'http://clsi.example.com', - outputUrlPrefix: '/zone/b', - downloadHost: 'http://localhost:3013', - }, - clsiCache: { - enabled: false, - url: 'http://localhost:3044', - }, - }, - }), - '@overleaf/metrics': { - Timer: sinon.stub().returns({ done: sinon.stub() }), - }, - './ProjectPersistenceManager': (this.ProjectPersistenceManager = {}), - './CLSICacheHandler': { - notifyCLSICacheAboutBuild: sinon.stub(), - downloadLatestCompileCache: sinon.stub().resolves(), - downloadOutputDotSynctexFromCompileCache: sinon.stub().resolves(), - }, - './Errors': (this.Erros = Errors), - }, - }) - this.Settings.externalUrl = 'http://www.example.com' - this.req = {} - this.res = {} - this.next = sinon.stub() - }) - - describe('compile', function () { - beforeEach(function () { - this.req.body = { - compile: 'mock-body', - } - this.req.params = { project_id: (this.project_id = 'project-id-123') } - this.request = { - compile: 'mock-parsed-request', - } - this.request_with_project_id = { - compile: this.request.compile, - project_id: this.project_id, - } - this.output_files = [ - { - path: 'output.pdf', - type: 'pdf', - size: 1337, - build: 1234, - }, - { - path: 'output.log', - type: 'log', - build: 1234, - }, - ] - this.RequestParser.parse = sinon - .stub() - .callsArgWith(1, null, this.request) - this.ProjectPersistenceManager.markProjectAsJustAccessed = sinon - .stub() - .callsArg(1) - this.stats = { foo: 1 } - this.timings = { bar: 2 } - this.res.status = sinon.stub().returnsThis() - this.res.send = sinon.stub() - - this.CompileManager.doCompileWithLock = sinon - .stub() - .callsFake((_req, stats, timings, cb) => { - Object.assign(stats, this.stats) - Object.assign(timings, this.timings) - cb(null, { - outputFiles: this.output_files, - buildId: this.buildId, - }) - }) - }) - - describe('successfully', function () { - beforeEach(function () { - this.CompileController.compile(this.req, this.res) - }) - - it('should parse the request', function () { - this.RequestParser.parse.calledWith(this.req.body).should.equal(true) - }) - - it('should run the compile for the specified project', function () { - this.CompileManager.doCompileWithLock - .calledWith(this.request_with_project_id) - .should.equal(true) - }) - - it('should mark the project as accessed', function () { - this.ProjectPersistenceManager.markProjectAsJustAccessed - .calledWith(this.project_id) - .should.equal(true) - }) - - it('should return the JSON response', function () { - this.res.status.calledWith(200).should.equal(true) - this.res.send - .calledWith({ - compile: { - status: 'success', - error: null, - stats: this.stats, - timings: this.timings, - buildId: this.buildId, - outputUrlPrefix: '/zone/b', - outputFiles: this.output_files.map(file => ({ - url: `${this.Settings.apis.clsi.url}/project/${this.project_id}/build/${file.build}/output/${file.path}`, - ...file, - })), - clsiCacheShard: undefined, - }, - }) - .should.equal(true) - }) - }) - - describe('without a outputUrlPrefix', function () { - beforeEach(function () { - this.Settings.apis.clsi.outputUrlPrefix = '' - this.CompileController.compile(this.req, this.res) - }) - - it('should return the JSON response with empty outputUrlPrefix', function () { - this.res.status.calledWith(200).should.equal(true) - this.res.send - .calledWith({ - compile: { - status: 'success', - error: null, - stats: this.stats, - timings: this.timings, - buildId: this.buildId, - outputUrlPrefix: '', - outputFiles: this.output_files.map(file => ({ - url: `${this.Settings.apis.clsi.url}/project/${this.project_id}/build/${file.build}/output/${file.path}`, - ...file, - })), - clsiCacheShard: undefined, - }, - }) - .should.equal(true) - }) - }) - - describe('with user provided fake_output.pdf', function () { - beforeEach(function () { - this.output_files = [ - { - path: 'fake_output.pdf', - type: 'pdf', - build: 1234, - }, - { - path: 'output.log', - type: 'log', - build: 1234, - }, - ] - this.CompileManager.doCompileWithLock = sinon - .stub() - .callsFake((_req, stats, timings, cb) => { - Object.assign(stats, this.stats) - Object.assign(timings, this.timings) - cb(null, { - outputFiles: this.output_files, - buildId: this.buildId, - }) - }) - this.CompileController.compile(this.req, this.res) - }) - - it('should return the JSON response with status failure', function () { - this.res.status.calledWith(200).should.equal(true) - this.res.send.should.have.been.calledWith({ - compile: { - status: 'failure', - error: null, - stats: this.stats, - timings: this.timings, - outputUrlPrefix: '/zone/b', - buildId: this.buildId, - outputFiles: this.output_files.map(file => ({ - url: `${this.Settings.apis.clsi.url}/project/${this.project_id}/build/${file.build}/output/${file.path}`, - ...file, - })), - clsiCacheShard: undefined, - }, - }) - }) - }) - - describe('with an empty output.pdf', function () { - beforeEach(function () { - this.output_files = [ - { - path: 'output.pdf', - type: 'pdf', - size: 0, - build: 1234, - }, - { - path: 'output.log', - type: 'log', - build: 1234, - }, - ] - this.CompileManager.doCompileWithLock = sinon - .stub() - .callsFake((_req, stats, timings, cb) => { - Object.assign(stats, this.stats) - Object.assign(timings, this.timings) - cb(null, { - outputFiles: this.output_files, - buildId: this.buildId, - }) - }) - this.CompileController.compile(this.req, this.res) - }) - - it('should return the JSON response with status failure', function () { - this.res.status.calledWith(200).should.equal(true) - this.res.send.should.have.been.calledWith({ - compile: { - status: 'failure', - error: null, - stats: this.stats, - buildId: this.buildId, - timings: this.timings, - outputUrlPrefix: '/zone/b', - outputFiles: this.output_files.map(file => ({ - url: `${this.Settings.apis.clsi.url}/project/${this.project_id}/build/${file.build}/output/${file.path}`, - ...file, - })), - clsiCacheShard: undefined, - }, - }) - }) - }) - - describe('with an error', function () { - beforeEach(function () { - const error = new Error((this.message = 'error message')) - error.buildId = this.buildId - this.CompileManager.doCompileWithLock = sinon - .stub() - .callsFake((_req, stats, timings, cb) => { - Object.assign(stats, this.stats) - Object.assign(timings, this.timings) - cb(error) - }) - this.CompileController.compile(this.req, this.res) - }) - - it('should return the JSON response with the error', function () { - this.res.status.calledWith(500).should.equal(true) - this.res.send - .calledWith({ - compile: { - status: 'error', - error: this.message, - outputUrlPrefix: '/zone/b', - outputFiles: [], - buildId: this.buildId, - stats: this.stats, - timings: this.timings, - clsiCacheShard: undefined, - }, - }) - .should.equal(true) - }) - }) - - describe('with too many compile requests error', function () { - beforeEach(function () { - const error = new Errors.TooManyCompileRequestsError( - 'too many concurrent compile requests' - ) - this.CompileManager.doCompileWithLock = sinon - .stub() - .callsFake((_req, stats, timings, cb) => { - Object.assign(stats, this.stats) - Object.assign(timings, this.timings) - cb(error) - }) - this.CompileController.compile(this.req, this.res) - }) - - it('should return the JSON response with the error', function () { - this.res.status.calledWith(503).should.equal(true) - this.res.send - .calledWith({ - compile: { - status: 'unavailable', - error: 'too many concurrent compile requests', - outputUrlPrefix: '/zone/b', - outputFiles: [], - stats: this.stats, - timings: this.timings, - // JSON.stringify will omit these undefined values - buildId: undefined, - clsiCacheShard: undefined, - }, - }) - .should.equal(true) - }) - }) - - describe('when the request times out', function () { - beforeEach(function () { - this.error = new Error((this.message = 'container timed out')) - this.error.timedout = true - this.CompileManager.doCompileWithLock = sinon - .stub() - .callsFake((_req, stats, timings, cb) => { - Object.assign(stats, this.stats) - Object.assign(timings, this.timings) - cb(this.error) - }) - this.CompileController.compile(this.req, this.res) - }) - - it('should return the JSON response with the timeout status', function () { - this.res.status.calledWith(200).should.equal(true) - this.res.send - .calledWith({ - compile: { - status: 'timedout', - error: this.message, - outputUrlPrefix: '/zone/b', - outputFiles: [], - stats: this.stats, - timings: this.timings, - // JSON.stringify will omit these undefined values - buildId: undefined, - clsiCacheShard: undefined, - }, - }) - .should.equal(true) - }) - }) - - describe('when the request returns no output files', function () { - beforeEach(function () { - this.CompileManager.doCompileWithLock = sinon - .stub() - .callsFake((_req, stats, timings, cb) => { - Object.assign(stats, this.stats) - Object.assign(timings, this.timings) - cb(null, {}) - }) - this.CompileController.compile(this.req, this.res) - }) - - it('should return the JSON response with the failure status', function () { - this.res.status.calledWith(200).should.equal(true) - this.res.send - .calledWith({ - compile: { - error: null, - status: 'failure', - outputUrlPrefix: '/zone/b', - outputFiles: [], - stats: this.stats, - timings: this.timings, - // JSON.stringify will omit these undefined values - buildId: undefined, - clsiCacheShard: undefined, - }, - }) - .should.equal(true) - }) - }) - }) - - describe('syncFromCode', function () { - beforeEach(function () { - this.file = 'main.tex' - this.line = 42 - this.column = 5 - this.project_id = 'mock-project-id' - this.req.params = { project_id: this.project_id } - this.req.query = { - file: this.file, - line: this.line.toString(), - column: this.column.toString(), - } - this.res.json = sinon.stub() - - this.CompileManager.syncFromCode = sinon - .stub() - .yields(null, (this.pdfPositions = ['mock-positions']), true) - this.CompileController.syncFromCode(this.req, this.res, this.next) - }) - - it('should find the corresponding location in the PDF', function () { - this.CompileManager.syncFromCode - .calledWith( - this.project_id, - undefined, - this.file, - this.line, - this.column - ) - .should.equal(true) - }) - - it('should return the positions', function () { - this.res.json - .calledWith({ - pdf: this.pdfPositions, - downloadedFromCache: true, - }) - .should.equal(true) - }) - }) - - describe('syncFromPdf', function () { - beforeEach(function () { - this.page = 5 - this.h = 100.23 - this.v = 45.67 - this.project_id = 'mock-project-id' - this.req.params = { project_id: this.project_id } - this.req.query = { - page: this.page.toString(), - h: this.h.toString(), - v: this.v.toString(), - } - this.res.json = sinon.stub() - - this.CompileManager.syncFromPdf = sinon - .stub() - .yields(null, (this.codePositions = ['mock-positions']), true) - this.CompileController.syncFromPdf(this.req, this.res, this.next) - }) - - it('should find the corresponding location in the code', function () { - this.CompileManager.syncFromPdf - .calledWith(this.project_id, undefined, this.page, this.h, this.v) - .should.equal(true) - }) - - it('should return the positions', function () { - this.res.json - .calledWith({ - code: this.codePositions, - downloadedFromCache: true, - }) - .should.equal(true) - }) - }) - - describe('wordcount', function () { - beforeEach(function () { - this.file = 'main.tex' - this.project_id = 'mock-project-id' - this.req.params = { project_id: this.project_id } - this.req.query = { - file: this.file, - image: (this.image = 'example.com/image'), - } - this.res.json = sinon.stub() - - this.CompileManager.wordcount = sinon - .stub() - .callsArgWith(4, null, (this.texcount = ['mock-texcount'])) - }) - - it('should return the word count of a file', function () { - this.CompileController.wordcount(this.req, this.res, this.next) - this.CompileManager.wordcount - .calledWith(this.project_id, undefined, this.file, this.image) - .should.equal(true) - }) - - it('should return the texcount info', function () { - this.CompileController.wordcount(this.req, this.res, this.next) - this.res.json - .calledWith({ - texcount: this.texcount, - }) - .should.equal(true) - }) - }) -}) diff --git a/services/clsi/test/unit/js/CompileManager.test.js b/services/clsi/test/unit/js/CompileManager.test.js new file mode 100644 index 0000000000..150da1e61d --- /dev/null +++ b/services/clsi/test/unit/js/CompileManager.test.js @@ -0,0 +1,773 @@ +import { vi, expect, describe, beforeEach, it } from 'vitest' +import Path from 'node:path' +import sinon from 'sinon' +import Metrics from '../../../app/js/Metrics.js' + +const MODULE_PATH = Path.join( + import.meta.dirname, + '../../../app/js/CompileManager' +) + +describe('CompileManager', () => { + beforeEach(async ctx => { + ctx.projectId = 'project-id-123' + ctx.userId = '1234' + ctx.resources = 'mock-resources' + ctx.outputFiles = [ + { + path: 'output.log', + type: 'log', + }, + { + path: 'output.pdf', + type: 'pdf', + }, + ] + ctx.buildFiles = [ + { + path: 'output.log', + type: 'log', + build: 1234, + }, + { + path: 'output.pdf', + type: 'pdf', + build: 1234, + }, + ] + ctx.buildId = '00000000000-0000000000000000' + ctx.commandOutput = 'Dummy output' + ctx.compileBaseDir = '/compile/dir' + ctx.outputBaseDir = '/output/dir' + ctx.compileDir = `${ctx.compileBaseDir}/${ctx.projectId}-${ctx.userId}` + ctx.outputDir = `${ctx.outputBaseDir}/${ctx.projectId}-${ctx.userId}` + + ctx.LatexRunner = { + promises: { + runLatex: sinon.stub().resolves({}), + }, + } + ctx.ResourceWriter = { + promises: { + syncResourcesToDisk: sinon.stub().resolves(ctx.resources), + }, + } + ctx.OutputFileFinder = { + promises: { + findOutputFiles: sinon.stub().resolves({ + outputFiles: ctx.outputFiles, + allEntries: ctx.outputFiles.map(f => f.path).concat(['main.tex']), + }), + }, + } + ctx.OutputCacheManager = { + BUILD_REGEX: /^[0-9a-f]+-[0-9a-f]+$/, + CACHE_SUBDIR: 'generated-files', + promises: { + queueDirOperation: sinon.stub().callsArg(1), + saveOutputFiles: sinon + .stub() + .resolves({ outputFiles: ctx.buildFiles, buildId: ctx.buildId }), + }, + } + ctx.Settings = { + path: { + compilesDir: ctx.compileBaseDir, + outputDir: ctx.outputBaseDir, + synctexBaseDir: sinon.stub(), + }, + clsi: { + docker: { + image: 'SOMEIMAGE', + }, + }, + } + ctx.Settings.path.synctexBaseDir + .withArgs(`${ctx.projectId}-${ctx.userId}`) + .returns(ctx.compileDir) + ctx.child_process = { + exec: sinon.stub(), + execFile: sinon.stub().yields(), + } + ctx.CommandRunner = { + canRunSyncTeXInOutputDir: sinon.stub().returns(false), + promises: { + run: sinon.stub().callsFake((_1, _2, _3, _4, _5, _6, compileGroup) => { + if (compileGroup === 'synctex' || compileGroup === 'synctex-output') { + return Promise.resolve({ stdout: ctx.commandOutput }) + } else { + return Promise.resolve({ + stdout: 'Encoding: ascii\nWords in text: 2', + }) + } + }), + }, + } + ctx.DraftModeManager = { + promises: { + injectDraftMode: sinon.stub().resolves(), + }, + } + ctx.TikzManager = { + promises: { + checkMainFile: sinon.stub().resolves(false), + }, + } + ctx.lock = { + release: sinon.stub(), + } + ctx.LockManager = { + acquire: sinon.stub().returns(ctx.lock), + } + ctx.SynctexOutputParser = { + parseViewOutput: sinon.stub(), + parseEditOutput: sinon.stub(), + } + + ctx.dirStats = { + isDirectory: sinon.stub().returns(true), + } + ctx.fileStats = { + isFile: sinon.stub().returns(true), + } + ctx.fsPromises = { + lstat: sinon.stub(), + stat: sinon.stub(), + readFile: sinon.stub(), + mkdir: sinon.stub().resolves(), + rm: sinon.stub().resolves(), + unlink: sinon.stub().resolves(), + rmdir: sinon.stub().resolves(), + } + ctx.fsPromises.lstat.withArgs(ctx.compileDir).resolves(ctx.dirStats) + ctx.fsPromises.stat + .withArgs(Path.join(ctx.compileDir, 'output.synctex.gz')) + .resolves(ctx.fileStats) + + ctx.CLSICacheHandler = { + notifyCLSICacheAboutBuild: sinon.stub(), + downloadLatestCompileCache: sinon.stub().resolves(), + downloadOutputDotSynctexFromCompileCache: sinon.stub().resolves(), + } + + ctx.LatexMetrics = { enableLatexMkMetrics: sinon.stub() } + + ctx.StatsManager = { sampleRequest: sinon.stub().returns(false) } + + vi.doMock('../../../app/js/LatexRunner', () => ({ + default: ctx.LatexRunner, + })) + + vi.doMock('../../../app/js/ResourceWriter', () => ({ + default: ctx.ResourceWriter, + })) + + vi.doMock('../../../app/js/OutputFileFinder', () => ({ + default: ctx.OutputFileFinder, + })) + + vi.doMock('../../../app/js/OutputCacheManager', () => ({ + default: ctx.OutputCacheManager, + })) + + vi.doMock('@overleaf/settings', () => ({ + default: ctx.Settings, + })) + + vi.doMock('@overleaf/metrics', () => ({ + default: { + inc: sinon.stub(), + timing: sinon.stub(), + gauge: sinon.stub(), + Timer: sinon.stub().returns({ done: sinon.stub() }), + }, + })) + + vi.doMock('child_process', () => ({ + default: ctx.child_process, + })) + + vi.doMock('../../../app/js/CommandRunner', () => ({ + default: ctx.CommandRunner, + })) + + vi.doMock('../../../app/js/DraftModeManager', () => ({ + default: ctx.DraftModeManager, + })) + + vi.doMock('../../../app/js/TikzManager', () => ({ + default: ctx.TikzManager, + })) + + vi.doMock('../../../app/js/LockManager', () => ({ + default: ctx.LockManager, + })) + + vi.doMock('../../../app/js/SynctexOutputParser', () => ({ + default: ctx.SynctexOutputParser, + })) + + vi.doMock('fs/promises', () => ({ + default: ctx.fsPromises, + })) + + vi.doMock('../../../app/js/CLSICacheHandler', () => ({ + default: ctx.CLSICacheHandler, + })) + + vi.doMock('../../../app/js/LatexMetrics', () => ({ + default: ctx.LatexMetrics, + })) + + vi.doMock('../../../app/js/StatsManager', () => ({ + default: ctx.StatsManager, + })) + + vi.doMock('../../../app/js/Metrics', () => ({ + default: Metrics, + })) + + ctx.CompileManager = (await import(MODULE_PATH)).default + }) + + describe('doCompileWithLock', () => { + beforeEach(ctx => { + ctx.request = { + resources: ctx.resources, + rootResourcePath: (ctx.rootResourcePath = 'main.tex'), + project_id: ctx.projectId, + user_id: ctx.userId, + compiler: (ctx.compiler = 'pdflatex'), + timeout: (ctx.timeout = 42000), + imageName: (ctx.image = 'example.com/image'), + flags: (ctx.flags = ['-file-line-error']), + compileGroup: (ctx.compileGroup = 'compile-group'), + stopOnFirstError: false, + metricsOpts: { + path: 'clsi-perf', + method: 'minimal', + compile: 'initial', + }, + } + ctx.env = { + OVERLEAF_PROJECT_ID: ctx.projectId, + } + }) + + describe('when the project is locked', () => { + beforeEach(async ctx => { + const error = new Error('locked') + ctx.LockManager.acquire.throws(error) + await expect( + ctx.CompileManager.promises.doCompileWithLock(ctx.request, {}, {}) + ).to.be.rejectedWith(error) + }) + + it('should ensure that the compile directory exists', ctx => { + expect(ctx.fsPromises.mkdir).to.have.been.calledWith(ctx.compileDir, { + recursive: true, + }) + }) + + it('should not run LaTeX', ctx => { + expect(ctx.LatexRunner.promises.runLatex).not.to.have.been.called + }) + }) + + describe('normally', () => { + beforeEach(async ctx => { + ctx.result = await ctx.CompileManager.promises.doCompileWithLock( + ctx.request, + {}, + {} + ) + }) + + it('should ensure that the compile directory exists', ctx => { + expect(ctx.fsPromises.mkdir).to.have.been.calledWith(ctx.compileDir, { + recursive: true, + }) + }) + + it('should write the resources to disk', ctx => { + expect( + ctx.ResourceWriter.promises.syncResourcesToDisk + ).to.have.been.calledWith(ctx.request, ctx.compileDir) + }) + + it('should run LaTeX', ctx => { + expect(ctx.LatexRunner.promises.runLatex).to.have.been.calledWith( + `${ctx.projectId}-${ctx.userId}`, + { + directory: ctx.compileDir, + mainFile: ctx.rootResourcePath, + compiler: ctx.compiler, + timeout: ctx.timeout, + image: ctx.image, + flags: ctx.flags, + environment: ctx.env, + compileGroup: ctx.compileGroup, + stopOnFirstError: ctx.request.stopOnFirstError, + stats: sinon.match.object, + timings: sinon.match.object, + } + ) + }) + + it('should find the output files', ctx => { + expect( + ctx.OutputFileFinder.promises.findOutputFiles + ).to.have.been.calledWith(ctx.resources, ctx.compileDir) + }) + + it('should return the output files', ctx => { + expect(ctx.result.outputFiles).to.equal(ctx.buildFiles) + }) + + it('should not inject draft mode by default', ctx => { + expect(ctx.DraftModeManager.promises.injectDraftMode).not.to.have.been + .called + }) + }) + + describe('with performance metric collection', () => { + it('should enable latexmk metrics when sampleRequest returns true', async ctx => { + ctx.StatsManager.sampleRequest.returns(true) + await ctx.CompileManager.promises.doCompileWithLock(ctx.request, {}, {}) + expect(ctx.LatexMetrics.enableLatexMkMetrics).to.have.been.calledWith( + sinon.match.object + ) + }) + + it('should enable latexmk metrics when sampleRequest returns false', async ctx => { + ctx.StatsManager.sampleRequest.returns(false) + await ctx.CompileManager.promises.doCompileWithLock(ctx.request, {}, {}) + expect(ctx.LatexMetrics.enableLatexMkMetrics).to.have.been.calledWith( + sinon.match.object + ) + }) + + it('should enable latexmk metrics when sampleRequest returns undefined', async ctx => { + ctx.StatsManager.sampleRequest.returns(undefined) + await ctx.CompileManager.promises.doCompileWithLock(ctx.request, {}, {}) + expect(ctx.LatexMetrics.enableLatexMkMetrics).to.have.been.calledWith( + sinon.match.object + ) + }) + }) + + describe('with draft mode', () => { + beforeEach(async ctx => { + ctx.request.draft = true + await ctx.CompileManager.promises.doCompileWithLock(ctx.request, {}, {}) + }) + + it('should inject the draft mode header', ctx => { + expect( + ctx.DraftModeManager.promises.injectDraftMode + ).to.have.been.calledWith(ctx.compileDir + '/' + ctx.rootResourcePath) + }) + }) + + describe('with a check option', () => { + beforeEach(async ctx => { + ctx.request.check = 'error' + await ctx.CompileManager.promises.doCompileWithLock(ctx.request, {}, {}) + }) + + it('should run chktex', ctx => { + expect(ctx.LatexRunner.promises.runLatex).to.have.been.calledWith( + `${ctx.projectId}-${ctx.userId}`, + { + directory: ctx.compileDir, + mainFile: ctx.rootResourcePath, + compiler: ctx.compiler, + timeout: ctx.timeout, + image: ctx.image, + flags: ctx.flags, + environment: { + CHKTEX_OPTIONS: '-nall -e9 -e10 -w15 -w16', + CHKTEX_EXIT_ON_ERROR: 1, + CHKTEX_ULIMIT_OPTIONS: '-t 5 -v 64000', + OVERLEAF_PROJECT_ID: ctx.projectId, + }, + compileGroup: ctx.compileGroup, + stopOnFirstError: ctx.request.stopOnFirstError, + stats: sinon.match.object, + timings: sinon.match.object, + } + ) + }) + }) + + describe('with a knitr file and check options', () => { + beforeEach(async ctx => { + ctx.request.rootResourcePath = 'main.Rtex' + ctx.request.check = 'error' + await ctx.CompileManager.promises.doCompileWithLock(ctx.request, {}, {}) + }) + + it('should not run chktex', ctx => { + expect(ctx.LatexRunner.promises.runLatex).to.have.been.calledWith( + `${ctx.projectId}-${ctx.userId}`, + { + directory: ctx.compileDir, + mainFile: 'main.Rtex', + compiler: ctx.compiler, + timeout: ctx.timeout, + image: ctx.image, + flags: ctx.flags, + environment: ctx.env, + compileGroup: ctx.compileGroup, + stopOnFirstError: ctx.request.stopOnFirstError, + stats: sinon.match.object, + timings: sinon.match.object, + } + ) + }) + }) + + describe('when the compile times out', () => { + beforeEach(async ctx => { + const error = new Error('timed out!') + error.timedout = true + ctx.LatexRunner.promises.runLatex.rejects(error) + await expect( + ctx.CompileManager.promises.doCompileWithLock(ctx.request, {}, {}) + ).to.be.rejected + }) + + it('should clear the compile directory', ctx => { + for (const { path } of ctx.buildFiles) { + expect(ctx.fsPromises.unlink).to.have.been.calledWith( + ctx.compileDir + '/' + path + ) + } + expect(ctx.fsPromises.unlink).to.have.been.calledWith( + ctx.compileDir + '/main.tex' + ) + expect(ctx.fsPromises.rmdir).to.have.been.calledWith(ctx.compileDir) + }) + }) + + describe('when the compile is manually stopped', () => { + beforeEach(async ctx => { + const error = new Error('terminated!') + error.terminated = true + ctx.LatexRunner.promises.runLatex.rejects(error) + await expect( + ctx.CompileManager.promises.doCompileWithLock(ctx.request, {}, {}) + ).to.be.rejected + }) + + it('should clear the compile directory', ctx => { + for (const { path } of ctx.buildFiles) { + expect(ctx.fsPromises.unlink).to.have.been.calledWith( + ctx.compileDir + '/' + path + ) + } + expect(ctx.fsPromises.unlink).to.have.been.calledWith( + ctx.compileDir + '/main.tex' + ) + expect(ctx.fsPromises.rmdir).to.have.been.calledWith(ctx.compileDir) + }) + }) + }) + + describe('clearProject', () => { + it('should clear the compile directory', async ctx => { + await ctx.CompileManager.promises.clearProject(ctx.projectId, ctx.userId) + + expect(ctx.fsPromises.rm).to.have.been.calledWith(ctx.compileDir, { + force: true, + recursive: true, + }) + }) + }) + + describe('syncing', () => { + beforeEach(ctx => { + ctx.page = 1 + ctx.h = 42.23 + ctx.v = 87.56 + ctx.width = 100.01 + ctx.height = 234.56 + ctx.line = 5 + ctx.column = 3 + ctx.filename = 'main.tex' + }) + + describe('syncFromCode', () => { + beforeEach(ctx => { + ctx.records = [{ page: 1, h: 2, v: 3, width: 4, height: 5 }] + ctx.SynctexOutputParser.parseViewOutput + .withArgs(ctx.commandOutput) + .returns(ctx.records) + }) + + describe('normal case', () => { + beforeEach(async ctx => { + ctx.result = await ctx.CompileManager.promises.syncFromCode( + ctx.projectId, + ctx.userId, + ctx.filename, + ctx.line, + ctx.column, + '' + ) + }) + + it('should execute the synctex binary', ctx => { + const outputFilePath = `${ctx.compileDir}/output.pdf` + const inputFilePath = `${ctx.compileDir}/${ctx.filename}` + expect(ctx.CommandRunner.promises.run).to.have.been.calledWith( + `${ctx.projectId}-${ctx.userId}`, + [ + 'synctex', + 'view', + '-i', + `${ctx.line}:${ctx.column}:${inputFilePath}`, + '-o', + outputFilePath, + ], + ctx.compileDir, + ctx.Settings.clsi.docker.image, + 60000, + {}, + 'synctex' + ) + }) + + it('should return the parsed output', ctx => { + expect(ctx.result).to.deep.equal({ + codePositions: ctx.records, + downloadedFromCache: false, + }) + }) + }) + + describe('from cache in docker', () => { + beforeEach(async ctx => { + ctx.CommandRunner.canRunSyncTeXInOutputDir.returns(true) + ctx.Settings.path.synctexBaseDir + .withArgs(`${ctx.projectId}-${ctx.userId}`) + .returns('/compile') + + const errNotFound = new Error() + errNotFound.code = 'ENOENT' + ctx.outputDir = `${ctx.outputBaseDir}/${ctx.projectId}-${ctx.userId}/${ctx.OutputCacheManager.CACHE_SUBDIR}/${ctx.buildId}` + const filename = Path.join(ctx.outputDir, 'output.synctex.gz') + ctx.fsPromises.stat + .withArgs(ctx.outputDir) + .onFirstCall() + .rejects(errNotFound) + ctx.fsPromises.stat + .withArgs(ctx.outputDir) + .onSecondCall() + .resolves(ctx.dirStats) + ctx.fsPromises.stat.withArgs(filename).resolves(ctx.fileStats) + ctx.CLSICacheHandler.downloadOutputDotSynctexFromCompileCache.resolves( + true + ) + ctx.result = await ctx.CompileManager.promises.syncFromCode( + ctx.projectId, + ctx.userId, + ctx.filename, + ctx.line, + ctx.column, + { + imageName: 'image', + editorId: '00000000-0000-0000-0000-000000000000', + buildId: ctx.buildId, + compileFromClsiCache: true, + } + ) + }) + + it('should run in output dir', ctx => { + const outputFilePath = '/compile/output.pdf' + const inputFilePath = `/compile/${ctx.filename}` + expect(ctx.CommandRunner.promises.run).to.have.been.calledWith( + `${ctx.projectId}-${ctx.userId}`, + [ + 'synctex', + 'view', + '-i', + `${ctx.line}:${ctx.column}:${inputFilePath}`, + '-o', + outputFilePath, + ], + ctx.outputDir, + 'image', + 60000, + {}, + 'synctex-output' + ) + }) + + it('should return the parsed output', ctx => { + expect(ctx.result).to.deep.equal({ + codePositions: ctx.records, + downloadedFromCache: true, + }) + }) + }) + + describe('with a custom imageName', () => { + const customImageName = 'foo/bar:tag-0' + beforeEach(async ctx => { + await ctx.CompileManager.promises.syncFromCode( + ctx.projectId, + ctx.userId, + ctx.filename, + ctx.line, + ctx.column, + { imageName: customImageName } + ) + }) + + it('should execute the synctex binary in a custom docker image', ctx => { + const outputFilePath = `${ctx.compileDir}/output.pdf` + const inputFilePath = `${ctx.compileDir}/${ctx.filename}` + expect(ctx.CommandRunner.promises.run).to.have.been.calledWith( + `${ctx.projectId}-${ctx.userId}`, + [ + 'synctex', + 'view', + '-i', + `${ctx.line}:${ctx.column}:${inputFilePath}`, + '-o', + outputFilePath, + ], + ctx.compileDir, + customImageName, + 60000, + {}, + 'synctex' + ) + }) + }) + }) + + describe('syncFromPdf', () => { + beforeEach(ctx => { + ctx.records = [{ file: 'main.tex', line: 1, column: 1 }] + ctx.SynctexOutputParser.parseEditOutput + .withArgs(ctx.commandOutput, ctx.compileDir) + .returns(ctx.records) + }) + + describe('normal case', () => { + beforeEach(async ctx => { + ctx.result = await ctx.CompileManager.promises.syncFromPdf( + ctx.projectId, + ctx.userId, + ctx.page, + ctx.h, + ctx.v, + { imageName: '' } + ) + }) + + it('should execute the synctex binary', ctx => { + const outputFilePath = `${ctx.compileDir}/output.pdf` + expect(ctx.CommandRunner.promises.run).to.have.been.calledWith( + `${ctx.projectId}-${ctx.userId}`, + [ + 'synctex', + 'edit', + '-o', + `${ctx.page}:${ctx.h}:${ctx.v}:${outputFilePath}`, + ], + ctx.compileDir, + ctx.Settings.clsi.docker.image, + 60000, + {} + ) + }) + + it('should return the parsed output', ctx => { + expect(ctx.result).to.deep.equal({ + pdfPositions: ctx.records, + downloadedFromCache: false, + }) + }) + }) + + describe('with a custom imageName', () => { + const customImageName = 'foo/bar:tag-1' + beforeEach(async ctx => { + await ctx.CompileManager.promises.syncFromPdf( + ctx.projectId, + ctx.userId, + ctx.page, + ctx.h, + ctx.v, + { imageName: customImageName } + ) + }) + + it('should execute the synctex binary in a custom docker image', ctx => { + const outputFilePath = `${ctx.compileDir}/output.pdf` + expect(ctx.CommandRunner.promises.run).to.have.been.calledWith( + `${ctx.projectId}-${ctx.userId}`, + [ + 'synctex', + 'edit', + '-o', + `${ctx.page}:${ctx.h}:${ctx.v}:${outputFilePath}`, + ], + ctx.compileDir, + customImageName, + 60000, + {} + ) + }) + }) + }) + }) + + describe('wordcount', () => { + beforeEach(async ctx => { + ctx.timeout = 60 * 1000 + ctx.filename = 'main.tex' + ctx.image = 'example.com/image' + + ctx.result = await ctx.CompileManager.promises.wordcount( + ctx.projectId, + ctx.userId, + ctx.filename, + ctx.image + ) + }) + + it('should run the texcount command', ctx => { + ctx.filePath = `$COMPILE_DIR/${ctx.filename}` + ctx.command = ['texcount', '-nocol', '-inc', ctx.filePath] + + expect(ctx.CommandRunner.promises.run).to.have.been.calledWith( + `${ctx.projectId}-${ctx.userId}`, + ctx.command, + ctx.compileDir, + ctx.image, + ctx.timeout, + {} + ) + }) + + it('should return the parsed output', ctx => { + expect(ctx.result).to.deep.equal({ + encode: 'ascii', + textWords: 2, + headWords: 0, + outside: 0, + headers: 0, + elements: 0, + mathInline: 0, + mathDisplay: 0, + errors: 0, + messages: '', + }) + }) + }) +}) diff --git a/services/clsi/test/unit/js/CompileManagerTests.js b/services/clsi/test/unit/js/CompileManagerTests.js deleted file mode 100644 index d564ca230f..0000000000 --- a/services/clsi/test/unit/js/CompileManagerTests.js +++ /dev/null @@ -1,753 +0,0 @@ -const Path = require('node:path') -const SandboxedModule = require('sandboxed-module') -const { expect } = require('chai') -const sinon = require('sinon') -const Metrics = require('../../../app/js/Metrics') - -const MODULE_PATH = require('node:path').join( - __dirname, - '../../../app/js/CompileManager' -) - -describe('CompileManager', function () { - beforeEach(function () { - this.projectId = 'project-id-123' - this.userId = '1234' - this.resources = 'mock-resources' - this.outputFiles = [ - { - path: 'output.log', - type: 'log', - }, - { - path: 'output.pdf', - type: 'pdf', - }, - ] - this.buildFiles = [ - { - path: 'output.log', - type: 'log', - build: 1234, - }, - { - path: 'output.pdf', - type: 'pdf', - build: 1234, - }, - ] - this.buildId = '00000000000-0000000000000000' - this.commandOutput = 'Dummy output' - this.compileBaseDir = '/compile/dir' - this.outputBaseDir = '/output/dir' - this.compileDir = `${this.compileBaseDir}/${this.projectId}-${this.userId}` - this.outputDir = `${this.outputBaseDir}/${this.projectId}-${this.userId}` - - this.LatexRunner = { - promises: { - runLatex: sinon.stub().resolves({}), - }, - } - this.ResourceWriter = { - promises: { - syncResourcesToDisk: sinon.stub().resolves(this.resources), - }, - } - this.OutputFileFinder = { - promises: { - findOutputFiles: sinon.stub().resolves({ - outputFiles: this.outputFiles, - allEntries: this.outputFiles.map(f => f.path).concat(['main.tex']), - }), - }, - } - this.OutputCacheManager = { - BUILD_REGEX: /^[0-9a-f]+-[0-9a-f]+$/, - CACHE_SUBDIR: 'generated-files', - promises: { - queueDirOperation: sinon.stub().callsArg(1), - saveOutputFiles: sinon - .stub() - .resolves({ outputFiles: this.buildFiles, buildId: this.buildId }), - }, - } - this.Settings = { - path: { - compilesDir: this.compileBaseDir, - outputDir: this.outputBaseDir, - synctexBaseDir: sinon.stub(), - }, - clsi: { - docker: { - image: 'SOMEIMAGE', - }, - }, - } - this.Settings.path.synctexBaseDir - .withArgs(`${this.projectId}-${this.userId}`) - .returns(this.compileDir) - this.child_process = { - exec: sinon.stub(), - execFile: sinon.stub().yields(), - } - this.CommandRunner = { - canRunSyncTeXInOutputDir: sinon.stub().returns(false), - promises: { - run: sinon.stub().callsFake((_1, _2, _3, _4, _5, _6, compileGroup) => { - if (compileGroup === 'synctex' || compileGroup === 'synctex-output') { - return Promise.resolve({ stdout: this.commandOutput }) - } else { - return Promise.resolve({ - stdout: 'Encoding: ascii\nWords in text: 2', - }) - } - }), - }, - } - this.DraftModeManager = { - promises: { - injectDraftMode: sinon.stub().resolves(), - }, - } - this.TikzManager = { - promises: { - checkMainFile: sinon.stub().resolves(false), - }, - } - this.lock = { - release: sinon.stub(), - } - this.LockManager = { - acquire: sinon.stub().returns(this.lock), - } - this.SynctexOutputParser = { - parseViewOutput: sinon.stub(), - parseEditOutput: sinon.stub(), - } - - this.dirStats = { - isDirectory: sinon.stub().returns(true), - } - this.fileStats = { - isFile: sinon.stub().returns(true), - } - this.fsPromises = { - lstat: sinon.stub(), - stat: sinon.stub(), - readFile: sinon.stub(), - mkdir: sinon.stub().resolves(), - rm: sinon.stub().resolves(), - unlink: sinon.stub().resolves(), - rmdir: sinon.stub().resolves(), - } - this.fsPromises.lstat.withArgs(this.compileDir).resolves(this.dirStats) - this.fsPromises.stat - .withArgs(Path.join(this.compileDir, 'output.synctex.gz')) - .resolves(this.fileStats) - - this.CLSICacheHandler = { - notifyCLSICacheAboutBuild: sinon.stub(), - downloadLatestCompileCache: sinon.stub().resolves(), - downloadOutputDotSynctexFromCompileCache: sinon.stub().resolves(), - } - - this.LatexMetrics = { enableLatexMkMetrics: sinon.stub() } - - this.StatsManager = { sampleRequest: sinon.stub().returns(false) } - - this.CompileManager = SandboxedModule.require(MODULE_PATH, { - requires: { - './LatexRunner': this.LatexRunner, - './ResourceWriter': this.ResourceWriter, - './OutputFileFinder': this.OutputFileFinder, - './OutputCacheManager': this.OutputCacheManager, - '@overleaf/settings': this.Settings, - '@overleaf/metrics': { - inc: sinon.stub(), - timing: sinon.stub(), - gauge: sinon.stub(), - Timer: sinon.stub().returns({ done: sinon.stub() }), - }, - child_process: this.child_process, - './CommandRunner': this.CommandRunner, - './DraftModeManager': this.DraftModeManager, - './TikzManager': this.TikzManager, - './LockManager': this.LockManager, - './SynctexOutputParser': this.SynctexOutputParser, - 'fs/promises': this.fsPromises, - './CLSICacheHandler': this.CLSICacheHandler, - './LatexMetrics': this.LatexMetrics, - './StatsManager': this.StatsManager, - './Metrics': Metrics, - }, - }) - }) - - describe('doCompileWithLock', function () { - beforeEach(function () { - this.request = { - resources: this.resources, - rootResourcePath: (this.rootResourcePath = 'main.tex'), - project_id: this.projectId, - user_id: this.userId, - compiler: (this.compiler = 'pdflatex'), - timeout: (this.timeout = 42000), - imageName: (this.image = 'example.com/image'), - flags: (this.flags = ['-file-line-error']), - compileGroup: (this.compileGroup = 'compile-group'), - stopOnFirstError: false, - metricsOpts: { - path: 'clsi-perf', - method: 'minimal', - compile: 'initial', - }, - } - this.env = { - OVERLEAF_PROJECT_ID: this.projectId, - } - }) - - describe('when the project is locked', function () { - beforeEach(async function () { - const error = new Error('locked') - this.LockManager.acquire.throws(error) - await expect( - this.CompileManager.promises.doCompileWithLock(this.request, {}, {}) - ).to.be.rejectedWith(error) - }) - - it('should ensure that the compile directory exists', function () { - expect(this.fsPromises.mkdir).to.have.been.calledWith(this.compileDir, { - recursive: true, - }) - }) - - it('should not run LaTeX', function () { - expect(this.LatexRunner.promises.runLatex).not.to.have.been.called - }) - }) - - describe('normally', function () { - beforeEach(async function () { - this.result = await this.CompileManager.promises.doCompileWithLock( - this.request, - {}, - {} - ) - }) - - it('should ensure that the compile directory exists', function () { - expect(this.fsPromises.mkdir).to.have.been.calledWith(this.compileDir, { - recursive: true, - }) - }) - - it('should write the resources to disk', function () { - expect( - this.ResourceWriter.promises.syncResourcesToDisk - ).to.have.been.calledWith(this.request, this.compileDir) - }) - - it('should run LaTeX', function () { - expect(this.LatexRunner.promises.runLatex).to.have.been.calledWith( - `${this.projectId}-${this.userId}`, - { - directory: this.compileDir, - mainFile: this.rootResourcePath, - compiler: this.compiler, - timeout: this.timeout, - image: this.image, - flags: this.flags, - environment: this.env, - compileGroup: this.compileGroup, - stopOnFirstError: this.request.stopOnFirstError, - stats: sinon.match.object, - timings: sinon.match.object, - } - ) - }) - - it('should find the output files', function () { - expect( - this.OutputFileFinder.promises.findOutputFiles - ).to.have.been.calledWith(this.resources, this.compileDir) - }) - - it('should return the output files', function () { - expect(this.result.outputFiles).to.equal(this.buildFiles) - }) - - it('should not inject draft mode by default', function () { - expect(this.DraftModeManager.promises.injectDraftMode).not.to.have.been - .called - }) - }) - - describe('with performance metric collection', function () { - it('should enable latexmk metrics when sampleRequest returns true', async function () { - this.StatsManager.sampleRequest.returns(true) - await this.CompileManager.promises.doCompileWithLock( - this.request, - {}, - {} - ) - expect(this.LatexMetrics.enableLatexMkMetrics).to.have.been.calledWith( - sinon.match.object - ) - }) - - it('should enable latexmk metrics when sampleRequest returns false', async function () { - this.StatsManager.sampleRequest.returns(false) - await this.CompileManager.promises.doCompileWithLock( - this.request, - {}, - {} - ) - expect(this.LatexMetrics.enableLatexMkMetrics).to.have.been.calledWith( - sinon.match.object - ) - }) - - it('should enable latexmk metrics when sampleRequest returns undefined', async function () { - this.StatsManager.sampleRequest.returns(undefined) - await this.CompileManager.promises.doCompileWithLock( - this.request, - {}, - {} - ) - expect(this.LatexMetrics.enableLatexMkMetrics).to.have.been.calledWith( - sinon.match.object - ) - }) - }) - - describe('with draft mode', function () { - beforeEach(async function () { - this.request.draft = true - await this.CompileManager.promises.doCompileWithLock( - this.request, - {}, - {} - ) - }) - - it('should inject the draft mode header', function () { - expect( - this.DraftModeManager.promises.injectDraftMode - ).to.have.been.calledWith(this.compileDir + '/' + this.rootResourcePath) - }) - }) - - describe('with a check option', function () { - beforeEach(async function () { - this.request.check = 'error' - await this.CompileManager.promises.doCompileWithLock( - this.request, - {}, - {} - ) - }) - - it('should run chktex', function () { - expect(this.LatexRunner.promises.runLatex).to.have.been.calledWith( - `${this.projectId}-${this.userId}`, - { - directory: this.compileDir, - mainFile: this.rootResourcePath, - compiler: this.compiler, - timeout: this.timeout, - image: this.image, - flags: this.flags, - environment: { - CHKTEX_OPTIONS: '-nall -e9 -e10 -w15 -w16', - CHKTEX_EXIT_ON_ERROR: 1, - CHKTEX_ULIMIT_OPTIONS: '-t 5 -v 64000', - OVERLEAF_PROJECT_ID: this.projectId, - }, - compileGroup: this.compileGroup, - stopOnFirstError: this.request.stopOnFirstError, - stats: sinon.match.object, - timings: sinon.match.object, - } - ) - }) - }) - - describe('with a knitr file and check options', function () { - beforeEach(async function () { - this.request.rootResourcePath = 'main.Rtex' - this.request.check = 'error' - await this.CompileManager.promises.doCompileWithLock( - this.request, - {}, - {} - ) - }) - - it('should not run chktex', function () { - expect(this.LatexRunner.promises.runLatex).to.have.been.calledWith( - `${this.projectId}-${this.userId}`, - { - directory: this.compileDir, - mainFile: 'main.Rtex', - compiler: this.compiler, - timeout: this.timeout, - image: this.image, - flags: this.flags, - environment: this.env, - compileGroup: this.compileGroup, - stopOnFirstError: this.request.stopOnFirstError, - stats: sinon.match.object, - timings: sinon.match.object, - } - ) - }) - }) - - describe('when the compile times out', function () { - beforeEach(async function () { - const error = new Error('timed out!') - error.timedout = true - this.LatexRunner.promises.runLatex.rejects(error) - await expect( - this.CompileManager.promises.doCompileWithLock(this.request, {}, {}) - ).to.be.rejected - }) - - it('should clear the compile directory', function () { - for (const { path } of this.buildFiles) { - expect(this.fsPromises.unlink).to.have.been.calledWith( - this.compileDir + '/' + path - ) - } - expect(this.fsPromises.unlink).to.have.been.calledWith( - this.compileDir + '/main.tex' - ) - expect(this.fsPromises.rmdir).to.have.been.calledWith(this.compileDir) - }) - }) - - describe('when the compile is manually stopped', function () { - beforeEach(async function () { - const error = new Error('terminated!') - error.terminated = true - this.LatexRunner.promises.runLatex.rejects(error) - await expect( - this.CompileManager.promises.doCompileWithLock(this.request, {}, {}) - ).to.be.rejected - }) - - it('should clear the compile directory', function () { - for (const { path } of this.buildFiles) { - expect(this.fsPromises.unlink).to.have.been.calledWith( - this.compileDir + '/' + path - ) - } - expect(this.fsPromises.unlink).to.have.been.calledWith( - this.compileDir + '/main.tex' - ) - expect(this.fsPromises.rmdir).to.have.been.calledWith(this.compileDir) - }) - }) - }) - - describe('clearProject', function () { - it('should clear the compile directory', async function () { - await this.CompileManager.promises.clearProject( - this.projectId, - this.userId - ) - - expect(this.fsPromises.rm).to.have.been.calledWith(this.compileDir, { - force: true, - recursive: true, - }) - }) - }) - - describe('syncing', function () { - beforeEach(function () { - this.page = 1 - this.h = 42.23 - this.v = 87.56 - this.width = 100.01 - this.height = 234.56 - this.line = 5 - this.column = 3 - this.filename = 'main.tex' - }) - - describe('syncFromCode', function () { - beforeEach(function () { - this.records = [{ page: 1, h: 2, v: 3, width: 4, height: 5 }] - this.SynctexOutputParser.parseViewOutput - .withArgs(this.commandOutput) - .returns(this.records) - }) - - describe('normal case', function () { - beforeEach(async function () { - this.result = await this.CompileManager.promises.syncFromCode( - this.projectId, - this.userId, - this.filename, - this.line, - this.column, - '' - ) - }) - - it('should execute the synctex binary', function () { - const outputFilePath = `${this.compileDir}/output.pdf` - const inputFilePath = `${this.compileDir}/${this.filename}` - expect(this.CommandRunner.promises.run).to.have.been.calledWith( - `${this.projectId}-${this.userId}`, - [ - 'synctex', - 'view', - '-i', - `${this.line}:${this.column}:${inputFilePath}`, - '-o', - outputFilePath, - ], - this.compileDir, - this.Settings.clsi.docker.image, - 60000, - {}, - 'synctex' - ) - }) - - it('should return the parsed output', function () { - expect(this.result).to.deep.equal({ - codePositions: this.records, - downloadedFromCache: false, - }) - }) - }) - - describe('from cache in docker', function () { - beforeEach(async function () { - this.CommandRunner.canRunSyncTeXInOutputDir.returns(true) - this.Settings.path.synctexBaseDir - .withArgs(`${this.projectId}-${this.userId}`) - .returns('/compile') - - const errNotFound = new Error() - errNotFound.code = 'ENOENT' - this.outputDir = `${this.outputBaseDir}/${this.projectId}-${this.userId}/${this.OutputCacheManager.CACHE_SUBDIR}/${this.buildId}` - const filename = Path.join(this.outputDir, 'output.synctex.gz') - this.fsPromises.stat - .withArgs(this.outputDir) - .onFirstCall() - .rejects(errNotFound) - this.fsPromises.stat - .withArgs(this.outputDir) - .onSecondCall() - .resolves(this.dirStats) - this.fsPromises.stat.withArgs(filename).resolves(this.fileStats) - this.CLSICacheHandler.downloadOutputDotSynctexFromCompileCache.resolves( - true - ) - this.result = await this.CompileManager.promises.syncFromCode( - this.projectId, - this.userId, - this.filename, - this.line, - this.column, - { - imageName: 'image', - editorId: '00000000-0000-0000-0000-000000000000', - buildId: this.buildId, - compileFromClsiCache: true, - } - ) - }) - - it('should run in output dir', function () { - const outputFilePath = '/compile/output.pdf' - const inputFilePath = `/compile/${this.filename}` - expect(this.CommandRunner.promises.run).to.have.been.calledWith( - `${this.projectId}-${this.userId}`, - [ - 'synctex', - 'view', - '-i', - `${this.line}:${this.column}:${inputFilePath}`, - '-o', - outputFilePath, - ], - this.outputDir, - 'image', - 60000, - {}, - 'synctex-output' - ) - }) - - it('should return the parsed output', function () { - expect(this.result).to.deep.equal({ - codePositions: this.records, - downloadedFromCache: true, - }) - }) - }) - - describe('with a custom imageName', function () { - const customImageName = 'foo/bar:tag-0' - beforeEach(async function () { - await this.CompileManager.promises.syncFromCode( - this.projectId, - this.userId, - this.filename, - this.line, - this.column, - { imageName: customImageName } - ) - }) - - it('should execute the synctex binary in a custom docker image', function () { - const outputFilePath = `${this.compileDir}/output.pdf` - const inputFilePath = `${this.compileDir}/${this.filename}` - expect(this.CommandRunner.promises.run).to.have.been.calledWith( - `${this.projectId}-${this.userId}`, - [ - 'synctex', - 'view', - '-i', - `${this.line}:${this.column}:${inputFilePath}`, - '-o', - outputFilePath, - ], - this.compileDir, - customImageName, - 60000, - {}, - 'synctex' - ) - }) - }) - }) - - describe('syncFromPdf', function () { - beforeEach(function () { - this.records = [{ file: 'main.tex', line: 1, column: 1 }] - this.SynctexOutputParser.parseEditOutput - .withArgs(this.commandOutput, this.compileDir) - .returns(this.records) - }) - - describe('normal case', function () { - beforeEach(async function () { - this.result = await this.CompileManager.promises.syncFromPdf( - this.projectId, - this.userId, - this.page, - this.h, - this.v, - { imageName: '' } - ) - }) - - it('should execute the synctex binary', function () { - const outputFilePath = `${this.compileDir}/output.pdf` - expect(this.CommandRunner.promises.run).to.have.been.calledWith( - `${this.projectId}-${this.userId}`, - [ - 'synctex', - 'edit', - '-o', - `${this.page}:${this.h}:${this.v}:${outputFilePath}`, - ], - this.compileDir, - this.Settings.clsi.docker.image, - 60000, - {} - ) - }) - - it('should return the parsed output', function () { - expect(this.result).to.deep.equal({ - pdfPositions: this.records, - downloadedFromCache: false, - }) - }) - }) - - describe('with a custom imageName', function () { - const customImageName = 'foo/bar:tag-1' - beforeEach(async function () { - await this.CompileManager.promises.syncFromPdf( - this.projectId, - this.userId, - this.page, - this.h, - this.v, - { imageName: customImageName } - ) - }) - - it('should execute the synctex binary in a custom docker image', function () { - const outputFilePath = `${this.compileDir}/output.pdf` - expect(this.CommandRunner.promises.run).to.have.been.calledWith( - `${this.projectId}-${this.userId}`, - [ - 'synctex', - 'edit', - '-o', - `${this.page}:${this.h}:${this.v}:${outputFilePath}`, - ], - this.compileDir, - customImageName, - 60000, - {} - ) - }) - }) - }) - }) - - describe('wordcount', function () { - beforeEach(async function () { - this.timeout = 60 * 1000 - this.filename = 'main.tex' - this.image = 'example.com/image' - - this.result = await this.CompileManager.promises.wordcount( - this.projectId, - this.userId, - this.filename, - this.image - ) - }) - - it('should run the texcount command', function () { - this.filePath = `$COMPILE_DIR/${this.filename}` - this.command = ['texcount', '-nocol', '-inc', this.filePath] - - expect(this.CommandRunner.promises.run).to.have.been.calledWith( - `${this.projectId}-${this.userId}`, - this.command, - this.compileDir, - this.image, - this.timeout, - {} - ) - }) - - it('should return the parsed output', function () { - expect(this.result).to.deep.equal({ - encode: 'ascii', - textWords: 2, - headWords: 0, - outside: 0, - headers: 0, - elements: 0, - mathInline: 0, - mathDisplay: 0, - errors: 0, - messages: '', - }) - }) - }) -}) diff --git a/services/clsi/test/unit/js/ContentCacheManagerTests.js b/services/clsi/test/unit/js/ContentCacheManager.test.js similarity index 93% rename from services/clsi/test/unit/js/ContentCacheManagerTests.js rename to services/clsi/test/unit/js/ContentCacheManager.test.js index df3bce212b..d76f48e27f 100644 --- a/services/clsi/test/unit/js/ContentCacheManagerTests.js +++ b/services/clsi/test/unit/js/ContentCacheManager.test.js @@ -1,15 +1,15 @@ -const fs = require('node:fs') -const Path = require('node:path') -const { expect } = require('chai') +import fs from 'node:fs' +import Path from 'node:path' +import { beforeAll, describe, expect, it } from 'vitest' const MODULE_PATH = '../../../app/js/ContentCacheManager' describe('ContentCacheManager', function () { let contentDir, pdfPath, xrefPath let ContentCacheManager, files, Settings - before(function () { - Settings = require('@overleaf/settings') - ContentCacheManager = require(MODULE_PATH) + beforeAll(async function () { + Settings = (await import('@overleaf/settings')).default + ContentCacheManager = (await import(MODULE_PATH)).default }) let contentRanges, newContentRanges, reclaimed async function run(filePath, pdfSize, pdfCachingMinChunkSize) { @@ -35,7 +35,7 @@ describe('ContentCacheManager', function () { files[path] = await fs.promises.readFile(path) } } - before(function () { + beforeAll(function () { contentDir = '/overleaf/services/clsi/output/602cee6f6460fca0ba7921e6/content/1797a7f48f9-5abc1998509dea1f' pdfPath = @@ -47,7 +47,7 @@ describe('ContentCacheManager', function () { Settings.pdfCachingMinChunkSize = 1024 }) - before(async function () { + beforeAll(async function () { await fs.promises.rm(contentDir, { recursive: true, force: true }) await fs.promises.mkdir(contentDir, { recursive: true }) await fs.promises.mkdir(Path.dirname(pdfPath), { recursive: true }) @@ -66,7 +66,7 @@ describe('ContentCacheManager', function () { return Path.join('test/unit/js/snapshots/minimalCompile/chunks', hash) } let MINIMAL_SIZE, RANGE_1, RANGE_2, h1, h2, START_1, START_2, END_1, END_2 - before(async function () { + beforeAll(async function () { await fs.promises.copyFile(PATH_MINIMAL_PDF, pdfPath) await fs.promises.copyFile(PATH_MINIMAL_XREF, xrefPath) const MINIMAL = await fs.promises.readFile(PATH_MINIMAL_PDF) @@ -85,7 +85,7 @@ describe('ContentCacheManager', function () { } describe('with two ranges qualifying', function () { - before(async function () { + beforeAll(async function () { await runWithMinimal(500) }) it('should produce two ranges', function () { @@ -133,7 +133,7 @@ describe('ContentCacheManager', function () { }) describe('when re-running with one range too small', function () { - before(async function () { + beforeAll(async function () { await runWithMinimal(1024) }) @@ -177,7 +177,7 @@ describe('ContentCacheManager', function () { describe('when re-running 5 more times', function () { for (let i = 0; i < 5; i++) { - before(async function () { + beforeAll(async function () { await runWithMinimal(1024) }) } diff --git a/services/clsi/test/unit/js/ContentTypeMapper.test.js b/services/clsi/test/unit/js/ContentTypeMapper.test.js new file mode 100644 index 0000000000..0cc61c4928 --- /dev/null +++ b/services/clsi/test/unit/js/ContentTypeMapper.test.js @@ -0,0 +1,69 @@ +import { describe, beforeEach, it } from 'vitest' +import path from 'node:path' + +const modulePath = path.join( + import.meta.dirname, + '../../../app/js/ContentTypeMapper' +) + +describe('ContentTypeMapper', function () { + beforeEach(async function (ctx) { + return (ctx.ContentTypeMapper = (await import(modulePath)).default) + }) + + return describe('map', function () { + it('should map .txt to text/plain', function (ctx) { + const contentType = ctx.ContentTypeMapper.map('example.txt') + return contentType.should.equal('text/plain') + }) + + it('should map .csv to text/csv', function (ctx) { + const contentType = ctx.ContentTypeMapper.map('example.csv') + return contentType.should.equal('text/csv') + }) + + it('should map .pdf to application/pdf', function (ctx) { + const contentType = ctx.ContentTypeMapper.map('example.pdf') + return contentType.should.equal('application/pdf') + }) + + it('should fall back to octet-stream', function (ctx) { + const contentType = ctx.ContentTypeMapper.map('example.unknown') + return contentType.should.equal('application/octet-stream') + }) + + describe('coercing web files to plain text', function () { + it('should map .js to plain text', function (ctx) { + const contentType = ctx.ContentTypeMapper.map('example.js') + return contentType.should.equal('text/plain') + }) + + it('should map .html to plain text', function (ctx) { + const contentType = ctx.ContentTypeMapper.map('example.html') + return contentType.should.equal('text/plain') + }) + + return it('should map .css to plain text', function (ctx) { + const contentType = ctx.ContentTypeMapper.map('example.css') + return contentType.should.equal('text/plain') + }) + }) + + return describe('image files', function () { + it('should map .png to image/png', function (ctx) { + const contentType = ctx.ContentTypeMapper.map('example.png') + return contentType.should.equal('image/png') + }) + + it('should map .jpeg to image/jpeg', function (ctx) { + const contentType = ctx.ContentTypeMapper.map('example.jpeg') + return contentType.should.equal('image/jpeg') + }) + + return it('should map .svg to text/plain to protect against XSS (SVG can execute JS)', function (ctx) { + const contentType = ctx.ContentTypeMapper.map('example.svg') + return contentType.should.equal('text/plain') + }) + }) + }) +}) diff --git a/services/clsi/test/unit/js/ContentTypeMapperTests.js b/services/clsi/test/unit/js/ContentTypeMapperTests.js deleted file mode 100644 index a413337153..0000000000 --- a/services/clsi/test/unit/js/ContentTypeMapperTests.js +++ /dev/null @@ -1,79 +0,0 @@ -/* eslint-disable - no-return-assign, - no-unused-vars, -*/ -// TODO: This file was created by bulk-decaffeinate. -// Fix any style issues and re-enable lint. -/* - * decaffeinate suggestions: - * DS102: Remove unnecessary code created because of implicit returns - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ -const SandboxedModule = require('sandboxed-module') -const sinon = require('sinon') -const modulePath = require('node:path').join( - __dirname, - '../../../app/js/ContentTypeMapper' -) - -describe('ContentTypeMapper', function () { - beforeEach(function () { - return (this.ContentTypeMapper = SandboxedModule.require(modulePath)) - }) - - return describe('map', function () { - it('should map .txt to text/plain', function () { - const contentType = this.ContentTypeMapper.map('example.txt') - return contentType.should.equal('text/plain') - }) - - it('should map .csv to text/csv', function () { - const contentType = this.ContentTypeMapper.map('example.csv') - return contentType.should.equal('text/csv') - }) - - it('should map .pdf to application/pdf', function () { - const contentType = this.ContentTypeMapper.map('example.pdf') - return contentType.should.equal('application/pdf') - }) - - it('should fall back to octet-stream', function () { - const contentType = this.ContentTypeMapper.map('example.unknown') - return contentType.should.equal('application/octet-stream') - }) - - describe('coercing web files to plain text', function () { - it('should map .js to plain text', function () { - const contentType = this.ContentTypeMapper.map('example.js') - return contentType.should.equal('text/plain') - }) - - it('should map .html to plain text', function () { - const contentType = this.ContentTypeMapper.map('example.html') - return contentType.should.equal('text/plain') - }) - - return it('should map .css to plain text', function () { - const contentType = this.ContentTypeMapper.map('example.css') - return contentType.should.equal('text/plain') - }) - }) - - return describe('image files', function () { - it('should map .png to image/png', function () { - const contentType = this.ContentTypeMapper.map('example.png') - return contentType.should.equal('image/png') - }) - - it('should map .jpeg to image/jpeg', function () { - const contentType = this.ContentTypeMapper.map('example.jpeg') - return contentType.should.equal('image/jpeg') - }) - - return it('should map .svg to text/plain to protect against XSS (SVG can execute JS)', function () { - const contentType = this.ContentTypeMapper.map('example.svg') - return contentType.should.equal('text/plain') - }) - }) - }) -}) diff --git a/services/clsi/test/unit/js/DockerLockManager.test.js b/services/clsi/test/unit/js/DockerLockManager.test.js new file mode 100644 index 0000000000..d0bee59913 --- /dev/null +++ b/services/clsi/test/unit/js/DockerLockManager.test.js @@ -0,0 +1,259 @@ +import { vi, describe, beforeEach, it } from 'vitest' + +import sinon from 'sinon' +import path from 'node:path' +const modulePath = path.join( + import.meta.dirname, + '../../../app/js/DockerLockManager' +) + +describe('DockerLockManager', function () { + beforeEach(async function (ctx) { + vi.doMock('@overleaf/settings', () => ({ + default: (ctx.Settings = { clsi: { docker: {} } }), + })) + + return (ctx.LockManager = (await import(modulePath)).default) + }) + + return describe('runWithLock', function () { + describe('with a single lock', function () { + beforeEach(async function (ctx) { + await new Promise((resolve, reject) => { + ctx.callback = sinon.stub() + return ctx.LockManager.runWithLock( + 'lock-one', + releaseLock => + setTimeout(() => releaseLock(null, 'hello', 'world'), 100), + + (err, ...args) => { + ctx.callback(err, ...Array.from(args)) + return resolve() + } + ) + }) + }) + + return it('should call the callback', function (ctx) { + return ctx.callback + .calledWith(null, 'hello', 'world') + .should.equal(true) + }) + }) + + describe('with two locks', function () { + beforeEach(async function (ctx) { + await new Promise((resolve, reject) => { + ctx.callback1 = sinon.stub() + ctx.callback2 = sinon.stub() + ctx.LockManager.runWithLock( + 'lock-one', + releaseLock => + setTimeout(() => releaseLock(null, 'hello', 'world', 'one'), 100), + + (err, ...args) => { + return ctx.callback1(err, ...Array.from(args)) + } + ) + return ctx.LockManager.runWithLock( + 'lock-two', + releaseLock => + setTimeout(() => releaseLock(null, 'hello', 'world', 'two'), 200), + + (err, ...args) => { + ctx.callback2(err, ...Array.from(args)) + return resolve() + } + ) + }) + }) + + it('should call the first callback', function (ctx) { + return ctx.callback1 + .calledWith(null, 'hello', 'world', 'one') + .should.equal(true) + }) + + return it('should call the second callback', function (ctx) { + return ctx.callback2 + .calledWith(null, 'hello', 'world', 'two') + .should.equal(true) + }) + }) + + return describe('with lock contention', function () { + describe('where the first lock is released quickly', function () { + beforeEach(async function (ctx) { + await new Promise((resolve, reject) => { + ctx.LockManager.MAX_LOCK_WAIT_TIME = 1000 + ctx.LockManager.LOCK_TEST_INTERVAL = 100 + ctx.callback1 = sinon.stub() + ctx.callback2 = sinon.stub() + ctx.LockManager.runWithLock( + 'lock', + releaseLock => + setTimeout( + () => releaseLock(null, 'hello', 'world', 'one'), + 100 + ), + + (err, ...args) => { + return ctx.callback1(err, ...Array.from(args)) + } + ) + return ctx.LockManager.runWithLock( + 'lock', + releaseLock => + setTimeout( + () => releaseLock(null, 'hello', 'world', 'two'), + 200 + ), + + (err, ...args) => { + ctx.callback2(err, ...Array.from(args)) + return resolve() + } + ) + }) + }) + + it('should call the first callback', function (ctx) { + return ctx.callback1 + .calledWith(null, 'hello', 'world', 'one') + .should.equal(true) + }) + + return it('should call the second callback', function (ctx) { + return ctx.callback2 + .calledWith(null, 'hello', 'world', 'two') + .should.equal(true) + }) + }) + + describe('where the first lock is held longer than the waiting time', function () { + beforeEach(async function (ctx) { + await new Promise((resolve, reject) => { + let doneTwo + ctx.LockManager.MAX_LOCK_HOLD_TIME = 10000 + ctx.LockManager.MAX_LOCK_WAIT_TIME = 1000 + ctx.LockManager.LOCK_TEST_INTERVAL = 100 + ctx.callback1 = sinon.stub() + ctx.callback2 = sinon.stub() + let doneOne = (doneTwo = false) + const finish = function (key) { + if (key === 1) { + doneOne = true + } + if (key === 2) { + doneTwo = true + } + if (doneOne && doneTwo) { + return resolve() + } + } + ctx.LockManager.runWithLock( + 'lock', + releaseLock => + setTimeout( + () => releaseLock(null, 'hello', 'world', 'one'), + 1100 + ), + + (err, ...args) => { + ctx.callback1(err, ...Array.from(args)) + return finish(1) + } + ) + return ctx.LockManager.runWithLock( + 'lock', + releaseLock => + setTimeout( + () => releaseLock(null, 'hello', 'world', 'two'), + 100 + ), + + (err, ...args) => { + ctx.callback2(err, ...Array.from(args)) + return finish(2) + } + ) + }) + }) + + it('should call the first callback', function (ctx) { + return ctx.callback1 + .calledWith(null, 'hello', 'world', 'one') + .should.equal(true) + }) + + return it('should call the second callback with an error', function (ctx) { + const error = sinon.match.instanceOf(Error) + return ctx.callback2.calledWith(error).should.equal(true) + }) + }) + + return describe('where the first lock is held longer than the max holding time', function () { + beforeEach(async function (ctx) { + await new Promise((resolve, reject) => { + let doneTwo + ctx.LockManager.MAX_LOCK_HOLD_TIME = 1000 + ctx.LockManager.MAX_LOCK_WAIT_TIME = 2000 + ctx.LockManager.LOCK_TEST_INTERVAL = 100 + ctx.callback1 = sinon.stub() + ctx.callback2 = sinon.stub() + let doneOne = (doneTwo = false) + const finish = function (key) { + if (key === 1) { + doneOne = true + } + if (key === 2) { + doneTwo = true + } + if (doneOne && doneTwo) { + return resolve() + } + } + ctx.LockManager.runWithLock( + 'lock', + releaseLock => + setTimeout( + () => releaseLock(null, 'hello', 'world', 'one'), + 1500 + ), + + (err, ...args) => { + ctx.callback1(err, ...Array.from(args)) + return finish(1) + } + ) + return ctx.LockManager.runWithLock( + 'lock', + releaseLock => + setTimeout( + () => releaseLock(null, 'hello', 'world', 'two'), + 100 + ), + + (err, ...args) => { + ctx.callback2(err, ...Array.from(args)) + return finish(2) + } + ) + }) + }) + + it('should call the first callback', function (ctx) { + return ctx.callback1 + .calledWith(null, 'hello', 'world', 'one') + .should.equal(true) + }) + + return it('should call the second callback', function (ctx) { + return ctx.callback2 + .calledWith(null, 'hello', 'world', 'two') + .should.equal(true) + }) + }) + }) + }) +}) diff --git a/services/clsi/test/unit/js/DockerLockManagerTests.js b/services/clsi/test/unit/js/DockerLockManagerTests.js deleted file mode 100644 index f69179443c..0000000000 --- a/services/clsi/test/unit/js/DockerLockManagerTests.js +++ /dev/null @@ -1,246 +0,0 @@ -/* eslint-disable - no-return-assign, -*/ -// TODO: This file was created by bulk-decaffeinate. -// Fix any style issues and re-enable lint. -/* - * decaffeinate suggestions: - * DS101: Remove unnecessary use of Array.from - * DS102: Remove unnecessary code created because of implicit returns - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ -const SandboxedModule = require('sandboxed-module') -const sinon = require('sinon') -const modulePath = require('node:path').join( - __dirname, - '../../../app/js/DockerLockManager' -) - -describe('DockerLockManager', function () { - beforeEach(function () { - return (this.LockManager = SandboxedModule.require(modulePath, { - requires: { - '@overleaf/settings': (this.Settings = { clsi: { docker: {} } }), - }, - })) - }) - - return describe('runWithLock', function () { - describe('with a single lock', function () { - beforeEach(function (done) { - this.callback = sinon.stub() - return this.LockManager.runWithLock( - 'lock-one', - releaseLock => - setTimeout(() => releaseLock(null, 'hello', 'world'), 100), - - (err, ...args) => { - this.callback(err, ...Array.from(args)) - return done() - } - ) - }) - - return it('should call the callback', function () { - return this.callback - .calledWith(null, 'hello', 'world') - .should.equal(true) - }) - }) - - describe('with two locks', function () { - beforeEach(function (done) { - this.callback1 = sinon.stub() - this.callback2 = sinon.stub() - this.LockManager.runWithLock( - 'lock-one', - releaseLock => - setTimeout(() => releaseLock(null, 'hello', 'world', 'one'), 100), - - (err, ...args) => { - return this.callback1(err, ...Array.from(args)) - } - ) - return this.LockManager.runWithLock( - 'lock-two', - releaseLock => - setTimeout(() => releaseLock(null, 'hello', 'world', 'two'), 200), - - (err, ...args) => { - this.callback2(err, ...Array.from(args)) - return done() - } - ) - }) - - it('should call the first callback', function () { - return this.callback1 - .calledWith(null, 'hello', 'world', 'one') - .should.equal(true) - }) - - return it('should call the second callback', function () { - return this.callback2 - .calledWith(null, 'hello', 'world', 'two') - .should.equal(true) - }) - }) - - return describe('with lock contention', function () { - describe('where the first lock is released quickly', function () { - beforeEach(function (done) { - this.LockManager.MAX_LOCK_WAIT_TIME = 1000 - this.LockManager.LOCK_TEST_INTERVAL = 100 - this.callback1 = sinon.stub() - this.callback2 = sinon.stub() - this.LockManager.runWithLock( - 'lock', - releaseLock => - setTimeout(() => releaseLock(null, 'hello', 'world', 'one'), 100), - - (err, ...args) => { - return this.callback1(err, ...Array.from(args)) - } - ) - return this.LockManager.runWithLock( - 'lock', - releaseLock => - setTimeout(() => releaseLock(null, 'hello', 'world', 'two'), 200), - - (err, ...args) => { - this.callback2(err, ...Array.from(args)) - return done() - } - ) - }) - - it('should call the first callback', function () { - return this.callback1 - .calledWith(null, 'hello', 'world', 'one') - .should.equal(true) - }) - - return it('should call the second callback', function () { - return this.callback2 - .calledWith(null, 'hello', 'world', 'two') - .should.equal(true) - }) - }) - - describe('where the first lock is held longer than the waiting time', function () { - beforeEach(function (done) { - let doneTwo - this.LockManager.MAX_LOCK_HOLD_TIME = 10000 - this.LockManager.MAX_LOCK_WAIT_TIME = 1000 - this.LockManager.LOCK_TEST_INTERVAL = 100 - this.callback1 = sinon.stub() - this.callback2 = sinon.stub() - let doneOne = (doneTwo = false) - const finish = function (key) { - if (key === 1) { - doneOne = true - } - if (key === 2) { - doneTwo = true - } - if (doneOne && doneTwo) { - return done() - } - } - this.LockManager.runWithLock( - 'lock', - releaseLock => - setTimeout( - () => releaseLock(null, 'hello', 'world', 'one'), - 1100 - ), - - (err, ...args) => { - this.callback1(err, ...Array.from(args)) - return finish(1) - } - ) - return this.LockManager.runWithLock( - 'lock', - releaseLock => - setTimeout(() => releaseLock(null, 'hello', 'world', 'two'), 100), - - (err, ...args) => { - this.callback2(err, ...Array.from(args)) - return finish(2) - } - ) - }) - - it('should call the first callback', function () { - return this.callback1 - .calledWith(null, 'hello', 'world', 'one') - .should.equal(true) - }) - - return it('should call the second callback with an error', function () { - const error = sinon.match.instanceOf(Error) - return this.callback2.calledWith(error).should.equal(true) - }) - }) - - return describe('where the first lock is held longer than the max holding time', function () { - beforeEach(function (done) { - let doneTwo - this.LockManager.MAX_LOCK_HOLD_TIME = 1000 - this.LockManager.MAX_LOCK_WAIT_TIME = 2000 - this.LockManager.LOCK_TEST_INTERVAL = 100 - this.callback1 = sinon.stub() - this.callback2 = sinon.stub() - let doneOne = (doneTwo = false) - const finish = function (key) { - if (key === 1) { - doneOne = true - } - if (key === 2) { - doneTwo = true - } - if (doneOne && doneTwo) { - return done() - } - } - this.LockManager.runWithLock( - 'lock', - releaseLock => - setTimeout( - () => releaseLock(null, 'hello', 'world', 'one'), - 1500 - ), - - (err, ...args) => { - this.callback1(err, ...Array.from(args)) - return finish(1) - } - ) - return this.LockManager.runWithLock( - 'lock', - releaseLock => - setTimeout(() => releaseLock(null, 'hello', 'world', 'two'), 100), - - (err, ...args) => { - this.callback2(err, ...Array.from(args)) - return finish(2) - } - ) - }) - - it('should call the first callback', function () { - return this.callback1 - .calledWith(null, 'hello', 'world', 'one') - .should.equal(true) - }) - - return it('should call the second callback', function () { - return this.callback2 - .calledWith(null, 'hello', 'world', 'two') - .should.equal(true) - }) - }) - }) - }) -}) diff --git a/services/clsi/test/unit/js/DockerRunner.test.js b/services/clsi/test/unit/js/DockerRunner.test.js new file mode 100644 index 0000000000..1b1c33214c --- /dev/null +++ b/services/clsi/test/unit/js/DockerRunner.test.js @@ -0,0 +1,1032 @@ +import { vi, expect, describe, beforeEach, afterEach, it } from 'vitest' + +import sinon from 'sinon' +import Path from 'node:path' + +const modulePath = Path.join( + import.meta.dirname, + '../../../app/js/DockerRunner' +) + +describe('DockerRunner', () => { + beforeEach(async ctx => { + let container + ctx.container = container = {} + + ctx.Settings = { + clsi: { docker: {} }, + path: {}, + } + const Docker = (function () { + const Docker = class Docker { + static initClass() { + this.prototype.getContainer = sinon.stub().returns(container) + this.prototype.createContainer = sinon.stub().yields(null, container) + this.prototype.listContainers = sinon.stub() + } + } + Docker.initClass() + return Docker + })() + + vi.doMock('@overleaf/settings', () => ({ + default: ctx.Settings, + })) + + vi.doMock('dockerode', () => ({ + default: Docker, + })) + + vi.doMock('fs', () => ({ + default: (ctx.fs = { + stat: sinon.stub().yields(null, { + isDirectory() { + return true + }, + }), + }), + })) + + const Timer = class Timer { + done() {} + } + + vi.doMock('@overleaf/metrics', () => ({ + default: { + Timer, + }, + })) + + vi.doMock('../../../app/js/LockManager', () => ({ + default: { + runWithLock(key, runner, callback) { + return runner(callback) + }, + }, + })) + + ctx.DockerRunner = (await import(modulePath)).default + ctx.Docker = Docker + ctx.getContainer = Docker.prototype.getContainer + ctx.createContainer = Docker.prototype.createContainer + ctx.listContainers = Docker.prototype.listContainers + + ctx.directory = '/local/compile/directory' + ctx.mainFile = 'main-file.tex' + ctx.compiler = 'pdflatex' + ctx.image = 'example.com/overleaf/image:2016.2' + ctx.env = {} + ctx.callback = sinon.stub() + ctx.project_id = 'project-id-123' + ctx.volumes = { '/some/host/dir/compiles/directory': '/compile' } + ctx.Settings.clsi.docker.image = ctx.defaultImage = 'default-image' + ctx.Settings.path.sandboxedCompilesHostDirCompiles = + '/some/host/dir/compiles' + ctx.Settings.path.sandboxedCompilesHostDirOutput = '/some/host/dir/output' + ctx.compileGroup = 'compile-group' + return (ctx.Settings.clsi.docker.env = { PATH: 'mock-path' }) + }) + + afterEach(ctx => { + ctx.DockerRunner.stopContainerMonitor() + }) + + describe('run', () => { + beforeEach(async ctx => { + await new Promise((resolve, reject) => { + ctx.DockerRunner._getContainerOptions = sinon + .stub() + .returns((ctx.options = { mockoptions: 'foo' })) + ctx.DockerRunner._fingerprintContainer = sinon + .stub() + .returns((ctx.fingerprint = 'fingerprint')) + + ctx.containerName = `project-${ctx.project_id}-${ctx.fingerprint}` + + ctx.command = ['mock', 'command', '--outdir=$COMPILE_DIR'] + ctx.command_with_dir = ['mock', 'command', '--outdir=/compile'] + ctx.timeout = 42000 + return resolve() + }) + }) + + describe('successfully', () => { + beforeEach(async ctx => { + await new Promise((resolve, reject) => { + ctx.DockerRunner._runAndWaitForContainer = sinon + .stub() + .callsArgWith(3, null, (ctx.output = 'mock-output')) + return ctx.DockerRunner.run( + ctx.project_id, + ctx.command, + ctx.directory, + ctx.image, + ctx.timeout, + ctx.env, + ctx.compileGroup, + (err, output) => { + ctx.callback(err, output) + return resolve() + } + ) + }) + }) + + it('should generate the options for the container', ctx => { + return ctx.DockerRunner._getContainerOptions + .calledWith(ctx.command_with_dir, ctx.image, ctx.volumes, ctx.timeout) + .should.equal(true) + }) + + it('should generate the fingerprint from the returned options', ctx => { + return ctx.DockerRunner._fingerprintContainer + .calledWith(ctx.options) + .should.equal(true) + }) + + it('should do the run', ctx => { + return ctx.DockerRunner._runAndWaitForContainer + .calledWith(ctx.options, ctx.volumes, ctx.timeout) + .should.equal(true) + }) + + return it('should call the callback', ctx => { + return ctx.callback.calledWith(null, ctx.output).should.equal(true) + }) + }) + + describe('standard compile', () => { + beforeEach(ctx => { + ctx.directory = '/var/lib/overleaf/data/compiles/xyz' + ctx.DockerRunner._runAndWaitForContainer = sinon + .stub() + .callsArgWith(3, null, (ctx.output = 'mock-output')) + return ctx.DockerRunner.run( + ctx.project_id, + ctx.command, + ctx.directory, + ctx.image, + ctx.timeout, + ctx.env, + ctx.compileGroup, + ctx.callback + ) + }) + + it('should re-write the bind directory', ctx => { + const volumes = + ctx.DockerRunner._runAndWaitForContainer.lastCall.args[1] + return expect(volumes).to.deep.equal({ + '/some/host/dir/compiles/xyz': '/compile', + }) + }) + + return it('should call the callback', ctx => { + return ctx.callback.calledWith(null, ctx.output).should.equal(true) + }) + }) + + describe('synctex-output', () => { + beforeEach(ctx => { + ctx.directory = '/var/lib/overleaf/data/output/xyz/generated-files/id' + ctx.DockerRunner._runAndWaitForContainer = sinon + .stub() + .callsArgWith(3, null, (ctx.output = 'mock-output')) + ctx.DockerRunner.run( + ctx.project_id, + ctx.command, + ctx.directory, + ctx.image, + ctx.timeout, + ctx.env, + 'synctex-output', + ctx.callback + ) + }) + + it('should re-write the bind directory and set ro flag', ctx => { + const volumes = + ctx.DockerRunner._runAndWaitForContainer.lastCall.args[1] + expect(volumes).to.deep.equal({ + '/some/host/dir/output/xyz/generated-files/id': '/compile:ro', + }) + }) + + it('should call the callback', ctx => { + ctx.callback.calledWith(null, ctx.output).should.equal(true) + }) + }) + + describe('synctex', () => { + beforeEach(ctx => { + ctx.directory = '/var/lib/overleaf/data/compile/xyz' + ctx.DockerRunner._runAndWaitForContainer = sinon + .stub() + .callsArgWith(3, null, (ctx.output = 'mock-output')) + ctx.DockerRunner.run( + ctx.project_id, + ctx.command, + ctx.directory, + ctx.image, + ctx.timeout, + ctx.env, + 'synctex', + ctx.callback + ) + }) + + it('should re-write the bind directory', ctx => { + const volumes = + ctx.DockerRunner._runAndWaitForContainer.lastCall.args[1] + expect(volumes).to.deep.equal({ + '/some/host/dir/compiles/xyz': '/compile:ro', + }) + }) + + it('should call the callback', ctx => { + ctx.callback.calledWith(null, ctx.output).should.equal(true) + }) + }) + + describe('wordcount', () => { + beforeEach(ctx => { + ctx.directory = '/var/lib/overleaf/data/compile/xyz' + ctx.DockerRunner._runAndWaitForContainer = sinon + .stub() + .callsArgWith(3, null, (ctx.output = 'mock-output')) + ctx.DockerRunner.run( + ctx.project_id, + ctx.command, + ctx.directory, + ctx.image, + ctx.timeout, + ctx.env, + 'wordcount', + ctx.callback + ) + }) + + it('should re-write the bind directory', ctx => { + const volumes = + ctx.DockerRunner._runAndWaitForContainer.lastCall.args[1] + expect(volumes).to.deep.equal({ + '/some/host/dir/compiles/xyz': '/compile:ro', + }) + }) + + it('should call the callback', ctx => { + ctx.callback.calledWith(null, ctx.output).should.equal(true) + }) + }) + + describe('when the run throws an error', () => { + beforeEach(ctx => { + let firstTime = true + ctx.output = 'mock-output' + ctx.DockerRunner._runAndWaitForContainer = ( + options, + volumes, + timeout, + callback + ) => { + if (callback == null) { + callback = function () {} + } + if (firstTime) { + firstTime = false + const error = new Error('(HTTP code 500) server error - ...') + error.statusCode = 500 + callback(error) + } else { + callback(null, ctx.output) + } + } + sinon.spy(ctx.DockerRunner, '_runAndWaitForContainer') + ctx.DockerRunner.destroyContainer = sinon.stub().callsArg(3) + ctx.DockerRunner.run( + ctx.project_id, + ctx.command, + ctx.directory, + ctx.image, + ctx.timeout, + ctx.env, + ctx.compileGroup, + ctx.callback + ) + }) + + it('should do the run twice', ctx => { + ctx.DockerRunner._runAndWaitForContainer.calledTwice.should.equal(true) + }) + + it('should destroy the container in between', ctx => { + ctx.DockerRunner.destroyContainer + .calledWith(ctx.containerName, null) + .should.equal(true) + }) + + it('should call the callback', ctx => { + ctx.callback.calledWith(null, ctx.output).should.equal(true) + }) + }) + + describe('with no image', () => { + beforeEach(ctx => { + ctx.DockerRunner._runAndWaitForContainer = sinon + .stub() + .callsArgWith(3, null, (ctx.output = 'mock-output')) + ctx.DockerRunner.run( + ctx.project_id, + ctx.command, + ctx.directory, + null, + ctx.timeout, + ctx.env, + ctx.compileGroup, + ctx.callback + ) + }) + + it('should use the default image', ctx => { + ctx.DockerRunner._getContainerOptions + .calledWith( + ctx.command_with_dir, + ctx.defaultImage, + ctx.volumes, + ctx.timeout + ) + .should.equal(true) + }) + }) + + describe('with image override', () => { + beforeEach(ctx => { + ctx.Settings.texliveImageNameOveride = 'overrideimage.com/something' + ctx.DockerRunner._runAndWaitForContainer = sinon + .stub() + .callsArgWith(3, null, (ctx.output = 'mock-output')) + ctx.DockerRunner.run( + ctx.project_id, + ctx.command, + ctx.directory, + ctx.image, + ctx.timeout, + ctx.env, + ctx.compileGroup, + ctx.callback + ) + }) + + it('should use the override and keep the tag', ctx => { + const image = ctx.DockerRunner._getContainerOptions.args[0][1] + image.should.equal('overrideimage.com/something/image:2016.2') + }) + }) + + describe('with image restriction', () => { + beforeEach(ctx => { + ctx.Settings.clsi.docker.allowedImages = [ + 'repo/image:tag1', + 'repo/image:tag2', + ] + ctx.DockerRunner._runAndWaitForContainer = sinon + .stub() + .callsArgWith(3, null, (ctx.output = 'mock-output')) + }) + + describe('with a valid image', () => { + beforeEach(ctx => { + ctx.DockerRunner.run( + ctx.project_id, + ctx.command, + ctx.directory, + 'repo/image:tag1', + ctx.timeout, + ctx.env, + ctx.compileGroup, + ctx.callback + ) + }) + + it('should setup the container', ctx => { + ctx.DockerRunner._getContainerOptions.called.should.equal(true) + }) + }) + + describe('with a invalid image', () => { + beforeEach(ctx => { + ctx.DockerRunner.run( + ctx.project_id, + ctx.command, + ctx.directory, + 'something/different:evil', + ctx.timeout, + ctx.env, + ctx.compileGroup, + ctx.callback + ) + }) + + it('should call the callback with an error', ctx => { + const err = new Error('image not allowed') + ctx.callback.called.should.equal(true) + ctx.callback.args[0][0].message.should.equal(err.message) + }) + + it('should not setup the container', ctx => { + ctx.DockerRunner._getContainerOptions.called.should.equal(false) + }) + }) + }) + }) + + describe('run with _getOptions', () => { + beforeEach(async ctx => { + await new Promise((resolve, reject) => { + // this.DockerRunner._getContainerOptions = sinon + // .stub() + // .returns((this.options = { mockoptions: 'foo' })) + ctx.DockerRunner._fingerprintContainer = sinon + .stub() + .returns((ctx.fingerprint = 'fingerprint')) + + ctx.containerName = `project-${ctx.project_id}-${ctx.fingerprint}` + + ctx.command = ['mock', 'command', '--outdir=$COMPILE_DIR'] + ctx.command_with_dir = ['mock', 'command', '--outdir=/compile'] + ctx.timeout = 42000 + resolve() + }) + }) + + describe('when a compile group config is set', () => { + beforeEach(ctx => { + ctx.Settings.clsi.docker.compileGroupConfig = { + 'compile-group': { + 'HostConfig.newProperty': 'new-property', + }, + 'other-group': { otherProperty: 'other-property' }, + } + ctx.DockerRunner._runAndWaitForContainer = sinon + .stub() + .callsArgWith(3, null, (ctx.output = 'mock-output')) + ctx.DockerRunner.run( + ctx.project_id, + ctx.command, + ctx.directory, + ctx.image, + ctx.timeout, + ctx.env, + ctx.compileGroup, + ctx.callback + ) + }) + + it('should set the docker options for the compile group', ctx => { + const options = + ctx.DockerRunner._runAndWaitForContainer.lastCall.args[0] + expect(options.HostConfig).to.deep.include({ + Binds: ['/some/host/dir/compiles/directory:/compile:rw'], + LogConfig: { Type: 'none', Config: {} }, + CapDrop: ['ALL'], + SecurityOpt: ['no-new-privileges'], + newProperty: 'new-property', + }) + }) + + it('should call the callback', ctx => { + ctx.callback.calledWith(null, ctx.output).should.equal(true) + }) + }) + }) + + describe('_runAndWaitForContainer', () => { + beforeEach(ctx => { + ctx.options = { + mockoptions: 'foo', + name: (ctx.containerName = 'mock-name'), + } + ctx.DockerRunner.startContainer = ( + options, + volumes, + attachStreamHandler, + callback + ) => { + attachStreamHandler(null, (ctx.output = 'mock-output')) + callback(null, (ctx.containerId = 'container-id')) + } + sinon.spy(ctx.DockerRunner, 'startContainer') + ctx.DockerRunner.waitForContainer = sinon + .stub() + .callsArgWith(3, null, (ctx.exitCode = 42)) + ctx.DockerRunner._runAndWaitForContainer( + ctx.options, + ctx.volumes, + ctx.timeout, + ctx.callback + ) + }) + + it('should create/start the container', ctx => { + ctx.DockerRunner.startContainer + .calledWith(ctx.options, ctx.volumes) + .should.equal(true) + }) + + it('should wait for the container to finish', ctx => { + ctx.DockerRunner.waitForContainer + .calledWith(ctx.containerName, ctx.timeout) + .should.equal(true) + }) + + it('should call the callback with the output', ctx => { + ctx.callback.calledWith(null, ctx.output).should.equal(true) + }) + }) + + describe('startContainer', () => { + beforeEach(ctx => { + ctx.attachStreamHandler = sinon.stub() + ctx.attachStreamHandler.cock = true + ctx.options = { mockoptions: 'foo', name: 'mock-name' } + ctx.container.inspect = sinon.stub().callsArgWith(0) + ctx.DockerRunner.attachToContainer = ( + containerId, + attachStreamHandler, + cb + ) => { + attachStreamHandler() + cb() + } + sinon.spy(ctx.DockerRunner, 'attachToContainer') + }) + + describe('when the container exists', () => { + beforeEach(ctx => { + ctx.container.inspect = sinon.stub().callsArgWith(0) + ctx.container.start = sinon.stub().yields() + + ctx.DockerRunner.startContainer( + ctx.options, + ctx.volumes, + () => {}, + ctx.callback + ) + }) + + it('should start the container with the given name', ctx => { + ctx.getContainer.calledWith(ctx.options.name).should.equal(true) + ctx.container.start.called.should.equal(true) + }) + + it('should not try to create the container', ctx => { + ctx.createContainer.called.should.equal(false) + }) + + it('should attach to the container', ctx => { + ctx.DockerRunner.attachToContainer.called.should.equal(true) + }) + + it('should call the callback', ctx => { + ctx.callback.called.should.equal(true) + }) + + it('should attach before the container starts', ctx => { + sinon.assert.callOrder( + ctx.DockerRunner.attachToContainer, + ctx.container.start + ) + }) + }) + + describe('when the container does not exist', () => { + beforeEach(ctx => { + ctx.container.start = sinon.stub().yields() + ctx.container.inspect = sinon + .stub() + .callsArgWith(0, { statusCode: 404 }) + ctx.DockerRunner.startContainer( + ctx.options, + ctx.volumes, + ctx.attachStreamHandler, + ctx.callback + ) + }) + + it('should create the container', ctx => { + ctx.createContainer.calledWith(ctx.options).should.equal(true) + }) + + it('should call the callback and stream handler', ctx => { + ctx.attachStreamHandler.called.should.equal(true) + ctx.callback.called.should.equal(true) + }) + + it('should attach to the container', ctx => { + ctx.DockerRunner.attachToContainer.called.should.equal(true) + }) + + it('should attach before the container starts', ctx => { + sinon.assert.callOrder( + ctx.DockerRunner.attachToContainer, + ctx.container.start + ) + }) + }) + + describe('when the container is already running', () => { + beforeEach(ctx => { + const error = new Error( + `HTTP code is 304 which indicates error: server error - start: Cannot start container ${ctx.containerName}: The container MOCKID is already running.` + ) + error.statusCode = 304 + ctx.container.start = sinon.stub().yields(error) + ctx.container.inspect = sinon.stub().callsArgWith(0) + ctx.DockerRunner.startContainer( + ctx.options, + ctx.volumes, + ctx.attachStreamHandler, + ctx.callback + ) + }) + + it('should not try to create the container', ctx => { + ctx.createContainer.called.should.equal(false) + }) + + it('should call the callback and stream handler without an error', ctx => { + ctx.attachStreamHandler.called.should.equal(true) + ctx.callback.called.should.equal(true) + }) + }) + + describe.todo( + 'when the container tries to be created, but already has been (race condition)', + () => {} + ) + }) + + describe('waitForContainer', () => { + beforeEach(ctx => { + ctx.containerId = 'container-id' + ctx.timeout = 5000 + ctx.container.wait = sinon + .stub() + .yields(null, { StatusCode: (ctx.statusCode = 42) }) + ctx.container.kill = sinon.stub().yields() + }) + + describe('when the container returns in time', () => { + beforeEach(ctx => { + ctx.DockerRunner.waitForContainer( + ctx.containerId, + ctx.timeout, + {}, + ctx.callback + ) + }) + + it('should wait for the container', ctx => { + ctx.getContainer.calledWith(ctx.containerId).should.equal(true) + ctx.container.wait.called.should.equal(true) + }) + + it('should call the callback with the exit', ctx => { + ctx.callback.calledWith(null, ctx.statusCode).should.equal(true) + }) + }) + + describe('when the container is removed before waiting', () => { + const err = new Error('not found') + err.statusCode = 404 + beforeEach(ctx => { + ctx.container.wait = sinon.stub().yields(err) + }) + + describe('AutoRemove not set', () => { + beforeEach(ctx => { + ctx.DockerRunner.waitForContainer( + ctx.containerId, + ctx.timeout, + { HostConfig: {} }, + ctx.callback + ) + }) + it('should wait for the container', ctx => { + ctx.getContainer.calledWith(ctx.containerId).should.equal(true) + ctx.container.wait.called.should.equal(true) + }) + it('should call the callback with the error', ctx => { + ctx.callback.calledWith(err).should.equal(true) + }) + }) + describe('AutoRemove=true', () => { + beforeEach(ctx => { + ctx.DockerRunner.waitForContainer( + ctx.containerId, + ctx.timeout, + { HostConfig: { AutoRemove: true } }, + ctx.callback + ) + }) + it('should wait for the container', ctx => { + ctx.getContainer.calledWith(ctx.containerId).should.equal(true) + ctx.container.wait.called.should.equal(true) + }) + it('should call the callback with exit code 0', ctx => { + ctx.callback.calledWith(null, 0).should.equal(true) + }) + }) + }) + + describe('when the container does not return before the timeout', () => { + beforeEach(async ctx => { + await new Promise((resolve, reject) => { + ctx.container.wait = function (callback) { + if (callback == null) { + callback = function () {} + } + setTimeout(() => callback(null, { StatusCode: 42 }), 100) + } + ctx.timeout = 5 + ctx.DockerRunner.waitForContainer( + ctx.containerId, + ctx.timeout, + {}, + (...args) => { + ctx.callback(...Array.from(args || [])) + resolve() + } + ) + }) + }) + + it('should call kill on the container', ctx => { + ctx.getContainer.calledWith(ctx.containerId).should.equal(true) + ctx.container.kill.called.should.equal(true) + }) + + it('should call the callback with an error', ctx => { + ctx.callback.calledWith(sinon.match(Error)).should.equal(true) + + const errorObj = ctx.callback.args[0][0] + expect(errorObj.message).to.include('container timed out') + expect(errorObj.timedout).equal(true) + }) + }) + }) + + describe('destroyOldContainers', () => { + beforeEach(async ctx => { + await new Promise((resolve, reject) => { + const oneHourInSeconds = 60 * 60 + const oneHourInMilliseconds = oneHourInSeconds * 1000 + const nowInSeconds = Date.now() / 1000 + ctx.containers = [ + { + Name: '/project-old-container-name', + Id: 'old-container-id', + Created: nowInSeconds - oneHourInSeconds - 100, + }, + { + Name: '/project-new-container-name', + Id: 'new-container-id', + Created: nowInSeconds - oneHourInSeconds + 100, + }, + { + Name: '/totally-not-a-project-container', + Id: 'some-random-id', + Created: nowInSeconds - 2 * oneHourInSeconds, + }, + ] + ctx.DockerRunner.MAX_CONTAINER_AGE = oneHourInMilliseconds + ctx.listContainers.callsArgWith(1, null, ctx.containers) + ctx.DockerRunner.destroyContainer = sinon.stub().callsArg(3) + ctx.DockerRunner.destroyOldContainers(error => { + ctx.callback(error) + resolve() + }) + }) + }) + + it('should list all containers', ctx => { + ctx.listContainers.calledWith({ all: true }).should.equal(true) + }) + + it('should destroy old containers', ctx => { + ctx.DockerRunner.destroyContainer.callCount.should.equal(1) + ctx.DockerRunner.destroyContainer + .calledWith('project-old-container-name', 'old-container-id') + .should.equal(true) + }) + + it('should not destroy new containers', ctx => { + ctx.DockerRunner.destroyContainer + .calledWith('project-new-container-name', 'new-container-id') + .should.equal(false) + }) + + it('should not destroy non-project containers', ctx => { + ctx.DockerRunner.destroyContainer + .calledWith('totally-not-a-project-container', 'some-random-id') + .should.equal(false) + }) + + it('should callback the callback', ctx => { + ctx.callback.called.should.equal(true) + }) + }) + + describe('_destroyContainer', () => { + beforeEach(ctx => { + ctx.containerId = 'some_id' + ctx.fakeContainer = { remove: sinon.stub().callsArgWith(1, null) } + ctx.Docker.prototype.getContainer = sinon + .stub() + .returns(ctx.fakeContainer) + }) + + it('should get the container', async ctx => { + await new Promise((resolve, reject) => { + ctx.DockerRunner._destroyContainer(ctx.containerId, false, err => { + if (err) return reject(err) + ctx.Docker.prototype.getContainer.callCount.should.equal(1) + ctx.Docker.prototype.getContainer + .calledWith(ctx.containerId) + .should.equal(true) + resolve() + }) + }) + }) + + it('should try to force-destroy the container when shouldForce=true', async ctx => { + await new Promise((resolve, reject) => { + ctx.DockerRunner._destroyContainer(ctx.containerId, true, err => { + if (err) return reject(err) + ctx.fakeContainer.remove.callCount.should.equal(1) + ctx.fakeContainer.remove + .calledWith({ force: true, v: true }) + .should.equal(true) + resolve() + }) + }) + }) + + it('should not try to force-destroy the container when shouldForce=false', async ctx => { + await new Promise((resolve, reject) => { + ctx.DockerRunner._destroyContainer(ctx.containerId, false, err => { + if (err) return reject(err) + ctx.fakeContainer.remove.callCount.should.equal(1) + ctx.fakeContainer.remove + .calledWith({ force: false, v: true }) + .should.equal(true) + resolve() + }) + }) + }) + + it('should not produce an error', async ctx => { + await new Promise((resolve, reject) => { + ctx.DockerRunner._destroyContainer(ctx.containerId, false, err => { + expect(err).to.equal(null) + resolve() + }) + }) + }) + + describe('when the container is already gone', () => { + beforeEach(ctx => { + ctx.fakeError = new Error('woops') + ctx.fakeError.statusCode = 404 + ctx.fakeContainer = { + remove: sinon.stub().callsArgWith(1, ctx.fakeError), + } + ctx.Docker.prototype.getContainer = sinon + .stub() + .returns(ctx.fakeContainer) + }) + + it('should not produce an error', async ctx => { + await new Promise((resolve, reject) => { + ctx.DockerRunner._destroyContainer(ctx.containerId, false, err => { + expect(err).to.equal(null) + resolve() + }) + }) + }) + }) + + describe('when container.destroy produces an error', () => { + beforeEach(ctx => { + ctx.fakeError = new Error('woops') + ctx.fakeError.statusCode = 500 + ctx.fakeContainer = { + remove: sinon.stub().callsArgWith(1, ctx.fakeError), + } + ctx.Docker.prototype.getContainer = sinon + .stub() + .returns(ctx.fakeContainer) + }) + + it('should produce an error', async ctx => { + await new Promise((resolve, reject) => { + ctx.DockerRunner._destroyContainer(ctx.containerId, false, err => { + expect(err).to.not.equal(null) + expect(err).to.equal(ctx.fakeError) + resolve() + }) + }) + }) + }) + }) + + describe('kill', () => { + beforeEach(ctx => { + ctx.containerId = 'some_id' + ctx.fakeContainer = { kill: sinon.stub().callsArgWith(0, null) } + ctx.Docker.prototype.getContainer = sinon + .stub() + .returns(ctx.fakeContainer) + }) + + it('should get the container', async ctx => { + await new Promise((resolve, reject) => { + ctx.DockerRunner.kill(ctx.containerId, err => { + if (err) return reject(err) + ctx.Docker.prototype.getContainer.callCount.should.equal(1) + ctx.Docker.prototype.getContainer + .calledWith(ctx.containerId) + .should.equal(true) + resolve() + }) + }) + }) + + it('should try to force-destroy the container', async ctx => { + await new Promise((resolve, reject) => { + ctx.DockerRunner.kill(ctx.containerId, err => { + if (err) return reject(err) + ctx.fakeContainer.kill.callCount.should.equal(1) + resolve() + }) + }) + }) + + it('should not produce an error', async ctx => { + await new Promise((resolve, reject) => { + ctx.DockerRunner.kill(ctx.containerId, err => { + expect(err).to.equal(undefined) + resolve() + }) + }) + }) + + describe('when the container is not actually running', () => { + beforeEach(ctx => { + ctx.fakeError = new Error('woops') + ctx.fakeError.statusCode = 500 + ctx.fakeError.message = + 'Cannot kill container is not running' + ctx.fakeContainer = { + kill: sinon.stub().callsArgWith(0, ctx.fakeError), + } + ctx.Docker.prototype.getContainer = sinon + .stub() + .returns(ctx.fakeContainer) + }) + + it('should not produce an error', async ctx => { + await new Promise((resolve, reject) => { + ctx.DockerRunner.kill(ctx.containerId, err => { + expect(err).to.equal(undefined) + resolve() + }) + }) + }) + }) + + describe('when container.kill produces a legitimate error', () => { + beforeEach(ctx => { + ctx.fakeError = new Error('woops') + ctx.fakeError.statusCode = 500 + ctx.fakeError.message = 'Totally legitimate reason to throw an error' + ctx.fakeContainer = { + kill: sinon.stub().callsArgWith(0, ctx.fakeError), + } + ctx.Docker.prototype.getContainer = sinon + .stub() + .returns(ctx.fakeContainer) + }) + + it('should produce an error', async ctx => { + await new Promise((resolve, reject) => { + ctx.DockerRunner.kill(ctx.containerId, err => { + expect(err).to.not.equal(undefined) + expect(err).to.equal(ctx.fakeError) + resolve() + }) + }) + }) + }) + }) +}) diff --git a/services/clsi/test/unit/js/DockerRunnerTests.js b/services/clsi/test/unit/js/DockerRunnerTests.js deleted file mode 100644 index ea82810d60..0000000000 --- a/services/clsi/test/unit/js/DockerRunnerTests.js +++ /dev/null @@ -1,1026 +0,0 @@ -/* eslint-disable - no-return-assign, - no-unused-vars, -*/ -// TODO: This file was created by bulk-decaffeinate. -// Fix any style issues and re-enable lint. -/* - * decaffeinate suggestions: - * DS101: Remove unnecessary use of Array.from - * DS102: Remove unnecessary code created because of implicit returns - * DS206: Consider reworking classes to avoid initClass - * DS207: Consider shorter variations of null checks - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ -const SandboxedModule = require('sandboxed-module') -const sinon = require('sinon') -const { expect } = require('chai') -const modulePath = require('node:path').join( - __dirname, - '../../../app/js/DockerRunner' -) -const Path = require('node:path') - -describe('DockerRunner', function () { - beforeEach(function () { - let container, Docker, Timer - this.container = container = {} - this.DockerRunner = SandboxedModule.require(modulePath, { - requires: { - '@overleaf/settings': (this.Settings = { - clsi: { docker: {} }, - path: {}, - }), - dockerode: (Docker = (function () { - Docker = class Docker { - static initClass() { - this.prototype.getContainer = sinon.stub().returns(container) - this.prototype.createContainer = sinon - .stub() - .yields(null, container) - this.prototype.listContainers = sinon.stub() - } - } - Docker.initClass() - return Docker - })()), - fs: (this.fs = { - stat: sinon.stub().yields(null, { - isDirectory() { - return true - }, - }), - }), - '@overleaf/metrics': { - Timer: (Timer = class Timer { - done() {} - }), - }, - './LockManager': { - runWithLock(key, runner, callback) { - return runner(callback) - }, - }, - }, - globals: { Math }, // used by lodash - }) - this.Docker = Docker - this.getContainer = Docker.prototype.getContainer - this.createContainer = Docker.prototype.createContainer - this.listContainers = Docker.prototype.listContainers - - this.directory = '/local/compile/directory' - this.mainFile = 'main-file.tex' - this.compiler = 'pdflatex' - this.image = 'example.com/overleaf/image:2016.2' - this.env = {} - this.callback = sinon.stub() - this.project_id = 'project-id-123' - this.volumes = { '/some/host/dir/compiles/directory': '/compile' } - this.Settings.clsi.docker.image = this.defaultImage = 'default-image' - this.Settings.path.sandboxedCompilesHostDirCompiles = - '/some/host/dir/compiles' - this.Settings.path.sandboxedCompilesHostDirOutput = '/some/host/dir/output' - this.compileGroup = 'compile-group' - return (this.Settings.clsi.docker.env = { PATH: 'mock-path' }) - }) - - afterEach(function () { - this.DockerRunner.stopContainerMonitor() - }) - - describe('run', function () { - beforeEach(function (done) { - this.DockerRunner._getContainerOptions = sinon - .stub() - .returns((this.options = { mockoptions: 'foo' })) - this.DockerRunner._fingerprintContainer = sinon - .stub() - .returns((this.fingerprint = 'fingerprint')) - - this.name = `project-${this.project_id}-${this.fingerprint}` - - this.command = ['mock', 'command', '--outdir=$COMPILE_DIR'] - this.command_with_dir = ['mock', 'command', '--outdir=/compile'] - this.timeout = 42000 - return done() - }) - - describe('successfully', function () { - beforeEach(function (done) { - this.DockerRunner._runAndWaitForContainer = sinon - .stub() - .callsArgWith(3, null, (this.output = 'mock-output')) - return this.DockerRunner.run( - this.project_id, - this.command, - this.directory, - this.image, - this.timeout, - this.env, - this.compileGroup, - (err, output) => { - this.callback(err, output) - return done() - } - ) - }) - - it('should generate the options for the container', function () { - return this.DockerRunner._getContainerOptions - .calledWith( - this.command_with_dir, - this.image, - this.volumes, - this.timeout - ) - .should.equal(true) - }) - - it('should generate the fingerprint from the returned options', function () { - return this.DockerRunner._fingerprintContainer - .calledWith(this.options) - .should.equal(true) - }) - - it('should do the run', function () { - return this.DockerRunner._runAndWaitForContainer - .calledWith(this.options, this.volumes, this.timeout) - .should.equal(true) - }) - - return it('should call the callback', function () { - return this.callback.calledWith(null, this.output).should.equal(true) - }) - }) - - describe('standard compile', function () { - beforeEach(function () { - this.directory = '/var/lib/overleaf/data/compiles/xyz' - this.DockerRunner._runAndWaitForContainer = sinon - .stub() - .callsArgWith(3, null, (this.output = 'mock-output')) - return this.DockerRunner.run( - this.project_id, - this.command, - this.directory, - this.image, - this.timeout, - this.env, - this.compileGroup, - this.callback - ) - }) - - it('should re-write the bind directory', function () { - const volumes = - this.DockerRunner._runAndWaitForContainer.lastCall.args[1] - return expect(volumes).to.deep.equal({ - '/some/host/dir/compiles/xyz': '/compile', - }) - }) - - return it('should call the callback', function () { - return this.callback.calledWith(null, this.output).should.equal(true) - }) - }) - - describe('synctex-output', function () { - beforeEach(function () { - this.directory = '/var/lib/overleaf/data/output/xyz/generated-files/id' - this.DockerRunner._runAndWaitForContainer = sinon - .stub() - .callsArgWith(3, null, (this.output = 'mock-output')) - this.DockerRunner.run( - this.project_id, - this.command, - this.directory, - this.image, - this.timeout, - this.env, - 'synctex-output', - this.callback - ) - }) - - it('should re-write the bind directory and set ro flag', function () { - const volumes = - this.DockerRunner._runAndWaitForContainer.lastCall.args[1] - expect(volumes).to.deep.equal({ - '/some/host/dir/output/xyz/generated-files/id': '/compile:ro', - }) - }) - - it('should call the callback', function () { - this.callback.calledWith(null, this.output).should.equal(true) - }) - }) - - describe('synctex', function () { - beforeEach(function () { - this.directory = '/var/lib/overleaf/data/compile/xyz' - this.DockerRunner._runAndWaitForContainer = sinon - .stub() - .callsArgWith(3, null, (this.output = 'mock-output')) - this.DockerRunner.run( - this.project_id, - this.command, - this.directory, - this.image, - this.timeout, - this.env, - 'synctex', - this.callback - ) - }) - - it('should re-write the bind directory', function () { - const volumes = - this.DockerRunner._runAndWaitForContainer.lastCall.args[1] - expect(volumes).to.deep.equal({ - '/some/host/dir/compiles/xyz': '/compile:ro', - }) - }) - - it('should call the callback', function () { - this.callback.calledWith(null, this.output).should.equal(true) - }) - }) - - describe('wordcount', function () { - beforeEach(function () { - this.directory = '/var/lib/overleaf/data/compile/xyz' - this.DockerRunner._runAndWaitForContainer = sinon - .stub() - .callsArgWith(3, null, (this.output = 'mock-output')) - this.DockerRunner.run( - this.project_id, - this.command, - this.directory, - this.image, - this.timeout, - this.env, - 'wordcount', - this.callback - ) - }) - - it('should re-write the bind directory', function () { - const volumes = - this.DockerRunner._runAndWaitForContainer.lastCall.args[1] - expect(volumes).to.deep.equal({ - '/some/host/dir/compiles/xyz': '/compile:ro', - }) - }) - - it('should call the callback', function () { - this.callback.calledWith(null, this.output).should.equal(true) - }) - }) - - describe('when the run throws an error', function () { - beforeEach(function () { - let firstTime = true - this.output = 'mock-output' - this.DockerRunner._runAndWaitForContainer = ( - options, - volumes, - timeout, - callback - ) => { - if (callback == null) { - callback = function () {} - } - if (firstTime) { - firstTime = false - const error = new Error('(HTTP code 500) server error - ...') - error.statusCode = 500 - return callback(error) - } else { - return callback(null, this.output) - } - } - sinon.spy(this.DockerRunner, '_runAndWaitForContainer') - this.DockerRunner.destroyContainer = sinon.stub().callsArg(3) - return this.DockerRunner.run( - this.project_id, - this.command, - this.directory, - this.image, - this.timeout, - this.env, - this.compileGroup, - this.callback - ) - }) - - it('should do the run twice', function () { - return this.DockerRunner._runAndWaitForContainer.calledTwice.should.equal( - true - ) - }) - - it('should destroy the container in between', function () { - return this.DockerRunner.destroyContainer - .calledWith(this.name, null) - .should.equal(true) - }) - - return it('should call the callback', function () { - return this.callback.calledWith(null, this.output).should.equal(true) - }) - }) - - describe('with no image', function () { - beforeEach(function () { - this.DockerRunner._runAndWaitForContainer = sinon - .stub() - .callsArgWith(3, null, (this.output = 'mock-output')) - return this.DockerRunner.run( - this.project_id, - this.command, - this.directory, - null, - this.timeout, - this.env, - this.compileGroup, - this.callback - ) - }) - - return it('should use the default image', function () { - return this.DockerRunner._getContainerOptions - .calledWith( - this.command_with_dir, - this.defaultImage, - this.volumes, - this.timeout - ) - .should.equal(true) - }) - }) - - describe('with image override', function () { - beforeEach(function () { - this.Settings.texliveImageNameOveride = 'overrideimage.com/something' - this.DockerRunner._runAndWaitForContainer = sinon - .stub() - .callsArgWith(3, null, (this.output = 'mock-output')) - return this.DockerRunner.run( - this.project_id, - this.command, - this.directory, - this.image, - this.timeout, - this.env, - this.compileGroup, - this.callback - ) - }) - - return it('should use the override and keep the tag', function () { - const image = this.DockerRunner._getContainerOptions.args[0][1] - return image.should.equal('overrideimage.com/something/image:2016.2') - }) - }) - - describe('with image restriction', function () { - beforeEach(function () { - this.Settings.clsi.docker.allowedImages = [ - 'repo/image:tag1', - 'repo/image:tag2', - ] - this.DockerRunner._runAndWaitForContainer = sinon - .stub() - .callsArgWith(3, null, (this.output = 'mock-output')) - }) - - describe('with a valid image', function () { - beforeEach(function () { - this.DockerRunner.run( - this.project_id, - this.command, - this.directory, - 'repo/image:tag1', - this.timeout, - this.env, - this.compileGroup, - this.callback - ) - }) - - it('should setup the container', function () { - this.DockerRunner._getContainerOptions.called.should.equal(true) - }) - }) - - describe('with a invalid image', function () { - beforeEach(function () { - this.DockerRunner.run( - this.project_id, - this.command, - this.directory, - 'something/different:evil', - this.timeout, - this.env, - this.compileGroup, - this.callback - ) - }) - - it('should call the callback with an error', function () { - const err = new Error('image not allowed') - this.callback.called.should.equal(true) - this.callback.args[0][0].message.should.equal(err.message) - }) - - it('should not setup the container', function () { - this.DockerRunner._getContainerOptions.called.should.equal(false) - }) - }) - }) - }) - - describe('run with _getOptions', function () { - beforeEach(function (done) { - // this.DockerRunner._getContainerOptions = sinon - // .stub() - // .returns((this.options = { mockoptions: 'foo' })) - this.DockerRunner._fingerprintContainer = sinon - .stub() - .returns((this.fingerprint = 'fingerprint')) - - this.name = `project-${this.project_id}-${this.fingerprint}` - - this.command = ['mock', 'command', '--outdir=$COMPILE_DIR'] - this.command_with_dir = ['mock', 'command', '--outdir=/compile'] - this.timeout = 42000 - return done() - }) - - describe('when a compile group config is set', function () { - beforeEach(function () { - this.Settings.clsi.docker.compileGroupConfig = { - 'compile-group': { - 'HostConfig.newProperty': 'new-property', - }, - 'other-group': { otherProperty: 'other-property' }, - } - this.DockerRunner._runAndWaitForContainer = sinon - .stub() - .callsArgWith(3, null, (this.output = 'mock-output')) - return this.DockerRunner.run( - this.project_id, - this.command, - this.directory, - this.image, - this.timeout, - this.env, - this.compileGroup, - this.callback - ) - }) - - it('should set the docker options for the compile group', function () { - const options = - this.DockerRunner._runAndWaitForContainer.lastCall.args[0] - return expect(options.HostConfig).to.deep.include({ - Binds: ['/some/host/dir/compiles/directory:/compile:rw'], - LogConfig: { Type: 'none', Config: {} }, - CapDrop: ['ALL'], - SecurityOpt: ['no-new-privileges'], - newProperty: 'new-property', - }) - }) - - return it('should call the callback', function () { - return this.callback.calledWith(null, this.output).should.equal(true) - }) - }) - }) - - describe('_runAndWaitForContainer', function () { - beforeEach(function () { - this.options = { mockoptions: 'foo', name: (this.name = 'mock-name') } - this.DockerRunner.startContainer = ( - options, - volumes, - attachStreamHandler, - callback - ) => { - attachStreamHandler(null, (this.output = 'mock-output')) - return callback(null, (this.containerId = 'container-id')) - } - sinon.spy(this.DockerRunner, 'startContainer') - this.DockerRunner.waitForContainer = sinon - .stub() - .callsArgWith(3, null, (this.exitCode = 42)) - return this.DockerRunner._runAndWaitForContainer( - this.options, - this.volumes, - this.timeout, - this.callback - ) - }) - - it('should create/start the container', function () { - return this.DockerRunner.startContainer - .calledWith(this.options, this.volumes) - .should.equal(true) - }) - - it('should wait for the container to finish', function () { - return this.DockerRunner.waitForContainer - .calledWith(this.name, this.timeout) - .should.equal(true) - }) - - return it('should call the callback with the output', function () { - return this.callback.calledWith(null, this.output).should.equal(true) - }) - }) - - describe('startContainer', function () { - beforeEach(function () { - this.attachStreamHandler = sinon.stub() - this.attachStreamHandler.cock = true - this.options = { mockoptions: 'foo', name: 'mock-name' } - this.container.inspect = sinon.stub().callsArgWith(0) - this.DockerRunner.attachToContainer = ( - containerId, - attachStreamHandler, - cb - ) => { - attachStreamHandler() - return cb() - } - return sinon.spy(this.DockerRunner, 'attachToContainer') - }) - - describe('when the container exists', function () { - beforeEach(function () { - this.container.inspect = sinon.stub().callsArgWith(0) - this.container.start = sinon.stub().yields() - - return this.DockerRunner.startContainer( - this.options, - this.volumes, - () => {}, - this.callback - ) - }) - - it('should start the container with the given name', function () { - this.getContainer.calledWith(this.options.name).should.equal(true) - return this.container.start.called.should.equal(true) - }) - - it('should not try to create the container', function () { - return this.createContainer.called.should.equal(false) - }) - - it('should attach to the container', function () { - return this.DockerRunner.attachToContainer.called.should.equal(true) - }) - - it('should call the callback', function () { - return this.callback.called.should.equal(true) - }) - - return it('should attach before the container starts', function () { - return sinon.assert.callOrder( - this.DockerRunner.attachToContainer, - this.container.start - ) - }) - }) - - describe('when the container does not exist', function () { - beforeEach(function () { - const exists = false - this.container.start = sinon.stub().yields() - this.container.inspect = sinon - .stub() - .callsArgWith(0, { statusCode: 404 }) - return this.DockerRunner.startContainer( - this.options, - this.volumes, - this.attachStreamHandler, - this.callback - ) - }) - - it('should create the container', function () { - return this.createContainer.calledWith(this.options).should.equal(true) - }) - - it('should call the callback and stream handler', function () { - this.attachStreamHandler.called.should.equal(true) - return this.callback.called.should.equal(true) - }) - - it('should attach to the container', function () { - return this.DockerRunner.attachToContainer.called.should.equal(true) - }) - - return it('should attach before the container starts', function () { - return sinon.assert.callOrder( - this.DockerRunner.attachToContainer, - this.container.start - ) - }) - }) - - describe('when the container is already running', function () { - beforeEach(function () { - const error = new Error( - `HTTP code is 304 which indicates error: server error - start: Cannot start container ${this.name}: The container MOCKID is already running.` - ) - error.statusCode = 304 - this.container.start = sinon.stub().yields(error) - this.container.inspect = sinon.stub().callsArgWith(0) - return this.DockerRunner.startContainer( - this.options, - this.volumes, - this.attachStreamHandler, - this.callback - ) - }) - - it('should not try to create the container', function () { - return this.createContainer.called.should.equal(false) - }) - - return it('should call the callback and stream handler without an error', function () { - this.attachStreamHandler.called.should.equal(true) - return this.callback.called.should.equal(true) - }) - }) - - return describe('when the container tries to be created, but already has been (race condition)', function () {}) - }) - - describe('waitForContainer', function () { - beforeEach(function () { - this.containerId = 'container-id' - this.timeout = 5000 - this.container.wait = sinon - .stub() - .yields(null, { StatusCode: (this.statusCode = 42) }) - return (this.container.kill = sinon.stub().yields()) - }) - - describe('when the container returns in time', function () { - beforeEach(function () { - return this.DockerRunner.waitForContainer( - this.containerId, - this.timeout, - {}, - this.callback - ) - }) - - it('should wait for the container', function () { - this.getContainer.calledWith(this.containerId).should.equal(true) - return this.container.wait.called.should.equal(true) - }) - - return it('should call the callback with the exit', function () { - return this.callback - .calledWith(null, this.statusCode) - .should.equal(true) - }) - }) - - describe('when the container is removed before waiting', function () { - const err = new Error('not found') - err.statusCode = 404 - beforeEach(function () { - this.container.wait = sinon.stub().yields(err) - }) - - describe('AutoRemove not set', function () { - beforeEach(function () { - this.DockerRunner.waitForContainer( - this.containerId, - this.timeout, - { HostConfig: {} }, - this.callback - ) - }) - it('should wait for the container', function () { - this.getContainer.calledWith(this.containerId).should.equal(true) - this.container.wait.called.should.equal(true) - }) - it('should call the callback with the error', function () { - this.callback.calledWith(err).should.equal(true) - }) - }) - describe('AutoRemove=true', function () { - beforeEach(function () { - this.DockerRunner.waitForContainer( - this.containerId, - this.timeout, - { HostConfig: { AutoRemove: true } }, - this.callback - ) - }) - it('should wait for the container', function () { - this.getContainer.calledWith(this.containerId).should.equal(true) - this.container.wait.called.should.equal(true) - }) - it('should call the callback with exit code 0', function () { - this.callback.calledWith(null, 0).should.equal(true) - }) - }) - }) - - return describe('when the container does not return before the timeout', function () { - beforeEach(function (done) { - this.container.wait = function (callback) { - if (callback == null) { - callback = function () {} - } - return setTimeout(() => callback(null, { StatusCode: 42 }), 100) - } - this.timeout = 5 - return this.DockerRunner.waitForContainer( - this.containerId, - this.timeout, - {}, - (...args) => { - this.callback(...Array.from(args || [])) - return done() - } - ) - }) - - it('should call kill on the container', function () { - this.getContainer.calledWith(this.containerId).should.equal(true) - return this.container.kill.called.should.equal(true) - }) - - it('should call the callback with an error', function () { - this.callback.calledWith(sinon.match(Error)).should.equal(true) - - const errorObj = this.callback.args[0][0] - expect(errorObj.message).to.include('container timed out') - expect(errorObj.timedout).equal(true) - }) - }) - }) - - describe('destroyOldContainers', function () { - beforeEach(function (done) { - const oneHourInSeconds = 60 * 60 - const oneHourInMilliseconds = oneHourInSeconds * 1000 - const nowInSeconds = Date.now() / 1000 - this.containers = [ - { - Name: '/project-old-container-name', - Id: 'old-container-id', - Created: nowInSeconds - oneHourInSeconds - 100, - }, - { - Name: '/project-new-container-name', - Id: 'new-container-id', - Created: nowInSeconds - oneHourInSeconds + 100, - }, - { - Name: '/totally-not-a-project-container', - Id: 'some-random-id', - Created: nowInSeconds - 2 * oneHourInSeconds, - }, - ] - this.DockerRunner.MAX_CONTAINER_AGE = oneHourInMilliseconds - this.listContainers.callsArgWith(1, null, this.containers) - this.DockerRunner.destroyContainer = sinon.stub().callsArg(3) - return this.DockerRunner.destroyOldContainers(error => { - this.callback(error) - return done() - }) - }) - - it('should list all containers', function () { - return this.listContainers.calledWith({ all: true }).should.equal(true) - }) - - it('should destroy old containers', function () { - this.DockerRunner.destroyContainer.callCount.should.equal(1) - return this.DockerRunner.destroyContainer - .calledWith('project-old-container-name', 'old-container-id') - .should.equal(true) - }) - - it('should not destroy new containers', function () { - return this.DockerRunner.destroyContainer - .calledWith('project-new-container-name', 'new-container-id') - .should.equal(false) - }) - - it('should not destroy non-project containers', function () { - return this.DockerRunner.destroyContainer - .calledWith('totally-not-a-project-container', 'some-random-id') - .should.equal(false) - }) - - return it('should callback the callback', function () { - return this.callback.called.should.equal(true) - }) - }) - - describe('_destroyContainer', function () { - beforeEach(function () { - this.containerId = 'some_id' - this.fakeContainer = { remove: sinon.stub().callsArgWith(1, null) } - return (this.Docker.prototype.getContainer = sinon - .stub() - .returns(this.fakeContainer)) - }) - - it('should get the container', function (done) { - return this.DockerRunner._destroyContainer( - this.containerId, - false, - err => { - if (err) return done(err) - this.Docker.prototype.getContainer.callCount.should.equal(1) - this.Docker.prototype.getContainer - .calledWith(this.containerId) - .should.equal(true) - return done() - } - ) - }) - - it('should try to force-destroy the container when shouldForce=true', function (done) { - return this.DockerRunner._destroyContainer( - this.containerId, - true, - err => { - if (err) return done(err) - this.fakeContainer.remove.callCount.should.equal(1) - this.fakeContainer.remove - .calledWith({ force: true, v: true }) - .should.equal(true) - return done() - } - ) - }) - - it('should not try to force-destroy the container when shouldForce=false', function (done) { - return this.DockerRunner._destroyContainer( - this.containerId, - false, - err => { - if (err) return done(err) - this.fakeContainer.remove.callCount.should.equal(1) - this.fakeContainer.remove - .calledWith({ force: false, v: true }) - .should.equal(true) - return done() - } - ) - }) - - it('should not produce an error', function (done) { - return this.DockerRunner._destroyContainer( - this.containerId, - false, - err => { - expect(err).to.equal(null) - return done() - } - ) - }) - - describe('when the container is already gone', function () { - beforeEach(function () { - this.fakeError = new Error('woops') - this.fakeError.statusCode = 404 - this.fakeContainer = { - remove: sinon.stub().callsArgWith(1, this.fakeError), - } - return (this.Docker.prototype.getContainer = sinon - .stub() - .returns(this.fakeContainer)) - }) - - return it('should not produce an error', function (done) { - return this.DockerRunner._destroyContainer( - this.containerId, - false, - err => { - expect(err).to.equal(null) - return done() - } - ) - }) - }) - - return describe('when container.destroy produces an error', function (done) { - beforeEach(function () { - this.fakeError = new Error('woops') - this.fakeError.statusCode = 500 - this.fakeContainer = { - remove: sinon.stub().callsArgWith(1, this.fakeError), - } - return (this.Docker.prototype.getContainer = sinon - .stub() - .returns(this.fakeContainer)) - }) - - return it('should produce an error', function (done) { - return this.DockerRunner._destroyContainer( - this.containerId, - false, - err => { - expect(err).to.not.equal(null) - expect(err).to.equal(this.fakeError) - return done() - } - ) - }) - }) - }) - - return describe('kill', function () { - beforeEach(function () { - this.containerId = 'some_id' - this.fakeContainer = { kill: sinon.stub().callsArgWith(0, null) } - return (this.Docker.prototype.getContainer = sinon - .stub() - .returns(this.fakeContainer)) - }) - - it('should get the container', function (done) { - return this.DockerRunner.kill(this.containerId, err => { - if (err) return done(err) - this.Docker.prototype.getContainer.callCount.should.equal(1) - this.Docker.prototype.getContainer - .calledWith(this.containerId) - .should.equal(true) - return done() - }) - }) - - it('should try to force-destroy the container', function (done) { - return this.DockerRunner.kill(this.containerId, err => { - if (err) return done(err) - this.fakeContainer.kill.callCount.should.equal(1) - return done() - }) - }) - - it('should not produce an error', function (done) { - return this.DockerRunner.kill(this.containerId, err => { - expect(err).to.equal(undefined) - return done() - }) - }) - - describe('when the container is not actually running', function () { - beforeEach(function () { - this.fakeError = new Error('woops') - this.fakeError.statusCode = 500 - this.fakeError.message = - 'Cannot kill container is not running' - this.fakeContainer = { - kill: sinon.stub().callsArgWith(0, this.fakeError), - } - return (this.Docker.prototype.getContainer = sinon - .stub() - .returns(this.fakeContainer)) - }) - - return it('should not produce an error', function (done) { - return this.DockerRunner.kill(this.containerId, err => { - expect(err).to.equal(undefined) - return done() - }) - }) - }) - - return describe('when container.kill produces a legitimate error', function (done) { - beforeEach(function () { - this.fakeError = new Error('woops') - this.fakeError.statusCode = 500 - this.fakeError.message = 'Totally legitimate reason to throw an error' - this.fakeContainer = { - kill: sinon.stub().callsArgWith(0, this.fakeError), - } - return (this.Docker.prototype.getContainer = sinon - .stub() - .returns(this.fakeContainer)) - }) - - return it('should produce an error', function (done) { - return this.DockerRunner.kill(this.containerId, err => { - expect(err).to.not.equal(undefined) - expect(err).to.equal(this.fakeError) - return done() - }) - }) - }) - }) -}) diff --git a/services/clsi/test/unit/js/DraftModeManager.test.js b/services/clsi/test/unit/js/DraftModeManager.test.js new file mode 100644 index 0000000000..0e8755b26d --- /dev/null +++ b/services/clsi/test/unit/js/DraftModeManager.test.js @@ -0,0 +1,46 @@ +import { vi, expect, describe, beforeEach, afterEach, it } from 'vitest' +import Path from 'node:path' +import fsPromises from 'node:fs/promises' +import mockFs from 'mock-fs' + +const MODULE_PATH = Path.join( + import.meta.dirname, + '../../../app/js/DraftModeManager' +) + +describe('DraftModeManager', () => { + beforeEach(async ctx => { + vi.doMock('node:fs/promises', () => ({ + default: fsPromises, + })) + + ctx.DraftModeManager = (await import(MODULE_PATH)).default + ctx.filename = '/mock/filename.tex' + ctx.contents = `\ +\\documentclass{article} +\\begin{document} +Hello world +\\end{document}\ +` + mockFs({ + [ctx.filename]: ctx.contents, + }) + }) + + afterEach(() => { + mockFs.restore() + }) + + describe('injectDraftMode', () => { + it('prepends a special command to the beginning of the file', async ctx => { + await ctx.DraftModeManager.promises.injectDraftMode(ctx.filename) + const contents = await fsPromises.readFile(ctx.filename, { + encoding: 'utf8', + }) + expect(contents).to.equal( + '\\PassOptionsToPackage{draft}{graphicx}\\PassOptionsToPackage{draft}{graphics}' + + ctx.contents + ) + }) + }) +}) diff --git a/services/clsi/test/unit/js/DraftModeManagerTests.js b/services/clsi/test/unit/js/DraftModeManagerTests.js deleted file mode 100644 index eda83380e7..0000000000 --- a/services/clsi/test/unit/js/DraftModeManagerTests.js +++ /dev/null @@ -1,44 +0,0 @@ -const Path = require('node:path') -const fsPromises = require('node:fs/promises') -const { expect } = require('chai') -const mockFs = require('mock-fs') -const SandboxedModule = require('sandboxed-module') - -const MODULE_PATH = Path.join(__dirname, '../../../app/js/DraftModeManager') - -describe('DraftModeManager', function () { - beforeEach(function () { - this.DraftModeManager = SandboxedModule.require(MODULE_PATH, { - requires: { - 'fs/promises': fsPromises, - }, - }) - this.filename = '/mock/filename.tex' - this.contents = `\ -\\documentclass{article} -\\begin{document} -Hello world -\\end{document}\ -` - mockFs({ - [this.filename]: this.contents, - }) - }) - - afterEach(function () { - mockFs.restore() - }) - - describe('injectDraftMode', function () { - it('prepends a special command to the beginning of the file', async function () { - await this.DraftModeManager.promises.injectDraftMode(this.filename) - const contents = await fsPromises.readFile(this.filename, { - encoding: 'utf8', - }) - expect(contents).to.equal( - '\\PassOptionsToPackage{draft}{graphicx}\\PassOptionsToPackage{draft}{graphics}' + - this.contents - ) - }) - }) -}) diff --git a/services/clsi/test/unit/js/LatexMetricsTests.js b/services/clsi/test/unit/js/LatexMetrics.test.js similarity index 77% rename from services/clsi/test/unit/js/LatexMetricsTests.js rename to services/clsi/test/unit/js/LatexMetrics.test.js index e8bb1543cc..a63fff3339 100644 --- a/services/clsi/test/unit/js/LatexMetricsTests.js +++ b/services/clsi/test/unit/js/LatexMetrics.test.js @@ -1,32 +1,34 @@ -const fs = require('node:fs') -const path = require('node:path') -const { expect } = require('chai') -const { addLatexFdbMetrics } = require('../../../app/js/LatexMetrics') +import fs from 'node:fs' +import path from 'node:path' +import { expect, describe, beforeEach, it } from 'vitest' +import LatexMetrics from '../../../app/js/LatexMetrics.js' + +const { addLatexFdbMetrics } = LatexMetrics describe('LatexMetrics', function () { describe('addLatexFdbMetrics', function () { - beforeEach(function () { - this.stats = {} - Object.defineProperty(this.stats, 'latexmk', { + beforeEach(function (ctx) { + ctx.stats = {} + Object.defineProperty(ctx.stats, 'latexmk', { value: {}, enumerable: false, }) }) - it('should do nothing if fdbContent is null or empty', function () { - addLatexFdbMetrics(null, this.stats) - expect(this.stats.latexmk).to.deep.equal({}) - addLatexFdbMetrics('', this.stats) - expect(this.stats.latexmk).to.deep.equal({}) + it('should do nothing if fdbContent is null or empty', function (ctx) { + addLatexFdbMetrics(null, ctx.stats) + expect(ctx.stats.latexmk).to.deep.equal({}) + addLatexFdbMetrics('', ctx.stats) + expect(ctx.stats.latexmk).to.deep.equal({}) }) - it('should parse v3 fdb content and add to stats', function () { + it('should parse v3 fdb content and add to stats', function (ctx) { const fdbContent = fs.readFileSync( - path.join(__dirname, 'fixtures', 'v3.fdb_latexmk'), + path.join(import.meta.dirname, 'fixtures', 'v3.fdb_latexmk'), 'utf8' ) - addLatexFdbMetrics(fdbContent, this.stats) - expect(this.stats.latexmk['fdb-file-types']).to.deep.equal({ + addLatexFdbMetrics(fdbContent, ctx.stats) + expect(ctx.stats.latexmk['fdb-file-types']).to.deep.equal({ system: [ { ext: 'fmt', count: 1, size: 3847283 }, { ext: 'map', count: 2, size: 1644257 }, @@ -64,13 +66,13 @@ describe('LatexMetrics', function () { }) }) - it('should parse v4 fdb content and add to stats', function () { + it('should parse v4 fdb content and add to stats', function (ctx) { const fdbContent = fs.readFileSync( - path.join(__dirname, 'fixtures', 'v4.fdb_latexmk'), + path.join(import.meta.dirname, 'fixtures', 'v4.fdb_latexmk'), 'utf8' ) - addLatexFdbMetrics(fdbContent, this.stats) - expect(this.stats.latexmk['fdb-file-types']).to.deep.equal({ + addLatexFdbMetrics(fdbContent, ctx.stats) + expect(ctx.stats.latexmk['fdb-file-types']).to.deep.equal({ system: [ { ext: 'fmt', count: 1, size: 8172536 }, { ext: 'map', count: 2, size: 4652176 }, diff --git a/services/clsi/test/unit/js/LatexRunner.test.js b/services/clsi/test/unit/js/LatexRunner.test.js new file mode 100644 index 0000000000..7475f52cb0 --- /dev/null +++ b/services/clsi/test/unit/js/LatexRunner.test.js @@ -0,0 +1,335 @@ +import { vi, expect, describe, beforeEach, it } from 'vitest' +import sinon from 'sinon' +import fs from 'node:fs' +import path from 'node:path' + +const MODULE_PATH = path.join( + import.meta.dirname, + '../../../app/js/LatexRunner' +) + +describe('LatexRunner', () => { + beforeEach(async ctx => { + ctx.Settings = { + docker: { + socketPath: '/var/run/docker.sock', + }, + } + ctx.commandRunnerOutput = { + stdout: 'this is stdout', + stderr: 'this is stderr', + } + ctx.CommandRunner = { + run: sinon.stub().yields(null, ctx.commandRunnerOutput), + } + ctx.fs = { + writeFile: sinon.stub().yields(), + unlink: sinon + .stub() + .yields(new Error('ENOENT: no such file or directory, unlink ...')), + } + + vi.doMock('@overleaf/settings', () => ({ + default: ctx.Settings, + })) + + vi.doMock('../../../app/js/CommandRunner', () => ({ + default: ctx.CommandRunner, + })) + + vi.doMock('fs', () => ({ + default: ctx.fs, + })) + + ctx.LatexRunner = (await import(MODULE_PATH)).default + + ctx.directory = '/local/compile/directory' + ctx.mainFile = 'main-file.tex' + ctx.compiler = 'pdflatex' + ctx.image = 'example.com/image' + ctx.compileGroup = 'compile-group' + ctx.callback = sinon.stub() + ctx.project_id = 'project-id-123' + ctx.env = { foo: '123' } + ctx.timeout = 42000 + ctx.flags = [] + ctx.stopOnFirstError = false + ctx.stats = {} + ctx.timings = {} + + ctx.call = function (callback) { + this.LatexRunner.runLatex( + this.project_id, + { + directory: this.directory, + mainFile: this.mainFile, + compiler: this.compiler, + timeout: this.timeout, + image: this.image, + environment: this.env, + compileGroup: this.compileGroup, + flags: this.flags, + stopOnFirstError: this.stopOnFirstError, + timings: this.timings, + stats: this.stats, + }, + callback + ) + } + }) + + describe('runLatex', () => { + describe('normally', () => { + beforeEach(async ctx => { + await new Promise((resolve, reject) => { + ctx.call(err => { + if (err) reject(err) + resolve() + }) + }) + }) + + it('should run the latex command', ctx => { + ctx.CommandRunner.run.should.have.been.calledWith( + ctx.project_id, + [ + 'latexmk', + '-cd', + '-jobname=output', + '-auxdir=$COMPILE_DIR', + '-outdir=$COMPILE_DIR', + '-synctex=1', + '-interaction=batchmode', + '-time', + '-f', + '-pdf', + '$COMPILE_DIR/main-file.tex', + ], + ctx.directory, + ctx.image, + ctx.timeout, + ctx.env, + ctx.compileGroup + ) + }) + + it('should record the stdout and stderr', ctx => { + ctx.fs.writeFile.should.have.been.calledWith( + ctx.directory + '/' + 'output.stdout', + 'this is stdout', + { flag: 'wx' } + ) + ctx.fs.writeFile.should.have.been.calledWith( + ctx.directory + '/' + 'output.stderr', + 'this is stderr', + { flag: 'wx' } + ) + ctx.fs.unlink.should.have.been.calledWith( + ctx.directory + '/' + 'output.stdout' + ) + ctx.fs.unlink.should.have.been.calledWith( + ctx.directory + '/' + 'output.stderr' + ) + }) + + it('should not record cpu metrics', ctx => { + expect(ctx.timings['cpu-percent']).to.not.exist + expect(ctx.timings['cpu-time']).to.not.exist + expect(ctx.timings['sys-time']).to.not.exist + }) + }) + + describe('with a different compiler', () => { + beforeEach(async ctx => { + await new Promise((resolve, reject) => { + ctx.compiler = 'lualatex' + ctx.call(err => { + if (err) reject(err) + resolve() + }) + }) + }) + + it('should set the appropriate latexmk flag', ctx => { + ctx.CommandRunner.run.should.have.been.calledWith(ctx.project_id, [ + 'latexmk', + '-cd', + '-jobname=output', + '-auxdir=$COMPILE_DIR', + '-outdir=$COMPILE_DIR', + '-synctex=1', + '-interaction=batchmode', + '-time', + '-f', + '-lualatex', + '$COMPILE_DIR/main-file.tex', + ]) + }) + }) + + describe('with time -v', () => { + beforeEach(async ctx => { + await new Promise((resolve, reject) => { + ctx.commandRunnerOutput.stderr = + '\tCommand being timed: "sh -c timeout 1 yes > /dev/null"\n' + + '\tUser time (seconds): 0.28\n' + + '\tSystem time (seconds): 0.70\n' + + '\tPercent of CPU this job got: 98%\n' + ctx.call(err => { + if (err) reject(err) + resolve() + }) + }) + }) + + it('should record cpu metrics', ctx => { + expect(ctx.timings['cpu-percent']).to.equal(98) + expect(ctx.timings['cpu-time']).to.equal(0.28) + expect(ctx.timings['sys-time']).to.equal(0.7) + }) + }) + + describe('with an .Rtex main file', () => { + beforeEach(async ctx => { + await new Promise((resolve, reject) => { + ctx.mainFile = 'main-file.Rtex' + ctx.call(err => { + if (err) reject(err) + resolve() + }) + }) + }) + + it('should run the latex command on the equivalent .tex file', ctx => { + const command = ctx.CommandRunner.run.args[0][1] + const mainFile = command.slice(-1)[0] + mainFile.should.equal('$COMPILE_DIR/main-file.tex') + }) + }) + + describe('with a flags option', () => { + beforeEach(async ctx => { + await new Promise((resolve, reject) => { + ctx.flags = ['-shell-restricted', '-halt-on-error'] + ctx.call(err => { + if (err) reject(err) + resolve() + }) + }) + }) + + it('should include the flags in the command', ctx => { + const command = ctx.CommandRunner.run.args[0][1] + const flags = command.filter( + arg => arg === '-shell-restricted' || arg === '-halt-on-error' + ) + flags.length.should.equal(2) + flags[0].should.equal('-shell-restricted') + flags[1].should.equal('-halt-on-error') + }) + }) + + describe('with the stopOnFirstError option', () => { + beforeEach(async ctx => { + await new Promise((resolve, reject) => { + ctx.stopOnFirstError = true + ctx.call(err => { + if (err) reject(err) + resolve() + }) + }) + }) + + it('should set the appropriate flags', ctx => { + ctx.CommandRunner.run.should.have.been.calledWith(ctx.project_id, [ + 'latexmk', + '-cd', + '-jobname=output', + '-auxdir=$COMPILE_DIR', + '-outdir=$COMPILE_DIR', + '-synctex=1', + '-interaction=batchmode', + '-time', + '-halt-on-error', + '-pdf', + '$COMPILE_DIR/main-file.tex', + ]) + }) + }) + + describe('with old latexmk timing output', () => { + beforeEach(async ctx => { + await new Promise((resolve, reject) => { + ctx.commandRunnerOutput.stdout = fs.readFileSync( + path.join(import.meta.dirname, 'fixtures', 'latexmk1.txt'), + 'utf-8' + ) + // pass in the `latexmk` property to signal that we want to receive parsed stats + ctx.stats.latexmk = {} + ctx.call(err => { + if (err) reject(err) + resolve() + }) + }) + }) + + it('should parse latexmk 4.52c (2017) timing information', ctx => { + expect(ctx.stats.latexmk).to.deep.equal({ + 'latexmk-rule-times': [ + { rule: 'makeindex', time_ms: 30 }, + { rule: 'bibtex', time_ms: 40 }, + { rule: 'latex', time_ms: 690 }, + { rule: 'makeindex', time_ms: 40 }, + { rule: 'bibtex', time_ms: 39 }, + { rule: 'latex', time_ms: 750 }, + { rule: 'makeindex', time_ms: 39 }, + { rule: 'bibtex', time_ms: 20 }, + { rule: 'latex', time_ms: 770 }, + ], + 'latexmk-rule-signature': + 'makeindex,bibtex,latex,makeindex,bibtex,latex,makeindex,bibtex,latex', + 'latexmk-rules-run': 9, + 'latexmk-time': { total: 2930 }, + 'latexmk-img-times': [], + }) + }) + }) + + describe('with modern latexmk timing output', () => { + beforeEach(async ctx => { + await new Promise((resolve, reject) => { + ctx.commandRunnerOutput.stdout = fs.readFileSync( + path.join(import.meta.dirname, 'fixtures', 'latexmk2.txt'), + 'utf-8' + ) + // pass in the `latexmk` property to signal that we want to receive parsed stats + ctx.stats.latexmk = {} + ctx.call(err => { + if (err) reject(err) + resolve() + }) + }) + }) + + it('should parse latexmk 4.83 (2024) timing information', ctx => { + expect(ctx.stats.latexmk).to.deep.equal({ + 'latexmk-rule-times': [ + { rule: 'latex', time_ms: 1880 }, + { rule: 'makeindex', time_ms: 50 }, + { rule: 'bibtex', time_ms: 50 }, + { rule: 'latex', time_ms: 2180 }, + ], + 'latexmk-rule-signature': 'latex,makeindex,bibtex,latex', + 'latexmk-time': { + total: 4770, + invoked: 4160, + other: 610, + }, + 'latexmk-clock-time': 4870, + 'latexmk-rules-run': 4, + 'latexmk-img-times': [], + }) + }) + }) + }) +}) diff --git a/services/clsi/test/unit/js/LatexRunnerTests.js b/services/clsi/test/unit/js/LatexRunnerTests.js deleted file mode 100644 index fca58b63f8..0000000000 --- a/services/clsi/test/unit/js/LatexRunnerTests.js +++ /dev/null @@ -1,289 +0,0 @@ -const SandboxedModule = require('sandboxed-module') -const sinon = require('sinon') -const { expect } = require('chai') -const fs = require('node:fs') -const path = require('node:path') - -const MODULE_PATH = require('node:path').join( - __dirname, - '../../../app/js/LatexRunner' -) - -describe('LatexRunner', function () { - beforeEach(function () { - this.Settings = { - docker: { - socketPath: '/var/run/docker.sock', - }, - } - this.commandRunnerOutput = { - stdout: 'this is stdout', - stderr: 'this is stderr', - } - this.CommandRunner = { - run: sinon.stub().yields(null, this.commandRunnerOutput), - } - this.fs = { - writeFile: sinon.stub().yields(), - unlink: sinon - .stub() - .yields(new Error('ENOENT: no such file or directory, unlink ...')), - } - this.LatexRunner = SandboxedModule.require(MODULE_PATH, { - requires: { - '@overleaf/settings': this.Settings, - './CommandRunner': this.CommandRunner, - fs: this.fs, - }, - }) - - this.directory = '/local/compile/directory' - this.mainFile = 'main-file.tex' - this.compiler = 'pdflatex' - this.image = 'example.com/image' - this.compileGroup = 'compile-group' - this.callback = sinon.stub() - this.project_id = 'project-id-123' - this.env = { foo: '123' } - this.timeout = 42000 - this.flags = [] - this.stopOnFirstError = false - this.stats = {} - this.timings = {} - - this.call = function (callback) { - this.LatexRunner.runLatex( - this.project_id, - { - directory: this.directory, - mainFile: this.mainFile, - compiler: this.compiler, - timeout: this.timeout, - image: this.image, - environment: this.env, - compileGroup: this.compileGroup, - flags: this.flags, - stopOnFirstError: this.stopOnFirstError, - timings: this.timings, - stats: this.stats, - }, - callback - ) - } - }) - - describe('runLatex', function () { - describe('normally', function () { - beforeEach(function (done) { - this.call(done) - }) - - it('should run the latex command', function () { - this.CommandRunner.run.should.have.been.calledWith( - this.project_id, - [ - 'latexmk', - '-cd', - '-jobname=output', - '-auxdir=$COMPILE_DIR', - '-outdir=$COMPILE_DIR', - '-synctex=1', - '-interaction=batchmode', - '-time', - '-f', - '-pdf', - '$COMPILE_DIR/main-file.tex', - ], - this.directory, - this.image, - this.timeout, - this.env, - this.compileGroup - ) - }) - - it('should record the stdout and stderr', function () { - this.fs.writeFile.should.have.been.calledWith( - this.directory + '/' + 'output.stdout', - 'this is stdout', - { flag: 'wx' } - ) - this.fs.writeFile.should.have.been.calledWith( - this.directory + '/' + 'output.stderr', - 'this is stderr', - { flag: 'wx' } - ) - this.fs.unlink.should.have.been.calledWith( - this.directory + '/' + 'output.stdout' - ) - this.fs.unlink.should.have.been.calledWith( - this.directory + '/' + 'output.stderr' - ) - }) - - it('should not record cpu metrics', function () { - expect(this.timings['cpu-percent']).to.not.exist - expect(this.timings['cpu-time']).to.not.exist - expect(this.timings['sys-time']).to.not.exist - }) - }) - - describe('with a different compiler', function () { - beforeEach(function (done) { - this.compiler = 'lualatex' - this.call(done) - }) - - it('should set the appropriate latexmk flag', function () { - this.CommandRunner.run.should.have.been.calledWith(this.project_id, [ - 'latexmk', - '-cd', - '-jobname=output', - '-auxdir=$COMPILE_DIR', - '-outdir=$COMPILE_DIR', - '-synctex=1', - '-interaction=batchmode', - '-time', - '-f', - '-lualatex', - '$COMPILE_DIR/main-file.tex', - ]) - }) - }) - - describe('with time -v', function () { - beforeEach(function (done) { - this.commandRunnerOutput.stderr = - '\tCommand being timed: "sh -c timeout 1 yes > /dev/null"\n' + - '\tUser time (seconds): 0.28\n' + - '\tSystem time (seconds): 0.70\n' + - '\tPercent of CPU this job got: 98%\n' - this.call(done) - }) - - it('should record cpu metrics', function () { - expect(this.timings['cpu-percent']).to.equal(98) - expect(this.timings['cpu-time']).to.equal(0.28) - expect(this.timings['sys-time']).to.equal(0.7) - }) - }) - - describe('with an .Rtex main file', function () { - beforeEach(function (done) { - this.mainFile = 'main-file.Rtex' - this.call(done) - }) - - it('should run the latex command on the equivalent .tex file', function () { - const command = this.CommandRunner.run.args[0][1] - const mainFile = command.slice(-1)[0] - mainFile.should.equal('$COMPILE_DIR/main-file.tex') - }) - }) - - describe('with a flags option', function () { - beforeEach(function (done) { - this.flags = ['-shell-restricted', '-halt-on-error'] - this.call(done) - }) - - it('should include the flags in the command', function () { - const command = this.CommandRunner.run.args[0][1] - const flags = command.filter( - arg => arg === '-shell-restricted' || arg === '-halt-on-error' - ) - flags.length.should.equal(2) - flags[0].should.equal('-shell-restricted') - flags[1].should.equal('-halt-on-error') - }) - }) - - describe('with the stopOnFirstError option', function () { - beforeEach(function (done) { - this.stopOnFirstError = true - this.call(done) - }) - - it('should set the appropriate flags', function () { - this.CommandRunner.run.should.have.been.calledWith(this.project_id, [ - 'latexmk', - '-cd', - '-jobname=output', - '-auxdir=$COMPILE_DIR', - '-outdir=$COMPILE_DIR', - '-synctex=1', - '-interaction=batchmode', - '-time', - '-halt-on-error', - '-pdf', - '$COMPILE_DIR/main-file.tex', - ]) - }) - }) - - describe('with old latexmk timing output', function () { - beforeEach(function (done) { - this.commandRunnerOutput.stdout = fs.readFileSync( - path.join(__dirname, 'fixtures', 'latexmk1.txt'), - 'utf-8' - ) - // pass in the `latexmk` property to signal that we want to receive parsed stats - this.stats.latexmk = {} - this.call(done) - }) - - it('should parse latexmk 4.52c (2017) timing information', function () { - expect(this.stats.latexmk).to.deep.equal({ - 'latexmk-rule-times': [ - { rule: 'makeindex', time_ms: 30 }, - { rule: 'bibtex', time_ms: 40 }, - { rule: 'latex', time_ms: 690 }, - { rule: 'makeindex', time_ms: 40 }, - { rule: 'bibtex', time_ms: 39 }, - { rule: 'latex', time_ms: 750 }, - { rule: 'makeindex', time_ms: 39 }, - { rule: 'bibtex', time_ms: 20 }, - { rule: 'latex', time_ms: 770 }, - ], - 'latexmk-rule-signature': - 'makeindex,bibtex,latex,makeindex,bibtex,latex,makeindex,bibtex,latex', - 'latexmk-rules-run': 9, - 'latexmk-time': { total: 2930 }, - 'latexmk-img-times': [], - }) - }) - }) - - describe('with modern latexmk timing output', function () { - beforeEach(function (done) { - this.commandRunnerOutput.stdout = fs.readFileSync( - path.join(__dirname, 'fixtures', 'latexmk2.txt'), - 'utf-8' - ) - // pass in the `latexmk` property to signal that we want to receive parsed stats - this.stats.latexmk = {} - this.call(done) - }) - - it('should parse latexmk 4.83 (2024) timing information', function () { - expect(this.stats.latexmk).to.deep.equal({ - 'latexmk-rule-times': [ - { rule: 'latex', time_ms: 1880 }, - { rule: 'makeindex', time_ms: 50 }, - { rule: 'bibtex', time_ms: 50 }, - { rule: 'latex', time_ms: 2180 }, - ], - 'latexmk-rule-signature': 'latex,makeindex,bibtex,latex', - 'latexmk-time': { - total: 4770, - invoked: 4160, - other: 610, - }, - 'latexmk-clock-time': 4870, - 'latexmk-rules-run': 4, - 'latexmk-img-times': [], - }) - }) - }) - }) -}) diff --git a/services/clsi/test/unit/js/LockManager.test.js b/services/clsi/test/unit/js/LockManager.test.js new file mode 100644 index 0000000000..defc28e4b1 --- /dev/null +++ b/services/clsi/test/unit/js/LockManager.test.js @@ -0,0 +1,124 @@ +import { vi, expect, describe, beforeEach, afterEach, it } from 'vitest' +import sinon from 'sinon' +import * as Errors from '../../../app/js/Errors.js' +import path from 'node:path' + +const modulePath = path.join(import.meta.dirname, '../../../app/js/LockManager') + +describe('LockManager', () => { + beforeEach(async ctx => { + ctx.key = '/local/compile/directory' + ctx.clock = sinon.useFakeTimers() + + vi.doMock('@overleaf/metrics', () => ({ + default: (ctx.Metrics = { + inc: sinon.stub(), + gauge: sinon.stub(), + }), + })) + + vi.doMock('@overleaf/settings', () => ({ + default: (ctx.Settings = { + compileConcurrencyLimit: 5, + }), + })) + + vi.doMock('../../../app/js/Errors', () => ({ + default: (ctx.Erros = Errors), + })) + + vi.doMock('../../../app/js/RequestParser', () => ({ + default: { MAX_TIMEOUT: 600 }, + })) + + ctx.LockManager = (await import(modulePath)).default + }) + + afterEach(ctx => { + ctx.clock.restore() + }) + + describe('when the lock is available', () => { + it('the lock can be acquired', ctx => { + const lock = ctx.LockManager.acquire(ctx.key) + expect(lock).to.exist + lock.release() + }) + }) + + describe('after the lock is acquired', () => { + beforeEach(ctx => { + ctx.lock = ctx.LockManager.acquire(ctx.key) + }) + + afterEach(ctx => { + if (ctx.lock != null) { + ctx.lock.release() + } + }) + + it("the lock can't be acquired again", ctx => { + expect(() => ctx.LockManager.acquire(ctx.key)).to.throw( + Errors.AlreadyCompilingError + ) + }) + + it('another lock can be acquired', ctx => { + const lock = ctx.LockManager.acquire('another key') + expect(lock).to.exist + lock.release() + }) + + it('the lock can be acquired again after an expiry period', ctx => { + // The expiry time is a little bit over 10 minutes. Let's wait 15 minutes. + ctx.clock.tick(15 * 60 * 1000) + ctx.lock = ctx.LockManager.acquire(ctx.key) + expect(ctx.lock).to.exist + }) + + it('the lock can be acquired again after it was released', ctx => { + ctx.lock.release() + ctx.lock = ctx.LockManager.acquire(ctx.key) + expect(ctx.lock).to.exist + }) + }) + + describe('concurrency limit', () => { + it('exceeding the limit', ctx => { + for (let i = 0; i <= ctx.Settings.compileConcurrencyLimit; i++) { + ctx.LockManager.acquire('test_key' + i) + } + ctx.Metrics.inc + .calledWith('exceeded-compilier-concurrency-limit') + .should.equal(false) + expect(() => + ctx.LockManager.acquire( + 'test_key_' + (ctx.Settings.compileConcurrencyLimit + 1), + false + ) + ).to.throw(Errors.TooManyCompileRequestsError) + + ctx.Metrics.inc + .calledWith('exceeded-compilier-concurrency-limit') + .should.equal(true) + }) + + it('within the limit', ctx => { + for (let i = 0; i <= ctx.Settings.compileConcurrencyLimit - 1; i++) { + ctx.LockManager.acquire('test_key' + i) + } + ctx.Metrics.inc + .calledWith('exceeded-compilier-concurrency-limit') + .should.equal(false) + + const lock = ctx.LockManager.acquire( + 'test_key_' + ctx.Settings.compileConcurrencyLimit, + false + ) + + expect(lock.key).to.equal( + 'test_key_' + ctx.Settings.compileConcurrencyLimit + ) + }) + }) +}) diff --git a/services/clsi/test/unit/js/LockManagerTests.js b/services/clsi/test/unit/js/LockManagerTests.js deleted file mode 100644 index 7005b3e5a3..0000000000 --- a/services/clsi/test/unit/js/LockManagerTests.js +++ /dev/null @@ -1,116 +0,0 @@ -const { expect } = require('chai') -const sinon = require('sinon') -const SandboxedModule = require('sandboxed-module') -const modulePath = require('node:path').join( - __dirname, - '../../../app/js/LockManager' -) -const Errors = require('../../../app/js/Errors') - -describe('LockManager', function () { - beforeEach(function () { - this.key = '/local/compile/directory' - this.clock = sinon.useFakeTimers() - this.LockManager = SandboxedModule.require(modulePath, { - requires: { - '@overleaf/metrics': (this.Metrics = { - inc: sinon.stub(), - gauge: sinon.stub(), - }), - '@overleaf/settings': (this.Settings = { - compileConcurrencyLimit: 5, - }), - './Errors': (this.Erros = Errors), - './RequestParser': { MAX_TIMEOUT: 600 }, - }, - }) - }) - - afterEach(function () { - this.clock.restore() - }) - - describe('when the lock is available', function () { - it('the lock can be acquired', function () { - const lock = this.LockManager.acquire(this.key) - expect(lock).to.exist - lock.release() - }) - }) - - describe('after the lock is acquired', function () { - beforeEach(function () { - this.lock = this.LockManager.acquire(this.key) - }) - - afterEach(function () { - if (this.lock != null) { - this.lock.release() - } - }) - - it("the lock can't be acquired again", function () { - expect(() => this.LockManager.acquire(this.key)).to.throw( - Errors.AlreadyCompilingError - ) - }) - - it('another lock can be acquired', function () { - const lock = this.LockManager.acquire('another key') - expect(lock).to.exist - lock.release() - }) - - it('the lock can be acquired again after an expiry period', function () { - // The expiry time is a little bit over 10 minutes. Let's wait 15 minutes. - this.clock.tick(15 * 60 * 1000) - this.lock = this.LockManager.acquire(this.key) - expect(this.lock).to.exist - }) - - it('the lock can be acquired again after it was released', function () { - this.lock.release() - this.lock = this.LockManager.acquire(this.key) - expect(this.lock).to.exist - }) - }) - - describe('concurrency limit', function () { - it('exceeding the limit', function () { - for (let i = 0; i <= this.Settings.compileConcurrencyLimit; i++) { - this.LockManager.acquire('test_key' + i) - } - this.Metrics.inc - .calledWith('exceeded-compilier-concurrency-limit') - .should.equal(false) - expect(() => - this.LockManager.acquire( - 'test_key_' + (this.Settings.compileConcurrencyLimit + 1), - false - ) - ).to.throw(Errors.TooManyCompileRequestsError) - - this.Metrics.inc - .calledWith('exceeded-compilier-concurrency-limit') - .should.equal(true) - }) - - it('within the limit', function () { - for (let i = 0; i <= this.Settings.compileConcurrencyLimit - 1; i++) { - this.LockManager.acquire('test_key' + i) - } - this.Metrics.inc - .calledWith('exceeded-compilier-concurrency-limit') - .should.equal(false) - - const lock = this.LockManager.acquire( - 'test_key_' + this.Settings.compileConcurrencyLimit, - false - ) - - expect(lock.key).to.equal( - 'test_key_' + this.Settings.compileConcurrencyLimit - ) - }) - }) -}) diff --git a/services/clsi/test/unit/js/OutputController.test.js b/services/clsi/test/unit/js/OutputController.test.js new file mode 100644 index 0000000000..95c02970e4 --- /dev/null +++ b/services/clsi/test/unit/js/OutputController.test.js @@ -0,0 +1,111 @@ +import { vi, describe, beforeEach, it } from 'vitest' +import sinon from 'sinon' +import path from 'node:path' +const MODULE_PATH = path.join( + import.meta.dirname, + '../../../app/js/OutputController' +) + +describe('OutputController', () => { + describe('createOutputZip', () => { + beforeEach(async ctx => { + ctx.archive = {} + + ctx.pipeline = sinon.stub().resolves() + + ctx.archiveFilesForBuild = sinon.stub().resolves(ctx.archive) + + vi.doMock('../../../app/js/OutputFileArchiveManager', () => ({ + default: { + archiveFilesForBuild: ctx.archiveFilesForBuild, + }, + })) + + vi.doMock('node:stream/promises', () => ({ + pipeline: ctx.pipeline, + })) + + ctx.OutputController = (await import(MODULE_PATH)).default + }) + + describe('when OutputFileArchiveManager creates an archive', () => { + beforeEach(async ctx => { + await new Promise((resolve, reject) => { + ctx.res = { + attachment: sinon.stub(), + setHeader: sinon.stub(), + } + ctx.req = { + params: { + project_id: 'project-id-123', + user_id: 'user-id-123', + build_id: 'build-id-123', + }, + query: { + files: ['output.tex'], + }, + } + ctx.pipeline.callsFake(() => { + resolve() + return Promise.resolve() + }) + ctx.OutputController.createOutputZip(ctx.req, ctx.res) + }) + }) + + it('creates a pipeline from the archive to the response', ctx => { + sinon.assert.calledWith(ctx.pipeline, ctx.archive, ctx.res) + }) + + it('calls the express convenience method to set attachment headers', ctx => { + sinon.assert.calledWith(ctx.res.attachment, 'output.zip') + }) + + it('sets the X-Content-Type-Options header to nosniff', ctx => { + sinon.assert.calledWith( + ctx.res.setHeader, + 'X-Content-Type-Options', + 'nosniff' + ) + }) + }) + + describe('when OutputFileArchiveManager throws an error', () => { + let error + + beforeEach(async ctx => { + await new Promise((resolve, reject) => { + error = new Error('error message') + + ctx.archiveFilesForBuild.rejects(error) + + ctx.res = { + status: sinon.stub().returnsThis(), + send: sinon.stub(), + } + ctx.req = { + params: { + project_id: 'project-id-123', + user_id: 'user-id-123', + build_id: 'build-id-123', + }, + query: { + files: ['output.tex'], + }, + } + ctx.OutputController.createOutputZip( + ctx.req, + ctx.res, + (ctx.next = sinon.stub().callsFake(() => { + resolve() + })) + ) + }) + }) + + it('calls next with the error', ctx => { + sinon.assert.calledWith(ctx.next, error) + }) + }) + }) +}) diff --git a/services/clsi/test/unit/js/OutputControllerTests.js b/services/clsi/test/unit/js/OutputControllerTests.js deleted file mode 100644 index ee5c9c2a7a..0000000000 --- a/services/clsi/test/unit/js/OutputControllerTests.js +++ /dev/null @@ -1,105 +0,0 @@ -const SandboxedModule = require('sandboxed-module') -const sinon = require('sinon') -const MODULE_PATH = require('node:path').join( - __dirname, - '../../../app/js/OutputController' -) - -describe('OutputController', function () { - describe('createOutputZip', function () { - beforeEach(function () { - this.archive = {} - - this.pipeline = sinon.stub().resolves() - - this.archiveFilesForBuild = sinon.stub().resolves(this.archive) - - this.OutputController = SandboxedModule.require(MODULE_PATH, { - requires: { - './OutputFileArchiveManager': { - archiveFilesForBuild: this.archiveFilesForBuild, - }, - 'stream/promises': { - pipeline: this.pipeline, - }, - }, - }) - }) - - describe('when OutputFileArchiveManager creates an archive', function () { - beforeEach(function (done) { - this.res = { - attachment: sinon.stub(), - setHeader: sinon.stub(), - } - this.req = { - params: { - project_id: 'project-id-123', - user_id: 'user-id-123', - build_id: 'build-id-123', - }, - query: { - files: ['output.tex'], - }, - } - this.pipeline.callsFake(() => { - done() - return Promise.resolve() - }) - this.OutputController.createOutputZip(this.req, this.res) - }) - - it('creates a pipeline from the archive to the response', function () { - sinon.assert.calledWith(this.pipeline, this.archive, this.res) - }) - - it('calls the express convenience method to set attachment headers', function () { - sinon.assert.calledWith(this.res.attachment, 'output.zip') - }) - - it('sets the X-Content-Type-Options header to nosniff', function () { - sinon.assert.calledWith( - this.res.setHeader, - 'X-Content-Type-Options', - 'nosniff' - ) - }) - }) - - describe('when OutputFileArchiveManager throws an error', function () { - let error - - beforeEach(function (done) { - error = new Error('error message') - - this.archiveFilesForBuild.rejects(error) - - this.res = { - status: sinon.stub().returnsThis(), - send: sinon.stub(), - } - this.req = { - params: { - project_id: 'project-id-123', - user_id: 'user-id-123', - build_id: 'build-id-123', - }, - query: { - files: ['output.tex'], - }, - } - this.OutputController.createOutputZip( - this.req, - this.res, - (this.next = sinon.stub().callsFake(() => { - done() - })) - ) - }) - - it('calls next with the error', function () { - sinon.assert.calledWith(this.next, error) - }) - }) - }) -}) diff --git a/services/clsi/test/unit/js/OutputFileArchiveManager.test.js b/services/clsi/test/unit/js/OutputFileArchiveManager.test.js new file mode 100644 index 0000000000..ca3a31009c --- /dev/null +++ b/services/clsi/test/unit/js/OutputFileArchiveManager.test.js @@ -0,0 +1,238 @@ +import { vi, assert, expect, describe, afterEach, beforeEach, it } from 'vitest' +import sinon from 'sinon' +import path from 'node:path' + +const MODULE_PATH = path.join( + import.meta.dirname, + '../../../app/js/OutputFileArchiveManager' +) + +describe('OutputFileArchiveManager', () => { + const userId = 'user-id-123' + const projectId = 'project-id-123' + const buildId = 'build-id-123' + + afterEach(() => { + sinon.restore() + }) + + beforeEach(async ctx => { + ctx.OutputFileFinder = { + promises: { + findOutputFiles: sinon.stub().resolves({ outputFiles: [] }), + }, + } + + ctx.OutputCacheManger = { + path: sinon.stub().callsFake((build, path) => { + return `${build}/${path}` + }), + } + + ctx.archive = { + append: sinon.stub(), + finalize: sinon.stub().resolves(), + on: sinon.stub(), + } + + ctx.archiver = sinon.stub().returns(ctx.archive) + + ctx.outputDir = '/output/dir' + + ctx.fs = { + open: sinon.stub().callsFake(file => ({ + createReadStream: sinon.stub().returns(`handle: ${file}`), + })), + } + + vi.doMock('../../../app/js/OutputFileFinder', () => ({ + default: ctx.OutputFileFinder, + })) + + vi.doMock('../../../app/js/OutputCacheManager', () => ({ + default: ctx.OutputCacheManger, + })) + + vi.doMock('archiver', () => ({ + default: ctx.archiver, + })) + + vi.doMock('node:fs/promises', () => ctx.fs) + + vi.doMock('@overleaf/settings', () => ({ + default: { + path: { + outputDir: ctx.outputDir, + }, + }, + })) + + ctx.OutputFileArchiveManager = (await import(MODULE_PATH)).default + }) + + describe('when the output cache directory contains only exportable files', () => { + beforeEach(async ctx => { + ctx.OutputFileFinder.promises.findOutputFiles.resolves({ + outputFiles: [ + { path: 'file_1' }, + { path: 'file_2' }, + { path: 'file_3' }, + { path: 'file_4' }, + ], + }) + await ctx.OutputFileArchiveManager.archiveFilesForBuild( + projectId, + userId, + buildId + ) + }) + + it('creates a zip archive', ctx => { + sinon.assert.calledWith(ctx.archiver, 'zip') + }) + + it('listens to errors from the archive', ctx => { + sinon.assert.calledWith(ctx.archive.on, 'error', sinon.match.func) + }) + + it('adds all the output files to the archive', ctx => { + expect(ctx.archive.append.callCount).to.equal(4) + sinon.assert.calledWith( + ctx.archive.append, + `handle: ${ctx.outputDir}/${projectId}-${userId}/${buildId}/file_1`, + sinon.match({ name: 'file_1' }) + ) + sinon.assert.calledWith( + ctx.archive.append, + `handle: ${ctx.outputDir}/${projectId}-${userId}/${buildId}/file_2`, + sinon.match({ name: 'file_2' }) + ) + sinon.assert.calledWith( + ctx.archive.append, + `handle: ${ctx.outputDir}/${projectId}-${userId}/${buildId}/file_3`, + sinon.match({ name: 'file_3' }) + ) + sinon.assert.calledWith( + ctx.archive.append, + `handle: ${ctx.outputDir}/${projectId}-${userId}/${buildId}/file_4`, + sinon.match({ name: 'file_4' }) + ) + }) + + it('finalizes the archive after all files are appended', ctx => { + sinon.assert.called(ctx.archive.finalize) + expect(ctx.archive.finalize.calledBefore(ctx.archive.append)).to.be.false + }) + }) + + describe('when the directory includes files ignored by web', () => { + beforeEach(async ctx => { + ctx.OutputFileFinder.promises.findOutputFiles.resolves({ + outputFiles: [ + { path: 'file_1' }, + { path: 'file_2' }, + { path: 'file_3' }, + { path: 'file_4' }, + { path: 'output.pdf' }, + ], + }) + await ctx.OutputFileArchiveManager.archiveFilesForBuild( + projectId, + userId, + buildId + ) + }) + + it('only includes the non-ignored files in the archive', ctx => { + expect(ctx.archive.append.callCount).to.equal(4) + sinon.assert.calledWith( + ctx.archive.append, + `handle: ${ctx.outputDir}/${projectId}-${userId}/${buildId}/file_1`, + sinon.match({ name: 'file_1' }) + ) + sinon.assert.calledWith( + ctx.archive.append, + `handle: ${ctx.outputDir}/${projectId}-${userId}/${buildId}/file_2`, + sinon.match({ name: 'file_2' }) + ) + sinon.assert.calledWith( + ctx.archive.append, + `handle: ${ctx.outputDir}/${projectId}-${userId}/${buildId}/file_3`, + sinon.match({ name: 'file_3' }) + ) + sinon.assert.calledWith( + ctx.archive.append, + `handle: ${ctx.outputDir}/${projectId}-${userId}/${buildId}/file_4`, + sinon.match({ name: 'file_4' }) + ) + }) + }) + + describe('when one of the files is called output.pdf', () => { + beforeEach(async ctx => { + ctx.OutputFileFinder.promises.findOutputFiles.resolves({ + outputFiles: [ + { path: 'file_1' }, + { path: 'file_2' }, + { path: 'file_3' }, + { path: 'file_4' }, + { path: 'output.pdf' }, + ], + }) + await ctx.OutputFileArchiveManager.archiveFilesForBuild( + projectId, + userId, + buildId + ) + }) + + it('does not include that file in the archive', ctx => { + expect(ctx.archive.append.callCount).to.equal(4) + sinon.assert.calledWith( + ctx.archive.append, + `handle: ${ctx.outputDir}/${projectId}-${userId}/${buildId}/file_1`, + sinon.match({ name: 'file_1' }) + ) + sinon.assert.calledWith( + ctx.archive.append, + `handle: ${ctx.outputDir}/${projectId}-${userId}/${buildId}/file_2`, + sinon.match({ name: 'file_2' }) + ) + sinon.assert.calledWith( + ctx.archive.append, + `handle: ${ctx.outputDir}/${projectId}-${userId}/${buildId}/file_3`, + sinon.match({ name: 'file_3' }) + ) + sinon.assert.calledWith( + ctx.archive.append, + `handle: ${ctx.outputDir}/${projectId}-${userId}/${buildId}/file_4`, + sinon.match({ name: 'file_4' }) + ) + }) + }) + + describe('when the output directory cannot be accessed', () => { + beforeEach(async ctx => { + ctx.OutputFileFinder.promises.findOutputFiles.rejects({ + code: 'ENOENT', + }) + }) + + it('rejects with a NotFoundError', async ctx => { + try { + await ctx.OutputFileArchiveManager.archiveFilesForBuild( + projectId, + userId, + buildId + ) + assert.fail('should have thrown a NotFoundError') + } catch (err) { + expect(err).to.haveOwnProperty('name', 'NotFoundError') + } + }) + + it('does not create an archive', ctx => { + expect(ctx.archiver.called).to.be.false + }) + }) +}) diff --git a/services/clsi/test/unit/js/OutputFileArchiveManagerTests.js b/services/clsi/test/unit/js/OutputFileArchiveManagerTests.js deleted file mode 100644 index d6817f3559..0000000000 --- a/services/clsi/test/unit/js/OutputFileArchiveManagerTests.js +++ /dev/null @@ -1,229 +0,0 @@ -const SandboxedModule = require('sandboxed-module') -const sinon = require('sinon') -const { assert, expect } = require('chai') - -const MODULE_PATH = require('node:path').join( - __dirname, - '../../../app/js/OutputFileArchiveManager' -) - -describe('OutputFileArchiveManager', function () { - const userId = 'user-id-123' - const projectId = 'project-id-123' - const buildId = 'build-id-123' - - afterEach(function () { - sinon.restore() - }) - - beforeEach(function () { - this.OutputFileFinder = { - promises: { - findOutputFiles: sinon.stub().resolves({ outputFiles: [] }), - }, - } - - this.OutputCacheManger = { - path: sinon.stub().callsFake((build, path) => { - return `${build}/${path}` - }), - } - - this.archive = { - append: sinon.stub(), - finalize: sinon.stub().resolves(), - on: sinon.stub(), - } - - this.archiver = sinon.stub().returns(this.archive) - - this.outputDir = '/output/dir' - - this.fs = { - open: sinon.stub().callsFake(file => ({ - createReadStream: sinon.stub().returns(`handle: ${file}`), - })), - } - - this.OutputFileArchiveManager = SandboxedModule.require(MODULE_PATH, { - requires: { - './OutputFileFinder': this.OutputFileFinder, - './OutputCacheManager': this.OutputCacheManger, - archiver: this.archiver, - 'fs/promises': this.fs, - '@overleaf/settings': { - path: { - outputDir: this.outputDir, - }, - }, - }, - }) - }) - - describe('when the output cache directory contains only exportable files', function () { - beforeEach(async function () { - this.OutputFileFinder.promises.findOutputFiles.resolves({ - outputFiles: [ - { path: 'file_1' }, - { path: 'file_2' }, - { path: 'file_3' }, - { path: 'file_4' }, - ], - }) - await this.OutputFileArchiveManager.archiveFilesForBuild( - projectId, - userId, - buildId - ) - }) - - it('creates a zip archive', function () { - sinon.assert.calledWith(this.archiver, 'zip') - }) - - it('listens to errors from the archive', function () { - sinon.assert.calledWith(this.archive.on, 'error', sinon.match.func) - }) - - it('adds all the output files to the archive', function () { - expect(this.archive.append.callCount).to.equal(4) - sinon.assert.calledWith( - this.archive.append, - `handle: ${this.outputDir}/${projectId}-${userId}/${buildId}/file_1`, - sinon.match({ name: 'file_1' }) - ) - sinon.assert.calledWith( - this.archive.append, - `handle: ${this.outputDir}/${projectId}-${userId}/${buildId}/file_2`, - sinon.match({ name: 'file_2' }) - ) - sinon.assert.calledWith( - this.archive.append, - `handle: ${this.outputDir}/${projectId}-${userId}/${buildId}/file_3`, - sinon.match({ name: 'file_3' }) - ) - sinon.assert.calledWith( - this.archive.append, - `handle: ${this.outputDir}/${projectId}-${userId}/${buildId}/file_4`, - sinon.match({ name: 'file_4' }) - ) - }) - - it('finalizes the archive after all files are appended', function () { - sinon.assert.called(this.archive.finalize) - expect(this.archive.finalize.calledBefore(this.archive.append)).to.be - .false - }) - }) - - describe('when the directory includes files ignored by web', function () { - beforeEach(async function () { - this.OutputFileFinder.promises.findOutputFiles.resolves({ - outputFiles: [ - { path: 'file_1' }, - { path: 'file_2' }, - { path: 'file_3' }, - { path: 'file_4' }, - { path: 'output.pdf' }, - ], - }) - await this.OutputFileArchiveManager.archiveFilesForBuild( - projectId, - userId, - buildId - ) - }) - - it('only includes the non-ignored files in the archive', function () { - expect(this.archive.append.callCount).to.equal(4) - sinon.assert.calledWith( - this.archive.append, - `handle: ${this.outputDir}/${projectId}-${userId}/${buildId}/file_1`, - sinon.match({ name: 'file_1' }) - ) - sinon.assert.calledWith( - this.archive.append, - `handle: ${this.outputDir}/${projectId}-${userId}/${buildId}/file_2`, - sinon.match({ name: 'file_2' }) - ) - sinon.assert.calledWith( - this.archive.append, - `handle: ${this.outputDir}/${projectId}-${userId}/${buildId}/file_3`, - sinon.match({ name: 'file_3' }) - ) - sinon.assert.calledWith( - this.archive.append, - `handle: ${this.outputDir}/${projectId}-${userId}/${buildId}/file_4`, - sinon.match({ name: 'file_4' }) - ) - }) - }) - - describe('when one of the files is called output.pdf', function () { - beforeEach(async function () { - this.OutputFileFinder.promises.findOutputFiles.resolves({ - outputFiles: [ - { path: 'file_1' }, - { path: 'file_2' }, - { path: 'file_3' }, - { path: 'file_4' }, - { path: 'output.pdf' }, - ], - }) - await this.OutputFileArchiveManager.archiveFilesForBuild( - projectId, - userId, - buildId - ) - }) - - it('does not include that file in the archive', function () { - expect(this.archive.append.callCount).to.equal(4) - sinon.assert.calledWith( - this.archive.append, - `handle: ${this.outputDir}/${projectId}-${userId}/${buildId}/file_1`, - sinon.match({ name: 'file_1' }) - ) - sinon.assert.calledWith( - this.archive.append, - `handle: ${this.outputDir}/${projectId}-${userId}/${buildId}/file_2`, - sinon.match({ name: 'file_2' }) - ) - sinon.assert.calledWith( - this.archive.append, - `handle: ${this.outputDir}/${projectId}-${userId}/${buildId}/file_3`, - sinon.match({ name: 'file_3' }) - ) - sinon.assert.calledWith( - this.archive.append, - `handle: ${this.outputDir}/${projectId}-${userId}/${buildId}/file_4`, - sinon.match({ name: 'file_4' }) - ) - }) - }) - - describe('when the output directory cannot be accessed', function () { - beforeEach(async function () { - this.OutputFileFinder.promises.findOutputFiles.rejects({ - code: 'ENOENT', - }) - }) - - it('rejects with a NotFoundError', async function () { - try { - await this.OutputFileArchiveManager.archiveFilesForBuild( - projectId, - userId, - buildId - ) - assert.fail('should have thrown a NotFoundError') - } catch (err) { - expect(err).to.haveOwnProperty('name', 'NotFoundError') - } - }) - - it('does not create an archive', function () { - expect(this.archiver.called).to.be.false - }) - }) -}) diff --git a/services/clsi/test/unit/js/OutputFileFinder.test.js b/services/clsi/test/unit/js/OutputFileFinder.test.js new file mode 100644 index 0000000000..466f150812 --- /dev/null +++ b/services/clsi/test/unit/js/OutputFileFinder.test.js @@ -0,0 +1,73 @@ +import sinon from 'sinon' +import { expect, describe, beforeEach, afterEach, it } from 'vitest' +import mockFs from 'mock-fs' +import path from 'node:path' + +const modulePath = path.join( + import.meta.dirname, + '../../../app/js/OutputFileFinder' +) + +describe('OutputFileFinder', function () { + beforeEach(async function (ctx) { + ctx.OutputFileFinder = (await import(modulePath)).default + ctx.directory = '/test/dir' + ctx.callback = sinon.stub() + + mockFs({ + [ctx.directory]: { + resource: { + 'path.tex': 'a source file', + }, + 'output.pdf': 'a generated pdf file', + extra: { + 'file.tex': 'a generated tex file', + }, + 'sneaky-file': mockFs.symlink({ + path: '../foo', + }), + }, + }) + }) + + afterEach(function () { + mockFs.restore() + }) + + describe('findOutputFiles', function () { + beforeEach(async function (ctx) { + ctx.resource_path = 'resource/path.tex' + ctx.output_paths = ['output.pdf', 'extra/file.tex'] + ctx.all_paths = ctx.output_paths.concat([ctx.resource_path]) + ctx.resources = [{ path: (ctx.resource_path = 'resource/path.tex') }] + const { outputFiles, allEntries } = + await ctx.OutputFileFinder.promises.findOutputFiles( + ctx.resources, + ctx.directory + ) + ctx.outputFiles = outputFiles + ctx.allEntries = allEntries + }) + + it('should only return the output files, not directories or resource paths', function (ctx) { + expect(ctx.outputFiles).to.have.deep.members([ + { + path: 'output.pdf', + type: 'pdf', + }, + { + path: 'extra/file.tex', + type: 'tex', + }, + ]) + expect(ctx.allEntries).to.deep.equal([ + 'extra/file.tex', + 'extra/', + 'output.pdf', + 'resource/path.tex', + 'resource/', + 'sneaky-file', + ]) + }) + }) +}) diff --git a/services/clsi/test/unit/js/OutputFileFinderTests.js b/services/clsi/test/unit/js/OutputFileFinderTests.js deleted file mode 100644 index c9e1b443be..0000000000 --- a/services/clsi/test/unit/js/OutputFileFinderTests.js +++ /dev/null @@ -1,72 +0,0 @@ -const SandboxedModule = require('sandboxed-module') -const sinon = require('sinon') -const modulePath = require('node:path').join( - __dirname, - '../../../app/js/OutputFileFinder' -) -const { expect } = require('chai') -const mockFs = require('mock-fs') - -describe('OutputFileFinder', function () { - beforeEach(function () { - this.OutputFileFinder = SandboxedModule.require(modulePath, {}) - this.directory = '/test/dir' - this.callback = sinon.stub() - - mockFs({ - [this.directory]: { - resource: { - 'path.tex': 'a source file', - }, - 'output.pdf': 'a generated pdf file', - extra: { - 'file.tex': 'a generated tex file', - }, - 'sneaky-file': mockFs.symlink({ - path: '../foo', - }), - }, - }) - }) - - afterEach(function () { - mockFs.restore() - }) - - describe('findOutputFiles', function () { - beforeEach(async function () { - this.resource_path = 'resource/path.tex' - this.output_paths = ['output.pdf', 'extra/file.tex'] - this.all_paths = this.output_paths.concat([this.resource_path]) - this.resources = [{ path: (this.resource_path = 'resource/path.tex') }] - const { outputFiles, allEntries } = - await this.OutputFileFinder.promises.findOutputFiles( - this.resources, - this.directory - ) - this.outputFiles = outputFiles - this.allEntries = allEntries - }) - - it('should only return the output files, not directories or resource paths', function () { - expect(this.outputFiles).to.have.deep.members([ - { - path: 'output.pdf', - type: 'pdf', - }, - { - path: 'extra/file.tex', - type: 'tex', - }, - ]) - expect(this.allEntries).to.deep.equal([ - 'extra/file.tex', - 'extra/', - 'output.pdf', - 'resource/path.tex', - 'resource/', - 'sneaky-file', - ]) - }) - }) -}) diff --git a/services/clsi/test/unit/js/OutputFileOptimiser.test.js b/services/clsi/test/unit/js/OutputFileOptimiser.test.js new file mode 100644 index 0000000000..976004fb61 --- /dev/null +++ b/services/clsi/test/unit/js/OutputFileOptimiser.test.js @@ -0,0 +1,193 @@ +import { vi, describe, beforeEach, it } from 'vitest' +import sinon from 'sinon' +import path from 'node:path' + +const modulePath = path.join( + import.meta.dirname, + '../../../app/js/OutputFileOptimiser' +) + +describe('OutputFileOptimiser', function () { + beforeEach(async function (ctx) { + vi.doMock('fs', () => ({ + default: (ctx.fs = {}), + })) + + vi.doMock('path', () => ({ + default: (ctx.Path = {}), + })) + + vi.doMock('child_process', () => ({ + default: { spawn: (ctx.spawn = sinon.stub()) }, + })) + + vi.doMock('@overleaf/metrics', () => ({ + default: {}, + })) + + ctx.OutputFileOptimiser = (await import(modulePath)).default + ctx.directory = '/test/dir' + return (ctx.callback = sinon.stub()) + }) + + describe('optimiseFile', function () { + beforeEach(function (ctx) { + ctx.src = './output.pdf' + return (ctx.dst = './output.pdf') + }) + + describe('when the file is not a pdf file', function () { + beforeEach(async function (ctx) { + await new Promise((resolve, reject) => { + ctx.src = './output.log' + ctx.OutputFileOptimiser.checkIfPDFIsOptimised = sinon + .stub() + .callsArgWith(1, null, false) + ctx.OutputFileOptimiser.optimisePDF = sinon + .stub() + .callsArgWith(2, null) + ctx.OutputFileOptimiser.optimiseFile(ctx.src, ctx.dst, err => { + if (err) reject(err) + resolve() + }) + }) + }) + + it('should not check if the file is optimised', function (ctx) { + return ctx.OutputFileOptimiser.checkIfPDFIsOptimised + .calledWith(ctx.src) + .should.equal(false) + }) + + return it('should not optimise the file', function (ctx) { + return ctx.OutputFileOptimiser.optimisePDF + .calledWith(ctx.src, ctx.dst) + .should.equal(false) + }) + }) + + describe('when the pdf file is not optimised', function () { + beforeEach(async function (ctx) { + await new Promise((resolve, reject) => { + ctx.OutputFileOptimiser.checkIfPDFIsOptimised = sinon + .stub() + .callsArgWith(1, null, false) + ctx.OutputFileOptimiser.optimisePDF = sinon + .stub() + .callsArgWith(2, null) + ctx.OutputFileOptimiser.optimiseFile(ctx.src, ctx.dst, err => { + if (err) reject(err) + resolve() + }) + }) + }) + + it('should check if the pdf is optimised', function (ctx) { + return ctx.OutputFileOptimiser.checkIfPDFIsOptimised + .calledWith(ctx.src) + .should.equal(true) + }) + + return it('should optimise the pdf', function (ctx) { + return ctx.OutputFileOptimiser.optimisePDF + .calledWith(ctx.src, ctx.dst) + .should.equal(true) + }) + }) + + return describe('when the pdf file is optimised', function () { + beforeEach(async function (ctx) { + await new Promise((resolve, reject) => { + ctx.OutputFileOptimiser.checkIfPDFIsOptimised = sinon + .stub() + .callsArgWith(1, null, true) + ctx.OutputFileOptimiser.optimisePDF = sinon + .stub() + .callsArgWith(2, null) + ctx.OutputFileOptimiser.optimiseFile(ctx.src, ctx.dst, err => { + if (err) reject(err) + resolve() + }) + }) + }) + + it('should check if the pdf is optimised', function (ctx) { + ctx.OutputFileOptimiser.checkIfPDFIsOptimised + .calledWith(ctx.src) + .should.equal(true) + }) + + it('should not optimise the pdf', function (ctx) { + ctx.OutputFileOptimiser.optimisePDF + .calledWith(ctx.src, ctx.dst) + .should.equal(false) + }) + }) + }) + + describe('checkIfPDFISOptimised', function () { + beforeEach(function (ctx) { + ctx.callback = sinon.stub() + ctx.fd = 1234 + ctx.fs.open = sinon.stub().yields(null, ctx.fd) + ctx.fs.read = sinon + .stub() + .withArgs(ctx.fd) + .yields(null, 100, Buffer.from('hello /Linearized 1')) + ctx.fs.close = sinon.stub().withArgs(ctx.fd).yields(null) + ctx.OutputFileOptimiser.checkIfPDFIsOptimised(ctx.src, ctx.callback) + }) + + describe('for a linearised file', function () { + beforeEach(function (ctx) { + ctx.fs.read = sinon + .stub() + .withArgs(ctx.fd) + .yields(null, 100, Buffer.from('hello /Linearized 1')) + ctx.OutputFileOptimiser.checkIfPDFIsOptimised(ctx.src, ctx.callback) + }) + + it('should open the file', function (ctx) { + ctx.fs.open.calledWith(ctx.src, 'r').should.equal(true) + }) + + it('should read the header', function (ctx) { + ctx.fs.read.calledWith(ctx.fd).should.equal(true) + }) + + it('should close the file', function (ctx) { + ctx.fs.close.calledWith(ctx.fd).should.equal(true) + }) + + it('should call the callback with a true result', function (ctx) { + ctx.callback.calledWith(null, true).should.equal(true) + }) + }) + + describe('for an unlinearised file', function () { + beforeEach(function (ctx) { + ctx.fs.read = sinon + .stub() + .withArgs(ctx.fd) + .yields(null, 100, Buffer.from('hello not linearized 1')) + ctx.OutputFileOptimiser.checkIfPDFIsOptimised(ctx.src, ctx.callback) + }) + + it('should open the file', function (ctx) { + ctx.fs.open.calledWith(ctx.src, 'r').should.equal(true) + }) + + it('should read the header', function (ctx) { + ctx.fs.read.calledWith(ctx.fd).should.equal(true) + }) + + it('should close the file', function (ctx) { + ctx.fs.close.calledWith(ctx.fd).should.equal(true) + }) + + it('should call the callback with a false result', function (ctx) { + ctx.callback.calledWith(null, false).should.equal(true) + }) + }) + }) +}) diff --git a/services/clsi/test/unit/js/OutputFileOptimiserTests.js b/services/clsi/test/unit/js/OutputFileOptimiserTests.js deleted file mode 100644 index 1e67792c92..0000000000 --- a/services/clsi/test/unit/js/OutputFileOptimiserTests.js +++ /dev/null @@ -1,192 +0,0 @@ -/* eslint-disable - no-return-assign, - no-unused-vars, - n/no-deprecated-api, -*/ -// TODO: This file was created by bulk-decaffeinate. -// Fix any style issues and re-enable lint. -/* - * decaffeinate suggestions: - * DS102: Remove unnecessary code created because of implicit returns - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ -const SandboxedModule = require('sandboxed-module') -const sinon = require('sinon') -const modulePath = require('node:path').join( - __dirname, - '../../../app/js/OutputFileOptimiser' -) -const path = require('node:path') -const { expect } = require('chai') -const { EventEmitter } = require('node:events') - -describe('OutputFileOptimiser', function () { - beforeEach(function () { - this.OutputFileOptimiser = SandboxedModule.require(modulePath, { - requires: { - fs: (this.fs = {}), - path: (this.Path = {}), - child_process: { spawn: (this.spawn = sinon.stub()) }, - '@overleaf/metrics': {}, - }, - globals: { Math }, // used by lodash - }) - this.directory = '/test/dir' - return (this.callback = sinon.stub()) - }) - - describe('optimiseFile', function () { - beforeEach(function () { - this.src = './output.pdf' - return (this.dst = './output.pdf') - }) - - describe('when the file is not a pdf file', function () { - beforeEach(function (done) { - this.src = './output.log' - this.OutputFileOptimiser.checkIfPDFIsOptimised = sinon - .stub() - .callsArgWith(1, null, false) - this.OutputFileOptimiser.optimisePDF = sinon - .stub() - .callsArgWith(2, null) - return this.OutputFileOptimiser.optimiseFile(this.src, this.dst, done) - }) - - it('should not check if the file is optimised', function () { - return this.OutputFileOptimiser.checkIfPDFIsOptimised - .calledWith(this.src) - .should.equal(false) - }) - - return it('should not optimise the file', function () { - return this.OutputFileOptimiser.optimisePDF - .calledWith(this.src, this.dst) - .should.equal(false) - }) - }) - - describe('when the pdf file is not optimised', function () { - beforeEach(function (done) { - this.OutputFileOptimiser.checkIfPDFIsOptimised = sinon - .stub() - .callsArgWith(1, null, false) - this.OutputFileOptimiser.optimisePDF = sinon - .stub() - .callsArgWith(2, null) - return this.OutputFileOptimiser.optimiseFile(this.src, this.dst, done) - }) - - it('should check if the pdf is optimised', function () { - return this.OutputFileOptimiser.checkIfPDFIsOptimised - .calledWith(this.src) - .should.equal(true) - }) - - return it('should optimise the pdf', function () { - return this.OutputFileOptimiser.optimisePDF - .calledWith(this.src, this.dst) - .should.equal(true) - }) - }) - - return describe('when the pdf file is optimised', function () { - beforeEach(function (done) { - this.OutputFileOptimiser.checkIfPDFIsOptimised = sinon - .stub() - .callsArgWith(1, null, true) - this.OutputFileOptimiser.optimisePDF = sinon - .stub() - .callsArgWith(2, null) - return this.OutputFileOptimiser.optimiseFile(this.src, this.dst, done) - }) - - it('should check if the pdf is optimised', function () { - return this.OutputFileOptimiser.checkIfPDFIsOptimised - .calledWith(this.src) - .should.equal(true) - }) - - return it('should not optimise the pdf', function () { - return this.OutputFileOptimiser.optimisePDF - .calledWith(this.src, this.dst) - .should.equal(false) - }) - }) - }) - - return describe('checkIfPDFISOptimised', function () { - beforeEach(function () { - this.callback = sinon.stub() - this.fd = 1234 - this.fs.open = sinon.stub().yields(null, this.fd) - this.fs.read = sinon - .stub() - .withArgs(this.fd) - .yields(null, 100, Buffer.from('hello /Linearized 1')) - this.fs.close = sinon.stub().withArgs(this.fd).yields(null) - return this.OutputFileOptimiser.checkIfPDFIsOptimised( - this.src, - this.callback - ) - }) - - describe('for a linearised file', function () { - beforeEach(function () { - this.fs.read = sinon - .stub() - .withArgs(this.fd) - .yields(null, 100, Buffer.from('hello /Linearized 1')) - return this.OutputFileOptimiser.checkIfPDFIsOptimised( - this.src, - this.callback - ) - }) - - it('should open the file', function () { - return this.fs.open.calledWith(this.src, 'r').should.equal(true) - }) - - it('should read the header', function () { - return this.fs.read.calledWith(this.fd).should.equal(true) - }) - - it('should close the file', function () { - return this.fs.close.calledWith(this.fd).should.equal(true) - }) - - return it('should call the callback with a true result', function () { - return this.callback.calledWith(null, true).should.equal(true) - }) - }) - - return describe('for an unlinearised file', function () { - beforeEach(function () { - this.fs.read = sinon - .stub() - .withArgs(this.fd) - .yields(null, 100, Buffer.from('hello not linearized 1')) - return this.OutputFileOptimiser.checkIfPDFIsOptimised( - this.src, - this.callback - ) - }) - - it('should open the file', function () { - return this.fs.open.calledWith(this.src, 'r').should.equal(true) - }) - - it('should read the header', function () { - return this.fs.read.calledWith(this.fd).should.equal(true) - }) - - it('should close the file', function () { - return this.fs.close.calledWith(this.fd).should.equal(true) - }) - - return it('should call the callback with a false result', function () { - return this.callback.calledWith(null, false).should.equal(true) - }) - }) - }) -}) diff --git a/services/clsi/test/unit/js/ProjectPersistenceManager.test.js b/services/clsi/test/unit/js/ProjectPersistenceManager.test.js new file mode 100644 index 0000000000..ea77af44de --- /dev/null +++ b/services/clsi/test/unit/js/ProjectPersistenceManager.test.js @@ -0,0 +1,182 @@ +import { vi, describe, beforeEach, it } from 'vitest' +import sinon from 'sinon' +import path from 'node:path' + +const modulePath = path.join( + import.meta.dirname, + '../../../app/js/ProjectPersistenceManager' +) + +describe('ProjectPersistenceManager', () => { + beforeEach(async ctx => { + ctx.fsPromises = { + statfs: sinon.stub(), + } + + vi.doMock('@overleaf/metrics', () => ({ + default: (ctx.Metrics = { gauge: sinon.stub() }), + })) + + vi.doMock('../../../app/js/UrlCache', () => ({ + default: (ctx.UrlCache = {}), + })) + + vi.doMock('../../../app/js/CompileManager', () => ({ + default: (ctx.CompileManager = {}), + })) + + vi.doMock('fs', () => ({ + default: { promises: ctx.fsPromises }, + })) + + vi.doMock('@overleaf/settings', () => ({ + default: (ctx.settings = { + project_cache_length_ms: 1000, + path: { + compilesDir: '/compiles', + outputDir: '/output', + clsiCacheDir: '/cache', + }, + }), + })) + + ctx.ProjectPersistenceManager = (await import(modulePath)).default + ctx.callback = sinon.stub() + ctx.project_id = 'project-id-123' + return (ctx.user_id = '1234') + }) + + describe('refreshExpiryTimeout', () => { + it('should leave expiry alone if plenty of disk', async ctx => { + await new Promise((resolve, reject) => { + ctx.fsPromises.statfs.resolves({ + blocks: 100, + bsize: 1, + bavail: 40, + }) + + ctx.ProjectPersistenceManager.refreshExpiryTimeout(() => { + ctx.Metrics.gauge.should.have.been.calledWith( + 'disk_available_percent', + 40 + ) + ctx.ProjectPersistenceManager.EXPIRY_TIMEOUT.should.equal( + ctx.settings.project_cache_length_ms + ) + resolve() + }) + }) + }) + + it('should drop EXPIRY_TIMEOUT 10% if low disk usage', async ctx => { + await new Promise((resolve, reject) => { + ctx.fsPromises.statfs.resolves({ + blocks: 100, + bsize: 1, + bavail: 5, + }) + + ctx.ProjectPersistenceManager.refreshExpiryTimeout(() => { + ctx.Metrics.gauge.should.have.been.calledWith( + 'disk_available_percent', + 5 + ) + ctx.ProjectPersistenceManager.EXPIRY_TIMEOUT.should.equal(900) + resolve() + }) + }) + }) + + it('should not drop EXPIRY_TIMEOUT to below 50% of project_cache_length_ms', async ctx => { + await new Promise((resolve, reject) => { + ctx.fsPromises.statfs.resolves({ + blocks: 100, + bsize: 1, + bavail: 5, + }) + ctx.ProjectPersistenceManager.EXPIRY_TIMEOUT = 500 + ctx.ProjectPersistenceManager.refreshExpiryTimeout(() => { + ctx.Metrics.gauge.should.have.been.calledWith( + 'disk_available_percent', + 5 + ) + ctx.ProjectPersistenceManager.EXPIRY_TIMEOUT.should.equal(500) + resolve() + }) + }) + }) + + it('should not modify EXPIRY_TIMEOUT if there is an error getting disk values', async ctx => { + await new Promise((resolve, reject) => { + ctx.fsPromises.statfs.rejects(new Error()) + ctx.ProjectPersistenceManager.refreshExpiryTimeout(() => { + ctx.ProjectPersistenceManager.EXPIRY_TIMEOUT.should.equal(1000) + resolve() + }) + }) + }) + }) + + describe('clearExpiredProjects', () => { + beforeEach(ctx => { + ctx.project_ids = ['project-id-1', 'project-id-2'] + ctx.ProjectPersistenceManager._findExpiredProjectIds = sinon + .stub() + .callsArgWith(0, null, ctx.project_ids) + ctx.ProjectPersistenceManager.clearProjectFromCache = sinon + .stub() + .callsArg(2) + ctx.CompileManager.clearExpiredProjects = sinon.stub().callsArg(1) + return ctx.ProjectPersistenceManager.clearExpiredProjects(ctx.callback) + }) + + it('should clear each expired project', ctx => { + return Array.from(ctx.project_ids).map(projectId => + ctx.ProjectPersistenceManager.clearProjectFromCache + .calledWith(projectId) + .should.equal(true) + ) + }) + + return it('should call the callback', ctx => { + return ctx.callback.called.should.equal(true) + }) + }) + + return describe('clearProject', () => { + beforeEach(ctx => { + ctx.ProjectPersistenceManager._clearProjectFromDatabase = sinon + .stub() + .callsArg(1) + ctx.UrlCache.clearProject = sinon.stub().callsArg(2) + ctx.CompileManager.clearProject = sinon.stub().callsArg(2) + return ctx.ProjectPersistenceManager.clearProject( + ctx.project_id, + ctx.user_id, + ctx.callback + ) + }) + + it('should clear the project from the database', ctx => { + return ctx.ProjectPersistenceManager._clearProjectFromDatabase + .calledWith(ctx.project_id) + .should.equal(true) + }) + + it('should clear all the cached Urls for the project', ctx => { + return ctx.UrlCache.clearProject + .calledWith(ctx.project_id) + .should.equal(true) + }) + + it('should clear the project compile folder', ctx => { + return ctx.CompileManager.clearProject + .calledWith(ctx.project_id, ctx.user_id) + .should.equal(true) + }) + + return it('should call the callback', ctx => { + return ctx.callback.called.should.equal(true) + }) + }) +}) diff --git a/services/clsi/test/unit/js/ProjectPersistenceManagerTests.js b/services/clsi/test/unit/js/ProjectPersistenceManagerTests.js deleted file mode 100644 index 4f42411fba..0000000000 --- a/services/clsi/test/unit/js/ProjectPersistenceManagerTests.js +++ /dev/null @@ -1,174 +0,0 @@ -/* eslint-disable - no-return-assign, - no-unused-vars, -*/ -// TODO: This file was created by bulk-decaffeinate. -// Fix any style issues and re-enable lint. -/* - * decaffeinate suggestions: - * DS101: Remove unnecessary use of Array.from - * DS102: Remove unnecessary code created because of implicit returns - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ -const SandboxedModule = require('sandboxed-module') -const sinon = require('sinon') -const assert = require('chai').assert -const modulePath = require('node:path').join( - __dirname, - '../../../app/js/ProjectPersistenceManager' -) -const tk = require('timekeeper') - -describe('ProjectPersistenceManager', function () { - beforeEach(function () { - this.fsPromises = { - statfs: sinon.stub(), - } - - this.ProjectPersistenceManager = SandboxedModule.require(modulePath, { - requires: { - '@overleaf/metrics': (this.Metrics = { gauge: sinon.stub() }), - './UrlCache': (this.UrlCache = {}), - './CompileManager': (this.CompileManager = {}), - fs: { promises: this.fsPromises }, - '@overleaf/settings': (this.settings = { - project_cache_length_ms: 1000, - path: { - compilesDir: '/compiles', - outputDir: '/output', - clsiCacheDir: '/cache', - }, - }), - }, - }) - this.callback = sinon.stub() - this.project_id = 'project-id-123' - return (this.user_id = '1234') - }) - - describe('refreshExpiryTimeout', function () { - it('should leave expiry alone if plenty of disk', function (done) { - this.fsPromises.statfs.resolves({ - blocks: 100, - bsize: 1, - bavail: 40, - }) - - this.ProjectPersistenceManager.refreshExpiryTimeout(() => { - this.Metrics.gauge.should.have.been.calledWith( - 'disk_available_percent', - 40 - ) - this.ProjectPersistenceManager.EXPIRY_TIMEOUT.should.equal( - this.settings.project_cache_length_ms - ) - done() - }) - }) - - it('should drop EXPIRY_TIMEOUT 10% if low disk usage', function (done) { - this.fsPromises.statfs.resolves({ - blocks: 100, - bsize: 1, - bavail: 5, - }) - - this.ProjectPersistenceManager.refreshExpiryTimeout(() => { - this.Metrics.gauge.should.have.been.calledWith( - 'disk_available_percent', - 5 - ) - this.ProjectPersistenceManager.EXPIRY_TIMEOUT.should.equal(900) - done() - }) - }) - - it('should not drop EXPIRY_TIMEOUT to below 50% of project_cache_length_ms', function (done) { - this.fsPromises.statfs.resolves({ - blocks: 100, - bsize: 1, - bavail: 5, - }) - this.ProjectPersistenceManager.EXPIRY_TIMEOUT = 500 - this.ProjectPersistenceManager.refreshExpiryTimeout(() => { - this.Metrics.gauge.should.have.been.calledWith( - 'disk_available_percent', - 5 - ) - this.ProjectPersistenceManager.EXPIRY_TIMEOUT.should.equal(500) - done() - }) - }) - - it('should not modify EXPIRY_TIMEOUT if there is an error getting disk values', function (done) { - this.fsPromises.statfs.rejects(new Error()) - this.ProjectPersistenceManager.refreshExpiryTimeout(() => { - this.ProjectPersistenceManager.EXPIRY_TIMEOUT.should.equal(1000) - done() - }) - }) - }) - - describe('clearExpiredProjects', function () { - beforeEach(function () { - this.project_ids = ['project-id-1', 'project-id-2'] - this.ProjectPersistenceManager._findExpiredProjectIds = sinon - .stub() - .callsArgWith(0, null, this.project_ids) - this.ProjectPersistenceManager.clearProjectFromCache = sinon - .stub() - .callsArg(2) - this.CompileManager.clearExpiredProjects = sinon.stub().callsArg(1) - return this.ProjectPersistenceManager.clearExpiredProjects(this.callback) - }) - - it('should clear each expired project', function () { - return Array.from(this.project_ids).map(projectId => - this.ProjectPersistenceManager.clearProjectFromCache - .calledWith(projectId) - .should.equal(true) - ) - }) - - return it('should call the callback', function () { - return this.callback.called.should.equal(true) - }) - }) - - return describe('clearProject', function () { - beforeEach(function () { - this.ProjectPersistenceManager._clearProjectFromDatabase = sinon - .stub() - .callsArg(1) - this.UrlCache.clearProject = sinon.stub().callsArg(2) - this.CompileManager.clearProject = sinon.stub().callsArg(2) - return this.ProjectPersistenceManager.clearProject( - this.project_id, - this.user_id, - this.callback - ) - }) - - it('should clear the project from the database', function () { - return this.ProjectPersistenceManager._clearProjectFromDatabase - .calledWith(this.project_id) - .should.equal(true) - }) - - it('should clear all the cached Urls for the project', function () { - return this.UrlCache.clearProject - .calledWith(this.project_id) - .should.equal(true) - }) - - it('should clear the project compile folder', function () { - return this.CompileManager.clearProject - .calledWith(this.project_id, this.user_id) - .should.equal(true) - }) - - return it('should call the callback', function () { - return this.callback.called.should.equal(true) - }) - }) -}) diff --git a/services/clsi/test/unit/js/RequestParser.test.js b/services/clsi/test/unit/js/RequestParser.test.js new file mode 100644 index 0000000000..411bcc0a9d --- /dev/null +++ b/services/clsi/test/unit/js/RequestParser.test.js @@ -0,0 +1,502 @@ +import { vi, expect, describe, beforeEach, afterEach, it } from 'vitest' +import sinon from 'sinon' +import tk from 'timekeeper' +import path from 'node:path' + +const modulePath = path.join( + import.meta.dirname, + '../../../app/js/RequestParser' +) + +describe('RequestParser', () => { + beforeEach(async ctx => { + tk.freeze() + ctx.callback = sinon.stub() + ctx.validResource = { + path: 'main.tex', + date: '12:00 01/02/03', + content: 'Hello world', + } + ctx.validRequest = { + compile: { + token: 'token-123', + options: { + imageName: 'basicImageName/here:2017-1', + compiler: 'pdflatex', + timeout: 42, + }, + resources: [], + }, + } + + vi.doMock('@overleaf/settings', () => ({ + default: (ctx.settings = {}), + })) + + vi.doMock('../../../app/js/OutputCacheManager', () => ({ + default: { BUILD_REGEX: /^[0-9a-f]+-[0-9a-f]+$/ }, + })) + + ctx.RequestParser = (await import(modulePath)).default + }) + + afterEach(() => { + tk.reset() + }) + + describe('without a top level object', () => { + beforeEach(ctx => { + ctx.RequestParser.parse([], ctx.callback) + }) + + it('should return an error', ctx => { + expect(ctx.callback).to.have.been.called + expect(ctx.callback.args[0][0].message).to.equal( + 'top level object should have a compile attribute' + ) + }) + }) + + describe('without a compile attribute', () => { + beforeEach(ctx => { + ctx.RequestParser.parse({}, ctx.callback) + }) + + it('should return an error', ctx => { + expect(ctx.callback).to.have.been.called + expect(ctx.callback.args[0][0].message).to.equal( + 'top level object should have a compile attribute' + ) + }) + }) + + describe('without a valid compiler', () => { + beforeEach(ctx => { + ctx.validRequest.compile.options.compiler = 'not-a-compiler' + ctx.RequestParser.parse(ctx.validRequest, ctx.callback) + }) + + it('should return an error', ctx => { + ctx.callback + .calledWithMatch({ + message: + 'compiler attribute should be one of: pdflatex, latex, xelatex, lualatex', + }) + .should.equal(true) + }) + }) + + describe('without a compiler specified', () => { + beforeEach(async ctx => { + await new Promise((resolve, reject) => { + delete ctx.validRequest.compile.options.compiler + ctx.RequestParser.parse(ctx.validRequest, (error, data) => { + if (error) return reject(error) + ctx.data = data + resolve() + }) + }) + }) + + it('should set the compiler to pdflatex by default', ctx => { + ctx.data.compiler.should.equal('pdflatex') + }) + }) + + describe('with imageName set', () => { + beforeEach(async ctx => { + await new Promise((resolve, reject) => { + ctx.RequestParser.parse(ctx.validRequest, (error, data) => { + if (error) return reject(error) + ctx.data = data + resolve() + }) + }) + }) + + it('should set the imageName', ctx => { + ctx.data.imageName.should.equal('basicImageName/here:2017-1') + }) + }) + + describe('when image restrictions are present', () => { + beforeEach(ctx => { + ctx.settings.clsi = { docker: {} } + ctx.settings.clsi.docker.allowedImages = [ + 'repo/name:tag1', + 'repo/name:tag2', + ] + }) + + describe('with imageName set to something invalid', () => { + beforeEach(ctx => { + const request = ctx.validRequest + request.compile.options.imageName = 'something/different:latest' + ctx.RequestParser.parse(request, (error, data) => { + ctx.error = error + ctx.data = data + }) + }) + + it('should throw an error for imageName', ctx => { + expect(String(ctx.error)).to.include( + 'imageName attribute should be one of' + ) + }) + }) + + describe('with imageName set to something valid', () => { + beforeEach(ctx => { + const request = ctx.validRequest + request.compile.options.imageName = 'repo/name:tag1' + ctx.RequestParser.parse(request, (error, data) => { + ctx.error = error + ctx.data = data + }) + }) + + it('should set the imageName', ctx => { + ctx.data.imageName.should.equal('repo/name:tag1') + }) + }) + }) + + describe('with flags set', () => { + beforeEach(async ctx => { + await new Promise((resolve, reject) => { + ctx.validRequest.compile.options.flags = ['-file-line-error'] + ctx.RequestParser.parse(ctx.validRequest, (error, data) => { + if (error) return reject(error) + ctx.data = data + resolve() + }) + }) + }) + + it('should set the flags attribute', ctx => { + expect(ctx.data.flags).to.deep.equal(['-file-line-error']) + }) + }) + + describe('with flags not specified', () => { + beforeEach(async ctx => { + await new Promise((resolve, reject) => { + ctx.RequestParser.parse(ctx.validRequest, (error, data) => { + if (error) return reject(error) + ctx.data = data + resolve() + }) + }) + }) + + it('it should have an empty flags list', ctx => { + expect(ctx.data.flags).to.deep.equal([]) + }) + }) + + describe('without a timeout specified', () => { + beforeEach(async ctx => { + await new Promise((resolve, reject) => { + delete ctx.validRequest.compile.options.timeout + ctx.RequestParser.parse(ctx.validRequest, (error, data) => { + if (error) return reject(error) + ctx.data = data + resolve() + }) + }) + }) + + it('should set the timeout to MAX_TIMEOUT', ctx => { + ctx.data.timeout.should.equal(ctx.RequestParser.MAX_TIMEOUT * 1000) + }) + }) + + describe('with a timeout larger than the maximum', () => { + beforeEach(async ctx => { + await new Promise((resolve, reject) => { + ctx.validRequest.compile.options.timeout = + ctx.RequestParser.MAX_TIMEOUT + 1 + ctx.RequestParser.parse(ctx.validRequest, (error, data) => { + if (error) return reject(error) + ctx.data = data + resolve() + }) + }) + }) + + it('should set the timeout to MAX_TIMEOUT', ctx => { + ctx.data.timeout.should.equal(ctx.RequestParser.MAX_TIMEOUT * 1000) + }) + }) + + describe('with a timeout', () => { + beforeEach(async ctx => { + await new Promise((resolve, reject) => { + ctx.RequestParser.parse(ctx.validRequest, (error, data) => { + if (error) return reject(error) + ctx.data = data + resolve() + }) + }) + }) + + it('should set the timeout (in milliseconds)', ctx => { + ctx.data.timeout.should.equal( + ctx.validRequest.compile.options.timeout * 1000 + ) + }) + }) + + describe('with a resource without a path', () => { + beforeEach(ctx => { + delete ctx.validResource.path + ctx.validRequest.compile.resources.push(ctx.validResource) + ctx.RequestParser.parse(ctx.validRequest, ctx.callback) + }) + + it('should return an error', ctx => { + ctx.callback + .calledWithMatch({ + message: 'all resources should have a path attribute', + }) + .should.equal(true) + }) + }) + + describe('with a resource with a path', () => { + beforeEach(ctx => { + ctx.validResource.path = ctx.path = 'test.tex' + ctx.validRequest.compile.resources.push(ctx.validResource) + ctx.RequestParser.parse(ctx.validRequest, ctx.callback) + ctx.data = ctx.callback.args[0][1] + }) + + it('should return the path in the parsed response', ctx => { + ctx.data.resources[0].path.should.equal(ctx.path) + }) + }) + + describe('with a resource with a malformed modified date', () => { + beforeEach(ctx => { + ctx.validResource.modified = 'not-a-date' + ctx.validRequest.compile.resources.push(ctx.validResource) + ctx.RequestParser.parse(ctx.validRequest, ctx.callback) + }) + + it('should return an error', ctx => { + ctx.callback + .calledWithMatch({ + message: + 'resource modified date could not be understood: ' + + ctx.validResource.modified, + }) + .should.equal(true) + }) + }) + + describe('with a valid buildId', () => { + beforeEach(async ctx => { + await new Promise((resolve, reject) => { + ctx.validRequest.compile.options.buildId = + '195a4869176-a4ad60bee7bf35e4' + ctx.RequestParser.parse(ctx.validRequest, (error, data) => { + if (error) return reject(error) + ctx.data = data + resolve() + }) + }) + }) + + it('should return an error', ctx => { + ctx.data.buildId.should.equal('195a4869176-a4ad60bee7bf35e4') + }) + }) + + describe('with a bad buildId', () => { + beforeEach(ctx => { + ctx.validRequest.compile.options.buildId = 'foo/bar' + ctx.RequestParser.parse(ctx.validRequest, ctx.callback) + }) + + it('should return an error', ctx => { + ctx.callback + .calledWithMatch({ + message: + 'buildId attribute does not match regex /^[0-9a-f]+-[0-9a-f]+$/', + }) + .should.equal(true) + }) + }) + + describe('with a resource with a valid date', () => { + beforeEach(ctx => { + ctx.date = '12:00 01/02/03' + ctx.validResource.modified = ctx.date + ctx.validRequest.compile.resources.push(ctx.validResource) + ctx.RequestParser.parse(ctx.validRequest, ctx.callback) + ctx.data = ctx.callback.args[0][1] + }) + + it('should return the date as a Javascript Date object', ctx => { + ;(ctx.data.resources[0].modified instanceof Date).should.equal(true) + ctx.data.resources[0].modified + .getTime() + .should.equal(Date.parse(ctx.date)) + }) + }) + + describe('with a resource without either a content or URL attribute', () => { + beforeEach(ctx => { + delete ctx.validResource.url + delete ctx.validResource.content + ctx.validRequest.compile.resources.push(ctx.validResource) + ctx.RequestParser.parse(ctx.validRequest, ctx.callback) + }) + + it('should return an error', ctx => { + ctx.callback + .calledWithMatch({ + message: + 'all resources should have either a url or content attribute', + }) + .should.equal(true) + }) + }) + + describe('with a resource where the content is not a string', () => { + beforeEach(ctx => { + ctx.validResource.content = [] + ctx.validRequest.compile.resources.push(ctx.validResource) + ctx.RequestParser.parse(ctx.validRequest, ctx.callback) + }) + + it('should return an error', ctx => { + ctx.callback + .calledWithMatch({ message: 'content attribute should be a string' }) + .should.equal(true) + }) + }) + + describe('with a resource where the url is not a string', () => { + beforeEach(ctx => { + ctx.validResource.url = [] + ctx.validRequest.compile.resources.push(ctx.validResource) + ctx.RequestParser.parse(ctx.validRequest, ctx.callback) + }) + + it('should return an error', ctx => { + ctx.callback + .calledWithMatch({ message: 'url attribute should be a string' }) + .should.equal(true) + }) + }) + + describe('with a resource with a url', () => { + beforeEach(ctx => { + ctx.validResource.url = ctx.url = 'www.example.com' + ctx.validRequest.compile.resources.push(ctx.validResource) + ctx.RequestParser.parse(ctx.validRequest, ctx.callback) + ctx.data = ctx.callback.args[0][1] + }) + + it('should return the url in the parsed response', ctx => { + ctx.data.resources[0].url.should.equal(ctx.url) + }) + }) + + describe('with a resource with a content attribute', () => { + beforeEach(ctx => { + ctx.validResource.content = ctx.content = 'Hello world' + ctx.validRequest.compile.resources.push(ctx.validResource) + ctx.RequestParser.parse(ctx.validRequest, ctx.callback) + ctx.data = ctx.callback.args[0][1] + }) + + it('should return the content in the parsed response', ctx => { + ctx.data.resources[0].content.should.equal(ctx.content) + }) + }) + + describe('without a root resource path', () => { + beforeEach(ctx => { + delete ctx.validRequest.compile.rootResourcePath + ctx.RequestParser.parse(ctx.validRequest, ctx.callback) + ctx.data = ctx.callback.args[0][1] + }) + + it("should set the root resource path to 'main.tex' by default", ctx => { + ctx.data.rootResourcePath.should.equal('main.tex') + }) + }) + + describe('with a root resource path', () => { + beforeEach(ctx => { + ctx.validRequest.compile.rootResourcePath = ctx.path = 'test.tex' + ctx.RequestParser.parse(ctx.validRequest, ctx.callback) + ctx.data = ctx.callback.args[0][1] + }) + + it('should return the root resource path in the parsed response', ctx => { + ctx.data.rootResourcePath.should.equal(ctx.path) + }) + }) + + describe('with a root resource path that is not a string', () => { + beforeEach(ctx => { + ctx.validRequest.compile.rootResourcePath = [] + ctx.RequestParser.parse(ctx.validRequest, ctx.callback) + }) + + it('should return an error', ctx => { + ctx.callback + .calledWithMatch({ + message: 'rootResourcePath attribute should be a string', + }) + .should.equal(true) + }) + }) + + describe('with a root resource path that has a relative path', () => { + beforeEach(ctx => { + ctx.validRequest.compile.rootResourcePath = 'foo/../../bar.tex' + ctx.RequestParser.parse(ctx.validRequest, ctx.callback) + ctx.data = ctx.callback.args[0][1] + }) + + it('should return an error', ctx => { + ctx.callback + .calledWithMatch({ message: 'relative path in root resource' }) + .should.equal(true) + }) + }) + + describe('with a root resource path that has unescaped + relative path', () => { + beforeEach(ctx => { + ctx.validRequest.compile.rootResourcePath = 'foo/../bar.tex' + ctx.RequestParser.parse(ctx.validRequest, ctx.callback) + ctx.data = ctx.callback.args[0][1] + }) + + it('should return an error', ctx => { + ctx.callback + .calledWithMatch({ message: 'relative path in root resource' }) + .should.equal(true) + }) + }) + + describe('with an unknown syncType', () => { + beforeEach(ctx => { + ctx.validRequest.compile.options.syncType = 'unexpected' + ctx.RequestParser.parse(ctx.validRequest, ctx.callback) + ctx.data = ctx.callback.args[0][1] + }) + + it('should return an error', ctx => { + ctx.callback + .calledWithMatch({ + message: 'syncType attribute should be one of: full, incremental', + }) + .should.equal(true) + }) + }) +}) diff --git a/services/clsi/test/unit/js/RequestParserTests.js b/services/clsi/test/unit/js/RequestParserTests.js deleted file mode 100644 index 437c3c4fbe..0000000000 --- a/services/clsi/test/unit/js/RequestParserTests.js +++ /dev/null @@ -1,480 +0,0 @@ -const SandboxedModule = require('sandboxed-module') -const sinon = require('sinon') -const { expect } = require('chai') -const modulePath = require('node:path').join( - __dirname, - '../../../app/js/RequestParser' -) -const tk = require('timekeeper') - -describe('RequestParser', function () { - beforeEach(function () { - tk.freeze() - this.callback = sinon.stub() - this.validResource = { - path: 'main.tex', - date: '12:00 01/02/03', - content: 'Hello world', - } - this.validRequest = { - compile: { - token: 'token-123', - options: { - imageName: 'basicImageName/here:2017-1', - compiler: 'pdflatex', - timeout: 42, - }, - resources: [], - }, - } - this.RequestParser = SandboxedModule.require(modulePath, { - requires: { - '@overleaf/settings': (this.settings = {}), - './OutputCacheManager': { BUILD_REGEX: /^[0-9a-f]+-[0-9a-f]+$/ }, - }, - }) - }) - - afterEach(function () { - tk.reset() - }) - - describe('without a top level object', function () { - beforeEach(function () { - this.RequestParser.parse([], this.callback) - }) - - it('should return an error', function () { - expect(this.callback).to.have.been.called - expect(this.callback.args[0][0].message).to.equal( - 'top level object should have a compile attribute' - ) - }) - }) - - describe('without a compile attribute', function () { - beforeEach(function () { - this.RequestParser.parse({}, this.callback) - }) - - it('should return an error', function () { - expect(this.callback).to.have.been.called - expect(this.callback.args[0][0].message).to.equal( - 'top level object should have a compile attribute' - ) - }) - }) - - describe('without a valid compiler', function () { - beforeEach(function () { - this.validRequest.compile.options.compiler = 'not-a-compiler' - this.RequestParser.parse(this.validRequest, this.callback) - }) - - it('should return an error', function () { - this.callback - .calledWithMatch({ - message: - 'compiler attribute should be one of: pdflatex, latex, xelatex, lualatex', - }) - .should.equal(true) - }) - }) - - describe('without a compiler specified', function () { - beforeEach(function (done) { - delete this.validRequest.compile.options.compiler - this.RequestParser.parse(this.validRequest, (error, data) => { - if (error) return done(error) - this.data = data - done() - }) - }) - - it('should set the compiler to pdflatex by default', function () { - this.data.compiler.should.equal('pdflatex') - }) - }) - - describe('with imageName set', function () { - beforeEach(function (done) { - this.RequestParser.parse(this.validRequest, (error, data) => { - if (error) return done(error) - this.data = data - done() - }) - }) - - it('should set the imageName', function () { - this.data.imageName.should.equal('basicImageName/here:2017-1') - }) - }) - - describe('when image restrictions are present', function () { - beforeEach(function () { - this.settings.clsi = { docker: {} } - this.settings.clsi.docker.allowedImages = [ - 'repo/name:tag1', - 'repo/name:tag2', - ] - }) - - describe('with imageName set to something invalid', function () { - beforeEach(function () { - const request = this.validRequest - request.compile.options.imageName = 'something/different:latest' - this.RequestParser.parse(request, (error, data) => { - this.error = error - this.data = data - }) - }) - - it('should throw an error for imageName', function () { - expect(String(this.error)).to.include( - 'imageName attribute should be one of' - ) - }) - }) - - describe('with imageName set to something valid', function () { - beforeEach(function () { - const request = this.validRequest - request.compile.options.imageName = 'repo/name:tag1' - this.RequestParser.parse(request, (error, data) => { - this.error = error - this.data = data - }) - }) - - it('should set the imageName', function () { - this.data.imageName.should.equal('repo/name:tag1') - }) - }) - }) - - describe('with flags set', function () { - beforeEach(function (done) { - this.validRequest.compile.options.flags = ['-file-line-error'] - this.RequestParser.parse(this.validRequest, (error, data) => { - if (error) return done(error) - this.data = data - done() - }) - }) - - it('should set the flags attribute', function () { - expect(this.data.flags).to.deep.equal(['-file-line-error']) - }) - }) - - describe('with flags not specified', function () { - beforeEach(function (done) { - this.RequestParser.parse(this.validRequest, (error, data) => { - if (error) return done(error) - this.data = data - done() - }) - }) - - it('it should have an empty flags list', function () { - expect(this.data.flags).to.deep.equal([]) - }) - }) - - describe('without a timeout specified', function () { - beforeEach(function (done) { - delete this.validRequest.compile.options.timeout - this.RequestParser.parse(this.validRequest, (error, data) => { - if (error) return done(error) - this.data = data - done() - }) - }) - - it('should set the timeout to MAX_TIMEOUT', function () { - this.data.timeout.should.equal(this.RequestParser.MAX_TIMEOUT * 1000) - }) - }) - - describe('with a timeout larger than the maximum', function () { - beforeEach(function (done) { - this.validRequest.compile.options.timeout = - this.RequestParser.MAX_TIMEOUT + 1 - this.RequestParser.parse(this.validRequest, (error, data) => { - if (error) return done(error) - this.data = data - done() - }) - }) - - it('should set the timeout to MAX_TIMEOUT', function () { - this.data.timeout.should.equal(this.RequestParser.MAX_TIMEOUT * 1000) - }) - }) - - describe('with a timeout', function () { - beforeEach(function (done) { - this.RequestParser.parse(this.validRequest, (error, data) => { - if (error) return done(error) - this.data = data - done() - }) - }) - - it('should set the timeout (in milliseconds)', function () { - this.data.timeout.should.equal( - this.validRequest.compile.options.timeout * 1000 - ) - }) - }) - - describe('with a resource without a path', function () { - beforeEach(function () { - delete this.validResource.path - this.validRequest.compile.resources.push(this.validResource) - this.RequestParser.parse(this.validRequest, this.callback) - }) - - it('should return an error', function () { - this.callback - .calledWithMatch({ - message: 'all resources should have a path attribute', - }) - .should.equal(true) - }) - }) - - describe('with a resource with a path', function () { - beforeEach(function () { - this.validResource.path = this.path = 'test.tex' - this.validRequest.compile.resources.push(this.validResource) - this.RequestParser.parse(this.validRequest, this.callback) - this.data = this.callback.args[0][1] - }) - - it('should return the path in the parsed response', function () { - this.data.resources[0].path.should.equal(this.path) - }) - }) - - describe('with a resource with a malformed modified date', function () { - beforeEach(function () { - this.validResource.modified = 'not-a-date' - this.validRequest.compile.resources.push(this.validResource) - this.RequestParser.parse(this.validRequest, this.callback) - }) - - it('should return an error', function () { - this.callback - .calledWithMatch({ - message: - 'resource modified date could not be understood: ' + - this.validResource.modified, - }) - .should.equal(true) - }) - }) - - describe('with a valid buildId', function () { - beforeEach(function (done) { - this.validRequest.compile.options.buildId = '195a4869176-a4ad60bee7bf35e4' - this.RequestParser.parse(this.validRequest, (error, data) => { - if (error) return done(error) - this.data = data - done() - }) - }) - - it('should return an error', function () { - this.data.buildId.should.equal('195a4869176-a4ad60bee7bf35e4') - }) - }) - - describe('with a bad buildId', function () { - beforeEach(function () { - this.validRequest.compile.options.buildId = 'foo/bar' - this.RequestParser.parse(this.validRequest, this.callback) - }) - - it('should return an error', function () { - this.callback - .calledWithMatch({ - message: - 'buildId attribute does not match regex /^[0-9a-f]+-[0-9a-f]+$/', - }) - .should.equal(true) - }) - }) - - describe('with a resource with a valid date', function () { - beforeEach(function () { - this.date = '12:00 01/02/03' - this.validResource.modified = this.date - this.validRequest.compile.resources.push(this.validResource) - this.RequestParser.parse(this.validRequest, this.callback) - this.data = this.callback.args[0][1] - }) - - it('should return the date as a Javascript Date object', function () { - ;(this.data.resources[0].modified instanceof Date).should.equal(true) - this.data.resources[0].modified - .getTime() - .should.equal(Date.parse(this.date)) - }) - }) - - describe('with a resource without either a content or URL attribute', function () { - beforeEach(function () { - delete this.validResource.url - delete this.validResource.content - this.validRequest.compile.resources.push(this.validResource) - this.RequestParser.parse(this.validRequest, this.callback) - }) - - it('should return an error', function () { - this.callback - .calledWithMatch({ - message: - 'all resources should have either a url or content attribute', - }) - .should.equal(true) - }) - }) - - describe('with a resource where the content is not a string', function () { - beforeEach(function () { - this.validResource.content = [] - this.validRequest.compile.resources.push(this.validResource) - this.RequestParser.parse(this.validRequest, this.callback) - }) - - it('should return an error', function () { - this.callback - .calledWithMatch({ message: 'content attribute should be a string' }) - .should.equal(true) - }) - }) - - describe('with a resource where the url is not a string', function () { - beforeEach(function () { - this.validResource.url = [] - this.validRequest.compile.resources.push(this.validResource) - this.RequestParser.parse(this.validRequest, this.callback) - }) - - it('should return an error', function () { - this.callback - .calledWithMatch({ message: 'url attribute should be a string' }) - .should.equal(true) - }) - }) - - describe('with a resource with a url', function () { - beforeEach(function () { - this.validResource.url = this.url = 'www.example.com' - this.validRequest.compile.resources.push(this.validResource) - this.RequestParser.parse(this.validRequest, this.callback) - this.data = this.callback.args[0][1] - }) - - it('should return the url in the parsed response', function () { - this.data.resources[0].url.should.equal(this.url) - }) - }) - - describe('with a resource with a content attribute', function () { - beforeEach(function () { - this.validResource.content = this.content = 'Hello world' - this.validRequest.compile.resources.push(this.validResource) - this.RequestParser.parse(this.validRequest, this.callback) - this.data = this.callback.args[0][1] - }) - - it('should return the content in the parsed response', function () { - this.data.resources[0].content.should.equal(this.content) - }) - }) - - describe('without a root resource path', function () { - beforeEach(function () { - delete this.validRequest.compile.rootResourcePath - this.RequestParser.parse(this.validRequest, this.callback) - this.data = this.callback.args[0][1] - }) - - it("should set the root resource path to 'main.tex' by default", function () { - this.data.rootResourcePath.should.equal('main.tex') - }) - }) - - describe('with a root resource path', function () { - beforeEach(function () { - this.validRequest.compile.rootResourcePath = this.path = 'test.tex' - this.RequestParser.parse(this.validRequest, this.callback) - this.data = this.callback.args[0][1] - }) - - it('should return the root resource path in the parsed response', function () { - this.data.rootResourcePath.should.equal(this.path) - }) - }) - - describe('with a root resource path that is not a string', function () { - beforeEach(function () { - this.validRequest.compile.rootResourcePath = [] - this.RequestParser.parse(this.validRequest, this.callback) - }) - - it('should return an error', function () { - this.callback - .calledWithMatch({ - message: 'rootResourcePath attribute should be a string', - }) - .should.equal(true) - }) - }) - - describe('with a root resource path that has a relative path', function () { - beforeEach(function () { - this.validRequest.compile.rootResourcePath = 'foo/../../bar.tex' - this.RequestParser.parse(this.validRequest, this.callback) - this.data = this.callback.args[0][1] - }) - - it('should return an error', function () { - this.callback - .calledWithMatch({ message: 'relative path in root resource' }) - .should.equal(true) - }) - }) - - describe('with a root resource path that has unescaped + relative path', function () { - beforeEach(function () { - this.validRequest.compile.rootResourcePath = 'foo/../bar.tex' - this.RequestParser.parse(this.validRequest, this.callback) - this.data = this.callback.args[0][1] - }) - - it('should return an error', function () { - this.callback - .calledWithMatch({ message: 'relative path in root resource' }) - .should.equal(true) - }) - }) - - describe('with an unknown syncType', function () { - beforeEach(function () { - this.validRequest.compile.options.syncType = 'unexpected' - this.RequestParser.parse(this.validRequest, this.callback) - this.data = this.callback.args[0][1] - }) - - it('should return an error', function () { - this.callback - .calledWithMatch({ - message: 'syncType attribute should be one of: full, incremental', - }) - .should.equal(true) - }) - }) -}) diff --git a/services/clsi/test/unit/js/ResourceStateManager.test.js b/services/clsi/test/unit/js/ResourceStateManager.test.js new file mode 100644 index 0000000000..7cf9a6dba2 --- /dev/null +++ b/services/clsi/test/unit/js/ResourceStateManager.test.js @@ -0,0 +1,229 @@ +import { vi, expect, describe, beforeEach, it } from 'vitest' + +import sinon from 'sinon' +import Path from 'node:path' +import * as Errors from '../../../app/js/Errors.js' + +const modulePath = Path.join( + import.meta.dirname, + '../../../app/js/ResourceStateManager' +) + +describe('ResourceStateManager', () => { + beforeEach(async ctx => { + vi.doMock('fs', () => ({ + default: (ctx.fs = {}), + })) + + vi.doMock('../../../app/js/SafeReader', () => ({ + default: (ctx.SafeReader = {}), + })) + + ctx.ResourceStateManager = (await import(modulePath)).default + ctx.basePath = '/path/to/write/files/to' + ctx.resources = [ + { path: 'resource-1-mock' }, + { path: 'resource-2-mock' }, + { path: 'resource-3-mock' }, + ] + ctx.state = '1234567890' + ctx.resourceFileName = `${ctx.basePath}/.project-sync-state` + ctx.resourceFileContents = `${ctx.resources[0].path}\n${ctx.resources[1].path}\n${ctx.resources[2].path}\nstateHash:${ctx.state}` + ctx.callback = sinon.stub() + }) + + describe('saveProjectState', () => { + beforeEach(ctx => { + ctx.fs.writeFile = sinon.stub().callsArg(2) + }) + + describe('when the state is specified', () => { + beforeEach(ctx => { + ctx.ResourceStateManager.saveProjectState( + ctx.state, + ctx.resources, + ctx.basePath, + ctx.callback + ) + }) + + it('should write the resource list to disk', ctx => { + ctx.fs.writeFile + .calledWith(ctx.resourceFileName, ctx.resourceFileContents) + .should.equal(true) + }) + + it('should call the callback', ctx => { + ctx.callback.called.should.equal(true) + }) + }) + + describe('when the state is undefined', () => { + beforeEach(ctx => { + ctx.state = undefined + ctx.fs.unlink = sinon.stub().callsArg(1) + ctx.ResourceStateManager.saveProjectState( + ctx.state, + ctx.resources, + ctx.basePath, + ctx.callback + ) + }) + + it('should unlink the resource file', ctx => { + ctx.fs.unlink.calledWith(ctx.resourceFileName).should.equal(true) + }) + + it('should not write the resource list to disk', ctx => { + ctx.fs.writeFile.called.should.equal(false) + }) + + it('should call the callback', ctx => { + ctx.callback.called.should.equal(true) + }) + }) + }) + + describe('checkProjectStateMatches', () => { + describe('when the state matches', () => { + beforeEach(ctx => { + ctx.SafeReader.readFile = sinon + .stub() + .callsArgWith(3, null, ctx.resourceFileContents) + ctx.ResourceStateManager.checkProjectStateMatches( + ctx.state, + ctx.basePath, + ctx.callback + ) + }) + + it('should read the resource file', ctx => { + ctx.SafeReader.readFile + .calledWith(ctx.resourceFileName) + .should.equal(true) + }) + + it('should call the callback with the results', ctx => { + ctx.callback.calledWithMatch(null, ctx.resources).should.equal(true) + }) + }) + + describe('when the state file is not present', () => { + beforeEach(ctx => { + ctx.SafeReader.readFile = sinon.stub().callsArg(3) + ctx.ResourceStateManager.checkProjectStateMatches( + ctx.state, + ctx.basePath, + ctx.callback + ) + }) + + it('should read the resource file', ctx => { + ctx.SafeReader.readFile + .calledWith(ctx.resourceFileName) + .should.equal(true) + }) + + it('should call the callback with an error', ctx => { + ctx.callback + .calledWith(sinon.match(Errors.FilesOutOfSyncError)) + .should.equal(true) + + const message = ctx.callback.args[0][0].message + expect(message).to.include('invalid state for incremental update') + }) + }) + + describe('when the state does not match', () => { + beforeEach(ctx => { + ctx.SafeReader.readFile = sinon + .stub() + .callsArgWith(3, null, ctx.resourceFileContents) + ctx.ResourceStateManager.checkProjectStateMatches( + 'not-the-original-state', + ctx.basePath, + ctx.callback + ) + }) + + it('should call the callback with an error', ctx => { + ctx.callback + .calledWith(sinon.match(Errors.FilesOutOfSyncError)) + .should.equal(true) + + const message = ctx.callback.args[0][0].message + expect(message).to.include('invalid state for incremental update') + }) + }) + }) + + describe('checkResourceFiles', () => { + describe('when all the files are present', () => { + beforeEach(ctx => { + ctx.allFiles = [ + ctx.resources[0].path, + ctx.resources[1].path, + ctx.resources[2].path, + ] + ctx.ResourceStateManager.checkResourceFiles( + ctx.resources, + ctx.allFiles, + ctx.basePath, + ctx.callback + ) + }) + + it('should call the callback', ctx => { + ctx.callback.calledWithExactly().should.equal(true) + }) + }) + + describe('when there is a missing file', () => { + beforeEach(ctx => { + ctx.allFiles = [ctx.resources[0].path, ctx.resources[1].path] + ctx.fs.stat = sinon.stub().callsArgWith(1, new Error()) + ctx.ResourceStateManager.checkResourceFiles( + ctx.resources, + ctx.allFiles, + ctx.basePath, + ctx.callback + ) + }) + + it('should call the callback with an error', ctx => { + ctx.callback + .calledWith(sinon.match(Errors.FilesOutOfSyncError)) + .should.equal(true) + + const message = ctx.callback.args[0][0].message + expect(message).to.include( + 'resource files missing in incremental update' + ) + }) + }) + + describe('when a resource contains a relative path', () => { + beforeEach(ctx => { + ctx.resources[0].path = '../foo/bar.tex' + ctx.allFiles = [ + ctx.resources[0].path, + ctx.resources[1].path, + ctx.resources[2].path, + ] + ctx.ResourceStateManager.checkResourceFiles( + ctx.resources, + ctx.allFiles, + ctx.basePath, + ctx.callback + ) + }) + + it('should call the callback with an error', ctx => { + ctx.callback.calledWith(sinon.match(Error)).should.equal(true) + + const message = ctx.callback.args[0][0].message + expect(message).to.include('relative path in resource file list') + }) + }) + }) +}) diff --git a/services/clsi/test/unit/js/ResourceStateManagerTests.js b/services/clsi/test/unit/js/ResourceStateManagerTests.js deleted file mode 100644 index 823c81616f..0000000000 --- a/services/clsi/test/unit/js/ResourceStateManagerTests.js +++ /dev/null @@ -1,241 +0,0 @@ -/* eslint-disable - no-return-assign, - no-unused-vars, -*/ -// TODO: This file was created by bulk-decaffeinate. -// Fix any style issues and re-enable lint. -/* - * decaffeinate suggestions: - * DS102: Remove unnecessary code created because of implicit returns - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ -const SandboxedModule = require('sandboxed-module') -const sinon = require('sinon') -const { expect } = require('chai') -const modulePath = require('node:path').join( - __dirname, - '../../../app/js/ResourceStateManager' -) -const Path = require('node:path') -const Errors = require('../../../app/js/Errors') - -describe('ResourceStateManager', function () { - beforeEach(function () { - this.ResourceStateManager = SandboxedModule.require(modulePath, { - singleOnly: true, - requires: { - fs: (this.fs = {}), - './SafeReader': (this.SafeReader = {}), - }, - }) - this.basePath = '/path/to/write/files/to' - this.resources = [ - { path: 'resource-1-mock' }, - { path: 'resource-2-mock' }, - { path: 'resource-3-mock' }, - ] - this.state = '1234567890' - this.resourceFileName = `${this.basePath}/.project-sync-state` - this.resourceFileContents = `${this.resources[0].path}\n${this.resources[1].path}\n${this.resources[2].path}\nstateHash:${this.state}` - return (this.callback = sinon.stub()) - }) - - describe('saveProjectState', function () { - beforeEach(function () { - return (this.fs.writeFile = sinon.stub().callsArg(2)) - }) - - describe('when the state is specified', function () { - beforeEach(function () { - return this.ResourceStateManager.saveProjectState( - this.state, - this.resources, - this.basePath, - this.callback - ) - }) - - it('should write the resource list to disk', function () { - return this.fs.writeFile - .calledWith(this.resourceFileName, this.resourceFileContents) - .should.equal(true) - }) - - return it('should call the callback', function () { - return this.callback.called.should.equal(true) - }) - }) - - return describe('when the state is undefined', function () { - beforeEach(function () { - this.state = undefined - this.fs.unlink = sinon.stub().callsArg(1) - return this.ResourceStateManager.saveProjectState( - this.state, - this.resources, - this.basePath, - this.callback - ) - }) - - it('should unlink the resource file', function () { - return this.fs.unlink - .calledWith(this.resourceFileName) - .should.equal(true) - }) - - it('should not write the resource list to disk', function () { - return this.fs.writeFile.called.should.equal(false) - }) - - return it('should call the callback', function () { - return this.callback.called.should.equal(true) - }) - }) - }) - - describe('checkProjectStateMatches', function () { - describe('when the state matches', function () { - beforeEach(function () { - this.SafeReader.readFile = sinon - .stub() - .callsArgWith(3, null, this.resourceFileContents) - return this.ResourceStateManager.checkProjectStateMatches( - this.state, - this.basePath, - this.callback - ) - }) - - it('should read the resource file', function () { - return this.SafeReader.readFile - .calledWith(this.resourceFileName) - .should.equal(true) - }) - - return it('should call the callback with the results', function () { - return this.callback - .calledWithMatch(null, this.resources) - .should.equal(true) - }) - }) - - describe('when the state file is not present', function () { - beforeEach(function () { - this.SafeReader.readFile = sinon.stub().callsArg(3) - return this.ResourceStateManager.checkProjectStateMatches( - this.state, - this.basePath, - this.callback - ) - }) - - it('should read the resource file', function () { - return this.SafeReader.readFile - .calledWith(this.resourceFileName) - .should.equal(true) - }) - - it('should call the callback with an error', function () { - this.callback - .calledWith(sinon.match(Errors.FilesOutOfSyncError)) - .should.equal(true) - - const message = this.callback.args[0][0].message - expect(message).to.include('invalid state for incremental update') - }) - }) - - return describe('when the state does not match', function () { - beforeEach(function () { - this.SafeReader.readFile = sinon - .stub() - .callsArgWith(3, null, this.resourceFileContents) - return this.ResourceStateManager.checkProjectStateMatches( - 'not-the-original-state', - this.basePath, - this.callback - ) - }) - - it('should call the callback with an error', function () { - this.callback - .calledWith(sinon.match(Errors.FilesOutOfSyncError)) - .should.equal(true) - - const message = this.callback.args[0][0].message - expect(message).to.include('invalid state for incremental update') - }) - }) - }) - - return describe('checkResourceFiles', function () { - describe('when all the files are present', function () { - beforeEach(function () { - this.allFiles = [ - this.resources[0].path, - this.resources[1].path, - this.resources[2].path, - ] - return this.ResourceStateManager.checkResourceFiles( - this.resources, - this.allFiles, - this.basePath, - this.callback - ) - }) - - return it('should call the callback', function () { - return this.callback.calledWithExactly().should.equal(true) - }) - }) - - describe('when there is a missing file', function () { - beforeEach(function () { - this.allFiles = [this.resources[0].path, this.resources[1].path] - this.fs.stat = sinon.stub().callsArgWith(1, new Error()) - return this.ResourceStateManager.checkResourceFiles( - this.resources, - this.allFiles, - this.basePath, - this.callback - ) - }) - - it('should call the callback with an error', function () { - this.callback - .calledWith(sinon.match(Errors.FilesOutOfSyncError)) - .should.equal(true) - - const message = this.callback.args[0][0].message - expect(message).to.include( - 'resource files missing in incremental update' - ) - }) - }) - - return describe('when a resource contains a relative path', function () { - beforeEach(function () { - this.resources[0].path = '../foo/bar.tex' - this.allFiles = [ - this.resources[0].path, - this.resources[1].path, - this.resources[2].path, - ] - return this.ResourceStateManager.checkResourceFiles( - this.resources, - this.allFiles, - this.basePath, - this.callback - ) - }) - - it('should call the callback with an error', function () { - this.callback.calledWith(sinon.match(Error)).should.equal(true) - - const message = this.callback.args[0][0].message - expect(message).to.include('relative path in resource file list') - }) - }) - }) -}) diff --git a/services/clsi/test/unit/js/ResourceWriter.test.js b/services/clsi/test/unit/js/ResourceWriter.test.js new file mode 100644 index 0000000000..29bfbbeb5d --- /dev/null +++ b/services/clsi/test/unit/js/ResourceWriter.test.js @@ -0,0 +1,536 @@ +import { vi, expect, describe, beforeEach, it } from 'vitest' + +import sinon from 'sinon' +import path from 'node:path' + +const modulePath = path.join( + import.meta.dirname, + '../../../app/js/ResourceWriter' +) + +describe('ResourceWriter', () => { + beforeEach(async ctx => { + let Timer + + vi.doMock('fs', () => ({ + default: (ctx.fs = { + mkdir: sinon.stub().callsArg(1), + unlink: sinon.stub().callsArg(1), + }), + })) + + vi.doMock('../../../app/js/ResourceStateManager', () => ({ + default: (ctx.ResourceStateManager = {}), + })) + + vi.doMock('../../../app/js/UrlCache', () => ({ + default: (ctx.UrlCache = { + createProjectDir: sinon.stub().yields(), + }), + })) + + vi.doMock('../../../app/js/OutputFileFinder', () => ({ + default: (ctx.OutputFileFinder = {}), + })) + + vi.doMock('@overleaf/metrics', () => ({ + // Mocks allow us to import Metrics.js twice without getting errors. + prom: { + Gauge: sinon.stub(), + Histogram: sinon.stub(), + Counter: sinon.stub(), + }, + default: (ctx.Metrics = { + inc: sinon.stub(), + Timer: (Timer = (function () { + Timer = class Timer { + static initClass() { + this.prototype.done = sinon.stub() + } + } + Timer.initClass() + return Timer + })()), + }), + })) + + ctx.ResourceWriter = (await import(modulePath)).default + ctx.project_id = 'project-id-123' + ctx.basePath = '/path/to/write/files/to' + return (ctx.callback = sinon.stub()) + }) + + describe('syncResourcesToDisk on a full request', () => { + beforeEach(ctx => { + ctx.resources = ['resource-1-mock', 'resource-2-mock', 'resource-3-mock'] + ctx.request = { + project_id: ctx.project_id, + syncState: (ctx.syncState = '0123456789abcdef'), + resources: ctx.resources, + } + ctx.ResourceWriter._writeResourceToDisk = sinon.stub().callsArg(3) + ctx.ResourceWriter._removeExtraneousFiles = sinon.stub().yields(null) + ctx.ResourceStateManager.saveProjectState = sinon.stub().callsArg(3) + return ctx.ResourceWriter.syncResourcesToDisk( + ctx.request, + ctx.basePath, + ctx.callback + ) + }) + + it('should remove old files', ctx => { + return ctx.ResourceWriter._removeExtraneousFiles + .calledWith(ctx.request, ctx.resources, ctx.basePath) + .should.equal(true) + }) + + it('should write each resource to disk', ctx => { + return Array.from(ctx.resources).map(resource => + ctx.ResourceWriter._writeResourceToDisk + .calledWith(ctx.project_id, resource, ctx.basePath) + .should.equal(true) + ) + }) + + it('should store the sync state and resource list', ctx => { + return ctx.ResourceStateManager.saveProjectState + .calledWith(ctx.syncState, ctx.resources, ctx.basePath) + .should.equal(true) + }) + + return it('should call the callback', ctx => { + return ctx.callback.called.should.equal(true) + }) + }) + + describe('syncResourcesToDisk on an incremental update', () => { + beforeEach(ctx => { + ctx.resources = ['resource-1-mock'] + ctx.request = { + project_id: ctx.project_id, + syncType: 'incremental', + syncState: (ctx.syncState = '1234567890abcdef'), + resources: ctx.resources, + } + ctx.fullResources = ctx.resources.concat(['file-1']) + ctx.ResourceWriter._writeResourceToDisk = sinon.stub().callsArg(3) + ctx.ResourceWriter._removeExtraneousFiles = sinon + .stub() + .yields(null, (ctx.outputFiles = []), (ctx.allFiles = [])) + ctx.ResourceStateManager.checkProjectStateMatches = sinon + .stub() + .callsArgWith(2, null, ctx.fullResources) + ctx.ResourceStateManager.saveProjectState = sinon.stub().callsArg(3) + ctx.ResourceStateManager.checkResourceFiles = sinon.stub().callsArg(3) + return ctx.ResourceWriter.syncResourcesToDisk( + ctx.request, + ctx.basePath, + ctx.callback + ) + }) + + it('should check the sync state matches', ctx => { + return ctx.ResourceStateManager.checkProjectStateMatches + .calledWith(ctx.syncState, ctx.basePath) + .should.equal(true) + }) + + it('should remove old files', ctx => { + return ctx.ResourceWriter._removeExtraneousFiles + .calledWith(ctx.request, ctx.fullResources, ctx.basePath) + .should.equal(true) + }) + + it('should check each resource exists', ctx => { + return ctx.ResourceStateManager.checkResourceFiles + .calledWith(ctx.fullResources, ctx.allFiles, ctx.basePath) + .should.equal(true) + }) + + it('should write each resource to disk', ctx => { + return Array.from(ctx.resources).map(resource => + ctx.ResourceWriter._writeResourceToDisk + .calledWith(ctx.project_id, resource, ctx.basePath) + .should.equal(true) + ) + }) + + return it('should call the callback', ctx => { + return ctx.callback.called.should.equal(true) + }) + }) + + describe('syncResourcesToDisk on an incremental update when the state does not match', () => { + beforeEach(ctx => { + ctx.resources = ['resource-1-mock'] + ctx.request = { + project_id: ctx.project_id, + syncType: 'incremental', + syncState: (ctx.syncState = '1234567890abcdef'), + resources: ctx.resources, + } + ctx.ResourceStateManager.checkProjectStateMatches = sinon + .stub() + .callsArgWith(2, (ctx.error = new Error())) + return ctx.ResourceWriter.syncResourcesToDisk( + ctx.request, + ctx.basePath, + ctx.callback + ) + }) + + it('should check whether the sync state matches', ctx => { + return ctx.ResourceStateManager.checkProjectStateMatches + .calledWith(ctx.syncState, ctx.basePath) + .should.equal(true) + }) + + return it('should call the callback with an error', ctx => { + return ctx.callback.calledWith(ctx.error).should.equal(true) + }) + }) + + describe('_removeExtraneousFiles', () => { + beforeEach(ctx => { + ctx.output_files = [ + { + path: 'output.pdf', + type: 'pdf', + }, + { + path: 'extra/file.tex', + type: 'tex', + }, + { + path: 'extra.aux', + type: 'aux', + }, + { + path: 'cache/_chunk1', + }, + { + path: 'figures/image-eps-converted-to.pdf', + type: 'pdf', + }, + { + path: 'foo/main-figure0.md5', + type: 'md5', + }, + { + path: 'foo/main-figure0.dpth', + type: 'dpth', + }, + { + path: 'foo/main-figure0.pdf', + type: 'pdf', + }, + { + path: '_minted-main/default-pyg-prefix.pygstyle', + type: 'pygstyle', + }, + { + path: '_minted-main/default.pygstyle', + type: 'pygstyle', + }, + { + path: '_minted-main/35E248B60965545BD232AE9F0FE9750D504A7AF0CD3BAA7542030FC560DFCC45.pygtex', + type: 'pygtex', + }, + { + path: '_markdown_main/30893013dec5d869a415610079774c2f.md.tex', + type: 'tex', + }, + { + path: 'output.stdout', + }, + { + path: 'output.stderr', + }, + ] + ctx.resources = 'mock-resources' + ctx.request = { + project_id: ctx.project_id, + syncType: 'incremental', + syncState: (ctx.syncState = '1234567890abcdef'), + resources: ctx.resources, + metricsOpts: { path: 'foo' }, + } + ctx.OutputFileFinder.findOutputFiles = sinon + .stub() + .callsArgWith(2, null, ctx.output_files) + ctx.ResourceWriter._deleteFileIfNotDirectory = sinon.stub().callsArg(1) + return ctx.ResourceWriter._removeExtraneousFiles( + ctx.request, + ctx.resources, + ctx.basePath, + ctx.callback + ) + }) + + it('should find the existing output files', ctx => { + return ctx.OutputFileFinder.findOutputFiles + .calledWith(ctx.resources, ctx.basePath) + .should.equal(true) + }) + + it('should delete the output files', ctx => { + return ctx.ResourceWriter._deleteFileIfNotDirectory + .calledWith(path.join(ctx.basePath, 'output.pdf')) + .should.equal(true) + }) + + it('should delete the stdout log file', ctx => { + return ctx.ResourceWriter._deleteFileIfNotDirectory + .calledWith(path.join(ctx.basePath, 'output.stdout')) + .should.equal(true) + }) + + it('should delete the stderr log file', ctx => { + return ctx.ResourceWriter._deleteFileIfNotDirectory + .calledWith(path.join(ctx.basePath, 'output.stderr')) + .should.equal(true) + }) + + it('should delete the extra files', ctx => { + return ctx.ResourceWriter._deleteFileIfNotDirectory + .calledWith(path.join(ctx.basePath, 'extra/file.tex')) + .should.equal(true) + }) + + it('should not delete the extra aux files', ctx => { + return ctx.ResourceWriter._deleteFileIfNotDirectory + .calledWith(path.join(ctx.basePath, 'extra.aux')) + .should.equal(false) + }) + + it('should not delete the knitr cache file', ctx => { + return ctx.ResourceWriter._deleteFileIfNotDirectory + .calledWith(path.join(ctx.basePath, 'cache/_chunk1')) + .should.equal(false) + }) + + it('should not delete the epstopdf converted files', ctx => { + return ctx.ResourceWriter._deleteFileIfNotDirectory + .calledWith( + path.join(ctx.basePath, 'figures/image-eps-converted-to.pdf') + ) + .should.equal(false) + }) + + it('should not delete the tikz md5 files', ctx => { + return ctx.ResourceWriter._deleteFileIfNotDirectory + .calledWith(path.join(ctx.basePath, 'foo/main-figure0.md5')) + .should.equal(false) + }) + + it('should not delete the tikz dpth files', ctx => { + return ctx.ResourceWriter._deleteFileIfNotDirectory + .calledWith(path.join(ctx.basePath, 'foo/main-figure0.dpth')) + .should.equal(false) + }) + + it('should not delete the tikz pdf files', ctx => { + return ctx.ResourceWriter._deleteFileIfNotDirectory + .calledWith(path.join(ctx.basePath, 'foo/main-figure0.pdf')) + .should.equal(false) + }) + + it('should not delete the minted pygstyle files', ctx => { + return ctx.ResourceWriter._deleteFileIfNotDirectory + .calledWith( + path.join(ctx.basePath, '_minted-main/default-pyg-prefix.pygstyle') + ) + .should.equal(false) + }) + + it('should not delete the minted default pygstyle files', ctx => { + return ctx.ResourceWriter._deleteFileIfNotDirectory + .calledWith(path.join(ctx.basePath, '_minted-main/default.pygstyle')) + .should.equal(false) + }) + + it('should not delete the minted default pygtex files', ctx => { + return ctx.ResourceWriter._deleteFileIfNotDirectory + .calledWith( + path.join( + ctx.basePath, + '_minted-main/35E248B60965545BD232AE9F0FE9750D504A7AF0CD3BAA7542030FC560DFCC45.pygtex' + ) + ) + .should.equal(false) + }) + + it('should not delete the markdown md.tex files', ctx => { + return ctx.ResourceWriter._deleteFileIfNotDirectory + .calledWith( + path.join( + ctx.basePath, + '_markdown_main/30893013dec5d869a415610079774c2f.md.tex' + ) + ) + .should.equal(false) + }) + + it('should call the callback', ctx => { + return ctx.callback.called.should.equal(true) + }) + + return it('should time the request', ctx => { + return ctx.Metrics.Timer.prototype.done.called.should.equal(true) + }) + }) + + describe('_writeResourceToDisk', () => { + describe('with a url based resource', () => { + beforeEach(ctx => { + ctx.fs.mkdir = sinon.stub().callsArg(2) + ctx.resource = { + path: 'main.tex', + url: 'http://www.example.com/primary/main.tex', + fallbackURL: 'http://fallback.example.com/fallback/main.tex', + modified: Date.now(), + } + ctx.UrlCache.downloadUrlToFile = sinon + .stub() + .callsArgWith(5, 'fake error downloading file') + return ctx.ResourceWriter._writeResourceToDisk( + ctx.project_id, + ctx.resource, + ctx.basePath, + ctx.callback + ) + }) + + it('should ensure the directory exists', ctx => { + ctx.fs.mkdir + .calledWith(path.dirname(path.join(ctx.basePath, ctx.resource.path))) + .should.equal(true) + }) + + it('should write the URL from the cache', ctx => { + return ctx.UrlCache.downloadUrlToFile + .calledWith( + ctx.project_id, + ctx.resource.url, + ctx.resource.fallbackURL, + path.join(ctx.basePath, ctx.resource.path), + ctx.resource.modified + ) + .should.equal(true) + }) + + it('should call the callback', ctx => { + return ctx.callback.called.should.equal(true) + }) + + return it('should not return an error if the resource writer errored', ctx => { + return expect(ctx.callback.args[0][0]).not.to.exist + }) + }) + + describe('with a content based resource', () => { + beforeEach(ctx => { + ctx.resource = { + path: 'main.tex', + content: 'Hello world', + } + ctx.fs.writeFile = sinon.stub().callsArg(2) + ctx.fs.mkdir = sinon.stub().callsArg(2) + return ctx.ResourceWriter._writeResourceToDisk( + ctx.project_id, + ctx.resource, + ctx.basePath, + ctx.callback + ) + }) + + it('should ensure the directory exists', ctx => { + return ctx.fs.mkdir + .calledWith(path.dirname(path.join(ctx.basePath, ctx.resource.path))) + .should.equal(true) + }) + + it('should write the contents to disk', ctx => { + return ctx.fs.writeFile + .calledWith( + path.join(ctx.basePath, ctx.resource.path), + ctx.resource.content + ) + .should.equal(true) + }) + + return it('should call the callback', ctx => { + return ctx.callback.called.should.equal(true) + }) + }) + + return describe('with a file path that breaks out of the root folder', () => { + beforeEach(ctx => { + ctx.resource = { + path: '../../main.tex', + content: 'Hello world', + } + ctx.fs.writeFile = sinon.stub().callsArg(2) + return ctx.ResourceWriter._writeResourceToDisk( + ctx.project_id, + ctx.resource, + ctx.basePath, + ctx.callback + ) + }) + + it('should not write to disk', ctx => { + return ctx.fs.writeFile.called.should.equal(false) + }) + + it('should return an error', ctx => { + ctx.callback.calledWith(sinon.match(Error)).should.equal(true) + + const message = ctx.callback.args[0][0].message + expect(message).to.include('resource path is outside root directory') + }) + }) + }) + + return describe('checkPath', () => { + describe('with a valid path', () => { + beforeEach(ctx => { + return ctx.ResourceWriter.checkPath('foo', 'bar', ctx.callback) + }) + + return it('should return the joined path', ctx => { + return ctx.callback.calledWith(null, 'foo/bar').should.equal(true) + }) + }) + + describe('with an invalid path', () => { + beforeEach(ctx => { + ctx.ResourceWriter.checkPath('foo', 'baz/../../bar', ctx.callback) + }) + + it('should return an error', ctx => { + ctx.callback.calledWith(sinon.match(Error)).should.equal(true) + + const message = ctx.callback.args[0][0].message + expect(message).to.include('resource path is outside root directory') + }) + }) + + describe('with another invalid path matching on a prefix', () => { + beforeEach(ctx => { + return ctx.ResourceWriter.checkPath( + 'foo', + '../foobar/baz', + ctx.callback + ) + }) + + it('should return an error', ctx => { + ctx.callback.calledWith(sinon.match(Error)).should.equal(true) + + const message = ctx.callback.args[0][0].message + expect(message).to.include('resource path is outside root directory') + }) + }) + }) +}) diff --git a/services/clsi/test/unit/js/ResourceWriterTests.js b/services/clsi/test/unit/js/ResourceWriterTests.js deleted file mode 100644 index 73b47f0fcd..0000000000 --- a/services/clsi/test/unit/js/ResourceWriterTests.js +++ /dev/null @@ -1,533 +0,0 @@ -/* eslint-disable - no-return-assign, -*/ -// TODO: This file was created by bulk-decaffeinate. -// Fix any style issues and re-enable lint. -/* - * decaffeinate suggestions: - * DS101: Remove unnecessary use of Array.from - * DS102: Remove unnecessary code created because of implicit returns - * DS206: Consider reworking classes to avoid initClass - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ -const SandboxedModule = require('sandboxed-module') -const sinon = require('sinon') -const { expect } = require('chai') -const modulePath = require('node:path').join( - __dirname, - '../../../app/js/ResourceWriter' -) -const path = require('node:path') - -describe('ResourceWriter', function () { - beforeEach(function () { - let Timer - this.ResourceWriter = SandboxedModule.require(modulePath, { - singleOnly: true, - requires: { - fs: (this.fs = { - mkdir: sinon.stub().callsArg(1), - unlink: sinon.stub().callsArg(1), - }), - './ResourceStateManager': (this.ResourceStateManager = {}), - './UrlCache': (this.UrlCache = { - createProjectDir: sinon.stub().yields(), - }), - './OutputFileFinder': (this.OutputFileFinder = {}), - '@overleaf/metrics': (this.Metrics = { - inc: sinon.stub(), - Timer: (Timer = (function () { - Timer = class Timer { - static initClass() { - this.prototype.done = sinon.stub() - } - } - Timer.initClass() - return Timer - })()), - }), - }, - }) - this.project_id = 'project-id-123' - this.basePath = '/path/to/write/files/to' - return (this.callback = sinon.stub()) - }) - - describe('syncResourcesToDisk on a full request', function () { - beforeEach(function () { - this.resources = ['resource-1-mock', 'resource-2-mock', 'resource-3-mock'] - this.request = { - project_id: this.project_id, - syncState: (this.syncState = '0123456789abcdef'), - resources: this.resources, - } - this.ResourceWriter._writeResourceToDisk = sinon.stub().callsArg(3) - this.ResourceWriter._removeExtraneousFiles = sinon.stub().yields(null) - this.ResourceStateManager.saveProjectState = sinon.stub().callsArg(3) - return this.ResourceWriter.syncResourcesToDisk( - this.request, - this.basePath, - this.callback - ) - }) - - it('should remove old files', function () { - return this.ResourceWriter._removeExtraneousFiles - .calledWith(this.request, this.resources, this.basePath) - .should.equal(true) - }) - - it('should write each resource to disk', function () { - return Array.from(this.resources).map(resource => - this.ResourceWriter._writeResourceToDisk - .calledWith(this.project_id, resource, this.basePath) - .should.equal(true) - ) - }) - - it('should store the sync state and resource list', function () { - return this.ResourceStateManager.saveProjectState - .calledWith(this.syncState, this.resources, this.basePath) - .should.equal(true) - }) - - return it('should call the callback', function () { - return this.callback.called.should.equal(true) - }) - }) - - describe('syncResourcesToDisk on an incremental update', function () { - beforeEach(function () { - this.resources = ['resource-1-mock'] - this.request = { - project_id: this.project_id, - syncType: 'incremental', - syncState: (this.syncState = '1234567890abcdef'), - resources: this.resources, - } - this.fullResources = this.resources.concat(['file-1']) - this.ResourceWriter._writeResourceToDisk = sinon.stub().callsArg(3) - this.ResourceWriter._removeExtraneousFiles = sinon - .stub() - .yields(null, (this.outputFiles = []), (this.allFiles = [])) - this.ResourceStateManager.checkProjectStateMatches = sinon - .stub() - .callsArgWith(2, null, this.fullResources) - this.ResourceStateManager.saveProjectState = sinon.stub().callsArg(3) - this.ResourceStateManager.checkResourceFiles = sinon.stub().callsArg(3) - return this.ResourceWriter.syncResourcesToDisk( - this.request, - this.basePath, - this.callback - ) - }) - - it('should check the sync state matches', function () { - return this.ResourceStateManager.checkProjectStateMatches - .calledWith(this.syncState, this.basePath) - .should.equal(true) - }) - - it('should remove old files', function () { - return this.ResourceWriter._removeExtraneousFiles - .calledWith(this.request, this.fullResources, this.basePath) - .should.equal(true) - }) - - it('should check each resource exists', function () { - return this.ResourceStateManager.checkResourceFiles - .calledWith(this.fullResources, this.allFiles, this.basePath) - .should.equal(true) - }) - - it('should write each resource to disk', function () { - return Array.from(this.resources).map(resource => - this.ResourceWriter._writeResourceToDisk - .calledWith(this.project_id, resource, this.basePath) - .should.equal(true) - ) - }) - - return it('should call the callback', function () { - return this.callback.called.should.equal(true) - }) - }) - - describe('syncResourcesToDisk on an incremental update when the state does not match', function () { - beforeEach(function () { - this.resources = ['resource-1-mock'] - this.request = { - project_id: this.project_id, - syncType: 'incremental', - syncState: (this.syncState = '1234567890abcdef'), - resources: this.resources, - } - this.ResourceStateManager.checkProjectStateMatches = sinon - .stub() - .callsArgWith(2, (this.error = new Error())) - return this.ResourceWriter.syncResourcesToDisk( - this.request, - this.basePath, - this.callback - ) - }) - - it('should check whether the sync state matches', function () { - return this.ResourceStateManager.checkProjectStateMatches - .calledWith(this.syncState, this.basePath) - .should.equal(true) - }) - - return it('should call the callback with an error', function () { - return this.callback.calledWith(this.error).should.equal(true) - }) - }) - - describe('_removeExtraneousFiles', function () { - beforeEach(function () { - this.output_files = [ - { - path: 'output.pdf', - type: 'pdf', - }, - { - path: 'extra/file.tex', - type: 'tex', - }, - { - path: 'extra.aux', - type: 'aux', - }, - { - path: 'cache/_chunk1', - }, - { - path: 'figures/image-eps-converted-to.pdf', - type: 'pdf', - }, - { - path: 'foo/main-figure0.md5', - type: 'md5', - }, - { - path: 'foo/main-figure0.dpth', - type: 'dpth', - }, - { - path: 'foo/main-figure0.pdf', - type: 'pdf', - }, - { - path: '_minted-main/default-pyg-prefix.pygstyle', - type: 'pygstyle', - }, - { - path: '_minted-main/default.pygstyle', - type: 'pygstyle', - }, - { - path: '_minted-main/35E248B60965545BD232AE9F0FE9750D504A7AF0CD3BAA7542030FC560DFCC45.pygtex', - type: 'pygtex', - }, - { - path: '_markdown_main/30893013dec5d869a415610079774c2f.md.tex', - type: 'tex', - }, - { - path: 'output.stdout', - }, - { - path: 'output.stderr', - }, - ] - this.resources = 'mock-resources' - this.request = { - project_id: this.project_id, - syncType: 'incremental', - syncState: (this.syncState = '1234567890abcdef'), - resources: this.resources, - metricsOpts: { path: 'foo' }, - } - this.OutputFileFinder.findOutputFiles = sinon - .stub() - .callsArgWith(2, null, this.output_files) - this.ResourceWriter._deleteFileIfNotDirectory = sinon.stub().callsArg(1) - return this.ResourceWriter._removeExtraneousFiles( - this.request, - this.resources, - this.basePath, - this.callback - ) - }) - - it('should find the existing output files', function () { - return this.OutputFileFinder.findOutputFiles - .calledWith(this.resources, this.basePath) - .should.equal(true) - }) - - it('should delete the output files', function () { - return this.ResourceWriter._deleteFileIfNotDirectory - .calledWith(path.join(this.basePath, 'output.pdf')) - .should.equal(true) - }) - - it('should delete the stdout log file', function () { - return this.ResourceWriter._deleteFileIfNotDirectory - .calledWith(path.join(this.basePath, 'output.stdout')) - .should.equal(true) - }) - - it('should delete the stderr log file', function () { - return this.ResourceWriter._deleteFileIfNotDirectory - .calledWith(path.join(this.basePath, 'output.stderr')) - .should.equal(true) - }) - - it('should delete the extra files', function () { - return this.ResourceWriter._deleteFileIfNotDirectory - .calledWith(path.join(this.basePath, 'extra/file.tex')) - .should.equal(true) - }) - - it('should not delete the extra aux files', function () { - return this.ResourceWriter._deleteFileIfNotDirectory - .calledWith(path.join(this.basePath, 'extra.aux')) - .should.equal(false) - }) - - it('should not delete the knitr cache file', function () { - return this.ResourceWriter._deleteFileIfNotDirectory - .calledWith(path.join(this.basePath, 'cache/_chunk1')) - .should.equal(false) - }) - - it('should not delete the epstopdf converted files', function () { - return this.ResourceWriter._deleteFileIfNotDirectory - .calledWith( - path.join(this.basePath, 'figures/image-eps-converted-to.pdf') - ) - .should.equal(false) - }) - - it('should not delete the tikz md5 files', function () { - return this.ResourceWriter._deleteFileIfNotDirectory - .calledWith(path.join(this.basePath, 'foo/main-figure0.md5')) - .should.equal(false) - }) - - it('should not delete the tikz dpth files', function () { - return this.ResourceWriter._deleteFileIfNotDirectory - .calledWith(path.join(this.basePath, 'foo/main-figure0.dpth')) - .should.equal(false) - }) - - it('should not delete the tikz pdf files', function () { - return this.ResourceWriter._deleteFileIfNotDirectory - .calledWith(path.join(this.basePath, 'foo/main-figure0.pdf')) - .should.equal(false) - }) - - it('should not delete the minted pygstyle files', function () { - return this.ResourceWriter._deleteFileIfNotDirectory - .calledWith( - path.join(this.basePath, '_minted-main/default-pyg-prefix.pygstyle') - ) - .should.equal(false) - }) - - it('should not delete the minted default pygstyle files', function () { - return this.ResourceWriter._deleteFileIfNotDirectory - .calledWith(path.join(this.basePath, '_minted-main/default.pygstyle')) - .should.equal(false) - }) - - it('should not delete the minted default pygtex files', function () { - return this.ResourceWriter._deleteFileIfNotDirectory - .calledWith( - path.join( - this.basePath, - '_minted-main/35E248B60965545BD232AE9F0FE9750D504A7AF0CD3BAA7542030FC560DFCC45.pygtex' - ) - ) - .should.equal(false) - }) - - it('should not delete the markdown md.tex files', function () { - return this.ResourceWriter._deleteFileIfNotDirectory - .calledWith( - path.join( - this.basePath, - '_markdown_main/30893013dec5d869a415610079774c2f.md.tex' - ) - ) - .should.equal(false) - }) - - it('should call the callback', function () { - return this.callback.called.should.equal(true) - }) - - return it('should time the request', function () { - return this.Metrics.Timer.prototype.done.called.should.equal(true) - }) - }) - - describe('_writeResourceToDisk', function () { - describe('with a url based resource', function () { - beforeEach(function () { - this.fs.mkdir = sinon.stub().callsArg(2) - this.resource = { - path: 'main.tex', - url: 'http://www.example.com/primary/main.tex', - fallbackURL: 'http://fallback.example.com/fallback/main.tex', - modified: Date.now(), - } - this.UrlCache.downloadUrlToFile = sinon - .stub() - .callsArgWith(5, 'fake error downloading file') - return this.ResourceWriter._writeResourceToDisk( - this.project_id, - this.resource, - this.basePath, - this.callback - ) - }) - - it('should ensure the directory exists', function () { - this.fs.mkdir - .calledWith( - path.dirname(path.join(this.basePath, this.resource.path)) - ) - .should.equal(true) - }) - - it('should write the URL from the cache', function () { - return this.UrlCache.downloadUrlToFile - .calledWith( - this.project_id, - this.resource.url, - this.resource.fallbackURL, - path.join(this.basePath, this.resource.path), - this.resource.modified - ) - .should.equal(true) - }) - - it('should call the callback', function () { - return this.callback.called.should.equal(true) - }) - - return it('should not return an error if the resource writer errored', function () { - return expect(this.callback.args[0][0]).not.to.exist - }) - }) - - describe('with a content based resource', function () { - beforeEach(function () { - this.resource = { - path: 'main.tex', - content: 'Hello world', - } - this.fs.writeFile = sinon.stub().callsArg(2) - this.fs.mkdir = sinon.stub().callsArg(2) - return this.ResourceWriter._writeResourceToDisk( - this.project_id, - this.resource, - this.basePath, - this.callback - ) - }) - - it('should ensure the directory exists', function () { - return this.fs.mkdir - .calledWith( - path.dirname(path.join(this.basePath, this.resource.path)) - ) - .should.equal(true) - }) - - it('should write the contents to disk', function () { - return this.fs.writeFile - .calledWith( - path.join(this.basePath, this.resource.path), - this.resource.content - ) - .should.equal(true) - }) - - return it('should call the callback', function () { - return this.callback.called.should.equal(true) - }) - }) - - return describe('with a file path that breaks out of the root folder', function () { - beforeEach(function () { - this.resource = { - path: '../../main.tex', - content: 'Hello world', - } - this.fs.writeFile = sinon.stub().callsArg(2) - return this.ResourceWriter._writeResourceToDisk( - this.project_id, - this.resource, - this.basePath, - this.callback - ) - }) - - it('should not write to disk', function () { - return this.fs.writeFile.called.should.equal(false) - }) - - it('should return an error', function () { - this.callback.calledWith(sinon.match(Error)).should.equal(true) - - const message = this.callback.args[0][0].message - expect(message).to.include('resource path is outside root directory') - }) - }) - }) - - return describe('checkPath', function () { - describe('with a valid path', function () { - beforeEach(function () { - return this.ResourceWriter.checkPath('foo', 'bar', this.callback) - }) - - return it('should return the joined path', function () { - return this.callback.calledWith(null, 'foo/bar').should.equal(true) - }) - }) - - describe('with an invalid path', function () { - beforeEach(function () { - this.ResourceWriter.checkPath('foo', 'baz/../../bar', this.callback) - }) - - it('should return an error', function () { - this.callback.calledWith(sinon.match(Error)).should.equal(true) - - const message = this.callback.args[0][0].message - expect(message).to.include('resource path is outside root directory') - }) - }) - - describe('with another invalid path matching on a prefix', function () { - beforeEach(function () { - return this.ResourceWriter.checkPath( - 'foo', - '../foobar/baz', - this.callback - ) - }) - - it('should return an error', function () { - this.callback.calledWith(sinon.match(Error)).should.equal(true) - - const message = this.callback.args[0][0].message - expect(message).to.include('resource path is outside root directory') - }) - }) - }) -}) diff --git a/services/clsi/test/unit/js/StaticServerForbidSymlinks.test.js b/services/clsi/test/unit/js/StaticServerForbidSymlinks.test.js new file mode 100644 index 0000000000..81349b6c1f --- /dev/null +++ b/services/clsi/test/unit/js/StaticServerForbidSymlinks.test.js @@ -0,0 +1,270 @@ +import { vi, describe, beforeEach, it } from 'vitest' + +import path from 'node:path' +import sinon from 'sinon' +const modulePath = path.join( + import.meta.dirname, + '../../../app/js/StaticServerForbidSymlinks' +) + +describe('StaticServerForbidSymlinks', function () { + beforeEach(async function (ctx) { + ctx.settings = { + path: { + compilesDir: '/compiles/here', + }, + } + + ctx.fs = {} + + vi.doMock('@overleaf/settings', () => ({ + default: ctx.settings, + })) + + vi.doMock('fs', () => ({ + default: ctx.fs, + })) + + ctx.ForbidSymlinks = (await import(modulePath)).default + + ctx.dummyStatic = (rootDir, options) => (req, res, next) => + // console.log "dummyStatic serving file", rootDir, "called with", req.url + // serve it + next() + + ctx.StaticServerForbidSymlinks = ctx.ForbidSymlinks( + ctx.dummyStatic, + ctx.settings.path.compilesDir + ) + ctx.req = { + params: { + project_id: '12345', + }, + } + + ctx.res = {} + ctx.req.url = '/12345/output.pdf' + }) + + describe('sending a normal file through', function () { + beforeEach(function (ctx) { + ctx.fs.realpath = sinon + .stub() + .callsArgWith( + 1, + null, + `${ctx.settings.path.compilesDir}/${ctx.req.params.project_id}/output.pdf` + ) + }) + + it('should call next', async function (ctx) { + await new Promise((resolve, reject) => { + ctx.res.sendStatus = function (resCode) { + resCode.should.equal(200) + resolve() + } + ctx.StaticServerForbidSymlinks(ctx.req, ctx.res, err => { + if (err) reject(err) + resolve() + }) + }) + }) + }) + + describe('with a missing file', function () { + beforeEach(function (ctx) { + ctx.fs.realpath = sinon + .stub() + .callsArgWith( + 1, + { code: 'ENOENT' }, + `${ctx.settings.path.compilesDir}/${ctx.req.params.project_id}/unknown.pdf` + ) + }) + + it('should send a 404', async function (ctx) { + await new Promise((resolve, reject) => { + ctx.res.sendStatus = function (resCode) { + resCode.should.equal(404) + resolve() + } + ctx.StaticServerForbidSymlinks(ctx.req, ctx.res) + }) + }) + }) + + describe('with a new line', function () { + beforeEach(function (ctx) { + ctx.req.url = '/12345/output.pdf\nother file' + ctx.fs.realpath = sinon.stub().yields() + }) + + it('should process the correct file', async function (ctx) { + await new Promise((resolve, reject) => { + ctx.res.sendStatus = () => { + ctx.fs.realpath.should.have.been.calledWith( + `${ctx.settings.path.compilesDir}/12345/output.pdf\nother file` + ) + resolve() + } + ctx.StaticServerForbidSymlinks(ctx.req, ctx.res) + }) + }) + }) + + describe('with a symlink file', function () { + beforeEach(function (ctx) { + ctx.fs.realpath = sinon + .stub() + .callsArgWith(1, null, `/etc/${ctx.req.params.project_id}/output.pdf`) + }) + + it('should send a 404', async function (ctx) { + await new Promise((resolve, reject) => { + ctx.res.sendStatus = function (resCode) { + resCode.should.equal(404) + resolve() + } + ctx.StaticServerForbidSymlinks(ctx.req, ctx.res) + }) + }) + }) + + describe('with a relative file', function () { + beforeEach(function (ctx) { + ctx.req.url = '/12345/../67890/output.pdf' + }) + + it('should send a 404', async function (ctx) { + await new Promise((resolve, reject) => { + ctx.res.sendStatus = function (resCode) { + resCode.should.equal(404) + resolve() + } + ctx.StaticServerForbidSymlinks(ctx.req, ctx.res) + }) + }) + }) + + describe('with a unnormalized file containing .', function () { + beforeEach(function (ctx) { + ctx.req.url = '/12345/foo/./output.pdf' + }) + + it('should send a 404', async function (ctx) { + await new Promise((resolve, reject) => { + ctx.res.sendStatus = function (resCode) { + resCode.should.equal(404) + resolve() + } + ctx.StaticServerForbidSymlinks(ctx.req, ctx.res) + }) + }) + }) + + describe('with a file containing an empty path', function () { + beforeEach(function (ctx) { + ctx.req.url = '/12345/foo//output.pdf' + }) + + it('should send a 404', async function (ctx) { + await new Promise((resolve, reject) => { + ctx.res.sendStatus = function (resCode) { + resCode.should.equal(404) + resolve() + } + ctx.StaticServerForbidSymlinks(ctx.req, ctx.res) + }) + }) + }) + + describe('with a non-project file', function () { + beforeEach(function (ctx) { + ctx.req.url = '/.foo/output.pdf' + }) + + it('should send a 404', async function (ctx) { + await new Promise((resolve, reject) => { + ctx.res.sendStatus = function (resCode) { + resCode.should.equal(404) + resolve() + } + ctx.StaticServerForbidSymlinks(ctx.req, ctx.res) + }) + }) + }) + + describe('with a file outside the compiledir', function () { + beforeEach(function (ctx) { + ctx.req.url = '/../bar/output.pdf' + }) + + it('should send a 404', async function (ctx) { + await new Promise((resolve, reject) => { + ctx.res.sendStatus = function (resCode) { + resCode.should.equal(404) + resolve() + } + ctx.StaticServerForbidSymlinks(ctx.req, ctx.res) + }) + }) + }) + + describe('with a file with no leading /', function () { + beforeEach(function (ctx) { + ctx.req.url = './../bar/output.pdf' + }) + + it('should send a 404', async function (ctx) { + await new Promise((resolve, reject) => { + ctx.res.sendStatus = function (resCode) { + resCode.should.equal(404) + resolve() + } + ctx.StaticServerForbidSymlinks(ctx.req, ctx.res) + }) + }) + }) + + describe('with a github style path', function () { + beforeEach(function (ctx) { + ctx.req.url = '/henryoswald-latex_example/output/output.log' + ctx.fs.realpath = sinon + .stub() + .callsArgWith( + 1, + null, + `${ctx.settings.path.compilesDir}/henryoswald-latex_example/output/output.log` + ) + }) + + it('should call next', async function (ctx) { + await new Promise((resolve, reject) => { + ctx.res.sendStatus = function (resCode) { + resCode.should.equal(200) + resolve() + } + ctx.StaticServerForbidSymlinks(ctx.req, ctx.res, err => { + if (err) reject(err) + resolve() + }) + }) + }) + }) + + describe('with an error from fs.realpath', function () { + beforeEach(function (ctx) { + ctx.fs.realpath = sinon.stub().callsArgWith(1, 'error') + }) + + it('should send a 500', async function (ctx) { + await new Promise((resolve, reject) => { + ctx.res.sendStatus = function (resCode) { + resCode.should.equal(500) + resolve() + } + ctx.StaticServerForbidSymlinks(ctx.req, ctx.res) + }) + }) + }) +}) diff --git a/services/clsi/test/unit/js/StaticServerForbidSymlinksTests.js b/services/clsi/test/unit/js/StaticServerForbidSymlinksTests.js deleted file mode 100644 index 53507fe3f2..0000000000 --- a/services/clsi/test/unit/js/StaticServerForbidSymlinksTests.js +++ /dev/null @@ -1,248 +0,0 @@ -/* eslint-disable - no-return-assign, - no-unused-vars, -*/ -// TODO: This file was created by bulk-decaffeinate. -// Fix any style issues and re-enable lint. -/* - * decaffeinate suggestions: - * DS102: Remove unnecessary code created because of implicit returns - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ -const SandboxedModule = require('sandboxed-module') -const assert = require('node:assert') -const path = require('node:path') -const sinon = require('sinon') -const modulePath = path.join( - __dirname, - '../../../app/js/StaticServerForbidSymlinks' -) -const { expect } = require('chai') - -describe('StaticServerForbidSymlinks', function () { - beforeEach(function () { - this.settings = { - path: { - compilesDir: '/compiles/here', - }, - } - - this.fs = {} - this.ForbidSymlinks = SandboxedModule.require(modulePath, { - requires: { - '@overleaf/settings': this.settings, - fs: this.fs, - }, - }) - - this.dummyStatic = (rootDir, options) => (req, res, next) => - // console.log "dummyStatic serving file", rootDir, "called with", req.url - // serve it - next() - - this.StaticServerForbidSymlinks = this.ForbidSymlinks( - this.dummyStatic, - this.settings.path.compilesDir - ) - this.req = { - params: { - project_id: '12345', - }, - } - - this.res = {} - return (this.req.url = '/12345/output.pdf') - }) - - describe('sending a normal file through', function () { - beforeEach(function () { - return (this.fs.realpath = sinon - .stub() - .callsArgWith( - 1, - null, - `${this.settings.path.compilesDir}/${this.req.params.project_id}/output.pdf` - )) - }) - - return it('should call next', function (done) { - this.res.sendStatus = function (resCode) { - resCode.should.equal(200) - return done() - } - return this.StaticServerForbidSymlinks(this.req, this.res, done) - }) - }) - - describe('with a missing file', function () { - beforeEach(function () { - return (this.fs.realpath = sinon - .stub() - .callsArgWith( - 1, - { code: 'ENOENT' }, - `${this.settings.path.compilesDir}/${this.req.params.project_id}/unknown.pdf` - )) - }) - - return it('should send a 404', function (done) { - this.res.sendStatus = function (resCode) { - resCode.should.equal(404) - return done() - } - return this.StaticServerForbidSymlinks(this.req, this.res) - }) - }) - - describe('with a new line', function () { - beforeEach(function () { - this.req.url = '/12345/output.pdf\nother file' - this.fs.realpath = sinon.stub().yields() - }) - - it('should process the correct file', function (done) { - this.res.sendStatus = () => { - this.fs.realpath.should.have.been.calledWith( - `${this.settings.path.compilesDir}/12345/output.pdf\nother file` - ) - done() - } - this.StaticServerForbidSymlinks(this.req, this.res) - }) - }) - - describe('with a symlink file', function () { - beforeEach(function () { - return (this.fs.realpath = sinon - .stub() - .callsArgWith(1, null, `/etc/${this.req.params.project_id}/output.pdf`)) - }) - - return it('should send a 404', function (done) { - this.res.sendStatus = function (resCode) { - resCode.should.equal(404) - return done() - } - return this.StaticServerForbidSymlinks(this.req, this.res) - }) - }) - - describe('with a relative file', function () { - beforeEach(function () { - return (this.req.url = '/12345/../67890/output.pdf') - }) - - return it('should send a 404', function (done) { - this.res.sendStatus = function (resCode) { - resCode.should.equal(404) - return done() - } - return this.StaticServerForbidSymlinks(this.req, this.res) - }) - }) - - describe('with a unnormalized file containing .', function () { - beforeEach(function () { - return (this.req.url = '/12345/foo/./output.pdf') - }) - - return it('should send a 404', function (done) { - this.res.sendStatus = function (resCode) { - resCode.should.equal(404) - return done() - } - return this.StaticServerForbidSymlinks(this.req, this.res) - }) - }) - - describe('with a file containing an empty path', function () { - beforeEach(function () { - return (this.req.url = '/12345/foo//output.pdf') - }) - - return it('should send a 404', function (done) { - this.res.sendStatus = function (resCode) { - resCode.should.equal(404) - return done() - } - return this.StaticServerForbidSymlinks(this.req, this.res) - }) - }) - - describe('with a non-project file', function () { - beforeEach(function () { - return (this.req.url = '/.foo/output.pdf') - }) - - return it('should send a 404', function (done) { - this.res.sendStatus = function (resCode) { - resCode.should.equal(404) - return done() - } - return this.StaticServerForbidSymlinks(this.req, this.res) - }) - }) - - describe('with a file outside the compiledir', function () { - beforeEach(function () { - return (this.req.url = '/../bar/output.pdf') - }) - - return it('should send a 404', function (done) { - this.res.sendStatus = function (resCode) { - resCode.should.equal(404) - return done() - } - return this.StaticServerForbidSymlinks(this.req, this.res) - }) - }) - - describe('with a file with no leading /', function () { - beforeEach(function () { - return (this.req.url = './../bar/output.pdf') - }) - - return it('should send a 404', function (done) { - this.res.sendStatus = function (resCode) { - resCode.should.equal(404) - return done() - } - return this.StaticServerForbidSymlinks(this.req, this.res) - }) - }) - - describe('with a github style path', function () { - beforeEach(function () { - this.req.url = '/henryoswald-latex_example/output/output.log' - return (this.fs.realpath = sinon - .stub() - .callsArgWith( - 1, - null, - `${this.settings.path.compilesDir}/henryoswald-latex_example/output/output.log` - )) - }) - - return it('should call next', function (done) { - this.res.sendStatus = function (resCode) { - resCode.should.equal(200) - return done() - } - return this.StaticServerForbidSymlinks(this.req, this.res, done) - }) - }) - - return describe('with an error from fs.realpath', function () { - beforeEach(function () { - return (this.fs.realpath = sinon.stub().callsArgWith(1, 'error')) - }) - - return it('should send a 500', function (done) { - this.res.sendStatus = function (resCode) { - resCode.should.equal(500) - return done() - } - return this.StaticServerForbidSymlinks(this.req, this.res) - }) - }) -}) diff --git a/services/clsi/test/unit/js/StatsManagerTests.js b/services/clsi/test/unit/js/StatsManager.test.js similarity index 94% rename from services/clsi/test/unit/js/StatsManagerTests.js rename to services/clsi/test/unit/js/StatsManager.test.js index 7cae36ce53..483c261879 100644 --- a/services/clsi/test/unit/js/StatsManagerTests.js +++ b/services/clsi/test/unit/js/StatsManager.test.js @@ -1,5 +1,16 @@ -const { expect } = require('chai') -const { sampleByHash, sampleRequest } = require('../../../app/js/StatsManager') +import { expect, describe, it, vi } from 'vitest' +import StatsManager from '../../../app/js/StatsManager.js' + +const { sampleByHash, sampleRequest } = StatsManager + +// Mocks allow us to import Metrics.js twice without getting errors. +vi.mock('@overleaf/metrics', () => ({ + prom: { + Gauge: vi.fn(), + Counter: vi.fn(), + Histogram: vi.fn(), + }, +})) describe('StatsManager', function () { describe('sampleByHash', function () { diff --git a/services/clsi/test/unit/js/SynctexOutputParserTests.js b/services/clsi/test/unit/js/SynctexOutputParser.test.js similarity index 67% rename from services/clsi/test/unit/js/SynctexOutputParserTests.js rename to services/clsi/test/unit/js/SynctexOutputParser.test.js index b999a6adb1..e7cd88c195 100644 --- a/services/clsi/test/unit/js/SynctexOutputParserTests.js +++ b/services/clsi/test/unit/js/SynctexOutputParser.test.js @@ -1,16 +1,18 @@ -const Path = require('node:path') -const SandboxedModule = require('sandboxed-module') -const { expect } = require('chai') +import Path from 'node:path' +import { expect, describe, beforeEach, it } from 'vitest' -const MODULE_PATH = Path.join(__dirname, '../../../app/js/SynctexOutputParser') +const MODULE_PATH = Path.join( + import.meta.dirname, + '../../../app/js/SynctexOutputParser' +) describe('SynctexOutputParser', function () { - beforeEach(function () { - this.SynctexOutputParser = SandboxedModule.require(MODULE_PATH) + beforeEach(async function (ctx) { + ctx.SynctexOutputParser = (await import(MODULE_PATH)).default }) describe('parseViewOutput', function () { - it('parses valid output', function () { + it('parses valid output', function (ctx) { const output = `This is SyncTeX command line utility, version 1.5 SyncTeX result begin Output:/compile/output.pdf @@ -39,7 +41,7 @@ middle: after: SyncTeX result end ` - const records = this.SynctexOutputParser.parseViewOutput(output) + const records = ctx.SynctexOutputParser.parseViewOutput(output) expect(records).to.deep.equal([ { page: 1, @@ -58,15 +60,15 @@ SyncTeX result end ]) }) - it('handles garbage', function () { + it('handles garbage', function (ctx) { const output = 'This computer is on strike!' - const records = this.SynctexOutputParser.parseViewOutput(output) + const records = ctx.SynctexOutputParser.parseViewOutput(output) expect(records).to.deep.equal([]) }) }) describe('parseEditOutput', function () { - it('parses valid output', function () { + it('parses valid output', function (ctx) { const output = `This is SyncTeX command line utility, version 1.5 SyncTeX result begin Output:/compile/output.pdf @@ -77,7 +79,7 @@ Offset:0 Context: SyncTeX result end ` - const records = this.SynctexOutputParser.parseEditOutput( + const records = ctx.SynctexOutputParser.parseEditOutput( output, '/compile' ) @@ -86,7 +88,7 @@ SyncTeX result end ]) }) - it('handles values that contain colons', function () { + it('handles values that contain colons', function (ctx) { const output = `This is SyncTeX command line utility, version 1.5 SyncTeX result begin Output:/compile/output.pdf @@ -98,7 +100,7 @@ Context: SyncTeX result end ` - const records = this.SynctexOutputParser.parseEditOutput( + const records = ctx.SynctexOutputParser.parseEditOutput( output, '/compile' ) @@ -107,9 +109,9 @@ SyncTeX result end ]) }) - it('handles garbage', function () { + it('handles garbage', function (ctx) { const output = '2 + 2 = 4' - const records = this.SynctexOutputParser.parseEditOutput(output) + const records = ctx.SynctexOutputParser.parseEditOutput(output) expect(records).to.deep.equal([]) }) }) diff --git a/services/clsi/test/unit/js/TikzManager.js b/services/clsi/test/unit/js/TikzManager.js deleted file mode 100644 index ee651f6b1d..0000000000 --- a/services/clsi/test/unit/js/TikzManager.js +++ /dev/null @@ -1,185 +0,0 @@ -/* eslint-disable - no-return-assign, -*/ -// TODO: This file was created by bulk-decaffeinate. -// Fix any style issues and re-enable lint. -/* - * decaffeinate suggestions: - * DS102: Remove unnecessary code created because of implicit returns - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ -const SandboxedModule = require('sandboxed-module') -const sinon = require('sinon') -const modulePath = require('node:path').join( - __dirname, - '../../../app/js/TikzManager' -) - -describe('TikzManager', function () { - beforeEach(function () { - return (this.TikzManager = SandboxedModule.require(modulePath, { - requires: { - './ResourceWriter': (this.ResourceWriter = {}), - './SafeReader': (this.SafeReader = {}), - fs: (this.fs = {}), - }, - })) - }) - - describe('checkMainFile', function () { - beforeEach(function () { - this.compileDir = 'compile-dir' - this.mainFile = 'main.tex' - return (this.callback = sinon.stub()) - }) - - describe('if there is already an output.tex file in the resources', function () { - beforeEach(function () { - this.resources = [{ path: 'main.tex' }, { path: 'output.tex' }] - return this.TikzManager.checkMainFile( - this.compileDir, - this.mainFile, - this.resources, - this.callback - ) - }) - - return it('should call the callback with false ', function () { - return this.callback.calledWithExactly(null, false).should.equal(true) - }) - }) - - return describe('if there is no output.tex file in the resources', function () { - beforeEach(function () { - this.resources = [{ path: 'main.tex' }] - return (this.ResourceWriter.checkPath = sinon - .stub() - .withArgs(this.compileDir, this.mainFile) - .callsArgWith(2, null, `${this.compileDir}/${this.mainFile}`)) - }) - - describe('and the main file contains tikzexternalize', function () { - beforeEach(function () { - this.SafeReader.readFile = sinon - .stub() - .withArgs(`${this.compileDir}/${this.mainFile}`) - .callsArgWith(3, null, 'hello \\tikzexternalize') - return this.TikzManager.checkMainFile( - this.compileDir, - this.mainFile, - this.resources, - this.callback - ) - }) - - it('should look at the file on disk', function () { - return this.SafeReader.readFile - .calledWith(`${this.compileDir}/${this.mainFile}`) - .should.equal(true) - }) - - return it('should call the callback with true ', function () { - return this.callback.calledWithExactly(null, true).should.equal(true) - }) - }) - - describe('and the main file does not contain tikzexternalize', function () { - beforeEach(function () { - this.SafeReader.readFile = sinon - .stub() - .withArgs(`${this.compileDir}/${this.mainFile}`) - .callsArgWith(3, null, 'hello') - return this.TikzManager.checkMainFile( - this.compileDir, - this.mainFile, - this.resources, - this.callback - ) - }) - - it('should look at the file on disk', function () { - return this.SafeReader.readFile - .calledWith(`${this.compileDir}/${this.mainFile}`) - .should.equal(true) - }) - - return it('should call the callback with false', function () { - return this.callback.calledWithExactly(null, false).should.equal(true) - }) - }) - - return describe('and the main file contains \\usepackage{pstool}', function () { - beforeEach(function () { - this.SafeReader.readFile = sinon - .stub() - .withArgs(`${this.compileDir}/${this.mainFile}`) - .callsArgWith(3, null, 'hello \\usepackage[random-options]{pstool}') - return this.TikzManager.checkMainFile( - this.compileDir, - this.mainFile, - this.resources, - this.callback - ) - }) - - it('should look at the file on disk', function () { - return this.SafeReader.readFile - .calledWith(`${this.compileDir}/${this.mainFile}`) - .should.equal(true) - }) - - return it('should call the callback with true ', function () { - return this.callback.calledWithExactly(null, true).should.equal(true) - }) - }) - }) - }) - - return describe('injectOutputFile', function () { - beforeEach(function () { - this.rootDir = '/mock' - this.filename = 'filename.tex' - this.callback = sinon.stub() - this.content = `\ -\\documentclass{article} -\\usepackage{tikz} -\\tikzexternalize -\\begin{document} -Hello world -\\end{document}\ -` - this.fs.readFile = sinon.stub().callsArgWith(2, null, this.content) - this.fs.writeFile = sinon.stub().callsArg(3) - this.ResourceWriter.checkPath = sinon - .stub() - .callsArgWith(2, null, `${this.rootDir}/${this.filename}`) - return this.TikzManager.injectOutputFile( - this.rootDir, - this.filename, - this.callback - ) - }) - - it('sould check the path', function () { - return this.ResourceWriter.checkPath - .calledWith(this.rootDir, this.filename) - .should.equal(true) - }) - - it('should read the file', function () { - return this.fs.readFile - .calledWith(`${this.rootDir}/${this.filename}`, 'utf8') - .should.equal(true) - }) - - it('should write out the same file as output.tex', function () { - return this.fs.writeFile - .calledWith(`${this.rootDir}/output.tex`, this.content, { flag: 'wx' }) - .should.equal(true) - }) - - return it('should call the callback', function () { - return this.callback.called.should.equal(true) - }) - }) -}) diff --git a/services/clsi/test/unit/js/TikzManager.test.js b/services/clsi/test/unit/js/TikzManager.test.js new file mode 100644 index 0000000000..7f9943fa56 --- /dev/null +++ b/services/clsi/test/unit/js/TikzManager.test.js @@ -0,0 +1,177 @@ +import { vi, describe, beforeEach, it } from 'vitest' + +import sinon from 'sinon' +import path from 'node:path' + +const modulePath = path.join(import.meta.dirname, '../../../app/js/TikzManager') + +describe('TikzManager', () => { + beforeEach(async ctx => { + vi.doMock('../../../app/js/ResourceWriter', () => ({ + default: (ctx.ResourceWriter = {}), + })) + + vi.doMock('../../../app/js/SafeReader', () => ({ + default: (ctx.SafeReader = {}), + })) + + vi.doMock('fs', () => ({ + default: (ctx.fs = {}), + })) + + ctx.TikzManager = (await import(modulePath)).default + }) + + describe('checkMainFile', () => { + beforeEach(ctx => { + ctx.compileDir = 'compile-dir' + ctx.mainFile = 'main.tex' + ctx.callback = sinon.stub() + }) + + describe('if there is already an output.tex file in the resources', () => { + beforeEach(ctx => { + ctx.resources = [{ path: 'main.tex' }, { path: 'output.tex' }] + ctx.TikzManager.checkMainFile( + ctx.compileDir, + ctx.mainFile, + ctx.resources, + ctx.callback + ) + }) + + it('should call the callback with false ', ctx => { + ctx.callback.calledWithExactly(null, false).should.equal(true) + }) + }) + + describe('if there is no output.tex file in the resources', () => { + beforeEach(ctx => { + ctx.resources = [{ path: 'main.tex' }] + ctx.ResourceWriter.checkPath = sinon + .stub() + .withArgs(ctx.compileDir, ctx.mainFile) + .callsArgWith(2, null, `${ctx.compileDir}/${ctx.mainFile}`) + }) + + describe('and the main file contains tikzexternalize', () => { + beforeEach(ctx => { + ctx.SafeReader.readFile = sinon + .stub() + .withArgs(`${ctx.compileDir}/${ctx.mainFile}`) + .callsArgWith(3, null, 'hello \\tikzexternalize') + ctx.TikzManager.checkMainFile( + ctx.compileDir, + ctx.mainFile, + ctx.resources, + ctx.callback + ) + }) + + it('should look at the file on disk', ctx => { + ctx.SafeReader.readFile + .calledWith(`${ctx.compileDir}/${ctx.mainFile}`) + .should.equal(true) + }) + + it('should call the callback with true ', ctx => { + ctx.callback.calledWithExactly(null, true).should.equal(true) + }) + }) + + describe('and the main file does not contain tikzexternalize', () => { + beforeEach(ctx => { + ctx.SafeReader.readFile = sinon + .stub() + .withArgs(`${ctx.compileDir}/${ctx.mainFile}`) + .callsArgWith(3, null, 'hello') + ctx.TikzManager.checkMainFile( + ctx.compileDir, + ctx.mainFile, + ctx.resources, + ctx.callback + ) + }) + + it('should look at the file on disk', ctx => { + ctx.SafeReader.readFile + .calledWith(`${ctx.compileDir}/${ctx.mainFile}`) + .should.equal(true) + }) + + it('should call the callback with false', ctx => { + ctx.callback.calledWithExactly(null, false).should.equal(true) + }) + }) + + describe('and the main file contains \\usepackage{pstool}', () => { + beforeEach(ctx => { + ctx.SafeReader.readFile = sinon + .stub() + .withArgs(`${ctx.compileDir}/${ctx.mainFile}`) + .callsArgWith(3, null, 'hello \\usepackage[random-options]{pstool}') + ctx.TikzManager.checkMainFile( + ctx.compileDir, + ctx.mainFile, + ctx.resources, + ctx.callback + ) + }) + + it('should look at the file on disk', ctx => { + ctx.SafeReader.readFile + .calledWith(`${ctx.compileDir}/${ctx.mainFile}`) + .should.equal(true) + }) + + it('should call the callback with true ', ctx => { + ctx.callback.calledWithExactly(null, true).should.equal(true) + }) + }) + }) + }) + + describe('injectOutputFile', () => { + beforeEach(ctx => { + ctx.rootDir = '/mock' + ctx.filename = 'filename.tex' + ctx.callback = sinon.stub() + ctx.content = `\ +\\documentclass{article} +\\usepackage{tikz} +\\tikzexternalize +\\begin{document} +Hello world +\\end{document}\ +` + ctx.fs.readFile = sinon.stub().callsArgWith(2, null, ctx.content) + ctx.fs.writeFile = sinon.stub().callsArg(3) + ctx.ResourceWriter.checkPath = sinon + .stub() + .callsArgWith(2, null, `${ctx.rootDir}/${ctx.filename}`) + ctx.TikzManager.injectOutputFile(ctx.rootDir, ctx.filename, ctx.callback) + }) + + it('should check the path', ctx => { + ctx.ResourceWriter.checkPath + .calledWith(ctx.rootDir, ctx.filename) + .should.equal(true) + }) + + it('should read the file', ctx => { + ctx.fs.readFile + .calledWith(`${ctx.rootDir}/${ctx.filename}`, 'utf8') + .should.equal(true) + }) + + it('should write out the same file as output.tex', ctx => { + ctx.fs.writeFile + .calledWith(`${ctx.rootDir}/output.tex`, ctx.content, { flag: 'wx' }) + .should.equal(true) + }) + + it('should call the callback', ctx => { + ctx.callback.called.should.equal(true) + }) + }) +}) diff --git a/services/clsi/test/unit/js/UrlCache.test.js b/services/clsi/test/unit/js/UrlCache.test.js new file mode 100644 index 0000000000..8cff7337ab --- /dev/null +++ b/services/clsi/test/unit/js/UrlCache.test.js @@ -0,0 +1,153 @@ +import { vi, expect, describe, beforeEach, it } from 'vitest' + +import sinon from 'sinon' +import path from 'node:path' + +const modulePath = path.join(import.meta.dirname, '../../../app/js/UrlCache') + +describe('UrlCache', () => { + beforeEach(async ctx => { + ctx.callback = sinon.stub() + ctx.url = + 'http://filestore/project/60b0dd39c418bc00598a0d22/file/60ae721ffb1d920027d3201f' + ctx.fallbackURL = 'http://filestore/bucket/project-blobs/key/ab/cd/ef' + ctx.project_id = '60b0dd39c418bc00598a0d22' + + vi.doMock('../../../app/js/UrlFetcher', () => ({ + default: (ctx.UrlFetcher = { + promises: { pipeUrlToFileWithRetry: sinon.stub().resolves() }, + }), + })) + + vi.doMock('@overleaf/settings', () => ({ + default: (ctx.Settings = { + path: { clsiCacheDir: '/cache/dir' }, + }), + })) + + vi.doMock('@overleaf/metrics', () => ({ + default: { + Timer: sinon.stub().returns({ done: sinon.stub() }), + }, + })) + + ctx.fs = { + promises: { + rm: sinon.stub().resolves(), + copyFile: sinon.stub().resolves(), + }, + } + + vi.doMock('fs', () => ({ default: ctx.fs })) + + return (ctx.UrlCache = (await import(modulePath)).default) + }) + + describe('downloadUrlToFile', () => { + beforeEach(ctx => { + ctx.destPath = 'path/to/destination' + }) + + it('should not download on the happy path', async ctx => { + await new Promise((resolve, reject) => { + ctx.UrlCache.downloadUrlToFile( + ctx.project_id, + ctx.url, + ctx.fallbackURL, + ctx.destPath, + ctx.lastModified, + error => { + expect(error).to.not.exist + expect( + ctx.UrlFetcher.promises.pipeUrlToFileWithRetry.called + ).to.equal(false) + resolve() + } + ) + }) + }) + + it('should not download on the semi-happy path', async ctx => { + await new Promise((resolve, reject) => { + const codedError = new Error() + codedError.code = 'ENOENT' + ctx.fs.promises.copyFile.onCall(0).rejects(codedError) + ctx.fs.promises.copyFile.onCall(1).resolves() + + ctx.UrlCache.downloadUrlToFile( + ctx.project_id, + ctx.url, + ctx.fallbackURL, + ctx.destPath, + ctx.lastModified, + error => { + expect(error).to.not.exist + expect( + ctx.UrlFetcher.promises.pipeUrlToFileWithRetry.called + ).to.equal(false) + resolve() + } + ) + }) + }) + + it('should download on cache miss', async ctx => { + await new Promise((resolve, reject) => { + const codedError = new Error() + codedError.code = 'ENOENT' + ctx.fs.promises.copyFile.onCall(0).rejects(codedError) + ctx.fs.promises.copyFile.onCall(1).rejects(codedError) + ctx.fs.promises.copyFile.onCall(2).resolves() + + ctx.UrlCache.downloadUrlToFile( + ctx.project_id, + ctx.url, + ctx.fallbackURL, + ctx.destPath, + ctx.lastModified, + error => { + expect(error).to.not.exist + expect( + ctx.UrlFetcher.promises.pipeUrlToFileWithRetry.called + ).to.equal(true) + resolve() + } + ) + }) + }) + + it('should raise non cache-miss errors', async ctx => { + await new Promise((resolve, reject) => { + const codedError = new Error() + codedError.code = 'FOO' + ctx.fs.promises.copyFile.rejects(codedError) + ctx.UrlCache.downloadUrlToFile( + ctx.project_id, + ctx.url, + ctx.fallbackURL, + ctx.destPath, + ctx.lastModified, + error => { + expect(error).to.equal(codedError) + resolve() + } + ) + }) + }) + }) + + describe('clearProject', () => { + beforeEach(async ctx => { + await ctx.UrlCache.promises.clearProject(ctx.project_id) + }) + + it('should clear the cache in bulk', ctx => { + expect( + ctx.fs.promises.rm.calledWith('/cache/dir/' + ctx.project_id, { + force: true, + recursive: true, + }) + ).to.equal(true) + }) + }) +}) diff --git a/services/clsi/test/unit/js/UrlCacheTests.js b/services/clsi/test/unit/js/UrlCacheTests.js deleted file mode 100644 index a3dc2fac3c..0000000000 --- a/services/clsi/test/unit/js/UrlCacheTests.js +++ /dev/null @@ -1,148 +0,0 @@ -/* eslint-disable - no-return-assign, - no-unused-vars, -*/ -// TODO: This file was created by bulk-decaffeinate. -// Fix any style issues and re-enable lint. -/* - * decaffeinate suggestions: - * DS101: Remove unnecessary use of Array.from - * DS102: Remove unnecessary code created because of implicit returns - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ -const SandboxedModule = require('sandboxed-module') -const sinon = require('sinon') -const { expect } = require('chai') -const modulePath = require('node:path').join( - __dirname, - '../../../app/js/UrlCache' -) - -describe('UrlCache', function () { - beforeEach(function () { - this.callback = sinon.stub() - this.url = - 'http://filestore/project/60b0dd39c418bc00598a0d22/file/60ae721ffb1d920027d3201f' - this.fallbackURL = 'http://filestore/bucket/project-blobs/key/ab/cd/ef' - this.project_id = '60b0dd39c418bc00598a0d22' - return (this.UrlCache = SandboxedModule.require(modulePath, { - requires: { - './UrlFetcher': (this.UrlFetcher = { - promises: { pipeUrlToFileWithRetry: sinon.stub().resolves() }, - }), - '@overleaf/settings': (this.Settings = { - path: { clsiCacheDir: '/cache/dir' }, - }), - '@overleaf/metrics': { - Timer: sinon.stub().returns({ done: sinon.stub() }), - }, - fs: (this.fs = { - promises: { - rm: sinon.stub().resolves(), - copyFile: sinon.stub().resolves(), - }, - }), - }, - })) - }) - - describe('downloadUrlToFile', function () { - beforeEach(function () { - this.destPath = 'path/to/destination' - }) - - it('should not download on the happy path', function (done) { - this.UrlCache.downloadUrlToFile( - this.project_id, - this.url, - this.fallbackURL, - this.destPath, - this.lastModified, - error => { - expect(error).to.not.exist - expect( - this.UrlFetcher.promises.pipeUrlToFileWithRetry.called - ).to.equal(false) - done() - } - ) - }) - - it('should not download on the semi-happy path', function (done) { - const codedError = new Error() - codedError.code = 'ENOENT' - this.fs.promises.copyFile.onCall(0).rejects(codedError) - this.fs.promises.copyFile.onCall(1).resolves() - - this.UrlCache.downloadUrlToFile( - this.project_id, - this.url, - this.fallbackURL, - this.destPath, - this.lastModified, - error => { - expect(error).to.not.exist - expect( - this.UrlFetcher.promises.pipeUrlToFileWithRetry.called - ).to.equal(false) - done() - } - ) - }) - - it('should download on cache miss', function (done) { - const codedError = new Error() - codedError.code = 'ENOENT' - this.fs.promises.copyFile.onCall(0).rejects(codedError) - this.fs.promises.copyFile.onCall(1).rejects(codedError) - this.fs.promises.copyFile.onCall(2).resolves() - - this.UrlCache.downloadUrlToFile( - this.project_id, - this.url, - this.fallbackURL, - this.destPath, - this.lastModified, - error => { - expect(error).to.not.exist - expect( - this.UrlFetcher.promises.pipeUrlToFileWithRetry.called - ).to.equal(true) - done() - } - ) - }) - - it('should raise non cache-miss errors', function (done) { - const codedError = new Error() - codedError.code = 'FOO' - this.fs.promises.copyFile.rejects(codedError) - this.UrlCache.downloadUrlToFile( - this.project_id, - this.url, - this.fallbackURL, - this.destPath, - this.lastModified, - error => { - expect(error).to.equal(codedError) - done() - } - ) - }) - }) - - describe('clearProject', function () { - beforeEach(function (done) { - this.UrlCache.clearProject(this.project_id, done) - }) - - it('should clear the cache in bulk', function () { - expect( - this.fs.promises.rm.calledWith('/cache/dir/' + this.project_id, { - force: true, - recursive: true, - }) - ).to.equal(true) - }) - }) -}) diff --git a/services/clsi/test/unit/js/pdfjsTests.js b/services/clsi/test/unit/js/pdfjs.test.js similarity index 85% rename from services/clsi/test/unit/js/pdfjsTests.js rename to services/clsi/test/unit/js/pdfjs.test.js index bc8b775b43..7d672f705c 100644 --- a/services/clsi/test/unit/js/pdfjsTests.js +++ b/services/clsi/test/unit/js/pdfjs.test.js @@ -1,12 +1,14 @@ -const fs = require('node:fs') -const Path = require('node:path') -const { expect } = require('chai') -const { parseXrefTable } = require('../../../app/js/XrefParser') -const { NoXrefTableError } = require('../../../app/js/Errors') +import fs from 'node:fs' +import Path from 'node:path' +import { beforeAll, expect, describe, it } from 'vitest' +import XrefParser from '../../../app/js/XrefParser.js' +import { NoXrefTableError } from '../../../app/js/Errors.js' const PATH_EXAMPLES = 'test/acceptance/fixtures/examples/' const PATH_SNAPSHOTS = 'test/unit/js/snapshots/pdfjs/' const EXAMPLES = fs.readdirSync(PATH_EXAMPLES) +const { parseXrefTable } = XrefParser + function snapshotPath(example) { return Path.join(PATH_SNAPSHOTS, example, 'XrefTable.json') } @@ -62,13 +64,15 @@ describe('pdfjs', function () { for (const example of EXAMPLES) { describe(example, function () { let size, snapshot - before('load snapshot', async function () { + beforeAll(async function () { + // load snapshot const ctx = await loadContext(example) size = ctx.size snapshot = ctx.snapshot }) - before('back fill new snapshot', async function () { + beforeAll(async function (ctx) { + // back fill new snapshot if (snapshot === null) { console.error('back filling snapshot for', example) snapshot = await backFillSnapshot(example, size) diff --git a/services/clsi/test/unit/setup.js b/services/clsi/test/unit/setup.js new file mode 100644 index 0000000000..70a75b0b8b --- /dev/null +++ b/services/clsi/test/unit/setup.js @@ -0,0 +1,26 @@ +import { afterEach, beforeEach, chai, vi } from 'vitest' +import sinonChai from 'sinon-chai' +import chaiAsPromised from 'chai-as-promised' + +// Setup chai +chai.should() +chai.use(sinonChai) +chai.use(chaiAsPromised) + +beforeEach(() => { + vi.doMock('@overleaf/logger', () => ({ + default: { + debug() {}, + log() {}, + info() {}, + warn() {}, + error() {}, + err() {}, + }, + })) +}) + +afterEach(() => { + vi.restoreAllMocks() + vi.resetModules() +}) diff --git a/services/clsi/tsconfig.json b/services/clsi/tsconfig.json index c018d6e682..64bc0e874a 100644 --- a/services/clsi/tsconfig.json +++ b/services/clsi/tsconfig.json @@ -8,6 +8,7 @@ "config/**/*", "scripts/**/*", "test/**/*", - "types" + "types", + "vitest.config.unit.cjs" ] } diff --git a/services/clsi/vitest.config.unit.cjs b/services/clsi/vitest.config.unit.cjs new file mode 100644 index 0000000000..c9b162b45b --- /dev/null +++ b/services/clsi/vitest.config.unit.cjs @@ -0,0 +1,25 @@ +const { defineConfig } = require('vitest/config') + +let reporterOptions = {} +if (process.env.CI) { + reporterOptions = { + reporters: [ + 'default', + [ + 'junit', + { + classnameTemplate: `Unit tests.{filename}`, + }, + ], + ], + outputFile: 'reports/junit-vitest-unit.xml', + } +} +module.exports = defineConfig({ + test: { + include: ['test/unit/js/**/*.test.{js,ts}'], + setupFiles: ['./test/unit/setup.js'], + isolate: false, + ...reporterOptions, + }, +})