diff --git a/libraries/logger/logging-manager.js b/libraries/logger/logging-manager.js index 5179aaca86..06b439b097 100644 --- a/libraries/logger/logging-manager.js +++ b/libraries/logger/logging-manager.js @@ -1,6 +1,5 @@ const Stream = require('stream') const bunyan = require('bunyan') -const yn = require('yn') const GCPManager = require('./gcp-manager') const SentryManager = require('./sentry-manager') const Serializers = require('./serializers') @@ -74,20 +73,29 @@ const LoggingManager = { }, _getOutputStreamConfig() { - const gcpEnabled = yn(process.env.GCP_LOGGING) - if (gcpEnabled) { - const stream = new Stream.Writable({ - objectMode: true, - write(entry, encoding, callback) { - const gcpEntry = GCPManager.convertLogEntry(entry) - // eslint-disable-next-line no-console - console.log(JSON.stringify(gcpEntry, bunyan.safeCycles())) - setImmediate(callback) - }, - }) - return { level: this.defaultLevel, type: 'raw', stream } - } else { - return { level: this.defaultLevel, stream: process.stdout } + switch (process.env.LOGGING_FORMAT) { + case 'gke': { + const stream = new Stream.Writable({ + objectMode: true, + write(entry, encoding, callback) { + const gcpEntry = GCPManager.convertLogEntry(entry) + // eslint-disable-next-line no-console + console.log(JSON.stringify(gcpEntry, bunyan.safeCycles())) + setImmediate(callback) + }, + }) + return { level: this.defaultLevel, type: 'raw', stream } + } + case 'gce': { + const { LoggingBunyan } = require('@google-cloud/logging-bunyan') + return new LoggingBunyan({ + logName: this.loggerName, + serviceContext: { service: this.loggerName }, + }).stream(this.defaultLevel) + } + default: { + return { level: this.defaultLevel, stream: process.stdout } + } } }, diff --git a/libraries/logger/package.json b/libraries/logger/package.json index 78e9a74595..cffb6da107 100644 --- a/libraries/logger/package.json +++ b/libraries/logger/package.json @@ -17,11 +17,11 @@ "test:ci": "npm run test" }, "dependencies": { + "@google-cloud/logging-bunyan": "^3.2.0", "@overleaf/o-error": "^3.4.0", "@sentry/node": "^6.13.2", "bunyan": "^1.8.14", - "node-fetch": "^2.6.7", - "yn": "^4.0.0" + "node-fetch": "^2.6.7" }, "devDependencies": { "chai": "^4.3.6", diff --git a/package-lock.json b/package-lock.json index c3e5c8b336..24d7c2b4a0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -79,11 +79,11 @@ "version": "3.1.1", "license": "AGPL-3.0-only", "dependencies": { + "@google-cloud/logging-bunyan": "^3.2.0", "@overleaf/o-error": "^3.4.0", "@sentry/node": "^6.13.2", "bunyan": "^1.8.14", - "node-fetch": "^2.6.7", - "yn": "^4.0.0" + "node-fetch": "^2.6.7" }, "devDependencies": { "chai": "^4.3.6", @@ -41326,7 +41326,8 @@ "services/clsi": { "name": "@overleaf/clsi", "dependencies": { - "@overleaf/metrics": "^3.5.1", + "@overleaf/logger": "^3.1.0", + "@overleaf/metrics": "^4.0.0", "@overleaf/o-error": "^3.4.0", "@overleaf/settings": "^3.0.0", "async": "3.2.0", @@ -41338,7 +41339,6 @@ "fs-extra": "^10.0.0", "lockfile": "^1.0.4", "lodash": "^4.17.21", - "logger-sharelatex": "^2.2.0", "p-limit": "^3.1.0", "pdfjs-dist": "~2.7.570", "request": "^2.88.2", @@ -41386,19 +41386,6 @@ "node": ">= 0.6" } }, - "services/clsi/node_modules/@overleaf/metrics": { - "version": "3.5.2", - "resolved": "https://registry.npmjs.org/@overleaf/metrics/-/metrics-3.5.2.tgz", - "integrity": "sha512-rzYUb9JqyHcjU5tQfPF8AyDsJ7SbTFn71iHj98oRXsNCk8A6QwLjCI+0LUMpaUdSKZINKbujtuHynqzUawNklQ==", - "dependencies": { - "@google-cloud/debug-agent": "^5.1.2", - "@google-cloud/profiler": "^4.1.3", - "@google-cloud/trace-agent": "^5.1.1", - "compression": "^1.7.4", - "prom-client": "^11.1.3", - "yn": "^3.1.1" - } - }, "services/clsi/node_modules/diff": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz", @@ -41439,14 +41426,6 @@ "node": ">=8" } }, - "services/clsi/node_modules/yn": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz", - "integrity": "sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==", - "engines": { - "node": ">=6" - } - }, "services/contacts": { "name": "@overleaf/contacts", "dependencies": { @@ -48946,7 +48925,8 @@ "@overleaf/clsi": { "version": "file:services/clsi", "requires": { - "@overleaf/metrics": "^3.5.1", + "@overleaf/logger": "^3.1.0", + "@overleaf/metrics": "^4.0.0", "@overleaf/o-error": "^3.4.0", "@overleaf/settings": "^3.0.0", "async": "3.2.0", @@ -48960,7 +48940,6 @@ "fs-extra": "^10.0.0", "lockfile": "^1.0.4", "lodash": "^4.17.21", - "logger-sharelatex": "^2.2.0", "mocha": "^8.4.0", "nodemon": "^2.0.7", "p-limit": "^3.1.0", @@ -48974,19 +48953,6 @@ "workerpool": "^6.1.5" }, "dependencies": { - "@overleaf/metrics": { - "version": "3.5.2", - "resolved": "https://registry.npmjs.org/@overleaf/metrics/-/metrics-3.5.2.tgz", - "integrity": "sha512-rzYUb9JqyHcjU5tQfPF8AyDsJ7SbTFn71iHj98oRXsNCk8A6QwLjCI+0LUMpaUdSKZINKbujtuHynqzUawNklQ==", - "requires": { - "@google-cloud/debug-agent": "^5.1.2", - "@google-cloud/profiler": "^4.1.3", - "@google-cloud/trace-agent": "^5.1.1", - "compression": "^1.7.4", - "prom-client": "^11.1.3", - "yn": "^3.1.1" - } - }, "diff": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz", @@ -49016,11 +48982,6 @@ "requires": { "has-flag": "^4.0.0" } - }, - "yn": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz", - "integrity": "sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==" } } }, @@ -49706,6 +49667,7 @@ "@overleaf/logger": { "version": "file:libraries/logger", "requires": { + "@google-cloud/logging-bunyan": "^3.2.0", "@overleaf/o-error": "^3.4.0", "@sentry/node": "^6.13.2", "bunyan": "^1.8.14", @@ -49714,8 +49676,7 @@ "node-fetch": "^2.6.7", "sandboxed-module": "^2.0.4", "sinon": "^9.2.4", - "sinon-chai": "^3.7.0", - "yn": "^4.0.0" + "sinon-chai": "^3.7.0" } }, "@overleaf/metrics": { diff --git a/services/clsi/app.js b/services/clsi/app.js index d7eca0be5a..61b186d0c4 100644 --- a/services/clsi/app.js +++ b/services/clsi/app.js @@ -11,7 +11,7 @@ Metrics.initialize('clsi') const CompileController = require('./app/js/CompileController') const ContentController = require('./app/js/ContentController') const Settings = require('@overleaf/settings') -const logger = require('logger-sharelatex') +const logger = require('@overleaf/logger') logger.initialize('clsi') if ((Settings.sentry != null ? Settings.sentry.dsn : undefined) != null) { logger.initializeErrorReporting(Settings.sentry.dsn) diff --git a/services/clsi/app/js/CommandRunner.js b/services/clsi/app/js/CommandRunner.js index 782707b3a8..2dce56a942 100644 --- a/services/clsi/app/js/CommandRunner.js +++ b/services/clsi/app/js/CommandRunner.js @@ -7,7 +7,7 @@ */ let commandRunnerPath const Settings = require('@overleaf/settings') -const logger = require('logger-sharelatex') +const logger = require('@overleaf/logger') if ((Settings.clsi != null ? Settings.clsi.dockerRunner : undefined) === true) { commandRunnerPath = './DockerRunner' diff --git a/services/clsi/app/js/CompileController.js b/services/clsi/app/js/CompileController.js index 8b480a7fe2..add99623aa 100644 --- a/services/clsi/app/js/CompileController.js +++ b/services/clsi/app/js/CompileController.js @@ -17,7 +17,7 @@ const CompileManager = require('./CompileManager') const Settings = require('@overleaf/settings') const Metrics = require('./Metrics') const ProjectPersistenceManager = require('./ProjectPersistenceManager') -const logger = require('logger-sharelatex') +const logger = require('@overleaf/logger') const Errors = require('./Errors') function isImageNameAllowed(imageName) { diff --git a/services/clsi/app/js/CompileManager.js b/services/clsi/app/js/CompileManager.js index 6c455ab691..d56d75af82 100644 --- a/services/clsi/app/js/CompileManager.js +++ b/services/clsi/app/js/CompileManager.js @@ -4,7 +4,7 @@ const OutputFileFinder = require('./OutputFileFinder') const OutputCacheManager = require('./OutputCacheManager') const Settings = require('@overleaf/settings') const Path = require('path') -const logger = require('logger-sharelatex') +const logger = require('@overleaf/logger') const Metrics = require('./Metrics') const childProcess = require('child_process') const DraftModeManager = require('./DraftModeManager') diff --git a/services/clsi/app/js/ContentCacheMetrics.js b/services/clsi/app/js/ContentCacheMetrics.js index a7830982da..e1f4b02d47 100644 --- a/services/clsi/app/js/ContentCacheMetrics.js +++ b/services/clsi/app/js/ContentCacheMetrics.js @@ -1,4 +1,4 @@ -const logger = require('logger-sharelatex') +const logger = require('@overleaf/logger') const Metrics = require('./Metrics') const os = require('os') diff --git a/services/clsi/app/js/DbQueue.js b/services/clsi/app/js/DbQueue.js index ca2155d230..5fad4ba8ba 100644 --- a/services/clsi/app/js/DbQueue.js +++ b/services/clsi/app/js/DbQueue.js @@ -7,7 +7,7 @@ */ const async = require('async') const Settings = require('@overleaf/settings') -const logger = require('logger-sharelatex') +const logger = require('@overleaf/logger') const queue = async.queue( (task, cb) => task(cb), Settings.parallelSqlQueryLimit diff --git a/services/clsi/app/js/DockerLockManager.js b/services/clsi/app/js/DockerLockManager.js index 76208e3ef1..97804b7499 100644 --- a/services/clsi/app/js/DockerLockManager.js +++ b/services/clsi/app/js/DockerLockManager.js @@ -8,7 +8,7 @@ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ let LockManager -const logger = require('logger-sharelatex') +const logger = require('@overleaf/logger') const LockState = {} // locks for docker container operations, by container name diff --git a/services/clsi/app/js/DockerRunner.js b/services/clsi/app/js/DockerRunner.js index c6d409840f..d876705c8d 100644 --- a/services/clsi/app/js/DockerRunner.js +++ b/services/clsi/app/js/DockerRunner.js @@ -1,5 +1,5 @@ const Settings = require('@overleaf/settings') -const logger = require('logger-sharelatex') +const logger = require('@overleaf/logger') const Docker = require('dockerode') const dockerode = new Docker() const crypto = require('crypto') diff --git a/services/clsi/app/js/DraftModeManager.js b/services/clsi/app/js/DraftModeManager.js index f5bc679155..b421fb2e62 100644 --- a/services/clsi/app/js/DraftModeManager.js +++ b/services/clsi/app/js/DraftModeManager.js @@ -12,7 +12,7 @@ */ let DraftModeManager const fs = require('fs') -const logger = require('logger-sharelatex') +const logger = require('@overleaf/logger') module.exports = DraftModeManager = { injectDraftMode(filename, callback) { diff --git a/services/clsi/app/js/LatexRunner.js b/services/clsi/app/js/LatexRunner.js index 760ffe2ce6..fc8670047a 100644 --- a/services/clsi/app/js/LatexRunner.js +++ b/services/clsi/app/js/LatexRunner.js @@ -15,7 +15,7 @@ let LatexRunner const Path = require('path') const Settings = require('@overleaf/settings') -const logger = require('logger-sharelatex') +const logger = require('@overleaf/logger') const Metrics = require('./Metrics') const CommandRunner = require('./CommandRunner') const fs = require('fs') diff --git a/services/clsi/app/js/LocalCommandRunner.js b/services/clsi/app/js/LocalCommandRunner.js index 84208d1987..c1ae800847 100644 --- a/services/clsi/app/js/LocalCommandRunner.js +++ b/services/clsi/app/js/LocalCommandRunner.js @@ -15,7 +15,7 @@ let CommandRunner const { spawn } = require('child_process') const _ = require('lodash') -const logger = require('logger-sharelatex') +const logger = require('@overleaf/logger') logger.info('using standard command runner') diff --git a/services/clsi/app/js/LockManager.js b/services/clsi/app/js/LockManager.js index e4061380ce..d53872c342 100644 --- a/services/clsi/app/js/LockManager.js +++ b/services/clsi/app/js/LockManager.js @@ -12,7 +12,7 @@ */ let LockManager const Settings = require('@overleaf/settings') -const logger = require('logger-sharelatex') +const logger = require('@overleaf/logger') const Lockfile = require('lockfile') // from https://github.com/npm/lockfile const Errors = require('./Errors') const fs = require('fs') diff --git a/services/clsi/app/js/OutputCacheManager.js b/services/clsi/app/js/OutputCacheManager.js index 8ac2019619..1218cb5930 100644 --- a/services/clsi/app/js/OutputCacheManager.js +++ b/services/clsi/app/js/OutputCacheManager.js @@ -16,7 +16,7 @@ const async = require('async') const fs = require('fs') const fse = require('fs-extra') const Path = require('path') -const logger = require('logger-sharelatex') +const logger = require('@overleaf/logger') const _ = require('lodash') const Settings = require('@overleaf/settings') const crypto = require('crypto') diff --git a/services/clsi/app/js/OutputFileFinder.js b/services/clsi/app/js/OutputFileFinder.js index 9088215d1a..718fc1852a 100644 --- a/services/clsi/app/js/OutputFileFinder.js +++ b/services/clsi/app/js/OutputFileFinder.js @@ -2,7 +2,7 @@ let OutputFileFinder const Path = require('path') const _ = require('lodash') const { spawn } = require('child_process') -const logger = require('logger-sharelatex') +const logger = require('@overleaf/logger') module.exports = OutputFileFinder = { findOutputFiles(resources, directory, callback) { diff --git a/services/clsi/app/js/OutputFileOptimiser.js b/services/clsi/app/js/OutputFileOptimiser.js index b7722166d9..af3a460e7f 100644 --- a/services/clsi/app/js/OutputFileOptimiser.js +++ b/services/clsi/app/js/OutputFileOptimiser.js @@ -16,7 +16,7 @@ let OutputFileOptimiser const fs = require('fs') const Path = require('path') const { spawn } = require('child_process') -const logger = require('logger-sharelatex') +const logger = require('@overleaf/logger') const Metrics = require('./Metrics') const _ = require('lodash') diff --git a/services/clsi/app/js/ProjectPersistenceManager.js b/services/clsi/app/js/ProjectPersistenceManager.js index ddb9c1daa7..6798700f9f 100644 --- a/services/clsi/app/js/ProjectPersistenceManager.js +++ b/services/clsi/app/js/ProjectPersistenceManager.js @@ -14,7 +14,7 @@ let ProjectPersistenceManager const UrlCache = require('./UrlCache') const CompileManager = require('./CompileManager') const async = require('async') -const logger = require('logger-sharelatex') +const logger = require('@overleaf/logger') const oneDay = 24 * 60 * 60 * 1000 const Settings = require('@overleaf/settings') const diskusage = require('diskusage') diff --git a/services/clsi/app/js/ResourceStateManager.js b/services/clsi/app/js/ResourceStateManager.js index 7ae3557d51..2d92c6c4f6 100644 --- a/services/clsi/app/js/ResourceStateManager.js +++ b/services/clsi/app/js/ResourceStateManager.js @@ -1,6 +1,6 @@ const Path = require('path') const fs = require('fs') -const logger = require('logger-sharelatex') +const logger = require('@overleaf/logger') const Errors = require('./Errors') const SafeReader = require('./SafeReader') diff --git a/services/clsi/app/js/ResourceWriter.js b/services/clsi/app/js/ResourceWriter.js index 69b5021189..80e07f0ee3 100644 --- a/services/clsi/app/js/ResourceWriter.js +++ b/services/clsi/app/js/ResourceWriter.js @@ -21,7 +21,7 @@ const async = require('async') const OutputFileFinder = require('./OutputFileFinder') const ResourceStateManager = require('./ResourceStateManager') const Metrics = require('./Metrics') -const logger = require('logger-sharelatex') +const logger = require('@overleaf/logger') const settings = require('@overleaf/settings') const parallelFileDownloads = settings.parallelFileDownloads || 1 diff --git a/services/clsi/app/js/SafeReader.js b/services/clsi/app/js/SafeReader.js index 15c4498559..96e5761d87 100644 --- a/services/clsi/app/js/SafeReader.js +++ b/services/clsi/app/js/SafeReader.js @@ -13,7 +13,7 @@ */ let SafeReader const fs = require('fs') -const logger = require('logger-sharelatex') +const logger = require('@overleaf/logger') module.exports = SafeReader = { // safely read up to size bytes from a file and return result as a diff --git a/services/clsi/app/js/StaticServerForbidSymlinks.js b/services/clsi/app/js/StaticServerForbidSymlinks.js index 774178ee46..0fb32e091d 100644 --- a/services/clsi/app/js/StaticServerForbidSymlinks.js +++ b/services/clsi/app/js/StaticServerForbidSymlinks.js @@ -18,7 +18,7 @@ let ForbidSymlinks const Path = require('path') const fs = require('fs') const Settings = require('@overleaf/settings') -const logger = require('logger-sharelatex') +const logger = require('@overleaf/logger') module.exports = ForbidSymlinks = function (staticFn, root, options) { const expressStatic = staticFn(root, options) diff --git a/services/clsi/app/js/TikzManager.js b/services/clsi/app/js/TikzManager.js index 090302e8b4..7a221d8e68 100644 --- a/services/clsi/app/js/TikzManager.js +++ b/services/clsi/app/js/TikzManager.js @@ -15,7 +15,7 @@ const fs = require('fs') const Path = require('path') const ResourceWriter = require('./ResourceWriter') const SafeReader = require('./SafeReader') -const logger = require('logger-sharelatex') +const logger = require('@overleaf/logger') // for \tikzexternalize or pstool to work the main file needs to match the // jobname. Since we set the -jobname to output, we have to create a diff --git a/services/clsi/app/js/UrlFetcher.js b/services/clsi/app/js/UrlFetcher.js index 360d747a22..b4ebca2c57 100644 --- a/services/clsi/app/js/UrlFetcher.js +++ b/services/clsi/app/js/UrlFetcher.js @@ -14,7 +14,7 @@ let UrlFetcher const request = require('request').defaults({ jar: false }) const fs = require('fs') -const logger = require('logger-sharelatex') +const logger = require('@overleaf/logger') const settings = require('@overleaf/settings') const async = require('async') const { URL } = require('url') diff --git a/services/clsi/package.json b/services/clsi/package.json index f284500cb9..c0ca2dbe66 100644 --- a/services/clsi/package.json +++ b/services/clsi/package.json @@ -16,7 +16,8 @@ "lint:fix": "eslint --fix ." }, "dependencies": { - "@overleaf/metrics": "^3.5.1", + "@overleaf/logger": "^3.1.0", + "@overleaf/metrics": "^4.0.0", "@overleaf/o-error": "^3.4.0", "@overleaf/settings": "^3.0.0", "async": "3.2.0", @@ -28,7 +29,6 @@ "fs-extra": "^10.0.0", "lockfile": "^1.0.4", "lodash": "^4.17.21", - "logger-sharelatex": "^2.2.0", "p-limit": "^3.1.0", "pdfjs-dist": "~2.7.570", "request": "^2.88.2", diff --git a/services/clsi/test/acceptance/js/ExampleDocumentTests.js b/services/clsi/test/acceptance/js/ExampleDocumentTests.js index c132245b53..5407f9a0d7 100644 --- a/services/clsi/test/acceptance/js/ExampleDocumentTests.js +++ b/services/clsi/test/acceptance/js/ExampleDocumentTests.js @@ -19,7 +19,7 @@ const fs = require('fs') const fsExtra = require('fs-extra') const ChildProcess = require('child_process') const ClsiApp = require('./helpers/ClsiApp') -const logger = require('logger-sharelatex') +const logger = require('@overleaf/logger') const Path = require('path') const fixturePath = path => { if (path.slice(0, 3) === 'tmp') { diff --git a/services/clsi/test/acceptance/js/helpers/ClsiApp.js b/services/clsi/test/acceptance/js/helpers/ClsiApp.js index 370f198b87..cffc54bc58 100644 --- a/services/clsi/test/acceptance/js/helpers/ClsiApp.js +++ b/services/clsi/test/acceptance/js/helpers/ClsiApp.js @@ -10,8 +10,8 @@ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ const app = require('../../../../app') -require('logger-sharelatex').logger.level('info') -const logger = require('logger-sharelatex') +require('@overleaf/logger').logger.level('info') +const logger = require('@overleaf/logger') const Settings = require('@overleaf/settings') module.exports = { diff --git a/services/clsi/test/setup.js b/services/clsi/test/setup.js index c9d07f6fed..a889532022 100644 --- a/services/clsi/test/setup.js +++ b/services/clsi/test/setup.js @@ -9,7 +9,7 @@ chai.use(sinonChai) // Global SandboxedModule settings SandboxedModule.configure({ requires: { - 'logger-sharelatex': { + '@overleaf/logger': { debug() {}, log() {}, info() {},