diff --git a/package-lock.json b/package-lock.json index ba0401c1fc..1fee486624 100644 --- a/package-lock.json +++ b/package-lock.json @@ -25881,9 +25881,9 @@ } }, "node_modules/node-forge": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-1.2.1.tgz", - "integrity": "sha512-Fcvtbb+zBcZXbTTVwqGA5W+MKBj56UjVRevvchv5XrcyXbmNdesfZL37nlcWOfpgHhgmxApw3tQbTr4CqNmX4w==", + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-1.3.1.tgz", + "integrity": "sha512-dPEtOeMvF9VMcYV/1Wb8CPoVAXtp6MKMlcbAt4ddqmGqUJ6fQZFXkNZNkNlfevtNkGtaSoXf/vNNNSvgrdXwtA==", "engines": { "node": ">= 6.13.0" } @@ -38918,6 +38918,7 @@ "@codemirror/view": "^6.7.1", "@contentful/rich-text-html-renderer": "^16.0.2", "@contentful/rich-text-types": "^16.0.2", + "@google-cloud/bigquery": "^6.0.1", "@lezer/common": "^1.0.2", "@lezer/highlight": "^1.1.3", "@lezer/lr": "^1.2.5", @@ -39170,6 +39171,97 @@ "worker-loader": "^3.0.8" } }, + "services/web/node_modules/@google-cloud/bigquery": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/@google-cloud/bigquery/-/bigquery-6.0.3.tgz", + "integrity": "sha512-BP464228S9dqDCb4dR99h9D8+N498YZi/AZvoOJUaieg2H6qbiYBE1xlYuaMvyV1WEQT/2/yZTCJnCo5WiaY0Q==", + "dependencies": { + "@google-cloud/common": "^4.0.0", + "@google-cloud/paginator": "^4.0.0", + "@google-cloud/promisify": "^3.0.0", + "arrify": "^2.0.1", + "big.js": "^6.0.0", + "duplexify": "^4.0.0", + "extend": "^3.0.2", + "is": "^3.3.0", + "p-event": "^4.1.0", + "readable-stream": "^4.0.0", + "stream-events": "^1.0.5", + "uuid": "^8.0.0" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "services/web/node_modules/@google-cloud/bigquery/node_modules/readable-stream": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.2.0.tgz", + "integrity": "sha512-gJrBHsaI3lgBoGMW/jHZsQ/o/TIWiu5ENCJG1BB7fuCKzpFM8GaS2UoBVt9NO+oI+3FcrBNbUkl3ilDe09aY4A==", + "dependencies": { + "abort-controller": "^3.0.0", + "buffer": "^6.0.3", + "events": "^3.3.0", + "process": "^0.11.10" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + } + }, + "services/web/node_modules/@google-cloud/bigquery/node_modules/uuid": { + "version": "8.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", + "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", + "bin": { + "uuid": "dist/bin/uuid" + } + }, + "services/web/node_modules/@google-cloud/common": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/@google-cloud/common/-/common-4.0.3.tgz", + "integrity": "sha512-fUoMo5b8iAKbrYpneIRV3z95AlxVJPrjpevxs4SKoclngWZvTXBSGpNisF5+x5m+oNGve7jfB1e6vNBZBUs7Fw==", + "dependencies": { + "@google-cloud/projectify": "^3.0.0", + "@google-cloud/promisify": "^3.0.0", + "arrify": "^2.0.1", + "duplexify": "^4.1.1", + "ent": "^2.2.0", + "extend": "^3.0.2", + "google-auth-library": "^8.0.2", + "retry-request": "^5.0.0", + "teeny-request": "^8.0.0" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "services/web/node_modules/@google-cloud/paginator": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@google-cloud/paginator/-/paginator-4.0.1.tgz", + "integrity": "sha512-6G1ui6bWhNyHjmbYwavdN7mpVPRBtyDg/bfqBTAlwr413On2TnFNfDxc9UhTJctkgoCDgQXEKiRPLPR9USlkbQ==", + "dependencies": { + "arrify": "^2.0.0", + "extend": "^3.0.2" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "services/web/node_modules/@google-cloud/projectify": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@google-cloud/projectify/-/projectify-3.0.0.tgz", + "integrity": "sha512-HRkZsNmjScY6Li8/kb70wjGlDDyLkVk3KvoEo9uIoxSjYLJasGiCch9+PqRVDOCGUFvEIqyogl+BeqILL4OJHA==", + "engines": { + "node": ">=12.0.0" + } + }, + "services/web/node_modules/@google-cloud/promisify": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/@google-cloud/promisify/-/promisify-3.0.1.tgz", + "integrity": "sha512-z1CjRjtQyBOYL+5Qr9DdYIfrdLBe746jRTYfaYU6MeXkqp7UfYs/jX16lFFVzZ7PGEJvqZNqYUEtb1mvDww4pA==", + "engines": { + "node": ">=12" + } + }, "services/web/node_modules/@hapi/address": { "version": "2.1.4", "resolved": "https://registry.npmjs.org/@hapi/address/-/address-2.1.4.tgz", @@ -39263,6 +39355,14 @@ "lodash": "^4.17.15" } }, + "services/web/node_modules/@tootallnate/once": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-2.0.0.tgz", + "integrity": "sha512-XCuKFP5PS55gnMVu3dty8KPatLqUoy/ZYzDzAGCQ8JNFCkLXzmI7vNHCR+XpbZaMWQK/vQubr7PkYq8g470J/A==", + "engines": { + "node": ">= 10" + } + }, "services/web/node_modules/@uppy/utils": { "version": "4.0.7", "resolved": "https://registry.npmjs.org/@uppy/utils/-/utils-4.0.7.tgz", @@ -39331,6 +39431,29 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" }, + "services/web/node_modules/buffer": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", + "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.2.1" + } + }, "services/web/node_modules/busboy": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/busboy/-/busboy-1.6.0.tgz", @@ -39803,6 +39926,17 @@ "node": ">=0.3.1" } }, + "services/web/node_modules/duplexify": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-4.1.2.tgz", + "integrity": "sha512-fz3OjcNCHmRP12MJoZMPglx8m4rrFP8rovnk4vT8Fs+aonZoCwGg10dSsQsfP/E62eZcPTMSMP6686fu9Qlqtw==", + "dependencies": { + "end-of-stream": "^1.4.1", + "inherits": "^2.0.3", + "readable-stream": "^3.1.1", + "stream-shift": "^1.0.0" + } + }, "services/web/node_modules/emoji-regex": { "version": "9.2.2", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", @@ -40006,6 +40140,32 @@ "universalify": "^0.1.0" } }, + "services/web/node_modules/gaxios": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-5.0.2.tgz", + "integrity": "sha512-TjtV2AJOZoMQqRYoy5eM8cCQogYwazWNYLQ72QB0kwa6vHHruYkGmhhyrlzbmgNHK1dNnuP2WSH81urfzyN2Og==", + "dependencies": { + "extend": "^3.0.2", + "https-proxy-agent": "^5.0.0", + "is-stream": "^2.0.0", + "node-fetch": "^2.6.7" + }, + "engines": { + "node": ">=12" + } + }, + "services/web/node_modules/gcp-metadata": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-5.0.1.tgz", + "integrity": "sha512-jiRJ+Fk7e8FH68Z6TLaqwea307OktJpDjmYnU7/li6ziwvVvU2RlrCyQo5vkdeP94chm0kcSCOOszvmuaioq3g==", + "dependencies": { + "gaxios": "^5.0.0", + "json-bigint": "^1.0.0" + }, + "engines": { + "node": ">=12" + } + }, "services/web/node_modules/glob-parent": { "version": "6.0.2", "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", @@ -40018,6 +40178,63 @@ "node": ">=10.13.0" } }, + "services/web/node_modules/google-auth-library": { + "version": "8.7.0", + "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-8.7.0.tgz", + "integrity": "sha512-1M0NG5VDIvJZEnstHbRdckLZESoJwguinwN8Dhae0j2ZKIQFIV63zxm6Fo6nM4xkgqUr2bbMtV5Dgo+Hy6oo0Q==", + "dependencies": { + "arrify": "^2.0.0", + "base64-js": "^1.3.0", + "ecdsa-sig-formatter": "^1.0.11", + "fast-text-encoding": "^1.0.0", + "gaxios": "^5.0.0", + "gcp-metadata": "^5.0.0", + "gtoken": "^6.1.0", + "jws": "^4.0.0", + "lru-cache": "^6.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "services/web/node_modules/google-auth-library/node_modules/lru-cache": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", + "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "services/web/node_modules/google-p12-pem": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/google-p12-pem/-/google-p12-pem-4.0.1.tgz", + "integrity": "sha512-WPkN4yGtz05WZ5EhtlxNDWPhC4JIic6G8ePitwUWy4l+XPVYec+a0j0Ts47PDtW59y3RwAhUd9/h9ZZ63px6RQ==", + "dependencies": { + "node-forge": "^1.3.1" + }, + "bin": { + "gp12-pem": "build/src/bin/gp12-pem.js" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "services/web/node_modules/gtoken": { + "version": "6.1.2", + "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-6.1.2.tgz", + "integrity": "sha512-4ccGpzz7YAr7lxrT2neugmXQ3hP9ho2gcaityLVkiUecAiwiy60Ii8gRbZeOsXV19fYaRjgBSshs8kXw+NKCPQ==", + "dependencies": { + "gaxios": "^5.0.1", + "google-p12-pem": "^4.0.0", + "jws": "^4.0.0" + }, + "engines": { + "node": ">=12.0.0" + } + }, "services/web/node_modules/http-errors": { "version": "1.7.2", "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.7.2.tgz", @@ -40033,6 +40250,19 @@ "node": ">= 0.6" } }, + "services/web/node_modules/http-proxy-agent": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-5.0.0.tgz", + "integrity": "sha512-n2hY8YdoRE1i7r6M0w9DIw5GgZN0G25P8zLCRQ8rjXtTU3vsNFBI/vWK/UIeE6g5MUUz6avwAPXmL6Fy9D/90w==", + "dependencies": { + "@tootallnate/once": "2", + "agent-base": "6", + "debug": "4" + }, + "engines": { + "node": ">= 6" + } + }, "services/web/node_modules/icss-utils": { "version": "5.1.0", "resolved": "https://registry.npmjs.org/icss-utils/-/icss-utils-5.1.0.tgz", @@ -40045,6 +40275,25 @@ "postcss": "^8.1.0" } }, + "services/web/node_modules/ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, "services/web/node_modules/inherits": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", @@ -41009,6 +41258,18 @@ "url": "https://github.com/sponsors/ljharb" } }, + "services/web/node_modules/retry-request": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/retry-request/-/retry-request-5.0.2.tgz", + "integrity": "sha512-wfI3pk7EE80lCIXprqh7ym48IHYdwmAAzESdbU8Q9l7pnRCk9LEhpbOTNKjz6FARLm/Bl5m+4F0ABxOkYUujSQ==", + "dependencies": { + "debug": "^4.1.1", + "extend": "^3.0.2" + }, + "engines": { + "node": ">=12" + } + }, "services/web/node_modules/sandboxed-module": { "version": "2.3.0", "resolved": "https://github.com/overleaf/node-sandboxed-module/archive/cafa2d60f17ce75cc023e6f296eb8de79d92d35d.tar.gz", @@ -41215,6 +41476,29 @@ "node": ">=10.13.0" } }, + "services/web/node_modules/teeny-request": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/teeny-request/-/teeny-request-8.0.2.tgz", + "integrity": "sha512-34pe0a4zASseXZCKdeTiIZqSKA8ETHb1EwItZr01PAR3CLPojeAKgSjzeNS4373gi59hNulyDrPKEbh2zO9sCg==", + "dependencies": { + "http-proxy-agent": "^5.0.0", + "https-proxy-agent": "^5.0.0", + "node-fetch": "^2.6.1", + "stream-events": "^1.0.5", + "uuid": "^9.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "services/web/node_modules/teeny-request/node_modules/uuid": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.0.tgz", + "integrity": "sha512-MXcSTerfPa4uqyzStbRoTgt5XIe3x5+42+q1sDuy3R5MDk66URdLMOZe5aPX/SQd+kuYAh0FdP/pO28IkQyTeg==", + "bin": { + "uuid": "dist/bin/uuid" + } + }, "services/web/node_modules/toidentifier": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.0.tgz", @@ -41271,8 +41555,7 @@ "services/web/node_modules/yallist": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", - "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", - "dev": true + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" } }, "dependencies": { @@ -49185,6 +49468,7 @@ "@contentful/rich-text-html-renderer": "^16.0.2", "@contentful/rich-text-types": "^16.0.2", "@cypress/react": "^6.2.0", + "@google-cloud/bigquery": "^6.0.1", "@juggle/resize-observer": "^3.3.1", "@lezer/common": "^1.0.2", "@lezer/generator": "^1.1.3", @@ -49431,6 +49715,78 @@ "yup": "^0.32.11" }, "dependencies": { + "@google-cloud/bigquery": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/@google-cloud/bigquery/-/bigquery-6.0.3.tgz", + "integrity": "sha512-BP464228S9dqDCb4dR99h9D8+N498YZi/AZvoOJUaieg2H6qbiYBE1xlYuaMvyV1WEQT/2/yZTCJnCo5WiaY0Q==", + "requires": { + "@google-cloud/common": "^4.0.0", + "@google-cloud/paginator": "^4.0.0", + "@google-cloud/promisify": "^3.0.0", + "arrify": "^2.0.1", + "big.js": "^6.0.0", + "duplexify": "^4.0.0", + "extend": "^3.0.2", + "is": "^3.3.0", + "p-event": "^4.1.0", + "readable-stream": "^4.0.0", + "stream-events": "^1.0.5", + "uuid": "^8.0.0" + }, + "dependencies": { + "readable-stream": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.2.0.tgz", + "integrity": "sha512-gJrBHsaI3lgBoGMW/jHZsQ/o/TIWiu5ENCJG1BB7fuCKzpFM8GaS2UoBVt9NO+oI+3FcrBNbUkl3ilDe09aY4A==", + "requires": { + "abort-controller": "^3.0.0", + "buffer": "^6.0.3", + "events": "^3.3.0", + "process": "^0.11.10" + } + }, + "uuid": { + "version": "8.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", + "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==" + } + } + }, + "@google-cloud/common": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/@google-cloud/common/-/common-4.0.3.tgz", + "integrity": "sha512-fUoMo5b8iAKbrYpneIRV3z95AlxVJPrjpevxs4SKoclngWZvTXBSGpNisF5+x5m+oNGve7jfB1e6vNBZBUs7Fw==", + "requires": { + "@google-cloud/projectify": "^3.0.0", + "@google-cloud/promisify": "^3.0.0", + "arrify": "^2.0.1", + "duplexify": "^4.1.1", + "ent": "^2.2.0", + "extend": "^3.0.2", + "google-auth-library": "^8.0.2", + "retry-request": "^5.0.0", + "teeny-request": "^8.0.0" + } + }, + "@google-cloud/paginator": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@google-cloud/paginator/-/paginator-4.0.1.tgz", + "integrity": "sha512-6G1ui6bWhNyHjmbYwavdN7mpVPRBtyDg/bfqBTAlwr413On2TnFNfDxc9UhTJctkgoCDgQXEKiRPLPR9USlkbQ==", + "requires": { + "arrify": "^2.0.0", + "extend": "^3.0.2" + } + }, + "@google-cloud/projectify": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@google-cloud/projectify/-/projectify-3.0.0.tgz", + "integrity": "sha512-HRkZsNmjScY6Li8/kb70wjGlDDyLkVk3KvoEo9uIoxSjYLJasGiCch9+PqRVDOCGUFvEIqyogl+BeqILL4OJHA==" + }, + "@google-cloud/promisify": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/@google-cloud/promisify/-/promisify-3.0.1.tgz", + "integrity": "sha512-z1CjRjtQyBOYL+5Qr9DdYIfrdLBe746jRTYfaYU6MeXkqp7UfYs/jX16lFFVzZ7PGEJvqZNqYUEtb1mvDww4pA==" + }, "@hapi/address": { "version": "2.1.4", "resolved": "https://registry.npmjs.org/@hapi/address/-/address-2.1.4.tgz", @@ -49503,6 +49859,11 @@ "lodash": "^4.17.15" } }, + "@tootallnate/once": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-2.0.0.tgz", + "integrity": "sha512-XCuKFP5PS55gnMVu3dty8KPatLqUoy/ZYzDzAGCQ8JNFCkLXzmI7vNHCR+XpbZaMWQK/vQubr7PkYq8g470J/A==" + }, "@uppy/utils": { "version": "4.0.7", "resolved": "https://registry.npmjs.org/@uppy/utils/-/utils-4.0.7.tgz", @@ -49558,6 +49919,15 @@ } } }, + "buffer": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", + "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==", + "requires": { + "base64-js": "^1.3.1", + "ieee754": "^1.2.1" + } + }, "busboy": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/busboy/-/busboy-1.6.0.tgz", @@ -49888,6 +50258,17 @@ "integrity": "sha512-A46qtFgd+g7pDZinpnwiRJtxbC1hpgf0uzP3iG89scHk0AUC7A1TGxf5OiiOUv/JMZR8GOt8hL900hV0bOy5xA==", "dev": true }, + "duplexify": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-4.1.2.tgz", + "integrity": "sha512-fz3OjcNCHmRP12MJoZMPglx8m4rrFP8rovnk4vT8Fs+aonZoCwGg10dSsQsfP/E62eZcPTMSMP6686fu9Qlqtw==", + "requires": { + "end-of-stream": "^1.4.1", + "inherits": "^2.0.3", + "readable-stream": "^3.1.1", + "stream-shift": "^1.0.0" + } + }, "emoji-regex": { "version": "9.2.2", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", @@ -50040,6 +50421,26 @@ "universalify": "^0.1.0" } }, + "gaxios": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-5.0.2.tgz", + "integrity": "sha512-TjtV2AJOZoMQqRYoy5eM8cCQogYwazWNYLQ72QB0kwa6vHHruYkGmhhyrlzbmgNHK1dNnuP2WSH81urfzyN2Og==", + "requires": { + "extend": "^3.0.2", + "https-proxy-agent": "^5.0.0", + "is-stream": "^2.0.0", + "node-fetch": "^2.6.7" + } + }, + "gcp-metadata": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-5.0.1.tgz", + "integrity": "sha512-jiRJ+Fk7e8FH68Z6TLaqwea307OktJpDjmYnU7/li6ziwvVvU2RlrCyQo5vkdeP94chm0kcSCOOszvmuaioq3g==", + "requires": { + "gaxios": "^5.0.0", + "json-bigint": "^1.0.0" + } + }, "glob-parent": { "version": "6.0.2", "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", @@ -50049,6 +50450,50 @@ "is-glob": "^4.0.3" } }, + "google-auth-library": { + "version": "8.7.0", + "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-8.7.0.tgz", + "integrity": "sha512-1M0NG5VDIvJZEnstHbRdckLZESoJwguinwN8Dhae0j2ZKIQFIV63zxm6Fo6nM4xkgqUr2bbMtV5Dgo+Hy6oo0Q==", + "requires": { + "arrify": "^2.0.0", + "base64-js": "^1.3.0", + "ecdsa-sig-formatter": "^1.0.11", + "fast-text-encoding": "^1.0.0", + "gaxios": "^5.0.0", + "gcp-metadata": "^5.0.0", + "gtoken": "^6.1.0", + "jws": "^4.0.0", + "lru-cache": "^6.0.0" + }, + "dependencies": { + "lru-cache": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", + "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", + "requires": { + "yallist": "^4.0.0" + } + } + } + }, + "google-p12-pem": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/google-p12-pem/-/google-p12-pem-4.0.1.tgz", + "integrity": "sha512-WPkN4yGtz05WZ5EhtlxNDWPhC4JIic6G8ePitwUWy4l+XPVYec+a0j0Ts47PDtW59y3RwAhUd9/h9ZZ63px6RQ==", + "requires": { + "node-forge": "^1.3.1" + } + }, + "gtoken": { + "version": "6.1.2", + "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-6.1.2.tgz", + "integrity": "sha512-4ccGpzz7YAr7lxrT2neugmXQ3hP9ho2gcaityLVkiUecAiwiy60Ii8gRbZeOsXV19fYaRjgBSshs8kXw+NKCPQ==", + "requires": { + "gaxios": "^5.0.1", + "google-p12-pem": "^4.0.0", + "jws": "^4.0.0" + } + }, "http-errors": { "version": "1.7.2", "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.7.2.tgz", @@ -50061,6 +50506,16 @@ "toidentifier": "1.0.0" } }, + "http-proxy-agent": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-5.0.0.tgz", + "integrity": "sha512-n2hY8YdoRE1i7r6M0w9DIw5GgZN0G25P8zLCRQ8rjXtTU3vsNFBI/vWK/UIeE6g5MUUz6avwAPXmL6Fy9D/90w==", + "requires": { + "@tootallnate/once": "2", + "agent-base": "6", + "debug": "4" + } + }, "icss-utils": { "version": "5.1.0", "resolved": "https://registry.npmjs.org/icss-utils/-/icss-utils-5.1.0.tgz", @@ -50068,6 +50523,11 @@ "dev": true, "requires": {} }, + "ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==" + }, "inherits": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", @@ -50707,6 +51167,15 @@ "path-parse": "^1.0.6" } }, + "retry-request": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/retry-request/-/retry-request-5.0.2.tgz", + "integrity": "sha512-wfI3pk7EE80lCIXprqh7ym48IHYdwmAAzESdbU8Q9l7pnRCk9LEhpbOTNKjz6FARLm/Bl5m+4F0ABxOkYUujSQ==", + "requires": { + "debug": "^4.1.1", + "extend": "^3.0.2" + } + }, "sandboxed-module": { "version": "https://github.com/overleaf/node-sandboxed-module/archive/cafa2d60f17ce75cc023e6f296eb8de79d92d35d.tar.gz", "integrity": "sha512-8h+3BGQj0Xou2mRYrRpFvDQ5Ne0/obZH9X27m/5zVMrvckNQazn1YPab7OE45VMzOI25iE9e0602QY2QmEnANQ==", @@ -50873,6 +51342,25 @@ "stable": "^0.1.8" } }, + "teeny-request": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/teeny-request/-/teeny-request-8.0.2.tgz", + "integrity": "sha512-34pe0a4zASseXZCKdeTiIZqSKA8ETHb1EwItZr01PAR3CLPojeAKgSjzeNS4373gi59hNulyDrPKEbh2zO9sCg==", + "requires": { + "http-proxy-agent": "^5.0.0", + "https-proxy-agent": "^5.0.0", + "node-fetch": "^2.6.1", + "stream-events": "^1.0.5", + "uuid": "^9.0.0" + }, + "dependencies": { + "uuid": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.0.tgz", + "integrity": "sha512-MXcSTerfPa4uqyzStbRoTgt5XIe3x5+42+q1sDuy3R5MDk66URdLMOZe5aPX/SQd+kuYAh0FdP/pO28IkQyTeg==" + } + } + }, "toidentifier": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.0.tgz", @@ -50908,8 +51396,7 @@ "yallist": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", - "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", - "dev": true + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" } } }, @@ -65239,9 +65726,9 @@ } }, "node-forge": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-1.2.1.tgz", - "integrity": "sha512-Fcvtbb+zBcZXbTTVwqGA5W+MKBj56UjVRevvchv5XrcyXbmNdesfZL37nlcWOfpgHhgmxApw3tQbTr4CqNmX4w==" + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-1.3.1.tgz", + "integrity": "sha512-dPEtOeMvF9VMcYV/1Wb8CPoVAXtp6MKMlcbAt4ddqmGqUJ6fQZFXkNZNkNlfevtNkGtaSoXf/vNNNSvgrdXwtA==" }, "node-gyp-build": { "version": "3.9.0", diff --git a/services/web/package.json b/services/web/package.json index 6e2d9c2ef9..eb6790055a 100644 --- a/services/web/package.json +++ b/services/web/package.json @@ -81,6 +81,7 @@ "@codemirror/view": "^6.7.1", "@contentful/rich-text-html-renderer": "^16.0.2", "@contentful/rich-text-types": "^16.0.2", + "@google-cloud/bigquery": "^6.0.1", "@lezer/common": "^1.0.2", "@lezer/highlight": "^1.1.3", "@lezer/lr": "^1.2.5", diff --git a/services/web/scripts/analytics/helpers/GoogleBigQueryHelper.js b/services/web/scripts/analytics/helpers/GoogleBigQueryHelper.js new file mode 100644 index 0000000000..1a79ab0329 --- /dev/null +++ b/services/web/scripts/analytics/helpers/GoogleBigQueryHelper.js @@ -0,0 +1,30 @@ +const GoogleBigQuery = require('@google-cloud/bigquery').BigQuery + +let dataset = null + +function getDataset() { + if (!dataset) { + console.log( + 'Connecting to BigQuery dataset: ', + process.env.BQ_PROJECT_ID, + process.env.BQ_DATASET + ) + + dataset = new GoogleBigQuery({ + projectId: process.env.BQ_PROJECT_ID, + keyFilename: process.env.GCS_KEY_FILE, + }).dataset(process.env.BQ_DATASET) + } + + return dataset +} + +async function query(query) { + const [job] = await getDataset().createQueryJob({ query }) + const [rows] = await job.getQueryResults() + return rows +} + +module.exports = { + query, +} diff --git a/services/web/scripts/analytics/sync_group_subscription_memberships.js b/services/web/scripts/analytics/sync_group_subscription_memberships.js new file mode 100644 index 0000000000..b6d30fe854 --- /dev/null +++ b/services/web/scripts/analytics/sync_group_subscription_memberships.js @@ -0,0 +1,241 @@ +const GoogleBigQueryHelper = require('./helpers/GoogleBigQueryHelper') +const { Subscription } = require('../../app/src/models/Subscription') +const { waitForDb } = require('../../app/src/infrastructure/mongodb') +const AnalyticsManager = require('../../app/src/Features/Analytics/AnalyticsManager') +const { + DeletedSubscription, +} = require('../../app/src/models/DeletedSubscription') +const minimist = require('minimist') +const _ = require('lodash') + +let FETCH_LIMIT, COMMIT, VERBOSE + +async function main() { + await waitForDb() + + console.log('## Syncing group subscription memberships...') + + const subscriptionsCount = await Subscription.count({ groupPlan: true }) + const deletedSubscriptionsCount = await DeletedSubscription.count({ + 'subscription.groupPlan': true, + }) + + console.log( + `## Going to synchronize ${subscriptionsCount} subscriptions and ${deletedSubscriptionsCount} deleted subscriptions` + ) + + await checkActiveSubscriptions() + await checkDeletedSubscriptions() +} + +async function checkActiveSubscriptions() { + let totalSubscriptionsChecked = 0 + let subscriptions + do { + subscriptions = await Subscription.find( + { groupPlan: true }, + { recurlySubscription_id: 1, member_ids: 1 } + ) + .sort('_id') + .skip(totalSubscriptionsChecked) + .limit(FETCH_LIMIT) + .lean() + + if (subscriptions.length) { + const groupIds = subscriptions.map(sub => sub._id) + const bigQueryGroupMemberships = await fetchBigQueryMembershipStatuses( + groupIds + ) + const membershipsByGroupId = _.groupBy( + bigQueryGroupMemberships, + 'group_id' + ) + + for (const subscription of subscriptions) { + checkSubscriptionMemberships( + subscription, + membershipsByGroupId[subscription._id.toString()] + ) + } + totalSubscriptionsChecked += subscriptions.length + } + } while (subscriptions.length > 0) +} + +async function checkDeletedSubscriptions() { + let totalDeletedSubscriptionsChecked = 0 + let deletedSubscriptions + do { + deletedSubscriptions = ( + await DeletedSubscription.find( + { 'subscription.groupPlan': true }, + { subscription: 1 } + ) + .sort('deletedAt') + .skip(totalDeletedSubscriptionsChecked) + .limit(FETCH_LIMIT) + ).map(sub => sub.toObject().subscription) + + if (deletedSubscriptions.length) { + const groupIds = deletedSubscriptions.map(sub => sub._id.toString()) + const bigQueryGroupMemberships = await fetchBigQueryMembershipStatuses( + groupIds + ) + + for (const deletedSubscription of deletedSubscriptions) { + checkDeletedSubscriptionMemberships( + deletedSubscription, + _.filter(bigQueryGroupMemberships, { + group_id: deletedSubscription._id.toString(), + }) + ) + } + totalDeletedSubscriptionsChecked += deletedSubscriptions.length + } + } while (deletedSubscriptions.length > 0) +} + +function checkSubscriptionMemberships(subscription, membershipStatuses) { + if (VERBOSE) { + console.log( + '\n###########################################################################################', + '\n# Subscription (mongo): ', + '\n# _id: \t\t\t\t', + subscription._id.toString(), + '\n# member_ids: \t\t\t', + subscription.member_ids, + '\n# recurlySubscription_id: \t', + subscription.recurlySubscription_id + ) + console.log('#\n# Membership statuses found in BigQuery: ') + console.table(membershipStatuses) + } + // create missing `joined` events when membership status is missing + for (const memberId of subscription.member_ids) { + if ( + !_.find(membershipStatuses, { + user_id: memberId.toString(), + is_member: true, + }) + ) { + sendCorrectiveEvent(memberId, 'group-subscription-joined', subscription) + } + } + // create missing `left` events if user is not a member of the group anymore + for (const { user_id: userId, is_member: isMember } of membershipStatuses) { + if ( + isMember && + !subscription.member_ids.some(sub => sub.id.toString() === userId) + ) { + sendCorrectiveEvent(userId, 'group-subscription-left', subscription) + } + } +} + +function checkDeletedSubscriptionMemberships(subscription, membershipStatuses) { + if (VERBOSE) { + console.log( + '\n###########################################################################################', + '\n# Deleted subscription (mongo): ', + '\n# _id: \t\t\t\t', + subscription._id.toString(), + '\n# member_ids: \t\t\t', + subscription.member_ids, + '\n# recurlySubscription_id: \t', + subscription.recurlySubscription_id + ) + console.log('#\n# Membership statuses found in BigQuery: ') + console.table(membershipStatuses) + } + + const updatedUserIds = new Set() + // create missing `left` events if user was a member of the group in BQ and status is not up-to-date + for (const memberId of subscription.member_ids.map(id => id.toString())) { + if ( + _.find(membershipStatuses, { + user_id: memberId, + is_member: true, + }) + ) { + sendCorrectiveEvent(memberId, 'group-subscription-left', subscription) + updatedUserIds.add(memberId) + } + } + // for cases where the user has been removed from the subscription before it was deleted and status is not up-to-date + for (const { user_id: userId, is_member: isMember } of membershipStatuses) { + if (isMember && !updatedUserIds.has(userId)) { + sendCorrectiveEvent(userId, 'group-subscription-left', subscription) + updatedUserIds.add(userId) + } + } +} + +function sendCorrectiveEvent(userId, event, subscription) { + const segmentation = { + groupId: subscription._id.toString(), + subscriptionId: subscription.recurlySubscription_id, + source: 'sync', + } + if (COMMIT) { + console.log( + `Sending event '${event}' for user ${userId} with segmentation: ${JSON.stringify( + segmentation + )}` + ) + AnalyticsManager.recordEventForUser(userId, event, segmentation) + } else { + console.log( + `Dry run - would send event '${event}' for user ${userId} with segmentation: ${JSON.stringify( + segmentation + )}` + ) + } +} + +async function fetchBigQueryMembershipStatuses(groupIds) { + const joinedGroupIds = groupIds.map(id => `"${id}"`).join(',') + const query = `\ + WITH memberships AS ( + SELECT + user_id, group_id, is_member, created_at, + ROW_NUMBER() OVER(PARTITION BY group_id, user_id ORDER BY created_at DESC) AS row_number + FROM analytics.user_group_memberships + WHERE group_id IN (${joinedGroupIds}) + ) + + SELECT + group_id, + COALESCE(user_aliases.user_id, memberships.user_id) AS user_id, + is_member, + memberships.created_at + FROM memberships + LEFT JOIN analytics.user_aliases ON memberships.user_id = user_aliases.analytics_id + WHERE row_number = 1; + ` + + return GoogleBigQueryHelper.query(query) +} + +const setup = () => { + const argv = minimist(process.argv.slice(2)) + FETCH_LIMIT = argv.fetch ? argv.fetch : 100 + COMMIT = argv.commit !== undefined + VERBOSE = argv.debug !== undefined + if (!COMMIT) { + console.warn('Doing dry run without --commit') + } + if (VERBOSE) { + console.log('Running in verbose mode') + } +} + +setup() +main() + .then(() => { + console.error('Done.') + process.exit(0) + }) + .catch(error => { + console.error({ error }) + process.exit(1) + })