From 5d68b00a7c47f4d8b28d8f429e243634cea27497 Mon Sep 17 00:00:00 2001 From: yu-i-i Date: Tue, 3 Dec 2024 01:14:41 +0100 Subject: [PATCH] Enable autocomplete of reference keys feature --- develop/dev.env | 1 + develop/docker-compose.dev.yml | 11 +++++ develop/docker-compose.yml | 8 ++++ server-ce/config/env.sh | 1 + server-ce/runit/references-overleaf/run | 12 ++++++ services/references/README.md | 10 +++++ services/references/app.js | 40 ++++++++++++++++++ .../app/js/ReferencesAPIController.js | 42 +++++++++++++++++++ .../references/config/settings.defaults.cjs | 9 ++++ services/references/package.json | 26 ++++++++++++ services/web/config/settings.defaults.js | 3 ++ 11 files changed, 163 insertions(+) create mode 100755 server-ce/runit/references-overleaf/run create mode 100644 services/references/README.md create mode 100644 services/references/app.js create mode 100644 services/references/app/js/ReferencesAPIController.js create mode 100644 services/references/config/settings.defaults.cjs create mode 100644 services/references/package.json diff --git a/develop/dev.env b/develop/dev.env index f817987f8e..e8a656f844 100644 --- a/develop/dev.env +++ b/develop/dev.env @@ -13,6 +13,7 @@ NOTIFICATIONS_HOST=notifications PROJECT_HISTORY_HOST=project-history REALTIME_HOST=real-time REDIS_HOST=redis +REFERENCES_HOST=references SPELLING_HOST=spelling WEBPACK_HOST=webpack WEB_API_PASSWORD=overleaf diff --git a/develop/docker-compose.dev.yml b/develop/docker-compose.dev.yml index 4432a24162..cc1e094a6c 100644 --- a/develop/docker-compose.dev.yml +++ b/develop/docker-compose.dev.yml @@ -112,6 +112,17 @@ services: - ../services/real-time/app.js:/overleaf/services/real-time/app.js - ../services/real-time/config:/overleaf/services/real-time/config + references: + command: ["node", "--watch", "app.js"] + environment: + - NODE_OPTIONS=--inspect=0.0.0.0:9229 + ports: + - "127.0.0.1:9236:9229" + volumes: + - ../services/references/app:/overleaf/services/references/app + - ../services/references/config:/overleaf/services/references/config + - ../services/references/app.js:/overleaf/services/references/app.js + web: command: ["node", "--watch", "app.js", "--watch-locales"] environment: diff --git a/develop/docker-compose.yml b/develop/docker-compose.yml index f9053c21ce..8323c32d4b 100644 --- a/develop/docker-compose.yml +++ b/develop/docker-compose.yml @@ -124,6 +124,13 @@ services: volumes: - redis-data:/data + references: + build: + context: .. + dockerfile: services/references/Dockerfile + env_file: + - dev.env + spelling: build: context: .. @@ -163,6 +170,7 @@ services: - notifications - project-history - real-time + - references - spelling webpack: diff --git a/server-ce/config/env.sh b/server-ce/config/env.sh index 2dee36a151..f5ebd97401 100644 --- a/server-ce/config/env.sh +++ b/server-ce/config/env.sh @@ -9,6 +9,7 @@ export HISTORY_V1_HOST=127.0.0.1 export NOTIFICATIONS_HOST=127.0.0.1 export PROJECT_HISTORY_HOST=127.0.0.1 export REALTIME_HOST=127.0.0.1 +export REFERENCES_HOST=127.0.0.1 export SPELLING_HOST=127.0.0.1 export WEB_HOST=127.0.0.1 export WEB_API_HOST=127.0.0.1 diff --git a/server-ce/runit/references-overleaf/run b/server-ce/runit/references-overleaf/run new file mode 100755 index 0000000000..875023df9f --- /dev/null +++ b/server-ce/runit/references-overleaf/run @@ -0,0 +1,12 @@ +#!/bin/bash + +NODE_PARAMS="" +if [ "$DEBUG_NODE" == "true" ]; then + echo "running debug - references" + NODE_PARAMS="--inspect=0.0.0.0:30560" +fi + +source /etc/overleaf/env.sh +export LISTEN_ADDRESS=127.0.0.1 + +exec /sbin/setuser www-data /usr/bin/node $NODE_PARAMS /overleaf/services/references/app.js >> /var/log/overleaf/references.log 2>&1 diff --git a/services/references/README.md b/services/references/README.md new file mode 100644 index 0000000000..41844d259a --- /dev/null +++ b/services/references/README.md @@ -0,0 +1,10 @@ +overleaf/references +=============== + +An API for providing citation-keys from user bib-files + +License +======= +The code in this repository is released under the GNU AFFERO GENERAL PUBLIC LICENSE, version 3. + +Based on https://github.com/overleaf/overleaf/commit/9964aebc794f9fd7ce1373ab3484f6b33b061af3 diff --git a/services/references/app.js b/services/references/app.js new file mode 100644 index 0000000000..a7da8720ed --- /dev/null +++ b/services/references/app.js @@ -0,0 +1,40 @@ +import '@overleaf/metrics/initialize.js' + +import express from 'express' +import Settings from '@overleaf/settings' +import logger from '@overleaf/logger' +import metrics from '@overleaf/metrics' +import ReferencesAPIController from './app/js/ReferencesAPIController.js' +import bodyParser from 'body-parser' + +const app = express() +metrics.injectMetricsRoute(app) + +app.use(bodyParser.json({ limit: '2mb' })) +app.use(metrics.http.monitor(logger)) + +app.post('/project/:project_id/index', ReferencesAPIController.index) +app.get('/status', (req, res) => res.send({ status: 'references api is up' })) + +const settings = + Settings.internal && Settings.internal.references + ? Settings.internal.references + : undefined +const host = settings && settings.host ? settings.host : 'localhost' +const port = settings && settings.port ? settings.port : 3056 + +logger.debug('Listening at', { host, port }) + +const server = app.listen(port, host, function (error) { + if (error) { + throw error + } + logger.info({ host, port }, 'references HTTP server starting up') +}) + +process.on('SIGTERM', () => { + server.close(() => { + logger.info({ host, port }, 'references HTTP server closed') + metrics.close() + }) +}) diff --git a/services/references/app/js/ReferencesAPIController.js b/services/references/app/js/ReferencesAPIController.js new file mode 100644 index 0000000000..68565c1b52 --- /dev/null +++ b/services/references/app/js/ReferencesAPIController.js @@ -0,0 +1,42 @@ +import logger from '@overleaf/logger' +import BibtexParser from '../../../web/app/src/util/bib2json.js' + +export default { + async index(req, res) { + const { docUrls, fullIndex } = req.body + try { + const responses = await Promise.all( + docUrls.map(async (docUrl) => { + try { + const response = await fetch(docUrl) + if (!response.ok) { + throw new Error(`HTTP error! status: ${response.status}`) + } + return response.text() + } catch (error) { + logger.error({ error }, "Failed to fetch document from URL: " + docUrl) + return null + } + }) + ) + const keys = [] + for (const body of responses) { + if (!body) continue + + try { + const parsedEntries = BibtexParser(body).entries + const ks = parsedEntries + .filter(entry => entry.EntryKey) + .map(entry => entry.EntryKey) + keys.push(...ks) + } catch (error) { + logger.error({ error }, "bib file skipped.") + } + } + res.status(200).json({ keys }) + } catch (error) { + logger.error({ error }, "Unexpected error during indexing process.") + res.status(500).json({ error: "Failed to process bib files." }) + } + } +} diff --git a/services/references/config/settings.defaults.cjs b/services/references/config/settings.defaults.cjs new file mode 100644 index 0000000000..2551f99f09 --- /dev/null +++ b/services/references/config/settings.defaults.cjs @@ -0,0 +1,9 @@ +module.exports = { + internal: { + references: { + port: 3056, + host: process.env.REFERENCES_HOST || '127.0.0.1', + }, + }, +} + diff --git a/services/references/package.json b/services/references/package.json new file mode 100644 index 0000000000..11be2ab68c --- /dev/null +++ b/services/references/package.json @@ -0,0 +1,26 @@ +{ + "name": "@overleaf/references", + "description": "An API for providing citation-keys", + "private": true, + "type": "module", + "main": "app.js", + "scripts": { + "start": "node app.js" + }, + "version": "0.1.0", + "dependencies": { + "@overleaf/settings": "*", + "@overleaf/logger": "*", + "@overleaf/metrics": "*", + "async": "^3.2.5", + "express": "^4.21.0" + }, + "devDependencies": { + "chai": "^4.3.6", + "chai-as-promised": "^7.1.1", + "esmock": "^2.6.9", + "mocha": "^10.2.0", + "sinon": "^9.2.4", + "typescript": "^5.0.4" + } +} diff --git a/services/web/config/settings.defaults.js b/services/web/config/settings.defaults.js index ab223daa88..3c23e40d77 100644 --- a/services/web/config/settings.defaults.js +++ b/services/web/config/settings.defaults.js @@ -259,6 +259,9 @@ module.exports = { notifications: { url: `http://${process.env.NOTIFICATIONS_HOST || '127.0.0.1'}:3042`, }, + references: { + url: `http://${process.env.REFERENCES_HOST || '127.0.0.1'}:3056`, + }, webpack: { url: `http://${process.env.WEBPACK_HOST || '127.0.0.1'}:3808`, },