Enable autocomplete of reference keys feature

develop/dev.env

@@ -13,6 +13,7 @@ NOTIFICATIONS_HOST=notifications
 PROJECT_HISTORY_HOST=project-history
 REALTIME_HOST=real-time
 REDIS_HOST=redis
+REFERENCES_HOST=references
 SPELLING_HOST=spelling
 WEBPACK_HOST=webpack
 WEB_API_PASSWORD=overleaf

develop/docker-compose.dev.yml

@@ -112,6 +112,17 @@ services:
       - ../services/real-time/app.js:/overleaf/services/real-time/app.js
       - ../services/real-time/config:/overleaf/services/real-time/config
 
+  references:
+    command: ["node", "--watch", "app.js"]
+    environment:
+      - NODE_OPTIONS=--inspect=0.0.0.0:9229
+    ports:
+      - "127.0.0.1:9236:9229"
+    volumes:
+      - ../services/references/app:/overleaf/services/references/app
+      - ../services/references/config:/overleaf/services/references/config
+      - ../services/references/app.js:/overleaf/services/references/app.js
+
   web:
     command: ["node", "--watch", "app.js", "--watch-locales"]
     environment:

develop/docker-compose.yml

@@ -124,6 +124,13 @@ services:
     volumes:
       - redis-data:/data
 
+  references:
+    build:
+      context: ..
+      dockerfile: services/references/Dockerfile
+    env_file:
+      - dev.env
+
   spelling:
     build:
       context: ..
@@ -163,6 +170,7 @@ services:
       - notifications
       - project-history
       - real-time
+      - references
       - spelling
 
   webpack:

server-ce/config/env.sh

@@ -9,6 +9,7 @@ export HISTORY_V1_HOST=127.0.0.1
 export NOTIFICATIONS_HOST=127.0.0.1
 export PROJECT_HISTORY_HOST=127.0.0.1
 export REALTIME_HOST=127.0.0.1
+export REFERENCES_HOST=127.0.0.1
 export SPELLING_HOST=127.0.0.1
 export WEB_HOST=127.0.0.1
 export WEB_API_HOST=127.0.0.1

server-ce/runit/references-overleaf/run

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+NODE_PARAMS=""
+if [ "$DEBUG_NODE" == "true" ]; then
+    echo "running debug - references"
+    NODE_PARAMS="--inspect=0.0.0.0:30560"
+fi
+
+source /etc/overleaf/env.sh
+export LISTEN_ADDRESS=127.0.0.1
+
+exec /sbin/setuser www-data /usr/bin/node $NODE_PARAMS /overleaf/services/references/app.js >> /var/log/overleaf/references.log 2>&1

services/references/README.md

@@ -0,0 +1,10 @@
+overleaf/references
+===============
+
+An API for providing citation-keys from user bib-files
+
+License
+=======
+The code in this repository is released under the GNU AFFERO GENERAL PUBLIC LICENSE, version 3.
+
+Based on https://github.com/overleaf/overleaf/commit/9964aebc794f9fd7ce1373ab3484f6b33b061af3

services/references/app.js

@@ -0,0 +1,40 @@
+import '@overleaf/metrics/initialize.js'
+
+import express from 'express'
+import Settings from '@overleaf/settings'
+import logger from '@overleaf/logger'
+import metrics from '@overleaf/metrics'
+import ReferencesAPIController from './app/js/ReferencesAPIController.js'
+import bodyParser from 'body-parser'
+
+const app = express()
+metrics.injectMetricsRoute(app)
+
+app.use(bodyParser.json({ limit: '2mb' }))
+app.use(metrics.http.monitor(logger))
+
+app.post('/project/:project_id/index', ReferencesAPIController.index)
+app.get('/status', (req, res) => res.send({ status: 'references api is up' }))
+
+const settings =
+  Settings.internal && Settings.internal.references
+    ? Settings.internal.references
+    : undefined
+const host = settings && settings.host ? settings.host : 'localhost'
+const port = settings && settings.port ? settings.port : 3056
+
+logger.debug('Listening at', { host, port })
+
+const server = app.listen(port, host, function (error) {
+  if (error) {
+    throw error
+  }
+  logger.info({ host, port }, 'references HTTP server starting up')
+})
+
+process.on('SIGTERM', () => {
+  server.close(() => {
+    logger.info({ host, port }, 'references HTTP server closed')
+    metrics.close()
+  })
+})

services/references/app/js/ReferencesAPIController.js

@@ -0,0 +1,42 @@
+import logger from '@overleaf/logger'
+import BibtexParser from '../../../web/app/src/util/bib2json.js'
+
+export default {
+  async index(req, res) {
+    const { docUrls, fullIndex } = req.body
+    try {
+      const responses = await Promise.all(
+        docUrls.map(async (docUrl) => {
+          try {
+            const response = await fetch(docUrl)
+            if (!response.ok) {
+              throw new Error(`HTTP error! status: ${response.status}`)
+            }
+            return response.text()
+          } catch (error) {
+            logger.error({ error }, "Failed to fetch document from URL: " + docUrl)
+            return null
+          }
+        })
+      )
+      const keys = []
+      for (const body of responses) {
+        if (!body) continue
+
+        try {
+          const parsedEntries = BibtexParser(body).entries
+          const ks = parsedEntries
+            .filter(entry => entry.EntryKey)
+            .map(entry => entry.EntryKey)
+          keys.push(...ks)
+        } catch (error) {
+          logger.error({ error }, "bib file skipped.")
+        }
+      }
+      res.status(200).json({ keys })
+    } catch (error) {
+      logger.error({ error }, "Unexpected error during indexing process.")
+      res.status(500).json({ error: "Failed to process bib files." })
+    }
+  }
+}

services/references/config/settings.defaults.cjs

@@ -0,0 +1,9 @@
+module.exports = {
+  internal: {
+    references: {
+      port: 3056,
+      host: process.env.REFERENCES_HOST || '127.0.0.1',
+    },
+  },
+}
+

services/references/package.json

@@ -0,0 +1,26 @@
+{
+  "name": "@overleaf/references",
+  "description": "An API for providing citation-keys",
+  "private": true,
+  "type": "module",
+  "main": "app.js",
+  "scripts": {
+    "start": "node app.js"
+  },
+  "version": "0.1.0",
+  "dependencies": {
+    "@overleaf/settings": "*",
+    "@overleaf/logger": "*",
+    "@overleaf/metrics": "*",
+    "async": "^3.2.5",
+    "express": "^4.21.0"
+  },
+  "devDependencies": {
+    "chai": "^4.3.6",
+    "chai-as-promised": "^7.1.1",
+    "esmock": "^2.6.9",
+    "mocha": "^10.2.0",
+    "sinon": "^9.2.4",
+    "typescript": "^5.0.4"
+  }
+}

services/web/config/settings.defaults.js

@@ -259,6 +259,9 @@ module.exports = {
     notifications: {
       url: `http://${process.env.NOTIFICATIONS_HOST || '127.0.0.1'}:3042`,
     },
+    references: {
+      url: `http://${process.env.REFERENCES_HOST || '127.0.0.1'}:3056`,
+    },
     webpack: {
       url: `http://${process.env.WEBPACK_HOST || '127.0.0.1'}:3808`,
     },