diff --git a/package-lock.json b/package-lock.json index 0e68e69fcd..ba453cb422 100644 --- a/package-lock.json +++ b/package-lock.json @@ -27,7 +27,6 @@ "services/notifications", "services/project-history", "services/real-time", - "services/references", "services/templates", "services/third-party-datastore", "services/third-party-references", @@ -16084,10 +16083,6 @@ "resolved": "libraries/redis-wrapper", "link": true }, - "node_modules/@overleaf/references": { - "resolved": "services/references", - "link": true - }, "node_modules/@overleaf/saas-e2e": { "resolved": "tools/saas-e2e", "link": true @@ -59877,33 +59872,6 @@ "url": "https://github.com/sponsors/eemeli" } }, - "services/references": { - "name": "@overleaf/references", - "dependencies": { - "@overleaf/fetch-utils": "*", - "@overleaf/logger": "*", - "@overleaf/metrics": "*", - "@overleaf/o-error": "*", - "@overleaf/redis-wrapper": "*", - "@overleaf/settings": "*", - "async": "^3.2.5", - "bunyan": "^1.8.15", - "express": "4.22.1", - "ioredis": "^4.16.1", - "lodash": "^4.17.19" - }, - "devDependencies": { - "chai": "^4.3.6", - "chai-as-promised": "^7.1.1", - "esmock": "^2.6.9", - "mocha": "^11.1.0", - "mocha-junit-reporter": "^2.2.1", - "mocha-multi-reporters": "^1.5.1", - "mongodb": "6.12.0", - "sinon": "^9.2.4", - "typescript": "^5.0.4" - } - }, "services/templates": { "name": "@overleaf/templates", "dependencies": { diff --git a/package.json b/package.json index a876d88648..49c67c921d 100644 --- a/package.json +++ b/package.json @@ -70,7 +70,6 @@ "services/notifications", "services/project-history", "services/real-time", - "services/references", "services/templates", "services/third-party-datastore", "services/third-party-references", diff --git a/server-ce/config/settings.js b/server-ce/config/settings.js index 640f1da0ce..01851043dd 100644 --- a/server-ce/config/settings.js +++ b/server-ce/config/settings.js @@ -140,7 +140,6 @@ const settings = { api: redisConfig, pubsub: redisConfig, project_history: redisConfig, - references: redisConfig, project_history_migration: { host: redisConfig.host, @@ -293,7 +292,6 @@ const settings = { ), }, }, - references: {}, notifications: undefined, defaultFeatures: { @@ -426,14 +424,6 @@ if ( } } -// /References -// ----------- -if (process.env.OVERLEAF_ELASTICSEARCH_URL != null) { - settings.references.elasticsearch = { - host: process.env.OVERLEAF_ELASTICSEARCH_URL, - } -} - // filestore switch (process.env.OVERLEAF_FILESTORE_BACKEND) { case 's3': diff --git a/services/web/app/src/infrastructure/Features.mjs b/services/web/app/src/infrastructure/Features.mjs index f887457cce..229312f38c 100644 --- a/services/web/app/src/infrastructure/Features.mjs +++ b/services/web/app/src/infrastructure/Features.mjs @@ -72,8 +72,6 @@ const Features = { case 'affiliations': case 'analytics': return Boolean(_.get(Settings, ['apis', 'v1', 'url'])) - case 'references': - return Boolean(_.get(Settings, ['apis', 'references', 'url'])) case 'saml': return Boolean(Settings.enableSaml) case 'linked-project-file': diff --git a/services/web/frontend/js/features/ide-react/references/bib2json.js b/services/web/frontend/js/features/ide-react/references/bib2json.js index b105ff3f7e..eaa6bcff15 100644 --- a/services/web/frontend/js/features/ide-react/references/bib2json.js +++ b/services/web/frontend/js/features/ide-react/references/bib2json.js @@ -59,6 +59,9 @@ function BibtexParser(arg0, allowedKeys) { this.ALLOWEDKEYS_ = allowedKeys || [] this.reset_(arg0) this.initMacros_() + // Force parsing of all entries, even if they arent in our list of ENTRY_TYPES since new types are being created, + // and users can create their own custom type anyways + this.FORCE_PARSING = true return this } @@ -131,6 +134,9 @@ BibtexParser.prototype.reset_ = function (arg0) { mvproceedings: 39, mvreference: 40, online: 41, + webpage: 41, + electronic: 41, + www: 41, patent: 42, performance: 43, reference: 44, @@ -335,7 +341,7 @@ BibtexParser.prototype.processCharacter_ = function (c) { if (ot == 'preamble') { this.STATE_ = this.STATES_.ENTRY_OR_JUNK } else { - if (ot in this.ENTRY_TYPES_) { + if (ot in this.ENTRY_TYPES_ || this.FORCE_PARSING) { // SUCCESS: Parsed a valid object type. // NEXT_STATE: ENTRY_KEY this.DATA_.ObjectType = 'entry' diff --git a/services/web/frontend/js/features/ide-react/references/types.ts b/services/web/frontend/js/features/ide-react/references/types.ts index ace908ab2a..96c47a1367 100644 --- a/services/web/frontend/js/features/ide-react/references/types.ts +++ b/services/web/frontend/js/features/ide-react/references/types.ts @@ -6,7 +6,7 @@ export type Bib2JsonEntry = { journal: string title: string year: string - } + } & Record } export type AdvancedReferenceSearchResult = { diff --git a/services/web/test/frontend/features/ide-react/unit/references/bib2json.spec.ts b/services/web/test/frontend/features/ide-react/unit/references/bib2json.spec.ts new file mode 100644 index 0000000000..411061748b --- /dev/null +++ b/services/web/test/frontend/features/ide-react/unit/references/bib2json.spec.ts @@ -0,0 +1,201 @@ +import { Bib2JsonEntry } from '@/features/ide-react/references/types' +import BibtexParserImport from '../../../../../../frontend/js/features/ide-react/references/bib2json' +import { expect } from 'chai' + +const BibtexParser = BibtexParserImport as unknown as ( + content: string, + allowedKeys?: string[] +) => { + entries: Bib2JsonEntry[] + errors: any[] +} + +describe('Bib2JsonTests', function () { + describe('Upstream', function () { + // pulled in from the bib2json repository + // https://github.com/mayanklahiri/bib2json/blob/3d7c1f0d738c07d0e1c9a59f4cb9b96c74deb744/test/spec/ParserSpec.js + // Author: Mayank Lahiri + // License: BSD-2-Clause + it('Parse an empty string without errors or results', function () { + const result = BibtexParser(' \t\t\n \n\n') + expect(result.entries.length).to.equal(0) + expect(result.errors.length).to.equal(0) + }) + + it('Parse braces within braces', function () { + const result = BibtexParser( + '@book { pollock, title={{{A}} very {Big} Book.} }' + ) + expect(result.entries.length).to.equal(1) + expect(result.errors.length).to.equal(0) + expect(result.entries[0].Fields.title).to.equal('A very Big Book.') + }) + + it('Parse braces within quotes', function () { + const result = BibtexParser( + '@book { pollock, title="{A} very {Big} Book."}' + ) + expect(result.entries.length).to.equal(1) + expect(result.errors.length).to.equal(0) + expect(result.entries[0].Fields.title).to.equal('A very Big Book.') + }) + + it('Parse quotes within braces', function () { + const result = BibtexParser( + '@book { pollock, title="{A} very {"Big"} Book."}' + ) + expect(result.entries.length).to.equal(1) + expect(result.errors.length).to.equal(0) + expect(result.entries[0].Fields.title).to.equal('A very "Big" Book.') + }) + + it('Respect backslashes', function () { + const text = '@book { pollock, title="{A} \\\\very \\{{Big} \\"Book\\"."}' + const result = BibtexParser(text) + expect(result.entries.length).to.equal(1) + expect(result.errors.length).to.equal(0) + expect(result.entries[0].Fields.title).to.equal('A \\very {Big "Book".') + }) + + it('Convert some Latex characters to UTF-8', function () { + const result = BibtexParser( + '@book { pollock, title="\\"{o}\\AA \\^{I}\\alpha " }' + ) + expect(result.entries.length).to.equal(1) + expect(result.errors.length).to.equal(0) + expect(result.entries[0].Fields.title).to.equal( + '\u00f6\u00c5\u00ce\u03b1' + ) + }) + + it('Expand a predefined macro in the middle an entry', function () { + const text = '@book{ pollock, month = jan, title="A title!" }' + const result = BibtexParser(text) + expect(result.entries.length).to.equal(1) + expect(result.errors.length).to.equal(0) + expect(result.entries[0].Fields.month).to.equal('January') + }) + + return it('Expand a macro at the end of an entry', function () { + const text = + '@string \n{ howdy = "well, hello!" }@book{ pollock, title=howdy }' + const result = BibtexParser(text) + expect(result.entries.length).to.equal(1) + expect(result.errors.length).to.equal(0) + expect(result.entries[0].Fields.title).to.equal('well, hello!') + }) + }) + + describe('normalize keys to lower case', function () { + it('should lower the case of RaNDoM', function () { + const text = '@book{ id, RaNDoM = "VALUE" }' + const result = BibtexParser(text) + expect(result.entries.length).to.equal(1) + expect(result.errors.length).to.equal(0) + expect(result.entries[0].Fields).to.deep.equal({ random: 'VALUE' }) + }) + + return it('should preserve the marco usage of RaNDoM', function () { + const text = '@string{ RaNDoM="MACRO"}@book{ id, Title = RaNDoM }' + const result = BibtexParser(text) + expect(result.entries.length).to.equal(1) + expect(result.errors.length).to.equal(0) + expect(result.entries[0].Fields).to.deep.equal({ title: 'MACRO' }) + }) + }) + + // sometimes imported bib files contain {\\}, {\\\}, {\\\\} etc to replace whitespace in values + describe('handles backslash braces from imported bibs', function () { + it('handles one backslash {\\}', function () { + const text = + '@book { pollock, title="{\\}{A} \\\\very \\{{Big} \\"Book\\"."}' + const result = BibtexParser(text) + expect(result.entries.length).to.equal(1) + expect(result.errors.length).to.equal(0) + expect(result.entries[0].Fields.title).to.equal('}A \\very {Big "Book".') + }) + it('handles two backslashes {\\\\}', function () { + const text = + '@book { pollock, title="{\\\\}{A} \\\\very \\{{Big} \\"Book\\"."}' + const result = BibtexParser(text) + expect(result.entries.length).to.equal(1) + expect(result.errors.length).to.equal(0) + expect(result.entries[0].Fields.title).to.equal('\\A \\very {Big "Book".') + }) + it('handles three backslashes {\\\\\\}', function () { + const text = + '@book { pollock, title="{\\\\\\}{A} \\\\very \\{{Big} \\"Book\\"."}' + const result = BibtexParser(text) + expect(result.entries.length).to.equal(1) + expect(result.errors.length).to.equal(0) + expect(result.entries[0].Fields.title).to.equal( + '\\}A \\very {Big "Book".' + ) + }) + it('handles four backslashes {\\\\\\\\}', function () { + const text = + '@book { pollock, title="{\\\\\\\\}{A} \\\\very \\{{Big} \\"Book\\"."}' + const result = BibtexParser(text) + expect(result.entries.length).to.equal(1) + expect(result.errors.length).to.equal(0) + expect(result.entries[0].Fields.title).to.equal( + '\\\\A \\very {Big "Book".' + ) + }) + it('keeps parsing future entries after an even amount of backslashes {\\\\}', function () { + const text = + '@book { pollock, title="{\\\\}{A} \\\\very \\{{Big} \\"Book\\"."}' + + '@book { pollock, title="{\\\\}{A} \\\\very \\{{Big} secondary \\"Book\\".", author="Muräkämi, Häruki"}' + const result = BibtexParser(text) + expect(result.entries.length).to.equal(2) + expect(result.errors.length).to.equal(0) + expect(result.entries[0].Fields.title).to.equal('\\A \\very {Big "Book".') + expect(result.entries[1].Fields.title).to.equal( + '\\A \\very {Big secondary "Book".' + ) + expect(result.entries[1].Fields.author).to.equal('Muräkämi, Häruki') + }) + return it('keeps parsing future entries after an odd amount of backslashes {\\\\\\}', function () { + const text = + '@book { pollock, title="{\\\\\\}{A} \\\\very \\{{Big} \\"Book\\"."}' + + '@book { pollock, title="{\\\\}{A} \\\\very \\{{Big} secondary \\"Book\\".", author="Muräkämi, Häruki"}' + const result = BibtexParser(text) + expect(result.entries.length).to.equal(2) + expect(result.errors.length).to.equal(0) + expect(result.entries[0].Fields.title).to.equal( + '\\}A \\very {Big "Book".' + ) + expect(result.entries[1].Fields.title).to.equal( + '\\A \\very {Big secondary "Book".' + ) + expect(result.entries[1].Fields.author).to.equal('Muräkämi, Häruki') + }) + }) + + return describe('with allowedKeys set', function () { + it('should skip the unknown key Random', function () { + const text = '@book{ id, Random = "ABC", title="VALUE" }' + const result = BibtexParser(text, ['title']) + expect(result.entries.length).to.equal(1) + expect(result.errors.length).to.equal(0) + expect(result.entries[0].Fields).to.deep.equal({ title: 'VALUE' }) + }) + + it('should still parse unknown entry types', function () { + const text = '@myCustomType{ id, Random = "ABC", title="VALUE" }' + const result = BibtexParser(text, ['title']) + expect(result.entries.length).to.equal(1) + expect(result.errors.length).to.equal(0) + expect(result.entries[0].Fields).to.deep.equal({ title: 'VALUE' }) + }) + + return it('should preserve the marco usage of RaNDoM', function () { + const text = + '@string{ RaNDoM="MACRO"}@book{ id, Random = "ABC", Title = RaNDoM }' + const result = BibtexParser(text, ['title']) + expect(result.entries.length).to.equal(1) + expect(result.errors.length).to.equal(0) + expect(result.entries[0].Fields).to.deep.equal({ title: 'MACRO' }) + }) + }) +}) diff --git a/services/web/test/unit/src/infrastructure/Features.test.mjs b/services/web/test/unit/src/infrastructure/Features.test.mjs index 0e01b23433..8f0ee593ff 100644 --- a/services/web/test/unit/src/infrastructure/Features.test.mjs +++ b/services/web/test/unit/src/infrastructure/Features.test.mjs @@ -59,7 +59,6 @@ describe('Features', function () { expect(ctx.Features.hasFeature('link-url')).to.be.false expect(ctx.Features.hasFeature('oauth')).to.be.false expect(ctx.Features.hasFeature('saas')).to.be.false - expect(ctx.Features.hasFeature('references')).to.be.false expect(ctx.Features.hasFeature('saml')).to.be.false expect(ctx.Features.hasFeature('templates-server-pro')).to.be.false }) @@ -82,7 +81,6 @@ describe('Features', function () { expect(ctx.Features.hasFeature('homepage')).to.be.false expect(ctx.Features.hasFeature('link-url')).to.be.false expect(ctx.Features.hasFeature('oauth')).to.be.false - expect(ctx.Features.hasFeature('references')).to.be.false expect(ctx.Features.hasFeature('saml')).to.be.false expect(ctx.Features.hasFeature('templates-server-pro')).to.be.false }) @@ -104,7 +102,6 @@ describe('Features', function () { expect(ctx.Features.hasFeature('affiliations')).to.be.true expect(ctx.Features.hasFeature('analytics')).to.be.true expect(ctx.Features.hasFeature('saas')).to.be.true - expect(ctx.Features.hasFeature('references')).to.be.true expect(ctx.Features.hasFeature('registration')).to.be.true }) it('should return false', function (ctx) { @@ -135,7 +132,6 @@ describe('Features', function () { expect(ctx.Features.hasFeature('link-url')).to.be.true expect(ctx.Features.hasFeature('oauth')).to.be.true expect(ctx.Features.hasFeature('saas')).to.be.true - expect(ctx.Features.hasFeature('references')).to.be.true expect(ctx.Features.hasFeature('registration')).to.be.true expect(ctx.Features.hasFeature('saml')).to.be.true })