Files
overleaf-cep/tools/grammar-llm-container/test/server.test.mjs
T

738 lines
25 KiB
JavaScript

// HTTP API tests with a deterministic fake engine (no LLM, no network).
// Mirrors how Overleaf's LanguageToolController talks to a LT-compatible box:
// GET /v2/languages
// POST /v2/check (x-www-form-urlencoded: language, level, data/annotation,
// disabledRules — see LanguageToolController.mjs)
import test from 'node:test';
import assert from 'node:assert/strict';
import { LtServer } from '../dist/server.mjs';
import { CheckEngine } from '../dist/server.mjs';
import { loadConfig } from '../dist/config.mjs';
import http from 'node:http';
function makeConfig(over = {}) {
const env = {
PORT: '8010',
HOST: '127.0.0.1',
GRAMMAR_LLM_MODEL_DIR: '/tmp',
GRAMMAR_LLM_PRELOAD: 'false',
GRAMMAR_LLM_LOG_LEVEL: 'warn',
...over,
};
return loadConfig(env);
}
/** Deterministic "engine": corrects the literal word 'teh' → 'the' wherever it
* appears, and inserts a comma after 'quick'. */
class FakeEngine {
state = 'ready';
statusError = null;
checkCalls = 0;
lastLevel = null;
constructor() {}
status() {
return {
state: this.state,
errorMessage: this.statusError,
modelPath: '/tmp/fake.gguf',
checksCompleted: 0,
checksFailed: 0,
};
}
async ensureReady() {
if (this.state !== 'ready') throw new Error('model not ready');
}
async check(runs, opts = {}) {
if (this.state !== 'ready') throw new Error('not ready');
this.checkCalls++;
this.lastLevel = opts.level ?? null;
const results = [];
for (const run of runs) {
const out = { suggestions: [], correctedText: run.text, incomplete: false };
const t = run.text;
const idx = t.indexOf('teh');
if (idx >= 0) {
const corrected = t.slice(0, idx) + 'the' + t.slice(idx + 3);
out.correctedText = corrected;
out.suggestions = [
{
original: t,
corrected,
sentence: 'Sentence 1',
start_index: 0,
end_index: t.length,
original_highlighted: '',
corrected_highlighted: '',
wordFixes: [{ original: 'teh', corrected: 'the', start: idx, end: idx + 3 }],
},
];
}
const kidx = t.indexOf('quick');
if (kidx >= 0 && out.suggestions.length === 0) {
// insertion after 'quick' (pos = kidx+5)
const pos = kidx + 5;
const corrected = t.slice(0, pos) + ',' + t.slice(pos);
out.correctedText = corrected;
out.suggestions = [
{
original: t,
corrected,
sentence: 'Sentence 1',
start_index: 0,
end_index: t.length,
original_highlighted: '',
corrected_highlighted: '',
wordFixes: [{ original: '', corrected: ',', start: pos, end: pos }],
},
];
}
results.push(out);
}
return { results, incomplete: false };
}
}
async function boot(_over = {}, engineOverride, cfgOver = {}) {
const cfg = makeConfig(cfgOver);
const engine = engineOverride ?? new FakeEngine();
const server = new LtServer({ cfg, engine });
await server.listen(0, '127.0.0.1');
const port = server.httpServer.address().port;
return { cfg, engine, server, base: `http://127.0.0.1:${port}` };
}
async function close(server) {
await server.close();
}
test('GET /v2/languages: LT shape (Overleaf reads name+code)', async (t) => {
const { server, base } = await boot();
try {
const res = await fetch(`${base}/v2/languages`);
assert.equal(res.status, 200);
const body = (await res.json());
assert.ok(Array.isArray(body));
assert.ok(body.some((l) => l.code === 'en-US'));
} finally {
await close(server);
}
});
test('POST /v2/check (Overleaf style: form + annotation data)', async (t) => {
const { server, base } = await boot();
try {
const source = 'He wrote \\textbf{report} about teh project.';
const annotation = [
{ text: 'He wrote ' },
{ markup: '\\textbf{report} ' },
{ text: 'about teh project.' },
];
const params = new URLSearchParams();
params.set('language', 'auto');
params.set('level', 'picky');
params.set('data', JSON.stringify({ annotation }));
params.set('disabledRules', 'WHITESPACE_RULE,CONSECUTIVE_SPACES');
const res = await fetch(`${base}/v2/check`, {
method: 'POST',
headers: { 'Content-Type': 'application/x-www-form-urlencoded', Accept: 'application/json' },
body: params.toString(),
});
assert.equal(res.status, 200);
const body = (await res.json());
assert.equal(body.software.name, 'LanguageTool');
assert.equal(body.software.apiVersion, 1);
assert.equal(body.warnings.incompleteResults, false);
assert.equal(body.language.code, 'en-US');
// 'teh' is at absolute offset: len('He wrote ') + len('\\textbf{report} ')
const baseOffset = 'He wrote '.length + '\\textbf{report} '.length;
assert.equal(body.matches.length, 1);
const m = body.matches[0];
assert.equal(m.offset, baseOffset + 'about '.length);
assert.equal(m.length, 3);
assert.deepEqual(m.replacements, [{ value: 'the' }]);
assert.equal(m.rule.category.id, 'GRAMMAR');
assert.ok(m.context.text.includes('teh'));
// overleaf invariant: offset within doc
assert.ok(m.offset + m.length <= source.length);
} finally {
await close(server);
}
});
test('POST /v2/check with JSON body', async (t) => {
const { server, base } = await boot();
try {
const res = await fetch(`${base}/v2/check`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ language: 'en-GB', level: 'default', text: 'I has teh book.' }),
});
assert.equal(res.status, 200);
const body = (await res.json());
assert.equal(body.language.code, 'en-GB');
assert.equal(body.matches.length, 1);
assert.equal(body.matches[0].offset, 6); // 'has ' before 'teh' → 'I has teh' → t@6
assert.deepEqual(body.matches[0].replacements, [{ value: 'the' }]);
} finally {
await close(server);
}
});
test('POST /v2/check: no text and no data → 400', async (t) => {
const { server, base } = await boot();
try {
const res = await fetch(`${base}/v2/check`, {
method: 'POST',
headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
body: 'language=en',
});
assert.equal(res.status, 400);
const body = (await res.json());
assert.match(body.message, /Missing/);
} finally {
await close(server);
}
});
test('POST /v2/check: too-long text → 413', async (t) => {
const { server, base } = await boot({}, undefined, { GRAMMAR_LLM_MAX_TEXT_LENGTH: '3000' });
try {
const res = await fetch(`${base}/v2/check`, {
method: 'POST',
headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
body: 'language=en&text=' + encodeURIComponent('a'.repeat(3500)),
});
assert.equal(res.status, 413);
} finally {
await close(server);
}
});
test('POST /v2/check: unsupported language → empty matches (default mode)', async (t) => {
const { server, base } = await boot();
try {
const res = await fetch(`${base}/v2/check`, {
method: 'POST',
headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
body: 'language=de&text=' + encodeURIComponent('Ich habe teh Buch.'),
});
assert.equal(res.status, 200);
const body = (await res.json());
assert.deepEqual(body.matches, []);
} finally {
await close(server);
}
});
test('POST /v2/check: unsupported language → 400 (error mode)', async (t) => {
const { server, base } = await boot({}, undefined, { GRAMMAR_LLM_UNSUPPORTED_LANGUAGE: 'error' });
try {
const res = await fetch(`${base}/v2/check`, {
method: 'POST',
headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
body: 'language=de&text=' + encodeURIComponent('Ich habe teh Buch.'),
});
assert.equal(res.status, 400);
} finally {
await close(server);
}
});
test('POST /v2/check: engine not ready → 503 with message', async (t) => {
const engine = new FakeEngine();
engine.state = 'loading';
const { server, base } = await boot({}, engine);
try {
const res = await fetch(`${base}/v2/check`, {
method: 'POST',
headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
body: 'language=en&text=' + encodeURIComponent('I has teh book.'),
});
assert.equal(res.status, 503);
const body = (await res.json());
assert.match(body.message, /not ready/);
} finally {
await close(server);
}
});
test('GET /v2/health reflects engine state', async (t) => {
const { server, base } = await boot();
try {
const res = await fetch(`${base}/v2/health`);
assert.equal(res.status, 200);
const body = (await res.json());
assert.equal(body.status, 'ok');
assert.equal(body.state, 'ready');
} finally {
await close(server);
}
});
test('GET /v2/health: not ready → 503', async (t) => {
const engine = new FakeEngine();
engine.state = 'error';
engine.statusError = 'boom';
const { server, base } = await boot({}, engine);
try {
const res = await fetch(`${base}/v2/health`);
assert.equal(res.status, 503);
const body = (await res.json());
assert.equal(body.state, 'error');
} finally {
await close(server);
}
});
test('GET /v2/version and / info payloads', async (t) => {
const { server, base } = await boot();
try {
const v = (await (await fetch(`${base}/v2/version`)).json());
assert.equal(v.name, 'LanguageTool');
assert.ok(v.engine);
const i = (await (await fetch(`${base}/`)).json());
assert.ok(Array.isArray(i.endpoints));
} finally {
await close(server);
}
});
test('unknown /v2 endpoint → 404 JSON', async (t) => {
const { server, base } = await boot();
try {
const res = await fetch(`${base}/v2/nope`);
assert.equal(res.status, 404);
const body = (await res.json());
assert.match(body.message, /unknown v2 endpoint/);
} finally {
await close(server);
}
});
test('OPTIONS preflight → CORS headers', async (t) => {
const { server, base } = await boot();
try {
const res = await fetch(`${base}/v2/check`, { method: 'OPTIONS' });
assert.ok([200, 204].includes(res.status));
assert.ok((res.headers.get('access-control-allow-origin') ?? '').includes('*'));
} finally {
await close(server);
}
});
test('responses carry CORS + no-store headers (browser client compat)', async (t) => {
const { server, base } = await boot();
try {
const res = await fetch(`${base}/v2/languages`);
assert.ok((res.headers.get('access-control-allow-origin') ?? '').includes('*'));
assert.equal(res.headers.get('cache-control'), 'no-store');
assert.match(res.headers.get('content-type') ?? '', /application\/json/);
} finally {
await close(server);
}
});
// ---------------------------------------------------------------------------
// New: LT-first cascade, LLM level modes, OpenAI-compatible remote backend
// ---------------------------------------------------------------------------
/** In-process fake LanguageTool (v2 /check). Optional {delayMs} to simulate a slow LT. */
function fakeLt(handler, { delayMs = 0 } = {}) {
const seen = { bodies: [] };
const server = http.createServer((req, res) => {
let raw = '';
req.on('data', (c) => (raw += c));
req.on('end', () => {
const body = new URLSearchParams(raw);
seen.bodies.push(Object.fromEntries(body.entries()));
const done = () => {
const out = handler(body, seen.bodies.length - 1) ?? { matches: [] };
res.writeHead(200, { 'Content-Type': 'application/json' });
res.end(JSON.stringify(out));
};
if (delayMs > 0) setTimeout(done, delayMs);
else done();
});
});
return new Promise((resolve) => {
server.listen(0, '127.0.0.1', () => {
const port = server.address().port;
resolve({
url: `http://127.0.0.1:${port}`,
seen,
close: () => new Promise((r) => server.close(() => r())),
});
});
});
}
/** In-process fake OpenAI-compatible chat/completions endpoint. */
function fakeOpenAi(corrections = {}) {
const seen = { requests: [] };
const server = http.createServer((req, res) => {
if (req.method === 'GET' && req.url?.endsWith('/models')) {
res.writeHead(200, { 'Content-Type': 'application/json' });
res.end('{"data":[{"id":"fake"}]}');
return;
}
let raw = '';
req.on('data', (c) => (raw += c));
req.on('end', async () => {
const json = JSON.parse(raw || '{}');
seen.requests.push(json);
const user = json.messages?.[1]?.content ?? '';
const corrected = corrections[user] ?? user;
res.writeHead(200, { 'Content-Type': 'application/json' });
res.end(JSON.stringify({
choices: [{ message: { content: corrected } }],
}));
});
});
return new Promise((resolve) => {
server.listen(0, '127.0.0.1', () => {
const port = server.address().port;
resolve({
baseUrl: `http://127.0.0.1:${port}/v1`,
seen,
close: () => new Promise((r) => server.close(() => r())),
});
});
});
}
test('cascade: real LT produces matches → LT wins, LLM skipped, level forwarded', async () => {
const lt = await fakeLt(() => ({
software: { name: 'LanguageTool', version: '6.4' },
warnings: { incompleteResults: false },
matches: [
{
message: 'FROM-REAL-LT',
shortMessage: 'LT-match',
replacements: [{ value: 'the' }],
offset: 2,
length: 3,
context: { text: 'I teh has', offset: 0, length: 9 },
type: { typeName: 'Typographical' },
rule: { id: 'RULE_LT', description: 'x', issueType: 'Typo', category: { id: 'TYPOS', name: 'Typos' } },
},
],
}));
try {
const engine = new FakeEngine();
const { server, base } = await boot(
{},
engine,
{ GRAMMAR_LLM_LANGUAGETOOL_URL: lt.url, GRAMMAR_LLM_LANGUAGETOOL_TIMEOUT_MS: '2000' },
);
try {
const res = await fetch(`${base}/v2/check`, {
method: 'POST',
headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
body: 'language=en&level=picky&text=' + encodeURIComponent('I teh has cats'),
});
assert.equal(res.status, 200);
const body = await res.json();
assert.equal(body.matches.length, 1);
assert.equal(body.matches[0].message, 'FROM-REAL-LT');
assert.equal(body.software.version, '6.4');
assert.equal(engine.checkCalls, 0, 'LLM must be skipped when LT has matches');
// the picky flag must reach the real LT verbatim:
assert.equal(lt.seen.bodies[0].level, 'picky');
assert.equal(lt.seen.bodies[0].language, 'en');
} finally {
await close(server);
}
} finally {
await lt.close();
}
});
test('cascade: real LT finds nothing → LLM runs second (fallback)', async () => {
const lt = await fakeLt(() => ({ matches: [] }));
try {
const engine = new FakeEngine();
const { server, base } = await boot(
{},
engine,
{ GRAMMAR_LLM_LANGUAGETOOL_URL: lt.url, GRAMMAR_LLM_LANGUAGETOOL_TIMEOUT_MS: '2000' },
);
try {
const res = await fetch(`${base}/v2/check`, {
method: 'POST',
headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
body: 'language=en&level=default&text=' + encodeURIComponent('I teh has cats'),
});
assert.equal(res.status, 200);
const body = await res.json();
assert.equal(body.matches.length, 1, 'fallback LLM match expected');
assert.match(body.matches[0].message, /the/);
assert.equal(engine.checkCalls, 1, 'LLM must run exactly once as fallback');
assert.equal(lt.seen.bodies[0].level, 'default');
} finally {
await close(server);
}
} finally {
await lt.close();
}
});
test('cascade: real LT down → LLM fallback still returns 200', async () => {
const engine = new FakeEngine();
const { server, base } = await boot(
{},
engine,
{
GRAMMAR_LLM_LANGUAGETOOL_URL: 'http://127.0.0.1:9', // closed port
GRAMMAR_LLM_LANGUAGETOOL_TIMEOUT_MS: '800',
},
);
try {
const res = await fetch(`${base}/v2/check`, {
method: 'POST',
headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
body: 'language=en&text=' + encodeURIComponent('I teh has cats'),
});
assert.equal(res.status, 200);
const body = await res.json();
assert.equal(body.matches.length, 1);
assert.equal(engine.checkCalls, 1);
} finally {
await close(server);
}
});
test('cascade race: slow empty LT does NOT block the LLM (the 71 s bug)', async () => {
// The real-world bug: LT took (or was given) a long time and answered "0
// matches"; the LLM then ran AFTER it, pushing the total past Overleaf's
// 60 s controller timeout → the UI silently showed nothing.
// With the shared-budget concurrent cascade the LLM must be served as soon
// as IT is ready, even while the LT is still running.
const lt = await fakeLt(() => ({ matches: [] }), { delayMs: 2000 });
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
const engine = new FakeEngine();
const origCheck = engine.check.bind(engine);
// make the LLM take ~300 ms so it settles well before the 2 s LT
engine.check = async (runs, opts = {}) => {
await sleep(300);
return origCheck(runs, opts);
};
const { server, base } = await boot(
{},
engine,
{
GRAMMAR_LLM_LANGUAGETOOL_URL: lt.url,
GRAMMAR_LLM_LANGUAGETOOL_TIMEOUT_MS: '4000',
GRAMMAR_LLM_CASCADE_BUDGET_MS: '6000',
GRAMMAR_LLM_LLM_GRACE_MS: '50', // LLM starts quickly, as in production
},
);
try {
const t0 = Date.now();
const res = await fetch(`${base}/v2/check`, {
method: 'POST',
headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
body: 'language=en&text=' + encodeURIComponent('I teh has cats'),
});
const elapsed = Date.now() - t0;
const body = await res.json();
assert.equal(res.status, 200);
assert.ok(body.matches.length >= 1, 'LLM fallback match must be present');
assert.ok(
elapsed < 2000,
`LLM result must be served before the slow empty LT settles (elapsed ${elapsed} ms; LT was 2000 ms)`,
);
assert.equal(engine.checkCalls, 1, 'the LLM ran');
} finally {
await close(server);
await lt.close();
}
});
test('cascade race: LT with matches still wins even when slower than the LLM grace', async () => {
const lt = await fakeLt(() => ({
software: { name: 'LanguageTool', version: '6.4' },
warnings: { incompleteResults: false },
matches: [
{
message: 'FROM-REAL-LT',
replacements: [{ value: 'the' }],
offset: 2,
length: 3,
context: { text: 'I teh has', offset: 0, length: 9 },
type: { typeName: 'Typographical' },
rule: { id: 'RULE_LT', category: { id: 'TYPOS', name: 'Typos' } },
},
],
}), { delayMs: 700 });
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
const engine = new FakeEngine();
const origCheck = engine.check.bind(engine);
engine.check = async (runs, opts = {}) => {
await sleep(200); // LLM would "finish" first — LT (700 ms) must still win
return origCheck(runs, opts);
};
const { server, base } = await boot(
{},
engine,
{
GRAMMAR_LLM_LANGUAGETOOL_URL: lt.url,
GRAMMAR_LLM_LANGUAGETOOL_TIMEOUT_MS: '4000',
GRAMMAR_LLM_CASCADE_BUDGET_MS: '6000',
GRAMMAR_LLM_LLM_GRACE_MS: '50',
},
);
try {
const res = await fetch(`${base}/v2/check`, {
method: 'POST',
headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
body: 'language=en&level=picky&text=' + encodeURIComponent('I teh has cats'),
});
const body = await res.json();
assert.equal(res.status, 200);
assert.equal(body.matches.length, 1);
assert.equal(body.matches[0].message, 'FROM-REAL-LT', 'the LT match must win');
} finally {
await close(server);
await lt.close();
}
});
test('level: absent/invalid → cfg.defaultLevel; explicit respected (LLM receives it)', async () => {
const mk = (env) => {
const engine = new FakeEngine();
return { engine, env: { ...env } };
};
// absent level
{
const { engine } = mk({});
const { server, base } = await boot({}, engine, { GRAMMAR_LLM_DEFAULT_LEVEL: 'picky' });
await fetch(`${base}/v2/check`, {
method: 'POST',
headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
body: 'language=en&text=' + encodeURIComponent('I teh has'),
});
assert.equal(engine.lastLevel, 'picky');
await close(server);
}
// invalid level → falls back to defaultLevel
{
const { engine } = mk({});
const { server, base } = await boot({}, engine, { GRAMMAR_LLM_DEFAULT_LEVEL: 'picky' });
await fetch(`${base}/v2/check`, {
method: 'POST',
headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
body: 'language=en&level=bogus&text=' + encodeURIComponent('I teh has'),
});
assert.equal(engine.lastLevel, 'picky');
await close(server);
}
// explicit picky
{
const { engine } = mk({});
const { server, base } = await boot({}, engine, {});
await fetch(`${base}/v2/check`, {
method: 'POST',
headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
body: 'language=en&level=picky&text=' + encodeURIComponent('I teh has'),
});
assert.equal(engine.lastLevel, 'picky');
await close(server);
}
// explicit default
{
const { engine } = mk({});
const { server, base } = await boot({}, engine, {});
await fetch(`${base}/v2/check`, {
method: 'POST',
headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
body: 'language=en&level=default&text=' + encodeURIComponent('I teh has'),
});
assert.equal(engine.lastLevel, 'default');
await close(server);
}
void mk;
});
test('openai backend: real engine pipeline against a fake OpenAI endpoint (default level prompt)', async () => {
const fake = await fakeOpenAi({ 'He run fast to school.': 'He ran fast to school.' });
try {
const cfg = makeConfig({
GRAMMAR_LLM_BACKEND: 'openai',
GRAMMAR_LLM_OPENAI_MODEL: 'fake-model-123',
GRAMMAR_LLM_OPENAI_BASE_URL: fake.baseUrl,
GRAMMAR_LLM_OPENAI_TIMEOUT_MS: '5000',
GRAMMAR_LLM_DEFAULT_LEVEL: 'default',
});
const log = { debug: () => {}, info: () => {}, warn: () => {}, error: () => {} };
const engine = new CheckEngine(cfg, log);
const server = new LtServer({ cfg, engine, log });
await server.listen(0, '127.0.0.1');
const base = `http://127.0.0.1:${server.httpServer.address().port}`;
try {
const res = await fetch(`${base}/v2/check`, {
method: 'POST',
headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
body: 'language=en&text=' + encodeURIComponent('He run fast to school.'),
});
assert.equal(res.status, 200);
const body = await res.json();
assert.equal(body.matches.length, 1, 'core diff must produce the ran-fix');
assert.equal(body.matches[0].replacements[0].value, 'ran');
// request shape: model + system prompt
const req = fake.seen.requests[0];
assert.equal(req.model, 'fake-model-123');
assert.equal(req.messages[0].role, 'system');
assert.match(req.messages[0].content, /grammar correction assistant/);
assert.doesNotMatch(req.messages[0].content, /highly picky/);
} finally {
await server.close();
}
} finally {
await fake.close();
}
});
test('openai backend: picky level gets the stricter prompt; default does not', async () => {
const fake = await fakeOpenAi({ 'He run fast to school.': 'He ran fast to school.' });
try {
const mk = (defaultLevel) => {
const cfg = makeConfig({
GRAMMAR_LLM_BACKEND: 'openai',
GRAMMAR_LLM_OPENAI_MODEL: 'fake-model-123',
GRAMMAR_LLM_OPENAI_BASE_URL: fake.baseUrl,
GRAMMAR_LLM_OPENAI_TIMEOUT_MS: '5000',
GRAMMAR_LLM_DEFAULT_LEVEL: defaultLevel,
});
const log = { debug: () => {}, info: () => {}, warn: () => {}, error: () => {} };
const engine = new CheckEngine(cfg, log);
return new LtServer({ cfg, engine, log });
};
// explicit picky
{
const server = mk('default');
await server.listen(0, '127.0.0.1');
const base = `http://127.0.0.1:${server.httpServer.address().port}`;
await fetch(`${base}/v2/check`, {
method: 'POST',
headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
body: 'language=en&level=picky&text=' + encodeURIComponent('He run fast to school.'),
});
await server.close();
}
const pickyReq = fake.seen.requests[fake.seen.requests.length - 1];
assert.match(pickyReq.messages[0].content, /highly picky/);
assert.match(pickyReq.messages[0].content, /grammar correction assistant/);
} finally {
await fake.close();
}
});