mirror of
https://github.com/yu-i-i/overleaf-cep.git
synced 2026-11-14 06:03:45 +01:00
Move source editor out of module (#12457)
* Update Copybara options in preparation for open-sourcing the source editor * Move files * Update paths * Remove source-editor module and checks for its existence * Explicitly mention CM6 license in files that contain code adapted from CM6 GitOrigin-RevId: 89b7cc2b409db01ad103198ccbd1b126ab56349b
This commit is contained in:
Tim Down
@@ -0,0 +1,66 @@
|
||||
import { parser } from '../../frontend/js/features/source-editor/lezer-latex/latex.mjs'
|
||||
|
||||
import * as fs from 'fs'
|
||||
import * as path from 'path'
|
||||
import { fileURLToPath } from 'url'
|
||||
import minimist from 'minimist'
|
||||
|
||||
const argv = minimist(process.argv.slice(2))
|
||||
const NUMBER_OF_OPS = argv.ops || 100
|
||||
const CSV_OUTPUT = argv.csv || false
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url))
|
||||
|
||||
const examplesDir = path.join(
|
||||
__dirname,
|
||||
'../../test/unit/src/lezer-latex/examples'
|
||||
)
|
||||
|
||||
const strictParser = parser.configure({ strict: true }) // throw exception for invalid documents
|
||||
|
||||
if (!fs.existsSync(examplesDir)) {
|
||||
console.error('No examples directory')
|
||||
process.exit()
|
||||
}
|
||||
|
||||
function dumpParserStats(parser) {
|
||||
console.log('Parser size:')
|
||||
console.dir({
|
||||
states: parser.states.length,
|
||||
data: parser.data.length,
|
||||
goto: parser.goto.length,
|
||||
})
|
||||
}
|
||||
|
||||
dumpParserStats(strictParser)
|
||||
|
||||
const folder = examplesDir
|
||||
for (const file of fs.readdirSync(folder).sort()) {
|
||||
if (!/\.tex$/.test(file)) continue
|
||||
const name = /^[^.]*/.exec(file)[0]
|
||||
const content = fs.readFileSync(path.join(folder, file), 'utf8')
|
||||
|
||||
benchmark(name, content)
|
||||
}
|
||||
|
||||
function benchmark(name, content) {
|
||||
let timeSum = 0
|
||||
try {
|
||||
for (let i = 0; i < NUMBER_OF_OPS; ++i) {
|
||||
const startTime = performance.now()
|
||||
strictParser.parse(content)
|
||||
const endTime = performance.now()
|
||||
timeSum += endTime - startTime
|
||||
}
|
||||
const avgTime = timeSum / NUMBER_OF_OPS
|
||||
if (CSV_OUTPUT) {
|
||||
console.log(`${name},${avgTime.toFixed(2)},${content.length}`)
|
||||
} else {
|
||||
console.log(
|
||||
`${name.padEnd(20)} time to run (ms):\t ${avgTime.toFixed(2)}`
|
||||
)
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`${name.padEnd(20)} ${error}`)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
const { buildParserFile } = require('@lezer/generator')
|
||||
const { writeFileSync, readFileSync } = require('fs')
|
||||
const path = require('path')
|
||||
|
||||
const options = {
|
||||
grammarPath: path.resolve(
|
||||
__dirname,
|
||||
'../../frontend/js/features/source-editor/lezer-latex/latex.grammar'
|
||||
),
|
||||
parserOutputPath: path.resolve(
|
||||
__dirname,
|
||||
'../../frontend/js/features/source-editor/lezer-latex/latex.mjs'
|
||||
),
|
||||
termsOutputPath: path.resolve(
|
||||
__dirname,
|
||||
'../../frontend/js/features/source-editor/lezer-latex/latex.terms.mjs'
|
||||
),
|
||||
}
|
||||
|
||||
function compile() {
|
||||
const { grammarPath, termsOutputPath, parserOutputPath } = options
|
||||
const moduleStyle = 'es'
|
||||
console.info(`Compiling ${grammarPath}`)
|
||||
|
||||
const grammarText = readFileSync(grammarPath, 'utf8')
|
||||
console.info(`Loaded grammar from ${grammarPath}`)
|
||||
|
||||
const { parser, terms } = buildParserFile(grammarText, {
|
||||
fileName: grammarPath,
|
||||
moduleStyle,
|
||||
})
|
||||
console.info(`Built parser`)
|
||||
|
||||
writeFileSync(parserOutputPath, parser)
|
||||
console.info(`Wrote parser to ${parserOutputPath}`)
|
||||
|
||||
writeFileSync(termsOutputPath, terms)
|
||||
console.info(`Wrote terms to ${termsOutputPath}`)
|
||||
|
||||
console.info('Done!')
|
||||
}
|
||||
|
||||
module.exports = { compile, options }
|
||||
|
||||
if (require.main === module) {
|
||||
try {
|
||||
compile()
|
||||
process.exit(0)
|
||||
} catch (err) {
|
||||
console.error(err)
|
||||
process.exit(1)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
import { readFileSync } from 'fs'
|
||||
import { logTree } from '../../frontend/js/features/source-editor/lezer-latex/print-tree.mjs'
|
||||
import { parser } from '../../frontend/js/features/source-editor/lezer-latex/latex.mjs'
|
||||
|
||||
// Runs the lezer-latex parser on a supplied file, and prints the resulting
|
||||
// parse tree to stdout
|
||||
//
|
||||
// show parse tree: lezer-latex-run.js test/frontend/shared/lezer-latex/examples/amsmath.tex
|
||||
// show error summary: lezer-latex-run.js coverage test/frontend/shared/lezer-latex/examples/amsmath.tex
|
||||
|
||||
let files = process.argv.slice(2)
|
||||
if (!files.length) {
|
||||
files = ['test/unit/src/LezerLatex/examples/demo.tex']
|
||||
}
|
||||
|
||||
let coverage = false
|
||||
if (files[0] === 'coverage') {
|
||||
// count errors
|
||||
coverage = true
|
||||
files.shift()
|
||||
}
|
||||
|
||||
function reportErrorCounts(output) {
|
||||
if (coverage) process.stdout.write(output)
|
||||
}
|
||||
|
||||
function parseFile(filename) {
|
||||
const text = readFileSync(filename).toString()
|
||||
const t0 = process.hrtime()
|
||||
const tree = parser.parse(text)
|
||||
const dt = process.hrtime(t0)
|
||||
const timeTaken = dt[0] + dt[1] * 1e-9
|
||||
let errorCount = 0
|
||||
let nodeCount = 0
|
||||
tree.iterate({
|
||||
enter: syntaxNodeRef => {
|
||||
nodeCount++
|
||||
if (syntaxNodeRef.type.isError) {
|
||||
errorCount++
|
||||
}
|
||||
},
|
||||
})
|
||||
if (!coverage) logTree(tree, text)
|
||||
return { nodeCount, errorCount, timeTaken, bytes: text.length }
|
||||
}
|
||||
|
||||
let totalErrors = 0
|
||||
let totalTime = 0
|
||||
let totalBytes = 0
|
||||
for (const file of files) {
|
||||
const { nodeCount, errorCount, timeTaken, bytes } = parseFile(file)
|
||||
const errorRate = Math.round((100 * errorCount) / nodeCount)
|
||||
totalErrors += errorCount
|
||||
totalTime += timeTaken
|
||||
totalBytes += bytes
|
||||
reportErrorCounts(
|
||||
`${errorCount} errors`.padStart(12) +
|
||||
`${nodeCount} nodes`.padStart(12) +
|
||||
`(${errorRate}%)`.padStart(6) +
|
||||
`${(1000 * timeTaken).toFixed(1)} ms`.padStart(8) +
|
||||
`${(bytes / 1024).toFixed(1)} KB`.padStart(8) +
|
||||
` ${file}\n`
|
||||
)
|
||||
}
|
||||
const timeInMilliseconds = 1000 * totalTime
|
||||
const hundredKBs = totalBytes / (100 * 1024)
|
||||
|
||||
reportErrorCounts(
|
||||
`\ntotal errors ${totalErrors}, performance ${(
|
||||
timeInMilliseconds / hundredKBs
|
||||
).toFixed(1)} ms/100KB \n`
|
||||
)
|
||||
|
||||
if (totalErrors > 0) {
|
||||
process.exit(1) // return non-zero exit status for tests
|
||||
}
|
||||
@@ -0,0 +1,157 @@
|
||||
import { parser } from '../../frontend/js/features/source-editor/lezer-latex/latex.mjs'
|
||||
|
||||
import * as fs from 'fs'
|
||||
import * as path from 'path'
|
||||
import { fileURLToPath } from 'url'
|
||||
import { TreeFragment } from '@lezer/common'
|
||||
import minimist from 'minimist'
|
||||
|
||||
const argv = minimist(process.argv.slice(2))
|
||||
const NUMBER_OF_OPS = argv.ops || 1000
|
||||
const CSV_OUTPUT = argv.csv || false
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url))
|
||||
|
||||
const examplesDir = path.join(
|
||||
__dirname,
|
||||
'../../test/unit/src/lezer-latex/examples'
|
||||
)
|
||||
|
||||
const folder = examplesDir
|
||||
for (const file of fs.readdirSync(folder).sort()) {
|
||||
if (!/\.tex$/.test(file)) continue
|
||||
const name = /^[^.]*/.exec(file)[0]
|
||||
const content = fs.readFileSync(path.join(folder, file), 'utf8')
|
||||
runPerformanceTests(name, content)
|
||||
}
|
||||
|
||||
function runPerformanceTests(name, content) {
|
||||
const insertEnd = writeTextAt(
|
||||
content,
|
||||
content.length,
|
||||
content.substring(0, NUMBER_OF_OPS)
|
||||
)
|
||||
const insertBeginning = writeTextAt(
|
||||
content,
|
||||
0,
|
||||
content.substring(0, NUMBER_OF_OPS)
|
||||
)
|
||||
const insertMiddle = writeTextAt(
|
||||
content,
|
||||
Math.floor(content.length / 2),
|
||||
content.substring(0, NUMBER_OF_OPS)
|
||||
)
|
||||
const randomDelete = randomDeletions(content, NUMBER_OF_OPS)
|
||||
const middleDelete = deletionsFromMiddle(content, NUMBER_OF_OPS)
|
||||
const randomInsert = randomInsertions(content, NUMBER_OF_OPS)
|
||||
|
||||
if (CSV_OUTPUT) {
|
||||
console.log(
|
||||
[
|
||||
name,
|
||||
insertBeginning.average,
|
||||
insertMiddle.average,
|
||||
insertEnd.average,
|
||||
randomInsert.average,
|
||||
randomDelete.average,
|
||||
middleDelete.average,
|
||||
content.length,
|
||||
].join(',')
|
||||
)
|
||||
} else {
|
||||
console.log({
|
||||
name,
|
||||
insertAtEnd: insertEnd.average,
|
||||
insertAtBeginning: insertBeginning.average,
|
||||
insertAtMiddle: insertMiddle.average,
|
||||
randomDelete: randomDelete.average,
|
||||
middleDelete: middleDelete.average,
|
||||
randomInsert: randomInsert.average,
|
||||
docLength: content.length,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
function timedChanges(document, changes, changeFn) {
|
||||
let totalParseTime = 0
|
||||
|
||||
// Do a fresh parse to get TreeFragments
|
||||
const initialTree = parser.parse(document)
|
||||
let fragments = TreeFragment.addTree(initialTree)
|
||||
let currentDoc = document
|
||||
|
||||
for (let i = 0; i < changes; ++i) {
|
||||
const change = changeFn(currentDoc, i)
|
||||
currentDoc = change.text
|
||||
// Do a timed parse
|
||||
const start = performance.now()
|
||||
fragments = TreeFragment.applyChanges(fragments, [change.range])
|
||||
const tree = parser.parse(currentDoc, fragments)
|
||||
fragments = TreeFragment.addTree(tree, fragments)
|
||||
const end = performance.now()
|
||||
totalParseTime += end - start
|
||||
}
|
||||
return {
|
||||
total: totalParseTime,
|
||||
average: totalParseTime / changes,
|
||||
ops: changes,
|
||||
fragments: fragments.length,
|
||||
}
|
||||
}
|
||||
|
||||
// Write and parse after every character insertion
|
||||
function writeTextAt(document, position, text) {
|
||||
return timedChanges(document, text.length, (currentDoc, index) =>
|
||||
insertAt(currentDoc, position + index, text[index])
|
||||
)
|
||||
}
|
||||
|
||||
function randomInsertions(document, num) {
|
||||
return timedChanges(document, num, currentDoc =>
|
||||
insertAt(currentDoc, Math.floor(Math.random() * currentDoc.length), 'a')
|
||||
)
|
||||
}
|
||||
|
||||
function randomDeletions(document, num) {
|
||||
return timedChanges(document, num, currentDoc =>
|
||||
deleteAt(currentDoc, Math.floor(Math.random() * currentDoc.length), 1)
|
||||
)
|
||||
}
|
||||
|
||||
function deletionsFromMiddle(document, num) {
|
||||
const deletionPoint = Math.floor(document.length / 2)
|
||||
const deletions = Math.min(num, deletionPoint - 1)
|
||||
return timedChanges(document, deletions, (currentDoc, index) =>
|
||||
deleteAt(currentDoc, deletionPoint - index, 1)
|
||||
)
|
||||
}
|
||||
|
||||
function insertAt(document, position, text) {
|
||||
const start = document.substring(0, position)
|
||||
const end = document.substring(position)
|
||||
|
||||
return {
|
||||
text: start + text + end,
|
||||
range: {
|
||||
fromA: position,
|
||||
toA: position,
|
||||
fromB: position,
|
||||
toB: position + text.length,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
function deleteAt(document, position, length = 1) {
|
||||
const start = document.substring(0, position)
|
||||
const end = document.substring(position + length)
|
||||
|
||||
return {
|
||||
text: start + end,
|
||||
range: {
|
||||
fromA: position,
|
||||
toA: position + length,
|
||||
fromB: position,
|
||||
toB: position,
|
||||
},
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user