[ContentCacheManager] use PDF.js Xref table instead of stream detection (#242)

* make the content cache manager tests configurable

* extend stream content in unit tests

* [ContentCacheManagerTests] prepare for full object caching

* filesystem stream for pdfjs

* working??

* cleaning up

* handle overflow

* [misc] install pdfjs-dist

* [misc] move pdfjs code into app/lib/ and scripts/, also use CamelCase

* [misc] abstract the file loading and parsing of xRef tables into helper

* [misc] pdfjsTests: add snapshot based tests for the Xref table parser

* [misc] FSStream: throw proper error and drop commented code

* [misc] FSStream: integrate throwing of MissingDataException into getter

* [misc] pdfjs: fix eslint errors

* [misc] pdfjs: run format_fix

* [misc] pdfjs: allocate very small non empty dummy buffers explicitly

* [misc] install @overleaf/o-error

* [ContentCacheManager] use PDF.js Xref table instead of stream detection

Co-Authored-By: Brian Gough <brian.gough@overleaf.com>

* [pdfjs] parseXrefTable: handle empty PDF files gracefully

Co-authored-by: Brian Gough <brian.gough@overleaf.com>
This commit is contained in:
Jakob Ackermann
2021-05-31 09:20:25 +01:00
committed by GitHub
co-authored by Brian Gough
parent d657007291
commit 294088fb27
37 changed files with 3406 additions and 373 deletions
+78 -96
View File
@@ -7,45 +7,97 @@ const fs = require('fs')
const crypto = require('crypto')
const Path = require('path')
const Settings = require('settings-sharelatex')
const OError = require('@overleaf/o-error')
const pLimit = require('p-limit')
const MIN_CHUNK_SIZE = Settings.pdfCachingMinChunkSize
const START_OF_STREAM_MARKER = 'stream'
const END_OF_STREAM_MARKER = 'endstream'
const START_OF_STREAM_MARKER_LENGTH = START_OF_STREAM_MARKER.length
const END_OF_STREAM_MARKER_LENGTH = END_OF_STREAM_MARKER.length
const { parseXrefTable } = require('../lib/pdfjs/parseXrefTable')
/**
*
* @param {String} contentDir path to directory where content hash files are cached
* @param {String} filePath the pdf file to scan for streams
* @param {number} size the pdf size
*/
async function update(contentDir, filePath) {
const stream = fs.createReadStream(filePath)
const extractor = new PdfStreamsExtractor()
async function update(contentDir, filePath, size) {
const ranges = []
const newRanges = []
// keep track of hashes expire old ones when they reach a generation > N.
const tracker = await HashFileTracker.from(contentDir)
tracker.updateAge()
for await (const chunk of stream) {
const pdfStreams = extractor.consume(chunk)
for (const pdfStream of pdfStreams) {
if (pdfStream.end - pdfStream.start < MIN_CHUNK_SIZE) continue
const hash = pdfStreamHash(pdfStream.buffers)
const rawTable = await parseXrefTable(filePath, size)
rawTable.sort((a, b) => {
return a.offset - b.offset
})
rawTable.forEach((obj, idx) => {
obj.idx = idx
})
const range = { start: pdfStream.start, end: pdfStream.end, hash }
const uncompressedObjects = []
for (const object of rawTable) {
if (!object.uncompressed) {
continue
}
const nextObject = rawTable[object.idx + 1]
if (!nextObject) {
// Ignore this possible edge case.
// The last object should be part of the xRef table.
continue
} else {
object.endOffset = nextObject.offset
}
const size = object.endOffset - object.offset
object.size = size
if (size < Settings.pdfCachingMinChunkSize) {
continue
}
uncompressedObjects.push(object)
}
const handle = await fs.promises.open(filePath)
try {
for (const object of uncompressedObjects) {
let buffer = Buffer.alloc(object.size, 0)
const { bytesRead } = await handle.read(
buffer,
0,
object.size,
object.offset
)
if (bytesRead !== object.size) {
throw new OError('could not read full chunk', {
object,
bytesRead
})
}
const idxObj = buffer.indexOf('obj')
if (idxObj > 100) {
throw new OError('objectId is too large', {
object,
idxObj
})
}
const objectIdRaw = buffer.subarray(0, idxObj)
buffer = buffer.subarray(objectIdRaw.byteLength)
const hash = pdfStreamHash(buffer)
const range = {
objectId: objectIdRaw.toString(),
start: object.offset + objectIdRaw.byteLength,
end: object.endOffset,
hash
}
ranges.push(range)
// Optimization: Skip writing of duplicate streams.
if (tracker.track(range)) continue
await writePdfStream(contentDir, hash, pdfStream.buffers)
await writePdfStream(contentDir, hash, buffer)
newRanges.push(range)
}
} finally {
await handle.close()
}
const reclaimedSpace = await tracker.deleteStaleHashes(5)
await tracker.flush()
return [ranges, newRanges, reclaimedSpace]
@@ -142,94 +194,21 @@ class HashFileTracker {
}
}
class PdfStreamsExtractor {
constructor() {
this.fileIndex = 0
this.inStream = false
this.streamStartIndex = 0
this.buffers = []
this.lastChunk = Buffer.alloc(0)
}
consume(chunk) {
let chunkIndex = 0
const pdfStreams = []
chunk = Buffer.concat([this.lastChunk, chunk])
while (true) {
if (!this.inStream) {
// Not in a stream, look for stream start
const index = chunk.indexOf(START_OF_STREAM_MARKER, chunkIndex)
if (index === -1) {
// Couldn't find stream start
break
}
// Found stream start, start a stream
this.inStream = true
this.streamStartIndex = this.fileIndex + index
chunkIndex = index
} else {
// In a stream, look for stream end
const index = chunk.indexOf(END_OF_STREAM_MARKER, chunkIndex)
if (index === -1) {
break
}
// add "endstream" part
const endIndex = index + END_OF_STREAM_MARKER_LENGTH
this.buffers.push(chunk.slice(chunkIndex, endIndex))
pdfStreams.push({
start: this.streamStartIndex,
end: this.fileIndex + endIndex,
buffers: this.buffers
})
this.inStream = false
this.buffers = []
chunkIndex = endIndex
}
}
const remaining = chunk.length - chunkIndex
const nextMarkerLength = this.inStream
? END_OF_STREAM_MARKER_LENGTH
: START_OF_STREAM_MARKER_LENGTH
if (remaining > nextMarkerLength) {
const retainMarkerSection = chunk.length - nextMarkerLength
if (this.inStream) {
this.buffers.push(chunk.slice(chunkIndex, retainMarkerSection))
}
this.lastChunk = chunk.slice(retainMarkerSection)
this.fileIndex += retainMarkerSection
} else {
this.lastChunk = chunk.slice(chunkIndex)
this.fileIndex += chunkIndex
}
return pdfStreams
}
}
function pdfStreamHash(buffers) {
function pdfStreamHash(buffer) {
const hash = crypto.createHash('sha256')
for (const buffer of buffers) {
hash.update(buffer)
}
hash.update(buffer)
return hash.digest('hex')
}
async function writePdfStream(dir, hash, buffers) {
async function writePdfStream(dir, hash, buffer) {
const filename = Path.join(dir, hash)
const atomicWriteFilename = filename + '~'
const file = await fs.promises.open(atomicWriteFilename, 'w')
if (Settings.enablePdfCachingDark) {
// Write an empty file in dark mode.
buffers = []
buffer = Buffer.alloc(0)
}
try {
try {
for (const buffer of buffers) {
await file.write(buffer)
}
} finally {
await file.close()
}
await fs.promises.writeFile(atomicWriteFilename, buffer)
await fs.promises.rename(atomicWriteFilename, filename)
} catch (err) {
try {
@@ -247,5 +226,8 @@ function promiseMapWithLimit(concurrency, array, fn) {
module.exports = {
HASH_REGEX: /^[0-9a-f]{64}$/,
update: callbackify(update)
update: callbackify(update),
promises: {
update
}
}
+30 -27
View File
@@ -275,36 +275,39 @@ module.exports = OutputCacheManager = {
outputDir,
OutputCacheManager.path(outputFile.build, outputFile.path)
)
const pdfSize = outputFile.size
const timer = new Metrics.Timer('compute-pdf-ranges')
ContentCacheManager.update(contentDir, outputFilePath, function (
err,
result
) {
if (err) return callback(err, outputFiles)
const [contentRanges, newContentRanges, reclaimedSpace] = result
ContentCacheManager.update(
contentDir,
outputFilePath,
pdfSize,
function (err, result) {
if (err) return callback(err, outputFiles)
const [contentRanges, newContentRanges, reclaimedSpace] = result
if (Settings.enablePdfCachingDark) {
// In dark mode we are doing the computation only and do not emit
// any ranges to the frontend.
} else {
outputFile.contentId = Path.basename(contentDir)
outputFile.ranges = contentRanges
if (Settings.enablePdfCachingDark) {
// In dark mode we are doing the computation only and do not emit
// any ranges to the frontend.
} else {
outputFile.contentId = Path.basename(contentDir)
outputFile.ranges = contentRanges
}
timings['compute-pdf-caching'] = timer.done()
stats['pdf-caching-n-ranges'] = contentRanges.length
stats['pdf-caching-total-ranges-size'] = contentRanges.reduce(
(sum, next) => sum + (next.end - next.start),
0
)
stats['pdf-caching-n-new-ranges'] = newContentRanges.length
stats['pdf-caching-new-ranges-size'] = newContentRanges.reduce(
(sum, next) => sum + (next.end - next.start),
0
)
stats['pdf-caching-reclaimed-space'] = reclaimedSpace
callback(null, outputFiles)
}
timings['compute-pdf-caching'] = timer.done()
stats['pdf-caching-n-ranges'] = contentRanges.length
stats['pdf-caching-total-ranges-size'] = contentRanges.reduce(
(sum, next) => sum + (next.end - next.start),
0
)
stats['pdf-caching-n-new-ranges'] = newContentRanges.length
stats['pdf-caching-new-ranges-size'] = newContentRanges.reduce(
(sum, next) => sum + (next.end - next.start),
0
)
stats['pdf-caching-reclaimed-space'] = reclaimedSpace
callback(null, outputFiles)
})
)
} else {
callback(null, outputFiles)
}
@@ -0,0 +1,43 @@
const { PDFDocument } = require('pdfjs-dist/lib/core/document')
const { LocalPdfManager } = require('pdfjs-dist/lib/core/pdf_manager')
const { MissingDataException } = require('pdfjs-dist/lib/core/core_utils')
const { FSStream } = require('./FSStream')
class FSPdfManager extends LocalPdfManager {
constructor(docId, options) {
const nonEmptyDummyBuffer = Buffer.alloc(1, 0)
super(docId, nonEmptyDummyBuffer)
this.stream = new FSStream(options.fh, 0, options.size)
this.pdfDocument = new PDFDocument(this, this.stream)
}
async ensure(obj, prop, args) {
try {
const value = obj[prop]
if (typeof value === 'function') {
return value.apply(obj, args)
}
return value
} catch (ex) {
if (!(ex instanceof MissingDataException)) {
throw ex
}
await this.requestRange(ex.begin, ex.end)
return this.ensure(obj, prop, args)
}
}
requestRange(begin, end) {
return this.stream.requestRange(begin, end)
}
requestLoadedStream() {}
onLoadedStream() {}
terminate(reason) {}
}
module.exports = {
FSPdfManager
}
+138
View File
@@ -0,0 +1,138 @@
const { Stream } = require('pdfjs-dist/lib/core/stream')
const { MissingDataException } = require('pdfjs-dist/lib/core/core_utils')
const BUF_SIZE = 1024 // read from the file in 1024 byte pages
class FSStream extends Stream {
constructor(fh, start, length, dict, cachedBytes) {
const nonEmptyDummyBuffer = Buffer.alloc(1, 0)
super(nonEmptyDummyBuffer, start, length, dict)
delete this.bytes
this.fh = fh
this.cachedBytes = cachedBytes || []
}
get length() {
return this.end - this.start
}
get isEmpty() {
return this.length === 0
}
// Manage cached reads from the file
requestRange(begin, end) {
// expand small ranges to read a larger amount
if (end - begin < BUF_SIZE) {
end = begin + BUF_SIZE
}
end = Math.min(end, this.length)
// keep a cache of previous reads with {begin,end,buffer} values
const result = {
begin: begin,
end: end,
buffer: Buffer.alloc(end - begin, 0)
}
this.cachedBytes.push(result)
return this.fh.read(result.buffer, 0, end - begin, begin)
}
_ensureGetPos(pos) {
const found = this.cachedBytes.find((x) => {
return x.begin <= pos && pos < x.end
})
if (!found) {
throw new MissingDataException(pos, pos + 1)
}
return found
}
_ensureGetRange(begin, end) {
end = Math.min(end, this.length) // BG: handle overflow case
const found = this.cachedBytes.find((x) => {
return x.begin <= begin && end <= x.end
})
if (!found) {
throw new MissingDataException(begin, end)
}
return found
}
_readByte(found, pos) {
return found.buffer[pos - found.begin]
}
_readBytes(found, pos, end) {
return found.buffer.subarray(pos - found.begin, end - found.begin)
}
// handle accesses to the bytes
ensureByte(pos) {
this._ensureGetPos(pos) // may throw a MissingDataException
}
getByte() {
const pos = this.pos
if (this.pos >= this.end) {
return -1
}
const found = this._ensureGetPos(pos)
return this._readByte(found, this.pos++)
}
// BG: for a range, end is not included (see Buffer.subarray for example)
ensureBytes(length, forceClamped = false) {
const pos = this.pos
this._ensureGetRange(pos, pos + length)
}
getBytes(length, forceClamped = false) {
const pos = this.pos
const strEnd = this.end
const found = this._ensureGetRange(pos, pos + length)
if (!length) {
const subarray = this._readBytes(found, pos, strEnd)
// `this.bytes` is always a `Uint8Array` here.
return forceClamped ? new Uint8ClampedArray(subarray) : subarray
}
let end = pos + length
if (end > strEnd) {
end = strEnd
}
this.pos = end
const subarray = this._readBytes(found, pos, end)
// `this.bytes` is always a `Uint8Array` here.
return forceClamped ? new Uint8ClampedArray(subarray) : subarray
}
getByteRange() {
// BG: this isn't needed as far as I can tell
throw new Error('not implemented')
}
reset() {
this.pos = this.start
}
moveStart() {
this.start = this.pos
}
makeSubStream(start, length, dict = null) {
// BG: had to add this check for null length, it is being called with only
// the start value at one point in the xref decoding. The intent is clear
// enough
// - a null length means "to the end of the file" -- not sure how it is
// working in the existing pdfjs code without this.
if (!length) {
length = this.end - start
}
return new FSStream(this.fh, start, length, dict, this.cachedBytes)
}
}
module.exports = { FSStream }
@@ -0,0 +1,24 @@
const fs = require('fs')
const { FSPdfManager } = require('./FSPdfManager')
async function parseXrefTable(path, size) {
if (size === 0) {
return []
}
const file = await fs.promises.open(path)
try {
const manager = new FSPdfManager(0, { fh: file, size })
await manager.ensureDoc('checkHeader')
await manager.ensureDoc('parseStartXRef')
await manager.ensureDoc('parse')
return manager.pdfDocument.catalog.xref.entries
} finally {
file.close()
}
}
module.exports = {
parseXrefTable
}
+18 -3
View File
@@ -1008,9 +1008,19 @@
}
},
"@overleaf/o-error": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/@overleaf/o-error/-/o-error-3.0.0.tgz",
"integrity": "sha512-LsM2s6Iy9G97ktPo0ys4VxtI/m3ahc1ZHwjo5XnhXtjeIkkkVAehsrcRRoV/yWepPjymB0oZonhcfojpjYR/tg=="
"version": "3.3.1",
"resolved": "https://registry.npmjs.org/@overleaf/o-error/-/o-error-3.3.1.tgz",
"integrity": "sha512-1FRBYZO0lbJ0U+FRGZVS8ou6RhEw3e2B86WW/NbtBw554g0h5iC8ESf+juIfPMU/WDf/JDIFbg3eB/LnP2RSow==",
"requires": {
"core-js": "^3.8.3"
},
"dependencies": {
"core-js": {
"version": "3.13.0",
"resolved": "https://registry.npmjs.org/core-js/-/core-js-3.13.0.tgz",
"integrity": "sha512-iWDbiyha1M5vFwPFmQnvRv+tJzGbFAm6XimJUT0NgHYW3xZEs1SkCAcasWSVFxpI2Xb/V1DDJckq3v90+bQnog=="
}
}
},
"@protobufjs/aspromise": {
"version": "1.1.2",
@@ -5594,6 +5604,11 @@
"integrity": "sha1-uULm1L3mUwBe9rcTYd74cn0GReA=",
"dev": true
},
"pdfjs-dist": {
"version": "2.7.570",
"resolved": "https://registry.npmjs.org/pdfjs-dist/-/pdfjs-dist-2.7.570.tgz",
"integrity": "sha512-/ZkA1FwkEOyDaq11JhMLazdwQAA0F9uwrP7h/1L9Akt9KWh1G5/tkzS+bPuUELq2s2GDFnaT+kooN/aSjT7DXQ=="
},
"performance-now": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz",
+3 -1
View File
@@ -10,7 +10,7 @@
"start": "node $NODE_APP_OPTIONS app.js",
"test:acceptance:_run": "mocha --recursive --reporter spec --timeout 15000 --exit $@ test/acceptance/js",
"test:acceptance": "npm run test:acceptance:_run -- --grep=$MOCHA_GREP",
"test:unit:_run": "mocha --recursive --reporter spec $@ test/unit/js",
"test:unit:_run": "mocha --recursive --reporter spec $@ test/unit/js test/unit/lib",
"test:unit": "npm run test:unit:_run -- --grep=$MOCHA_GREP",
"nodemon": "nodemon --config nodemon.json",
"lint": "node_modules/.bin/eslint --max-warnings 0 .",
@@ -20,6 +20,7 @@
"author": "James Allen <james@sharelatex.com>",
"dependencies": {
"@overleaf/metrics": "^3.5.1",
"@overleaf/o-error": "^3.3.1",
"async": "3.2.0",
"body-parser": "^1.19.0",
"diskusage": "^1.1.3",
@@ -32,6 +33,7 @@
"logger-sharelatex": "^2.2.0",
"mysql": "^2.18.1",
"p-limit": "^3.1.0",
"pdfjs-dist": "^2.7.570",
"request": "^2.88.2",
"sequelize": "^5.21.5",
"settings-sharelatex": "^1.1.0",
+12
View File
@@ -0,0 +1,12 @@
const fs = require('fs')
const { parseXrefTable } = require('../app/lib/pdfjs/parseXrefTable')
const pdfPath = process.argv[2]
async function main() {
const size = (await fs.promises.stat(pdfPath)).size
const xRefEntries = await parseXrefTable(pdfPath, size)
console.log('Xref entries', xRefEntries)
}
main().catch(console.error)
Binary file not shown.
@@ -1,201 +1,123 @@
const fs = require('fs')
const Path = require('path')
const crypto = require('crypto')
const { Readable } = require('stream')
const SandboxedModule = require('sandboxed-module')
const sinon = require('sinon')
const { expect } = require('chai')
const MODULE_PATH = '../../../app/js/ContentCacheManager'
class FakeFile {
constructor() {
this.closed = false
this.contents = []
}
async write(blob) {
this.contents.push(blob)
return this
}
async close() {
this.closed = true
return this
}
toJSON() {
return {
contents: Buffer.concat(this.contents).toString(),
closed: this.closed
}
}
}
function hash(blob) {
const hash = crypto.createHash('sha256')
hash.update(blob)
return hash.digest('hex')
}
describe('ContentCacheManager', function () {
let contentDir, pdfPath
let ContentCacheManager, fs, files, Settings
function load() {
ContentCacheManager = SandboxedModule.require(MODULE_PATH, {
requires: {
fs,
'settings-sharelatex': Settings
}
})
}
let ContentCacheManager, files, Settings
before(function () {
Settings = require('settings-sharelatex')
ContentCacheManager = require(MODULE_PATH)
})
let contentRanges, newContentRanges, reclaimed
function run(filePath, done) {
ContentCacheManager.update(contentDir, filePath, (err, ranges) => {
if (err) return done(err)
let newlyReclaimed
;[contentRanges, newContentRanges, newlyReclaimed] = ranges
reclaimed += newlyReclaimed
done()
})
}
async function run(filePath, size) {
const result = await ContentCacheManager.promises.update(
contentDir,
filePath,
size
)
let newlyReclaimed
;[contentRanges, newContentRanges, newlyReclaimed] = result
reclaimed += newlyReclaimed
beforeEach(function () {
reclaimed = 0
const fileNames = await fs.promises.readdir(contentDir)
files = {}
for (const fileName of fileNames) {
const path = Path.join(contentDir, fileName)
files[path] = await fs.promises.readFile(path)
}
}
before(function () {
contentDir =
'/app/output/602cee6f6460fca0ba7921e6/content/1797a7f48f9-5abc1998509dea1f'
pdfPath =
'/app/output/602cee6f6460fca0ba7921e6/generated-files/1797a7f48ea-8ac6805139f43351/output.pdf'
Settings = {
pdfCachingMinChunkSize: 1024,
enablePdfCachingDark: false
}
files = {}
fs = {
createReadStream: sinon.stub().returns(Readable.from([])),
promises: {
async writeFile(name, blob) {
const file = new FakeFile()
await file.write(Buffer.from(blob))
await file.close()
files[name] = file
},
async readFile(name) {
if (!files[name]) {
throw new Error()
}
return files[name].toJSON().contents
},
async open(name) {
files[name] = new FakeFile()
return files[name]
},
async stat(name) {
if (!files[name]) {
throw new Error()
}
},
async rename(oldName, newName) {
if (!files[oldName]) {
throw new Error()
}
files[newName] = files[oldName]
delete files[oldName]
},
async unlink(name) {
if (!files[name]) {
throw new Error()
}
delete files[name]
}
}
}
reclaimed = 0
Settings.pdfCachingMinChunkSize = 1024
})
describe('with a small minChunkSize', function () {
beforeEach(function () {
Settings.pdfCachingMinChunkSize = 1
load()
before(async function () {
await fs.promises.rmdir(contentDir, { recursive: true })
await fs.promises.mkdir(contentDir, { recursive: true })
await fs.promises.mkdir(Path.dirname(pdfPath), { recursive: true })
})
describe('minimal', function () {
const PATH_MINIMAL = 'test/acceptance/fixtures/minimal.pdf'
const OBJECT_ID_1 = '9 0 '
const HASH_LARGE =
'd7cfc73ad2fba4578a437517923e3714927bbf35e63ea88bd93c7a8076cf1fcd'
const OBJECT_ID_2 = '10 0 '
const HASH_SMALL =
'896749b8343851b0dc385f71616916a7ba0434fcfb56d1fc7e27cd139eaa2f71'
function getChunkPath(hash) {
return Path.join('test/unit/js/snapshots/minimalCompile/chunks', hash)
}
let MINIMAL_SIZE, RANGE_1, RANGE_2, h1, h2, START_1, START_2, END_1, END_2
before(async function () {
await fs.promises.copyFile(PATH_MINIMAL, pdfPath)
const MINIMAL = await fs.promises.readFile(PATH_MINIMAL)
MINIMAL_SIZE = (await fs.promises.stat(PATH_MINIMAL)).size
RANGE_1 = await fs.promises.readFile(getChunkPath(HASH_LARGE))
RANGE_2 = await fs.promises.readFile(getChunkPath(HASH_SMALL))
h1 = HASH_LARGE
h2 = HASH_SMALL
START_1 = MINIMAL.indexOf(RANGE_1)
END_1 = START_1 + RANGE_1.byteLength
START_2 = MINIMAL.indexOf(RANGE_2)
END_2 = START_2 + RANGE_2.byteLength
})
async function runWithMinimal() {
await run(pdfPath, MINIMAL_SIZE)
}
describe('when the ranges are split across chunks', function () {
const RANGE_1 = 'stream123endstream'
const RANGE_2 = 'stream(||)endstream'
const RANGE_3 = 'stream!$%/=endstream'
const h1 = hash(RANGE_1)
const h2 = hash(RANGE_2)
const h3 = hash(RANGE_3)
function runWithSplitStream(done) {
fs.createReadStream
.withArgs(pdfPath)
.returns(
Readable.from([
Buffer.from('abcstr'),
Buffer.from('eam123endstreamABC'),
Buffer.from('str'),
Buffer.from('eam(||'),
Buffer.from(')end'),
Buffer.from('stream-_~stream!$%/=endstream')
])
)
run(pdfPath, done)
}
beforeEach(function (done) {
runWithSplitStream(done)
describe('with two ranges qualifying', function () {
before(function () {
Settings.pdfCachingMinChunkSize = 500
})
it('should produce three ranges', function () {
expect(contentRanges).to.have.length(3)
before(async function () {
await runWithMinimal()
})
it('should produce two ranges', function () {
expect(contentRanges).to.have.length(2)
})
it('should find the correct offsets', function () {
expect(contentRanges).to.deep.equal([
{
start: 3,
end: 21,
hash: hash(RANGE_1)
objectId: OBJECT_ID_1,
start: START_1,
end: END_1,
hash: h1
},
{
start: 24,
end: 43,
hash: hash(RANGE_2)
},
{
start: 46,
end: 66,
hash: hash(RANGE_3)
objectId: OBJECT_ID_2,
start: START_2,
end: END_2,
hash: h2
}
])
})
it('should store the contents', function () {
expect(JSON.parse(JSON.stringify(files))).to.deep.equal({
[Path.join(contentDir, h1)]: {
contents: RANGE_1,
closed: true
},
[Path.join(contentDir, h2)]: {
contents: RANGE_2,
closed: true
},
[Path.join(contentDir, h3)]: {
contents: RANGE_3,
closed: true
},
[Path.join(contentDir, '.state.v0.json')]: {
contents: JSON.stringify({
expect(files).to.deep.equal({
[Path.join(contentDir, h1)]: RANGE_1,
[Path.join(contentDir, h2)]: RANGE_2,
[Path.join(contentDir, '.state.v0.json')]: Buffer.from(
JSON.stringify({
hashAge: [
[h1, 0],
[h2, 0],
[h3, 0]
[h2, 0]
],
hashSize: [
[h1, 18],
[h2, 19],
[h3, 20]
[h1, RANGE_1.byteLength],
[h2, RANGE_2.byteLength]
]
}),
closed: true
}
})
)
})
})
@@ -203,71 +125,46 @@ describe('ContentCacheManager', function () {
expect(contentRanges).to.deep.equal(newContentRanges)
})
describe('when re-running with one stream removed', function () {
function runWithOneSplitStreamRemoved(done) {
fs.createReadStream
.withArgs(pdfPath)
.returns(
Readable.from([
Buffer.from('abcstr'),
Buffer.from('eam123endstreamABC'),
Buffer.from('stream!$%/=endstream')
])
)
run(pdfPath, done)
}
beforeEach(function (done) {
runWithOneSplitStreamRemoved(done)
describe('when re-running with one range too small', function () {
before(function () {
Settings.pdfCachingMinChunkSize = 1024
})
it('should produce two ranges', function () {
expect(contentRanges).to.have.length(2)
before(async function () {
await runWithMinimal()
})
it('should produce one range', function () {
expect(contentRanges).to.have.length(1)
})
it('should find the correct offsets', function () {
expect(contentRanges).to.deep.equal([
{
start: 3,
end: 21,
hash: hash(RANGE_1)
},
{
start: 24,
end: 44,
hash: hash(RANGE_3)
objectId: OBJECT_ID_1,
start: START_1,
end: END_1,
hash: h1
}
])
})
it('should update the age of the 2nd range', function () {
expect(JSON.parse(JSON.stringify(files))).to.deep.equal({
[Path.join(contentDir, h1)]: {
contents: RANGE_1,
closed: true
},
[Path.join(contentDir, h2)]: {
contents: RANGE_2,
closed: true
},
[Path.join(contentDir, h3)]: {
contents: RANGE_3,
closed: true
},
[Path.join(contentDir, '.state.v0.json')]: {
contents: JSON.stringify({
expect(files).to.deep.equal({
[Path.join(contentDir, h1)]: RANGE_1,
[Path.join(contentDir, h2)]: RANGE_2,
[Path.join(contentDir, '.state.v0.json')]: Buffer.from(
JSON.stringify({
hashAge: [
[h1, 0],
[h2, 1],
[h3, 0]
[h2, 1]
],
hashSize: [
[h1, 18],
[h2, 19],
[h3, 20]
[h1, RANGE_1.byteLength],
[h2, RANGE_2.byteLength]
]
}),
closed: true
}
})
)
})
})
@@ -277,53 +174,35 @@ describe('ContentCacheManager', function () {
describe('when re-running 5 more times', function () {
for (let i = 0; i < 5; i++) {
beforeEach(function (done) {
runWithOneSplitStreamRemoved(done)
before(async function () {
await runWithMinimal()
})
}
it('should still produce two ranges', function () {
expect(contentRanges).to.have.length(2)
it('should still produce one range', function () {
expect(contentRanges).to.have.length(1)
})
it('should still find the correct offsets', function () {
expect(contentRanges).to.deep.equal([
{
start: 3,
end: 21,
hash: hash(RANGE_1)
},
{
start: 24,
end: 44,
hash: hash(RANGE_3)
objectId: OBJECT_ID_1,
start: START_1,
end: END_1,
hash: h1
}
])
})
it('should delete the 2nd range', function () {
expect(JSON.parse(JSON.stringify(files))).to.deep.equal({
[Path.join(contentDir, h1)]: {
contents: RANGE_1,
closed: true
},
[Path.join(contentDir, h3)]: {
contents: RANGE_3,
closed: true
},
[Path.join(contentDir, '.state.v0.json')]: {
contents: JSON.stringify({
hashAge: [
[h1, 0],
[h3, 0]
],
hashSize: [
[h1, 18],
[h3, 20]
]
}),
closed: true
}
expect(files).to.deep.equal({
[Path.join(contentDir, h1)]: RANGE_1,
[Path.join(contentDir, '.state.v0.json')]: Buffer.from(
JSON.stringify({
hashAge: [[h1, 0]],
hashSize: [[h1, RANGE_1.byteLength]]
})
)
})
})
@@ -332,7 +211,7 @@ describe('ContentCacheManager', function () {
})
it('should yield the reclaimed space', function () {
expect(reclaimed).to.equal(RANGE_2.length)
expect(reclaimed).to.equal(RANGE_2.byteLength)
})
})
})
@@ -0,0 +1,7 @@
obj
<< /Type /ObjStm /Length 447 /Filter /FlateDecode /N 5 /First 32 >>
stream
xœ…RQkÛ0~߯¸Ç�K'É’ ¥Ð$„vkGIJ[(yð‘Ò8Ø*ëþýî$¯IûR°déîû>ÝI"H@¥¤Ï9@J!` Vú/gg fí>BZx†J˜Ãù9ˆÛ®]-B„'ZNg îÂk„%¥i\÷!ÅÍõüêÇôûäfŽ’4š®�“mÝ�©²ô4ô«®9ĶãCY]\ב@‹—ßñï!Ð 4cþ U‰‡f·=�ËJgOÚÊòg³î©>—’ËÿÌÛzzÆ>A)å§C9Wøw³K³qªPÜ#®”à¤/48•¢VX/ ¶TŒð¦p §-%2"³­*¡B;ä‡XâêÚ2£,9’õG¥z;ž†E–Î¥øØJj/‰c
ón­%¯�ˆÌ¢ê¨áµ¦f3æˆÎ]!µ¢«¤ç=µyšÍ ,Ós]ô«@ïe+COW.C³ÙÒÖkÚ’ Øc_Å¥X‹ v¢í>ˆNüù‚�ÃÌ2u¶«7=˜ìŒñ¸}¥— #•®HVér–9?kvªì6ÞÿªŸÃG«^Åz׬.ö‚ÒvÃó=UyjÀïŠÇ¡p£‘ÍõzìË2
endstream
endobj
+78
View File
@@ -0,0 +1,78 @@
const fs = require('fs')
const Path = require('path')
const { expect } = require('chai')
const { parseXrefTable } = require('../../../app/lib/pdfjs/parseXrefTable')
const PATH_EXAMPLES = 'test/acceptance/fixtures/examples/'
const PATH_SNAPSHOTS = 'test/unit/lib/snapshots/'
const EXAMPLES = fs.readdirSync(PATH_EXAMPLES)
function snapshotPath(example) {
return Path.join(PATH_SNAPSHOTS, example, 'XrefTable.json')
}
function pdfPath(example) {
return Path.join(PATH_EXAMPLES, example, 'output.pdf')
}
async function loadContext(example) {
const size = (await fs.promises.stat(pdfPath(example))).size
let blob
try {
blob = await fs.promises.readFile(snapshotPath(example))
} catch (e) {
if (e.code !== 'ENOENT') {
throw e
}
}
const snapshot = blob ? JSON.parse(blob) : null
return {
size,
snapshot
}
}
async function backFillSnapshot(example, size) {
const table = await parseXrefTable(pdfPath(example), size)
await fs.promises.mkdir(Path.dirname(snapshotPath(example)), {
recursive: true
})
await fs.promises.writeFile(
snapshotPath(example),
JSON.stringify(table, null, 2)
)
return table
}
describe('pdfjs', function () {
describe('when the pdf is an empty file', function () {
it('should yield no entries', async function () {
const path = 'does/not/matter.pdf'
const table = await parseXrefTable(path, 0)
expect(table).to.deep.equal([])
})
})
for (const example of EXAMPLES) {
describe(example, function () {
let size, snapshot
before('load snapshot', async function () {
const ctx = await loadContext(example)
size = ctx.size
snapshot = ctx.snapshot
})
before('back fill new snapshot', async function () {
if (snapshot === null) {
console.error('back filling snapshot for', example)
snapshot = await backFillSnapshot(example, size)
}
})
it('should produce the expected xRef table', async function () {
const table = await parseXrefTable(pdfPath(example), size)
expect(table).to.deep.equal(snapshot)
})
})
}
})
@@ -0,0 +1,356 @@
[
{
"offset": 0,
"gen": 0,
"free": true
},
{
"offset": 123086,
"gen": 0,
"uncompressed": true
},
{
"offset": 123405,
"gen": 0,
"uncompressed": true
},
{
"offset": 15,
"gen": 0,
"uncompressed": true
},
{
"offset": 216,
"gen": 0,
"uncompressed": true
},
{
"offset": 1084,
"gen": 0,
"uncompressed": true
},
{
"offset": 1244,
"gen": 0,
"uncompressed": true
},
{
"offset": 4001,
"gen": 0,
"uncompressed": true
},
{
"offset": 4155,
"gen": 0,
"uncompressed": true
},
{
"offset": 4297,
"gen": 0,
"uncompressed": true
},
{
"offset": 4932,
"gen": 0,
"uncompressed": true
},
{
"offset": 5307,
"gen": 0,
"uncompressed": true
},
{
"offset": 5495,
"gen": 0,
"uncompressed": true
},
{
"offset": 30246,
"gen": 0,
"uncompressed": true
},
{
"offset": 31466,
"gen": 0,
"uncompressed": true
},
{
"offset": 38398,
"gen": 0,
"uncompressed": true
},
{
"offset": 39039,
"gen": 0,
"uncompressed": true
},
{
"offset": 40158,
"gen": 0,
"uncompressed": true
},
{
"offset": 40897,
"gen": 0,
"uncompressed": true
},
{
"offset": 65550,
"gen": 0,
"uncompressed": true
},
{
"offset": 74691,
"gen": 0,
"uncompressed": true
},
{
"offset": 81693,
"gen": 0,
"uncompressed": true
},
{
"offset": 97169,
"gen": 0,
"uncompressed": true
},
{
"offset": 104103,
"gen": 0,
"uncompressed": true
},
{
"offset": 111180,
"gen": 0,
"uncompressed": true
},
{
"offset": 118555,
"gen": 0,
"uncompressed": true
},
{
"offset": 6,
"gen": 0
},
{
"offset": 6,
"gen": 1
},
{
"offset": 6,
"gen": 2
},
{
"offset": 6,
"gen": 3
},
{
"offset": 6,
"gen": 4
},
{
"offset": 6,
"gen": 5
},
{
"offset": 6,
"gen": 6
},
{
"offset": 6,
"gen": 7
},
{
"offset": 6,
"gen": 8
},
{
"offset": 6,
"gen": 9
},
{
"offset": 6,
"gen": 10
},
{
"offset": 6,
"gen": 11
},
{
"offset": 6,
"gen": 12
},
{
"offset": 6,
"gen": 13
},
{
"offset": 6,
"gen": 14
},
{
"offset": 6,
"gen": 15
},
{
"offset": 6,
"gen": 16
},
{
"offset": 6,
"gen": 17
},
{
"offset": 6,
"gen": 18
},
{
"offset": 6,
"gen": 19
},
{
"offset": 6,
"gen": 20
},
{
"offset": 6,
"gen": 21
},
{
"offset": 6,
"gen": 22
},
{
"offset": 6,
"gen": 23
},
{
"offset": 6,
"gen": 24
},
{
"offset": 6,
"gen": 25
},
{
"offset": 6,
"gen": 26
},
{
"offset": 6,
"gen": 27
},
{
"offset": 6,
"gen": 28
},
{
"offset": 6,
"gen": 29
},
{
"offset": 6,
"gen": 30
},
{
"offset": 6,
"gen": 31
},
{
"offset": 6,
"gen": 32
},
{
"offset": 6,
"gen": 33
},
{
"offset": 6,
"gen": 34
},
{
"offset": 6,
"gen": 35
},
{
"offset": 6,
"gen": 36
},
{
"offset": 6,
"gen": 37
},
{
"offset": 6,
"gen": 38
},
{
"offset": 6,
"gen": 39
},
{
"offset": 6,
"gen": 40
},
{
"offset": 6,
"gen": 41
},
{
"offset": 6,
"gen": 42
},
{
"offset": 6,
"gen": 43
},
{
"offset": 6,
"gen": 44
},
{
"offset": 6,
"gen": 45
},
{
"offset": 6,
"gen": 46
},
{
"offset": 6,
"gen": 47
},
{
"offset": 6,
"gen": 48
},
{
"offset": 6,
"gen": 49
},
{
"offset": 6,
"gen": 50
},
{
"offset": 6,
"gen": 51
},
{
"offset": 6,
"gen": 52
},
{
"offset": 6,
"gen": 53
},
{
"offset": 6,
"gen": 54
},
{
"offset": 6,
"gen": 55
}
]
@@ -0,0 +1,128 @@
[
{
"offset": 0,
"gen": 255,
"free": true
},
{
"offset": 9,
"gen": 1
},
{
"offset": 9,
"gen": 0
},
{
"offset": 15,
"gen": 0,
"uncompressed": true
},
{
"offset": 9,
"gen": 14
},
{
"offset": 9,
"gen": 12
},
{
"offset": 9,
"gen": 15
},
{
"offset": 9,
"gen": 13
},
{
"offset": 9,
"gen": 16
},
{
"offset": 57274,
"gen": 0,
"uncompressed": true
},
{
"offset": 9,
"gen": 17
},
{
"offset": 9,
"gen": 2
},
{
"offset": 9,
"gen": 3
},
{
"offset": 9,
"gen": 4
},
{
"offset": 9,
"gen": 5
},
{
"offset": 9,
"gen": 6
},
{
"offset": 522,
"gen": 0,
"uncompressed": true
},
{
"offset": 9,
"gen": 7
},
{
"offset": 8788,
"gen": 0,
"uncompressed": true
},
{
"offset": 9,
"gen": 8
},
{
"offset": 17289,
"gen": 0,
"uncompressed": true
},
{
"offset": 9,
"gen": 9
},
{
"offset": 32619,
"gen": 0,
"uncompressed": true
},
{
"offset": 9,
"gen": 10
},
{
"offset": 44596,
"gen": 0,
"uncompressed": true
},
{
"offset": 9,
"gen": 11
},
{
"offset": 9,
"gen": 18
},
{
"offset": 57027,
"gen": 0,
"uncompressed": true
},
{
"offset": 58655,
"gen": 0,
"uncompressed": true
}
]
@@ -0,0 +1,125 @@
[
{
"offset": 0,
"gen": 255,
"free": true
},
{
"offset": 208,
"gen": 0,
"uncompressed": true
},
{
"offset": 6,
"gen": 11
},
{
"offset": 6,
"gen": 0
},
{
"offset": 15,
"gen": 0,
"uncompressed": true
},
{
"offset": 6,
"gen": 14
},
{
"offset": 29580,
"gen": 0,
"uncompressed": true
},
{
"offset": 6,
"gen": 15
},
{
"offset": 6,
"gen": 1
},
{
"offset": 6,
"gen": 2
},
{
"offset": 6,
"gen": 3
},
{
"offset": 6,
"gen": 4
},
{
"offset": 6,
"gen": 5
},
{
"offset": 6,
"gen": 6
},
{
"offset": 6,
"gen": 7
},
{
"offset": 6,
"gen": 8
},
{
"offset": 18310,
"gen": 0,
"uncompressed": true
},
{
"offset": 6,
"gen": 9
},
{
"offset": 6,
"gen": 10
},
{
"offset": 18554,
"gen": 0,
"uncompressed": true
},
{
"offset": 18945,
"gen": 0,
"uncompressed": true
},
{
"offset": 19674,
"gen": 0,
"uncompressed": true
},
{
"offset": 6,
"gen": 12
},
{
"offset": 22318,
"gen": 0,
"uncompressed": true
},
{
"offset": 6,
"gen": 13
},
{
"offset": 6,
"gen": 16
},
{
"offset": 29321,
"gen": 0,
"uncompressed": true
},
{
"offset": 30697,
"gen": 0,
"uncompressed": true
}
]
@@ -0,0 +1,115 @@
[
{
"offset": 0,
"gen": 255,
"free": true
},
{
"offset": 8,
"gen": 1
},
{
"offset": 8,
"gen": 0
},
{
"offset": 15,
"gen": 0,
"uncompressed": true
},
{
"offset": 8,
"gen": 5
},
{
"offset": 8,
"gen": 12
},
{
"offset": 8,
"gen": 14
},
{
"offset": 8,
"gen": 13
},
{
"offset": 25628,
"gen": 0,
"uncompressed": true
},
{
"offset": 8,
"gen": 15
},
{
"offset": 8,
"gen": 2
},
{
"offset": 8,
"gen": 3
},
{
"offset": 8,
"gen": 4
},
{
"offset": 250,
"gen": 0,
"uncompressed": true
},
{
"offset": 8,
"gen": 6
},
{
"offset": 8,
"gen": 7
},
{
"offset": 8,
"gen": 8
},
{
"offset": 3854,
"gen": 0,
"uncompressed": true
},
{
"offset": 8,
"gen": 9
},
{
"offset": 11227,
"gen": 0,
"uncompressed": true
},
{
"offset": 8,
"gen": 10
},
{
"offset": 18230,
"gen": 0,
"uncompressed": true
},
{
"offset": 8,
"gen": 11
},
{
"offset": 8,
"gen": 16
},
{
"offset": 25381,
"gen": 0,
"uncompressed": true
},
{
"offset": 26423,
"gen": 0,
"uncompressed": true
}
]
@@ -0,0 +1,102 @@
[
{
"offset": 0,
"gen": 65535,
"free": true
},
{
"offset": 1094,
"gen": 0,
"uncompressed": true
},
{
"offset": 5512,
"gen": 0,
"uncompressed": true
},
{
"offset": 1035,
"gen": 0,
"uncompressed": true
},
{
"offset": 875,
"gen": 0,
"uncompressed": true
},
{
"offset": 15,
"gen": 0,
"uncompressed": true
},
{
"offset": 856,
"gen": 0,
"uncompressed": true
},
{
"offset": 1159,
"gen": 0,
"uncompressed": true
},
{
"offset": 1593,
"gen": 0,
"uncompressed": true
},
{
"offset": 3149,
"gen": 0,
"uncompressed": true
},
{
"offset": 1436,
"gen": 0,
"uncompressed": true
},
{
"offset": 2412,
"gen": 0,
"uncompressed": true
},
{
"offset": 1282,
"gen": 0,
"uncompressed": true
},
{
"offset": 1768,
"gen": 0,
"uncompressed": true
},
{
"offset": 1200,
"gen": 0,
"uncompressed": true
},
{
"offset": 1230,
"gen": 0,
"uncompressed": true
},
{
"offset": 2006,
"gen": 0,
"uncompressed": true
},
{
"offset": 2625,
"gen": 0,
"uncompressed": true
},
{
"offset": 3381,
"gen": 0,
"uncompressed": true
},
{
"offset": 4069,
"gen": 0,
"uncompressed": true
}
]
@@ -0,0 +1,93 @@
[
{
"offset": 0,
"gen": 255,
"free": true
},
{
"offset": 7,
"gen": 1
},
{
"offset": 7,
"gen": 0
},
{
"offset": 15,
"gen": 0,
"uncompressed": true
},
{
"offset": 7,
"gen": 11
},
{
"offset": 7,
"gen": 10
},
{
"offset": 7,
"gen": 9
},
{
"offset": 29476,
"gen": 0,
"uncompressed": true
},
{
"offset": 7,
"gen": 12
},
{
"offset": 7,
"gen": 7
},
{
"offset": 7,
"gen": 2
},
{
"offset": 7,
"gen": 8
},
{
"offset": 7,
"gen": 3
},
{
"offset": 7,
"gen": 4
},
{
"offset": 255,
"gen": 0,
"uncompressed": true
},
{
"offset": 7,
"gen": 5
},
{
"offset": 17910,
"gen": 0,
"uncompressed": true
},
{
"offset": 7,
"gen": 6
},
{
"offset": 7,
"gen": 13
},
{
"offset": 29228,
"gen": 0,
"uncompressed": true
},
{
"offset": 30448,
"gen": 0,
"uncompressed": true
}
]
@@ -0,0 +1,126 @@
[
{
"offset": 0,
"gen": 0,
"free": true
},
{
"offset": 6338,
"gen": 0,
"uncompressed": true
},
{
"offset": 15,
"gen": 0,
"uncompressed": true
},
{
"offset": 216,
"gen": 0,
"uncompressed": true
},
{
"offset": 707,
"gen": 0,
"uncompressed": true
},
{
"offset": 757,
"gen": 0,
"uncompressed": true
},
{
"offset": 888,
"gen": 0,
"uncompressed": true
},
{
"offset": 991,
"gen": 0,
"uncompressed": true
},
{
"offset": 1257,
"gen": 0,
"uncompressed": true
},
{
"offset": 1678,
"gen": 0,
"uncompressed": true
},
{
"offset": 2050,
"gen": 0,
"uncompressed": true
},
{
"offset": 4246,
"gen": 0,
"uncompressed": true
},
{
"offset": 4339,
"gen": 0,
"uncompressed": true
},
{
"offset": 5382,
"gen": 0,
"uncompressed": true
},
{
"offset": 5475,
"gen": 0,
"uncompressed": true
},
{
"offset": 5513,
"gen": 0,
"uncompressed": true
},
{
"offset": 15,
"gen": 0
},
{
"offset": 15,
"gen": 1
},
{
"offset": 15,
"gen": 2
},
{
"offset": 15,
"gen": 3
},
{
"offset": 15,
"gen": 4
},
{
"offset": 15,
"gen": 5
},
{
"offset": 15,
"gen": 6
},
{
"offset": 15,
"gen": 7
},
{
"offset": 15,
"gen": 8
},
{
"offset": 15,
"gen": 9
},
{
"offset": 15,
"gen": 10
}
]
@@ -0,0 +1,94 @@
[
{
"offset": 0,
"gen": 255,
"free": true
},
{
"offset": 7,
"gen": 1
},
{
"offset": 7,
"gen": 0
},
{
"offset": 15,
"gen": 0,
"uncompressed": true
},
{
"offset": 7,
"gen": 10
},
{
"offset": 7,
"gen": 9
},
{
"offset": 7,
"gen": 8
},
{
"offset": 33085,
"gen": 0,
"uncompressed": true
},
{
"offset": 7,
"gen": 11
},
{
"offset": 7,
"gen": 2
},
{
"offset": 7,
"gen": 3
},
{
"offset": 7,
"gen": 4
},
{
"offset": 445,
"gen": 0,
"uncompressed": true
},
{
"offset": 7,
"gen": 5
},
{
"offset": 10048,
"gen": 0,
"uncompressed": true
},
{
"offset": 7,
"gen": 6
},
{
"offset": 18151,
"gen": 0,
"uncompressed": true
},
{
"offset": 7,
"gen": 7
},
{
"offset": 7,
"gen": 12
},
{
"offset": 32838,
"gen": 0,
"uncompressed": true
},
{
"offset": 34157,
"gen": 0,
"uncompressed": true
}
]
@@ -0,0 +1,89 @@
[
{
"offset": 0,
"gen": 255,
"free": true
},
{
"offset": 4,
"gen": 0
},
{
"offset": 4,
"gen": 1
},
{
"offset": 4,
"gen": 2
},
{
"offset": 22047,
"gen": 0,
"uncompressed": true
},
{
"offset": 4,
"gen": 4
},
{
"offset": 4,
"gen": 3
},
{
"offset": 15,
"gen": 0,
"uncompressed": true
},
{
"offset": 4,
"gen": 10
},
{
"offset": 4,
"gen": 9
},
{
"offset": 4,
"gen": 11
},
{
"offset": 4,
"gen": 5
},
{
"offset": 4,
"gen": 6
},
{
"offset": 6451,
"gen": 0,
"uncompressed": true
},
{
"offset": 4,
"gen": 7
},
{
"offset": 14825,
"gen": 0,
"uncompressed": true
},
{
"offset": 4,
"gen": 8
},
{
"offset": 4,
"gen": 12
},
{
"offset": 21800,
"gen": 0,
"uncompressed": true
},
{
"offset": 22696,
"gen": 0,
"uncompressed": true
}
]
@@ -0,0 +1,81 @@
[
{
"offset": 0,
"gen": 255,
"free": true
},
{
"offset": 6,
"gen": 1
},
{
"offset": 6,
"gen": 0
},
{
"offset": 15,
"gen": 0,
"uncompressed": true
},
{
"offset": 6,
"gen": 8
},
{
"offset": 6,
"gen": 7
},
{
"offset": 22744,
"gen": 0,
"uncompressed": true
},
{
"offset": 6,
"gen": 9
},
{
"offset": 6,
"gen": 2
},
{
"offset": 6,
"gen": 6
},
{
"offset": 6,
"gen": 3
},
{
"offset": 364,
"gen": 0,
"uncompressed": true
},
{
"offset": 6,
"gen": 4
},
{
"offset": 12163,
"gen": 0,
"uncompressed": true
},
{
"offset": 6,
"gen": 5
},
{
"offset": 6,
"gen": 10
},
{
"offset": 22496,
"gen": 0,
"uncompressed": true
},
{
"offset": 23856,
"gen": 0,
"uncompressed": true
}
]
@@ -0,0 +1,145 @@
[
{
"offset": 0,
"gen": 0,
"free": true
},
{
"offset": 43543,
"gen": 0,
"uncompressed": true
},
{
"offset": 43792,
"gen": 0,
"uncompressed": true
},
{
"offset": 15,
"gen": 0,
"uncompressed": true
},
{
"offset": 216,
"gen": 0,
"uncompressed": true
},
{
"offset": 734,
"gen": 0,
"uncompressed": true
},
{
"offset": 784,
"gen": 0,
"uncompressed": true
},
{
"offset": 912,
"gen": 0,
"uncompressed": true
},
{
"offset": 1020,
"gen": 0,
"uncompressed": true
},
{
"offset": 1544,
"gen": 0,
"uncompressed": true
},
{
"offset": 5791,
"gen": 0,
"uncompressed": true
},
{
"offset": 12911,
"gen": 0,
"uncompressed": true
},
{
"offset": 23655,
"gen": 0,
"uncompressed": true
},
{
"offset": 30651,
"gen": 0,
"uncompressed": true
},
{
"offset": 42597,
"gen": 0,
"uncompressed": true
},
{
"offset": 14,
"gen": 0
},
{
"offset": 14,
"gen": 1
},
{
"offset": 14,
"gen": 2
},
{
"offset": 14,
"gen": 3
},
{
"offset": 14,
"gen": 4
},
{
"offset": 14,
"gen": 5
},
{
"offset": 14,
"gen": 6
},
{
"offset": 14,
"gen": 7
},
{
"offset": 14,
"gen": 8
},
{
"offset": 14,
"gen": 9
},
{
"offset": 14,
"gen": 10
},
{
"offset": 14,
"gen": 11
},
{
"offset": 14,
"gen": 12
},
{
"offset": 14,
"gen": 13
},
{
"offset": 14,
"gen": 14
},
{
"offset": 14,
"gen": 15
},
{
"offset": 14,
"gen": 16
}
]
@@ -0,0 +1,179 @@
[
{
"offset": 0,
"gen": 0,
"free": true
},
{
"offset": 75291,
"gen": 0,
"uncompressed": true
},
{
"offset": 75540,
"gen": 0,
"uncompressed": true
},
{
"offset": 15,
"gen": 0,
"uncompressed": true
},
{
"offset": 216,
"gen": 0,
"uncompressed": true
},
{
"offset": 790,
"gen": 0,
"uncompressed": true
},
{
"offset": 840,
"gen": 0,
"uncompressed": true
},
{
"offset": 975,
"gen": 0,
"uncompressed": true
},
{
"offset": 1083,
"gen": 0,
"uncompressed": true
},
{
"offset": 2127,
"gen": 0,
"uncompressed": true
},
{
"offset": 13797,
"gen": 0,
"uncompressed": true
},
{
"offset": 23679,
"gen": 0,
"uncompressed": true
},
{
"offset": 31863,
"gen": 0,
"uncompressed": true
},
{
"offset": 36111,
"gen": 0,
"uncompressed": true
},
{
"offset": 50346,
"gen": 0,
"uncompressed": true
},
{
"offset": 61562,
"gen": 0,
"uncompressed": true
},
{
"offset": 73508,
"gen": 0,
"uncompressed": true
},
{
"offset": 16,
"gen": 0
},
{
"offset": 16,
"gen": 1
},
{
"offset": 16,
"gen": 2
},
{
"offset": 16,
"gen": 3
},
{
"offset": 16,
"gen": 4
},
{
"offset": 16,
"gen": 5
},
{
"offset": 16,
"gen": 6
},
{
"offset": 16,
"gen": 7
},
{
"offset": 16,
"gen": 8
},
{
"offset": 16,
"gen": 9
},
{
"offset": 16,
"gen": 10
},
{
"offset": 16,
"gen": 11
},
{
"offset": 16,
"gen": 12
},
{
"offset": 16,
"gen": 13
},
{
"offset": 16,
"gen": 14
},
{
"offset": 16,
"gen": 15
},
{
"offset": 16,
"gen": 16
},
{
"offset": 16,
"gen": 17
},
{
"offset": 16,
"gen": 18
},
{
"offset": 16,
"gen": 19
},
{
"offset": 16,
"gen": 20
},
{
"offset": 16,
"gen": 21
},
{
"offset": 16,
"gen": 22
}
]
@@ -0,0 +1,132 @@
[
{
"offset": 0,
"gen": 65535,
"free": true
},
{
"offset": 17349,
"gen": 0,
"uncompressed": true
},
{
"offset": 25448,
"gen": 0,
"uncompressed": true
},
{
"offset": 17290,
"gen": 0,
"uncompressed": true
},
{
"offset": 17130,
"gen": 0,
"uncompressed": true
},
{
"offset": 15,
"gen": 0,
"uncompressed": true
},
{
"offset": 17109,
"gen": 0,
"uncompressed": true
},
{
"offset": 17414,
"gen": 0,
"uncompressed": true
},
{
"offset": 17548,
"gen": 0,
"uncompressed": true
},
{
"offset": 18912,
"gen": 0,
"uncompressed": true
},
{
"offset": 18381,
"gen": 0,
"uncompressed": true
},
{
"offset": 22406,
"gen": 0,
"uncompressed": true
},
{
"offset": 17897,
"gen": 0,
"uncompressed": true
},
{
"offset": 21796,
"gen": 0,
"uncompressed": true
},
{
"offset": 18758,
"gen": 0,
"uncompressed": true
},
{
"offset": 23361,
"gen": 0,
"uncompressed": true
},
{
"offset": 17455,
"gen": 0,
"uncompressed": true
},
{
"offset": 17485,
"gen": 0,
"uncompressed": true
},
{
"offset": 19218,
"gen": 0,
"uncompressed": true
},
{
"offset": 22021,
"gen": 0,
"uncompressed": true
},
{
"offset": 22638,
"gen": 0,
"uncompressed": true
},
{
"offset": 23599,
"gen": 0,
"uncompressed": true
},
{
"offset": 18051,
"gen": 0,
"uncompressed": true
},
{
"offset": 18142,
"gen": 0,
"uncompressed": true
},
{
"offset": 18657,
"gen": 0,
"uncompressed": true
},
{
"offset": 24005,
"gen": 0,
"uncompressed": true
}
]
@@ -0,0 +1,74 @@
[
{
"offset": 0,
"gen": 255,
"free": true
},
{
"offset": 6,
"gen": 1
},
{
"offset": 6,
"gen": 0
},
{
"offset": 15,
"gen": 0,
"uncompressed": true
},
{
"offset": 6,
"gen": 4
},
{
"offset": 6,
"gen": 6
},
{
"offset": 2320,
"gen": 0,
"uncompressed": true
},
{
"offset": 6,
"gen": 2
},
{
"offset": 202,
"gen": 0,
"uncompressed": true
},
{
"offset": 298,
"gen": 0,
"uncompressed": true
},
{
"offset": 6,
"gen": 3
},
{
"offset": 1642,
"gen": 0,
"uncompressed": true
},
{
"offset": 6,
"gen": 5
},
{
"offset": 6,
"gen": 7
},
{
"offset": 2120,
"gen": 0,
"uncompressed": true
},
{
"offset": 2892,
"gen": 0,
"uncompressed": true
}
]
@@ -0,0 +1,107 @@
[
{
"offset": 0,
"gen": 255,
"free": true
},
{
"offset": 5,
"gen": 1
},
{
"offset": 5,
"gen": 0
},
{
"offset": 15,
"gen": 0,
"uncompressed": true
},
{
"offset": 5,
"gen": 12
},
{
"offset": 30470,
"gen": 0,
"uncompressed": true
},
{
"offset": 5,
"gen": 13
},
{
"offset": 5,
"gen": 3
},
{
"offset": 5,
"gen": 2
},
{
"offset": 366,
"gen": 0,
"uncompressed": true
},
{
"offset": 5,
"gen": 11
},
{
"offset": 5,
"gen": 10
},
{
"offset": 5,
"gen": 4
},
{
"offset": 5,
"gen": 5
},
{
"offset": 5,
"gen": 6
},
{
"offset": 638,
"gen": 0,
"uncompressed": true
},
{
"offset": 5,
"gen": 7
},
{
"offset": 7838,
"gen": 0,
"uncompressed": true
},
{
"offset": 5,
"gen": 8
},
{
"offset": 15674,
"gen": 0,
"uncompressed": true
},
{
"offset": 5,
"gen": 9
},
{
"offset": 5,
"gen": 14
},
{
"offset": 30223,
"gen": 0,
"uncompressed": true
},
{
"offset": 31457,
"gen": 0,
"uncompressed": true
}
]
@@ -0,0 +1,90 @@
[
{
"offset": 0,
"gen": 255,
"free": true
},
{
"offset": 5,
"gen": 1
},
{
"offset": 5,
"gen": 0
},
{
"offset": 15,
"gen": 0,
"uncompressed": true
},
{
"offset": 5,
"gen": 9
},
{
"offset": 23352,
"gen": 0,
"uncompressed": true
},
{
"offset": 5,
"gen": 10
},
{
"offset": 5,
"gen": 3
},
{
"offset": 5,
"gen": 2
},
{
"offset": 366,
"gen": 0,
"uncompressed": true
},
{
"offset": 5,
"gen": 8
},
{
"offset": 5,
"gen": 4
},
{
"offset": 5,
"gen": 5
},
{
"offset": 589,
"gen": 0,
"uncompressed": true
},
{
"offset": 5,
"gen": 6
},
{
"offset": 8425,
"gen": 0,
"uncompressed": true
},
{
"offset": 5,
"gen": 7
},
{
"offset": 5,
"gen": 11
},
{
"offset": 23105,
"gen": 0,
"uncompressed": true
},
{
"offset": 24245,
"gen": 0,
"uncompressed": true
}
]
@@ -0,0 +1,77 @@
[
{
"offset": 0,
"gen": 255,
"free": true
},
{
"offset": 6,
"gen": 1
},
{
"offset": 6,
"gen": 0
},
{
"offset": 15,
"gen": 0,
"uncompressed": true
},
{
"offset": 6,
"gen": 7
},
{
"offset": 6,
"gen": 6
},
{
"offset": 19462,
"gen": 0,
"uncompressed": true
},
{
"offset": 6,
"gen": 8
},
{
"offset": 6,
"gen": 2
},
{
"offset": 6,
"gen": 3
},
{
"offset": 340,
"gen": 0,
"uncompressed": true
},
{
"offset": 6,
"gen": 4
},
{
"offset": 7343,
"gen": 0,
"uncompressed": true
},
{
"offset": 6,
"gen": 5
},
{
"offset": 6,
"gen": 9
},
{
"offset": 19215,
"gen": 0,
"uncompressed": true
},
{
"offset": 20089,
"gen": 0,
"uncompressed": true
}
]
@@ -0,0 +1,133 @@
[
{
"offset": 0,
"gen": 255,
"free": true
},
{
"offset": 6,
"gen": 1
},
{
"offset": 6,
"gen": 0
},
{
"offset": 15,
"gen": 0,
"uncompressed": true
},
{
"offset": 6,
"gen": 14
},
{
"offset": 6,
"gen": 15
},
{
"offset": 40591,
"gen": 0,
"uncompressed": true
},
{
"offset": 6,
"gen": 17
},
{
"offset": 6,
"gen": 3
},
{
"offset": 6,
"gen": 2
},
{
"offset": 290,
"gen": 0,
"uncompressed": true
},
{
"offset": 6,
"gen": 16
},
{
"offset": 6,
"gen": 5
},
{
"offset": 6,
"gen": 4
},
{
"offset": 595,
"gen": 0,
"uncompressed": true
},
{
"offset": 6,
"gen": 7
},
{
"offset": 6,
"gen": 6
},
{
"offset": 844,
"gen": 0,
"uncompressed": true
},
{
"offset": 6,
"gen": 8
},
{
"offset": 6,
"gen": 9
},
{
"offset": 6,
"gen": 10
},
{
"offset": 1107,
"gen": 0,
"uncompressed": true
},
{
"offset": 6,
"gen": 11
},
{
"offset": 11816,
"gen": 0,
"uncompressed": true
},
{
"offset": 6,
"gen": 12
},
{
"offset": 28180,
"gen": 0,
"uncompressed": true
},
{
"offset": 6,
"gen": 13
},
{
"offset": 6,
"gen": 18
},
{
"offset": 40344,
"gen": 0,
"uncompressed": true
},
{
"offset": 41851,
"gen": 0,
"uncompressed": true
}
]
@@ -0,0 +1,94 @@
[
{
"offset": 0,
"gen": 255,
"free": true
},
{
"offset": 7,
"gen": 1
},
{
"offset": 7,
"gen": 0
},
{
"offset": 15,
"gen": 0,
"uncompressed": true
},
{
"offset": 7,
"gen": 8
},
{
"offset": 7,
"gen": 9
},
{
"offset": 7,
"gen": 10
},
{
"offset": 32363,
"gen": 0,
"uncompressed": true
},
{
"offset": 7,
"gen": 11
},
{
"offset": 7,
"gen": 2
},
{
"offset": 7,
"gen": 3
},
{
"offset": 7,
"gen": 4
},
{
"offset": 565,
"gen": 0,
"uncompressed": true
},
{
"offset": 7,
"gen": 5
},
{
"offset": 10031,
"gen": 0,
"uncompressed": true
},
{
"offset": 7,
"gen": 6
},
{
"offset": 18203,
"gen": 0,
"uncompressed": true
},
{
"offset": 7,
"gen": 7
},
{
"offset": 7,
"gen": 12
},
{
"offset": 32116,
"gen": 0,
"uncompressed": true
},
{
"offset": 33492,
"gen": 0,
"uncompressed": true
}
]
@@ -0,0 +1,90 @@
[
{
"offset": 0,
"gen": 255,
"free": true
},
{
"offset": 5,
"gen": 1
},
{
"offset": 5,
"gen": 0
},
{
"offset": 15,
"gen": 0,
"uncompressed": true
},
{
"offset": 5,
"gen": 9
},
{
"offset": 15855,
"gen": 0,
"uncompressed": true
},
{
"offset": 5,
"gen": 10
},
{
"offset": 5,
"gen": 3
},
{
"offset": 5,
"gen": 2
},
{
"offset": 167,
"gen": 0,
"uncompressed": true
},
{
"offset": 5,
"gen": 8
},
{
"offset": 5,
"gen": 4
},
{
"offset": 5,
"gen": 5
},
{
"offset": 341,
"gen": 0,
"uncompressed": true
},
{
"offset": 5,
"gen": 6
},
{
"offset": 7911,
"gen": 0,
"uncompressed": true
},
{
"offset": 5,
"gen": 7
},
{
"offset": 5,
"gen": 11
},
{
"offset": 15608,
"gen": 0,
"uncompressed": true
},
{
"offset": 16597,
"gen": 0,
"uncompressed": true
}
]
@@ -0,0 +1,94 @@
[
{
"offset": 0,
"gen": 255,
"free": true
},
{
"offset": 7,
"gen": 1
},
{
"offset": 7,
"gen": 0
},
{
"offset": 15,
"gen": 0,
"uncompressed": true
},
{
"offset": 7,
"gen": 9
},
{
"offset": 7,
"gen": 8
},
{
"offset": 7,
"gen": 10
},
{
"offset": 35573,
"gen": 0,
"uncompressed": true
},
{
"offset": 7,
"gen": 11
},
{
"offset": 7,
"gen": 2
},
{
"offset": 7,
"gen": 3
},
{
"offset": 7,
"gen": 4
},
{
"offset": 373,
"gen": 0,
"uncompressed": true
},
{
"offset": 7,
"gen": 5
},
{
"offset": 8639,
"gen": 0,
"uncompressed": true
},
{
"offset": 7,
"gen": 6
},
{
"offset": 23349,
"gen": 0,
"uncompressed": true
},
{
"offset": 7,
"gen": 7
},
{
"offset": 7,
"gen": 12
},
{
"offset": 35326,
"gen": 0,
"uncompressed": true
},
{
"offset": 36668,
"gen": 0,
"uncompressed": true
}
]
@@ -0,0 +1,108 @@
[
{
"offset": 0,
"gen": 255,
"free": true
},
{
"offset": 548,
"gen": 0,
"uncompressed": true
},
{
"offset": 9,
"gen": 1
},
{
"offset": 9,
"gen": 2
},
{
"offset": 9,
"gen": 0
},
{
"offset": 15,
"gen": 0,
"uncompressed": true
},
{
"offset": 9,
"gen": 10
},
{
"offset": 9,
"gen": 9
},
{
"offset": 9,
"gen": 11
},
{
"offset": 46398,
"gen": 0,
"uncompressed": true
},
{
"offset": 9,
"gen": 12
},
{
"offset": 8100,
"gen": 0,
"uncompressed": true
},
{
"offset": 9,
"gen": 3
},
{
"offset": 9,
"gen": 4
},
{
"offset": 9,
"gen": 5
},
{
"offset": 9693,
"gen": 0,
"uncompressed": true
},
{
"offset": 9,
"gen": 6
},
{
"offset": 17959,
"gen": 0,
"uncompressed": true
},
{
"offset": 9,
"gen": 7
},
{
"offset": 34174,
"gen": 0,
"uncompressed": true
},
{
"offset": 9,
"gen": 8
},
{
"offset": 9,
"gen": 13
},
{
"offset": 46151,
"gen": 0,
"uncompressed": true
},
{
"offset": 47562,
"gen": 0,
"uncompressed": true
}
]
@@ -0,0 +1,145 @@
[
{
"offset": 0,
"gen": 255,
"free": true
},
{
"offset": 8,
"gen": 8
},
{
"offset": 8,
"gen": 9
},
{
"offset": 8,
"gen": 10
},
{
"offset": 8,
"gen": 1
},
{
"offset": 8,
"gen": 0
},
{
"offset": 15,
"gen": 0,
"uncompressed": true
},
{
"offset": 8,
"gen": 20
},
{
"offset": 33577,
"gen": 0,
"uncompressed": true
},
{
"offset": 8,
"gen": 3
},
{
"offset": 8,
"gen": 2
},
{
"offset": 1151,
"gen": 0,
"uncompressed": true
},
{
"offset": 8,
"gen": 17
},
{
"offset": 8,
"gen": 19
},
{
"offset": 8,
"gen": 18
},
{
"offset": 8,
"gen": 5
},
{
"offset": 8,
"gen": 4
},
{
"offset": 2721,
"gen": 0,
"uncompressed": true
},
{
"offset": 8,
"gen": 7
},
{
"offset": 8,
"gen": 6
},
{
"offset": 5757,
"gen": 0,
"uncompressed": true
},
{
"offset": 8,
"gen": 11
},
{
"offset": 8,
"gen": 12
},
{
"offset": 8,
"gen": 13
},
{
"offset": 9558,
"gen": 0,
"uncompressed": true
},
{
"offset": 8,
"gen": 14
},
{
"offset": 18967,
"gen": 0,
"uncompressed": true
},
{
"offset": 8,
"gen": 15
},
{
"offset": 26388,
"gen": 0,
"uncompressed": true
},
{
"offset": 8,
"gen": 16
},
{
"offset": 8,
"gen": 21
},
{
"offset": 33354,
"gen": 0,
"uncompressed": true
},
{
"offset": 34451,
"gen": 0,
"uncompressed": true
}
]
@@ -0,0 +1,73 @@
[
{
"offset": 0,
"gen": 65535,
"free": true
},
{
"offset": 8,
"gen": 4
},
{
"offset": 8,
"gen": 3
},
{
"offset": 8,
"gen": 1
},
{
"offset": 8,
"gen": 6
},
{
"offset": 8,
"gen": 8
},
{
"offset": 15,
"gen": 0,
"uncompressed": true
},
{
"offset": 8,
"gen": 0
},
{
"offset": 6857,
"gen": 0,
"uncompressed": true
},
{
"offset": 8,
"gen": 2
},
{
"offset": 265,
"gen": 0,
"uncompressed": true
},
{
"offset": 8,
"gen": 7
},
{
"offset": 8,
"gen": 5
},
{
"offset": 701,
"gen": 0,
"uncompressed": true
},
{
"offset": 6750,
"gen": 0,
"uncompressed": true
},
{
"offset": 7655,
"gen": 0,
"uncompressed": true
}
]