diff --git a/.gitignore b/.gitignore index bdbcabaa3..360713b19 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ /coverage /node_modules +/performance/lib /test/assets/QT3TS /test/assets/XQUTS /test/assets/runnablePerformanceTestNames.csv diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b2777ecc4..697abe2ce 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -153,7 +153,6 @@ FontoXPath contains different test sets: |The QT3 XQueryX tests|`npm run qt3testsxqueryx`| |The XQUTS tests|`npm run xqutstests`| |The XQUTS XQueryX tests|`npm run xqutstestsxqueryx`| -|The QT3 performance tests|`npm run performance`| They all run in Node. By running the tests with the `--inspect` flag, they can be debugged by the browser: `npm run test -- --inspect @@ -185,16 +184,24 @@ If you are adding a new feature, don't forget to edit the file `test/runnableTestSets.csv`. This file disables tests for features we have not yet implemented. -To check the performance of fontoxpath we pick a random subset of the -qt3tests as running all will take too long (hours). This random subset -is not checked in but can be generated using -`npm run performance -- --regenerate []`, this will create -and populate `test/runnablePerformanceTestNames.csv`. You can manually -edit this file to run specific tests. By storing these in a file you can -run the same set even when switching between different commits. With the -generated file present you can run the tests using `npm run performance`, -this will run a benchmark for each qt3 test using -[benchmarkjs](https://benchmarkjs.com/). +### Running benchmarks + +FontoXPath has 2 options to run benchmarks. + +In one we run benchmarks over scenarios defined in javascript which are located in the directory +`/performance`. These can be run using `npm run performance` which will run the benchmarks in the +console. Or you can start a server which hosts them using `npm run performance-server` which allows +you to test the performance in different browsers. Some of these benchmarks are set up as a +comparison which indicates the performance overhead of FontoXPath. Note that some tests use assets +from the QT3TS, see the steps in [Setting up a development environment](#setting-up-a-development-environment). + +To check the performance of fontoxpath with the qt3tests, we pick a random subset of the qt3tests as +running all will take too long (hours). This random subset is not checked in but can be generated +using `npm run qt3performance -- --regenerate []`, this will create and populate +`test/runnablePerformanceTestNames.csv`. You can manually edit this file to run specific tests. By +storing these in a file you can run the same set even when switching between different commits. With +the generated file present you can run the tests using `npm run qt3performance`, this will run a +benchmark for each qt3 test using [benchmarkjs](https://benchmarkjs.com/). ### Building diff --git a/package-lock.json b/package-lock.json index bccd39411..00d053532 100644 --- a/package-lock.json +++ b/package-lock.json @@ -312,6 +312,73 @@ "fastq": "^1.6.0" } }, + "@rollup/plugin-commonjs": { + "version": "11.1.0", + "resolved": "https://registry.npmjs.org/@rollup/plugin-commonjs/-/plugin-commonjs-11.1.0.tgz", + "integrity": "sha512-Ycr12N3ZPN96Fw2STurD21jMqzKwL9QuFhms3SD7KKRK7oaXUsBU9Zt0jL/rOPHiPYisI21/rXGO3jr9BnLHUA==", + "dev": true, + "requires": { + "@rollup/pluginutils": "^3.0.8", + "commondir": "^1.0.1", + "estree-walker": "^1.0.1", + "glob": "^7.1.2", + "is-reference": "^1.1.2", + "magic-string": "^0.25.2", + "resolve": "^1.11.0" + }, + "dependencies": { + "resolve": { + "version": "1.16.1", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.16.1.tgz", + "integrity": "sha512-rmAglCSqWWMrrBv/XM6sW0NuRFiKViw/W4d9EbC4pt+49H8JwHy+mcGmALTEg504AUDcLTvb1T2q3E9AnmY+ig==", + "dev": true, + "requires": { + "path-parse": "^1.0.6" + } + } + } + }, + "@rollup/plugin-node-resolve": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/@rollup/plugin-node-resolve/-/plugin-node-resolve-7.1.3.tgz", + "integrity": "sha512-RxtSL3XmdTAE2byxekYLnx+98kEUOrPHF/KRVjLH+DEIHy6kjIw7YINQzn+NXiH/NTrQLAwYs0GWB+csWygA9Q==", + "dev": true, + "requires": { + "@rollup/pluginutils": "^3.0.8", + "@types/resolve": "0.0.8", + "builtin-modules": "^3.1.0", + "is-module": "^1.0.0", + "resolve": "^1.14.2" + }, + "dependencies": { + "builtin-modules": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/builtin-modules/-/builtin-modules-3.1.0.tgz", + "integrity": "sha512-k0KL0aWZuBt2lrxrcASWDfwOLMnodeQjodT/1SxEQAXsHANgo6ZC/VEaSEHCXt7aSTZ4/4H5LKa+tBXmW7Vtvw==", + "dev": true + }, + "resolve": { + "version": "1.16.1", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.16.1.tgz", + "integrity": "sha512-rmAglCSqWWMrrBv/XM6sW0NuRFiKViw/W4d9EbC4pt+49H8JwHy+mcGmALTEg504AUDcLTvb1T2q3E9AnmY+ig==", + "dev": true, + "requires": { + "path-parse": "^1.0.6" + } + } + } + }, + "@rollup/pluginutils": { + "version": "3.0.9", + "resolved": "https://registry.npmjs.org/@rollup/pluginutils/-/pluginutils-3.0.9.tgz", + "integrity": "sha512-TLZavlfPAZYI7v33wQh4mTP6zojne14yok3DNSLcjoG/Hirxfkonn6icP5rrNWRn8nZsirJBFFpijVOJzkUHDg==", + "dev": true, + "requires": { + "@types/estree": "0.0.39", + "estree-walker": "^1.0.1", + "micromatch": "^4.0.2" + } + }, "@rushstack/node-core-library": { "version": "3.19.5", "resolved": "https://registry.npmjs.org/@rushstack/node-core-library/-/node-core-library-3.19.5.tgz", @@ -733,6 +800,29 @@ "integrity": "sha512-rr+OQyAjxze7GgWrSaJwydHStIhHq2lvY3BOC2Mj7KnzI7XK0Uw1TOOdI9lDoajEbSWLiYgoo4f1R51erQfhPQ==", "dev": true }, + "@types/estree": { + "version": "0.0.39", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-0.0.39.tgz", + "integrity": "sha512-EYNwp3bU+98cpU4lAWYYL7Zz+2gryWH1qbdDTidVd6hkiR6weksdbMadyXKXNPEkQFhXM+hVO9ZygomHXp+AIw==", + "dev": true + }, + "@types/events": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@types/events/-/events-3.0.0.tgz", + "integrity": "sha512-EaObqwIvayI5a8dCzhFrjKzVwKLxjoG9T6Ppd5CEo07LRKfQ8Yokw54r5+Wq7FaBQ+yXRvQAYPrHwya1/UFt9g==", + "dev": true + }, + "@types/glob": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/@types/glob/-/glob-7.1.1.tgz", + "integrity": "sha512-1Bh06cbWJUHMC97acuD6UMG29nMt0Aqz1vF3guLfG+kHHJhy3AyohZFFxYk2f7Q1SQIrNwvncxAE0N/9s70F2w==", + "dev": true, + "requires": { + "@types/events": "*", + "@types/minimatch": "*", + "@types/node": "*" + } + }, "@types/json5": { "version": "0.0.29", "resolved": "https://registry.npmjs.org/@types/json5/-/json5-0.0.29.tgz", @@ -745,6 +835,12 @@ "integrity": "sha512-FwI9gX75FgVBJ7ywgnq/P7tw+/o1GUbtP0KzbtusLigAOgIgNISRK0ZPl4qertvXSIE8YbsVJueQ90cDt9YYyw==", "dev": true }, + "@types/minimatch": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@types/minimatch/-/minimatch-3.0.3.tgz", + "integrity": "sha512-tHq6qdbT9U1IRSGf14CL0pUlULksvY9OZ+5eEgl1N7t+OA3tGvNpxJCzuKQlsNgCVwbAs670L1vcVQi8j9HjnA==", + "dev": true + }, "@types/mocha": { "version": "7.0.2", "resolved": "https://registry.npmjs.org/@types/mocha/-/mocha-7.0.2.tgz", @@ -767,6 +863,15 @@ "@types/node": "*" } }, + "@types/resolve": { + "version": "0.0.8", + "resolved": "https://registry.npmjs.org/@types/resolve/-/resolve-0.0.8.tgz", + "integrity": "sha512-auApPaJf3NPfe18hSoJkp8EbZzer2ISk7o8mCC3M9he/a04+gbMF97NkpD2S8riMGvm4BMRI59/SZQSaLTKpsQ==", + "dev": true, + "requires": { + "@types/node": "*" + } + }, "@types/sinon": { "version": "7.5.2", "resolved": "https://registry.npmjs.org/@types/sinon/-/sinon-7.5.2.tgz", @@ -1468,6 +1573,12 @@ "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", "dev": true }, + "estree-walker": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-1.0.1.tgz", + "integrity": "sha512-1fMXF3YP4pZZVozF8j/ZLfvnR8NSIljt56UhbZ5PeeDmmGHpgpdwQt7ITlGvYaQukCvuBRMLEiKiYC+oeIg4cg==", + "dev": true + }, "esutils": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", @@ -1688,9 +1799,9 @@ } }, "glob": { - "version": "7.1.3", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.3.tgz", - "integrity": "sha512-vcfuiIxogLV4DlGBHIUOwI0IbrJ8HWPc4MU7HzviGeNho/UJDfi6B5p3sHeWIQ0KGIU0Jpxi5ZHxemQfLkkAwQ==", + "version": "7.1.6", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz", + "integrity": "sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==", "dev": true, "requires": { "fs.realpath": "^1.0.0", @@ -1941,6 +2052,12 @@ "integrity": "sha512-2HvIEKRoqS62guEC+qBjpvRubdX910WCMuJTZ+I9yvqKU2/12eSL549HMwtabb4oupdj2sMP50k+XJfB/8JE6w==", "dev": true }, + "is-module": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-module/-/is-module-1.0.0.tgz", + "integrity": "sha1-Mlj7afeMFNW4FdZkM2tM/7ZEFZE=", + "dev": true + }, "is-negated-glob": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-negated-glob/-/is-negated-glob-1.0.0.tgz", @@ -1953,6 +2070,15 @@ "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", "dev": true }, + "is-reference": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/is-reference/-/is-reference-1.1.4.tgz", + "integrity": "sha512-uJA/CDPO3Tao3GTrxYn6AwkM4nUPJiGGYu5+cB8qbC7WGFlrKZbiRo7SFKxUAEpFUfiHofWCXBUNhvYJMh+6zw==", + "dev": true, + "requires": { + "@types/estree": "0.0.39" + } + }, "is-regex": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.0.5.tgz", @@ -2331,6 +2457,15 @@ "chalk": "^2.4.2" } }, + "magic-string": { + "version": "0.25.7", + "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.25.7.tgz", + "integrity": "sha512-4CrMT5DOHTDk4HYDlzmwu4FVCcIYI8gauveasrdCu2IKIFOJ3f0v/8MDGJCDL9oD2ppz/Av1b0Nj345H9M+XIA==", + "dev": true, + "requires": { + "sourcemap-codec": "^1.4.4" + } + }, "make-dir": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-3.0.0.tgz", @@ -2461,6 +2596,20 @@ "yargs-unparser": "1.6.0" }, "dependencies": { + "glob": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.3.tgz", + "integrity": "sha512-vcfuiIxogLV4DlGBHIUOwI0IbrJ8HWPc4MU7HzviGeNho/UJDfi6B5p3sHeWIQ0KGIU0Jpxi5ZHxemQfLkkAwQ==", + "dev": true, + "requires": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.0.4", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + } + }, "supports-color": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-6.0.0.tgz", @@ -3455,6 +3604,15 @@ "glob": "^7.1.3" } }, + "rollup": { + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/rollup/-/rollup-2.6.1.tgz", + "integrity": "sha512-1RhFDRJeg027YjBO6+JxmVWkEZY0ASztHhoEUEWxOwkh4mjO58TFD6Uo7T7Y3FbmDpRTfKhM5NVxJyimCn0Elg==", + "dev": true, + "requires": { + "fsevents": "~2.1.2" + } + }, "run-parallel": { "version": "1.1.9", "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.1.9.tgz", @@ -3585,6 +3743,12 @@ "source-map": "^0.6.0" } }, + "sourcemap-codec": { + "version": "1.4.8", + "resolved": "https://registry.npmjs.org/sourcemap-codec/-/sourcemap-codec-1.4.8.tgz", + "integrity": "sha512-9NykojV5Uih4lgo5So5dtw+f0JgJX30KCNI8gwhz2J9A15wD0Ml6tjHKwf6fTSa6fAdVBdZeNOs9eJ71qCk8vA==", + "dev": true + }, "spawn-command": { "version": "0.0.2-1", "resolved": "https://registry.npmjs.org/spawn-command/-/spawn-command-0.0.2-1.tgz", diff --git a/package.json b/package.json index bc0eda3ea..2a0a8bd19 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,9 @@ "coverage": "nyc report --reporter=text-lcov | coveralls", "integrationtests": "ts-mocha --require test/testhook.js \"test/specs/parsing/**/*.ts\" -p test/tsconfig.json", "prepare": "npm run build", - "performance": "ts-node -P test/tsconfig.json -r tsconfig-paths/register test/qt3testsBenchmark.ts", + "performance": "ts-node -P ./performance/tsconfig.json ./performance/runBenchmarks.ts", + "performance-server": "rimraf ./performance/lib && tsc -p ./performance/web.tsconfig.json && concurrently -k -p \"[{name}]\" -n \"tsc,rollup,server\" \"tsc -p ./performance/web.tsconfig.json -w\" \"rollup -c ./performance/rollup.config.js -w \" \"ts-node -P ./performance/tsconfig.json ./performance/server.ts\"", + "qt3performance": "ts-node -P test/tsconfig.json -r tsconfig-paths/register test/qt3testsBenchmark.ts", "qt3tests": "ts-mocha --paths -p test/tsconfig.json test/qt3tests.ts", "qt3testsxqueryx": "ts-mocha --paths -p test/tsconfig.json test/qt3testsXQueryX.ts", "start": "concurrently -k -p \"[{name}]\" -n \"TypeScript,Node\" \"tsc -p ./demo/tsconfig.json -w\" \"ts-node ./demo/server.ts --dist\"", @@ -61,9 +63,12 @@ "homepage": "https://github.com/FontoXML/fontoxpath", "devDependencies": { "@microsoft/api-extractor": "^7.7.10", + "@rollup/plugin-commonjs": "^11.1.0", + "@rollup/plugin-node-resolve": "^7.1.3", "@tscc/tscc": "^0.4.8", "@types/benchmark": "^1.0.31", "@types/chai": "^4.2.11", + "@types/glob": "^7.1.1", "@types/mocha": "^7.0.2", "@types/node": "^13.5.0", "@types/node-static": "^0.7.3", @@ -74,11 +79,13 @@ "coveralls": "^3.0.11", "cross-env": "^7.0.2", "fs-extra": "^9.0.0", + "glob": "^7.1.6", "mocha": "^7.1.1", "node-static": "^0.7.11", "nyc": "^15.0.0", "pegjs": "^0.10.0", "prettier": "^2.0.2", + "rollup": "^2.6.1", "sinon": "^9.0.1", "sinon-chai": "^3.5.0", "slimdom": "^2.3.2", diff --git a/performance/benchmarkRunner/BenchmarkCollection.ts b/performance/benchmarkRunner/BenchmarkCollection.ts new file mode 100644 index 000000000..3b6118231 --- /dev/null +++ b/performance/benchmarkRunner/BenchmarkCollection.ts @@ -0,0 +1,55 @@ +import Benchmark from 'benchmark'; + +type testFunction = () => void; +type setupFunction = () => void | Promise; +type teardownFunction = () => void | Promise; + +export default abstract class BenchmarkCollection { + protected readonly _benchmarks: { + benchmark: Benchmark; + setup?: setupFunction; + teardown?: teardownFunction; + }[] = []; + + protected readonly _comparisons: { + benchmarks: Benchmark[]; + name: string; + setup?: setupFunction; + teardown?: teardownFunction; + }[] = []; + + public addBenchmark( + name: string, + test: testFunction, + setup?: setupFunction, + teardown?: teardownFunction + ): void { + this._benchmarks.push({ + benchmark: new Benchmark(name, test), + // We do not use the setup and teardown which is offered within the API of benchmarkjs + // as several attempts to get this working did not yield any successful results. + setup, + teardown, + }); + } + + public compareBenchmarks( + name: string, + setup?: setupFunction, + teardown?: teardownFunction, + ...benchmarks: { + name: string; + test: testFunction; + }[] + ): void { + // We do not use the setup and teardown which is offered within the API of benchmarkjs + // as several attempts to get this working did not yield any successful results. We also + // allow only 1 setup and teardown as all functions which compare with one another should + // use the same data to test with. + const comparison = { name, benchmarks: [], setup, teardown }; + for (const benchmark of benchmarks) { + comparison.benchmarks.push(new Benchmark(benchmark.name, benchmark.test)); + } + this._comparisons.push(comparison); + } +} diff --git a/performance/benchmarkRunner/BenchmarkRunner.ts b/performance/benchmarkRunner/BenchmarkRunner.ts new file mode 100644 index 000000000..d3d378e0a --- /dev/null +++ b/performance/benchmarkRunner/BenchmarkRunner.ts @@ -0,0 +1,87 @@ +import BenchmarkCollection from './BenchmarkCollection'; +import OnlyBenchmarks from './OnlyBenchmarks'; + +class BenchmarkRunner extends BenchmarkCollection { + private readonly _only: OnlyBenchmarks = new OnlyBenchmarks(); + + public get only(): BenchmarkCollection { + return this._only; + } + + public async run(): Promise { + let benchmarks = this._benchmarks; + let comparisons = this._comparisons; + if (this._only.hasBenchmarks()) { + benchmarks = this._only.getBenchmarks(); + comparisons = this._only.getComparisons(); + } + + console.log(`Running ${benchmarks.length} benchmarks`); + for (const benchmark of benchmarks) { + if (benchmark.setup !== undefined) { + benchmark.setup(); + } + + benchmark.benchmark.on('complete', (event: Event) => { + console.log(String(event.target)); + + const error = (event.target as any).error; + if (error) { + console.error(error); + } + }); + + benchmark.benchmark.run({ async: false }); + + if (benchmark.teardown !== undefined) { + benchmark.teardown(); + } + } + + console.log(`Running ${comparisons.length} comparisons`); + for (const comparison of comparisons) { + console.log(`------------${comparison.name}------------`); + + const operationsPerSecond: { name: string; ops: number }[] = []; + for (const benchmark of comparison.benchmarks) { + if (comparison.setup !== undefined) { + await comparison.setup(); + } + + benchmark.on('complete', (event: Event) => { + console.log(String(event.target)); + + const error = (event.target as any).error; + if (error) { + console.error(error); + } else { + operationsPerSecond.push({ + name: (benchmark as any).name, + ops: (event.target as any).hz as number, + }); + } + }); + + benchmark.run({ async: false }); + + if (comparison.teardown !== undefined) { + await comparison.teardown(); + } + } + + operationsPerSecond.sort((a, b) => a.ops - b.ops); + const base = operationsPerSecond[0]; + for (let i = 0; i < operationsPerSecond.length; i++) { + const ops = operationsPerSecond[i]; + if (i === 0) { + console.log(`${ops.name} 100%`); + } else { + console.log(`${ops.name} ${(ops.ops / base.ops) * 100}%`); + } + } + } + } +} + +const runner = new BenchmarkRunner(); +export default runner; diff --git a/performance/benchmarkRunner/OnlyBenchmarks.ts b/performance/benchmarkRunner/OnlyBenchmarks.ts new file mode 100644 index 000000000..250a671d0 --- /dev/null +++ b/performance/benchmarkRunner/OnlyBenchmarks.ts @@ -0,0 +1,25 @@ +import Benchmark from 'benchmark'; +import BenchmarkCollection from './BenchmarkCollection'; + +export default class OnlyBenchmarks extends BenchmarkCollection { + public getBenchmarks(): { + benchmark: Benchmark; + setup?: () => void; + teardown?: () => void; + }[] { + return this._benchmarks; + } + + public getComparisons(): { + benchmarks: Benchmark[]; + name: string; + setup?: () => void; + teardown?: () => void; + }[] { + return this._comparisons; + } + + public hasBenchmarks(): boolean { + return this._benchmarks.length !== 0 || this._comparisons.length !== 0; + } +} diff --git a/performance/compare.benchmark.ts b/performance/compare.benchmark.ts new file mode 100644 index 000000000..8cc5d05df --- /dev/null +++ b/performance/compare.benchmark.ts @@ -0,0 +1,151 @@ +import { Document, Node } from 'slimdom'; +import * as slimdomSaxParser from 'slimdom-sax-parser'; +import { + domFacade, + evaluateXPathToFirstNode, + evaluateXPathToNumber, + evaluateXPath, +} from '../src/index'; +import runner from './benchmarkRunner/BenchmarkRunner'; +import jsonMlMapper from '../test/helpers/jsonMlMapper'; +import loadFile from './loadFile'; + +const testDocumentFilename = 'test/assets/QT3TS/app/XMark/XMarkAuction.xml'; + +let document: Document; + +runner.compareBenchmarks( + 'simple traversal to first descendant', + () => { + document = new Document(); + jsonMlMapper.parse(['r', ['a', ['b', ['c']]]], document); + }, + undefined, + { + name: 'xpath', + test: () => { + evaluateXPathToFirstNode('descendant::c', document, domFacade); + }, + }, + { + name: 'manual', + test: () => { + let node: Node = document; + while (node) { + if (node.nodeName === 'c') { + break; + } + node = node.firstChild; + } + }, + } +); + +runner.compareBenchmarks( + 'count 3190 text elements', + async () => { + const content = await loadFile(testDocumentFilename); + document = slimdomSaxParser.sync(content); + }, + undefined, + { + name: 'xpath', + test: () => { + evaluateXPathToNumber('count(//text)', document, domFacade); + }, + }, + { + name: 'manual', + test: () => { + function countParagraphs(node: Node): number { + let count = 0; + if (node.nodeName === 'text') { + count++; + } + node.childNodes.forEach((child) => { + count += countParagraphs(child); + }); + return count; + } + + countParagraphs(document); + }, + } +); + +runner.compareBenchmarks( + 'XMark-Q14, this is one of the more expensive tests in the qt3ts', + async () => { + const content = await loadFile(testDocumentFilename); + document = slimdomSaxParser.sync(content); + }, + undefined, + { + name: 'xpath', + test: () => { + evaluateXPathToFirstNode( + `(: Purpose: Return the names of all items whose description contains the word ;'gold'. :) + { + let $auction := (/) return + for $i in $auction/site//item + where contains(string(exactly-one($i/description)), "gold") + return $i/name/text() } + `, + document, + domFacade, + {}, + { language: evaluateXPath.XQUERY_3_1_LANGUAGE } + ); + }, + }, + { + name: 'manual', + test: () => { + function getDescendantItems(node: Node, nameTest: string): Node[] { + const results: Node[] = []; + node.childNodes.forEach((child) => { + if (child.nodeName === nameTest) { + results.push(child); + } + results.push(...getDescendantItems(child, nameTest)); + }); + return results; + } + + const auction = document; + const nameContents = []; + auction.childNodes.forEach((child) => { + if (child.nodeName !== 'site') { + return; + } + + const items = getDescendantItems(child, 'item'); + for (const i of items) { + const descriptionElements = []; + i.childNodes.forEach((c) => { + if (c.nodeName === 'description') { + descriptionElements.push(c); + } + }); + + if (descriptionElements.length !== 1) { + throw new Error('FORG0005'); + } + + if (descriptionElements[0].textContent.includes('gold')) { + i.childNodes.forEach((c) => { + if (c.nodeName === 'name') { + nameContents.push(c.textContent); + } + }); + } + } + }); + + const textNode = document.createTextNode(nameContents.join('')); + + const res = document.createElement('XMark-result-Q14'); + res.append(textNode); + }, + } +); diff --git a/performance/evaluateXPath.benchmark.ts b/performance/evaluateXPath.benchmark.ts new file mode 100644 index 000000000..d4c98b008 --- /dev/null +++ b/performance/evaluateXPath.benchmark.ts @@ -0,0 +1,17 @@ +import { Document } from 'slimdom'; +import { domFacade, evaluateXPath } from '../src/index'; +import runner from './benchmarkRunner/BenchmarkRunner'; + +let document: Document; + +function setup() { + document = new Document(); +} + +runner.addBenchmark( + 'evaluateXPath', + () => { + evaluateXPath('true()', document, domFacade); + }, + setup +); diff --git a/performance/index.html b/performance/index.html new file mode 100644 index 000000000..bd1eb8415 --- /dev/null +++ b/performance/index.html @@ -0,0 +1,16 @@ + + + performance benchmarks + + + + + + + +
+

performance benchmarks

+
Check the console for the results.
+
+ + diff --git a/performance/loadBenchmarks.ts b/performance/loadBenchmarks.ts new file mode 100644 index 000000000..83707bca7 --- /dev/null +++ b/performance/loadBenchmarks.ts @@ -0,0 +1,23 @@ +import * as glob from 'glob'; + +export function getImports(): string[] { + const toImport: string[] = []; + const files = glob.sync('performance/**/*.benchmark.ts'); + toImport.push( + ...files.map(file => { + const fileWithoutExtension = file.slice(0, -3); + // We need to replace 'performance/' with './' + const relativeImport = '.' + fileWithoutExtension.substring('performance'.length); + return relativeImport; + }) + ); + + return toImport; +} + +export default async function loadBenchmarks() { + const imports = getImports(); + for (const relativeImport of imports) { + await import(relativeImport); + } +} diff --git a/performance/loadFile.ts b/performance/loadFile.ts new file mode 100644 index 000000000..af0d10c4c --- /dev/null +++ b/performance/loadFile.ts @@ -0,0 +1,12 @@ +declare var window: Window; + +export default async function loadFile(filename: string): Promise { + if (typeof window !== 'undefined' && window.fetch) { + const request = new Request(`${window.location}${filename}`); + const response = await window.fetch(request); + return response.text(); + } else { + const fs = await import('fs'); + return fs.readFileSync(filename).toString(); + } +} diff --git a/performance/rollup-plugin-import-benchmarks.js b/performance/rollup-plugin-import-benchmarks.js new file mode 100644 index 000000000..9e5987119 --- /dev/null +++ b/performance/rollup-plugin-import-benchmarks.js @@ -0,0 +1,26 @@ +// We can import the javascript version as we always compile the typescript before rollup. +import { getImports } from './lib/performance/loadBenchmarks'; + +export default function importBenchmarks() { + return { + name: 'import-benchmarks', + + load(id) { + if (!id.endsWith('runBenchmarks.js')) { + return null; + } + + // We want to overwrite the code of runBenchmarks. Instead of loading all benchmarks by + // searching for the files we'll search for the files now and generate import statements. + const toImport = getImports(); + let imports = ''; + for (const i of toImport) { + imports += `import '${i}';`; + } + + return `import runner from './benchmarkRunner/BenchmarkRunner'; + ${imports} + runner.run();`; + } + }; +} diff --git a/performance/rollup.config.js b/performance/rollup.config.js new file mode 100644 index 000000000..0a1a481c4 --- /dev/null +++ b/performance/rollup.config.js @@ -0,0 +1,25 @@ +import resolve from '@rollup/plugin-node-resolve'; +import commonjs from '@rollup/plugin-commonjs'; +import importBenchmarks from './rollup-plugin-import-benchmarks.js'; + +export default { + input: 'performance/lib/performance/runBenchmarks.js', + output: [ + { + name: 'fontoxpath', + file: 'performance/lib/fontoxpath-perf.js', + format: 'umd', + sourcemap: true + } + ], + onwarn(warning) { + // Ignore "this is undefined" warning triggered by typescript's __extends helper + if (warning.code === 'THIS_IS_UNDEFINED') { + return; + } + + console.error(warning.message); + }, + plugins: [importBenchmarks(), resolve(), commonjs()], + external: ['benchmark'] +}; diff --git a/performance/runBenchmarks.ts b/performance/runBenchmarks.ts new file mode 100644 index 000000000..e833c74ce --- /dev/null +++ b/performance/runBenchmarks.ts @@ -0,0 +1,9 @@ +import runner from './benchmarkRunner/BenchmarkRunner'; +import loadBenchmarks from './loadBenchmarks'; + +async function run() { + await loadBenchmarks(); + await runner.run(); +} + +run(); diff --git a/performance/server.ts b/performance/server.ts new file mode 100644 index 000000000..0adac13ff --- /dev/null +++ b/performance/server.ts @@ -0,0 +1,41 @@ +import http, { IncomingMessage, ServerResponse } from 'http'; +import * as staticAlias from 'node-static'; + +const builtFileServer = new staticAlias.Server('./performance/lib'); +const performanceFileServer = new staticAlias.Server('./performance'); +const rootFileServer = new staticAlias.Server('./'); +const modulesFileServer = new staticAlias.Server('./node_modules'); + +http.createServer((request: IncomingMessage, response: ServerResponse) => { + request + .addListener('end', () => { + // + // Serve files! + // + if (request.url.startsWith('/test')) { + console.log('Serving .' + request.url); + rootFileServer.serve(request, response); + return; + } + + switch (request.url) { + case '/fontoxpath-perf.js': + case '/fontoxpath-perf.js.map': + console.log('Serving ./performance/lib' + request.url); + builtFileServer.serve(request, response); + break; + case '/lodash/lodash.js': + case '/platform/platform.js': + case '/benchmark/benchmark.js': + console.log('Serving ./node_modules' + request.url); + modulesFileServer.serve(request, response); + break; + default: + console.log('Serving ./performance' + request.url); + performanceFileServer.serve(request, response); + break; + } + }) + .resume(); +}).listen(8080); +console.log('Now listening on localhost:8080'); diff --git a/performance/tsconfig.json b/performance/tsconfig.json new file mode 100644 index 000000000..2f6946d18 --- /dev/null +++ b/performance/tsconfig.json @@ -0,0 +1,13 @@ +{ + "extends": "../tsconfig.json", + "compilerOptions": { + "baseUrl": "..", + "esModuleInterop": true, + "lib": ["dom"], + "module": "commonjs", + "outDir": "lib", + "sourceMap": false, + "target": "esnext" + }, + "include": ["../src/**/*", "**/*"] +} diff --git a/performance/tslint.json b/performance/tslint.json new file mode 100644 index 000000000..16577d426 --- /dev/null +++ b/performance/tslint.json @@ -0,0 +1,7 @@ +{ + "extends": ["../tslint.json"], + "rules": { + "no-console": false, + "no-implicit-dependencies": false + } +} diff --git a/performance/web.tsconfig.json b/performance/web.tsconfig.json new file mode 100644 index 000000000..6399c52ba --- /dev/null +++ b/performance/web.tsconfig.json @@ -0,0 +1,12 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "module": "esnext", + "outDir": "lib", + "paths": { + "slimdom": ["node_modules/slimdom/dist/slimdom.d.ts"], + "slimdom-sax-parser": ["node_modules/slimdom-sax-parser/index.js"] + }, + "strict": false + } +}