From 989ddd20923eedbeca4d2840e8e12bcf49fb7363 Mon Sep 17 00:00:00 2001 From: Michael Perrotte Date: Tue, 5 Nov 2019 22:24:23 -0500 Subject: [PATCH 01/28] chore: added package files --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 7c2375c..ba0e3fe 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2049,4 +2049,4 @@ "dev": true } } -} \ No newline at end of file +} diff --git a/package.json b/package.json index b38d384..833d6b6 100644 --- a/package.json +++ b/package.json @@ -29,4 +29,4 @@ "url": "https://github.com/npm/benchmarks/issues" }, "homepage": "https://github.com/npm/benchmarks#readme" -} \ No newline at end of file +} From 050e625da840746c18d3e1a601b7094561360f65 Mon Sep 17 00:00:00 2001 From: Michael Perrotte Date: Tue, 5 Nov 2019 22:25:02 -0500 Subject: [PATCH 02/28] feat: added fixture data --- fixtures/angular-quickstart/package.json | 48 +++++++++ fixtures/app-large/package.json | 118 +++++++++++++++++++++++ fixtures/app-medium/package.json | 63 ++++++++++++ fixtures/ember-quickstart/package.json | 47 +++++++++ fixtures/react-app/package.json | 16 +++ 5 files changed, 292 insertions(+) create mode 100644 fixtures/angular-quickstart/package.json create mode 100644 fixtures/app-large/package.json create mode 100644 fixtures/app-medium/package.json create mode 100644 fixtures/ember-quickstart/package.json create mode 100644 fixtures/react-app/package.json diff --git a/fixtures/angular-quickstart/package.json b/fixtures/angular-quickstart/package.json new file mode 100644 index 0000000..9e2b771 --- /dev/null +++ b/fixtures/angular-quickstart/package.json @@ -0,0 +1,48 @@ +{ + "name": "angular-quickstart", + "version": "0.0.0", + "license": "MIT", + "scripts": { + "ng": "ng", + "start": "ng serve", + "build": "ng build --prod", + "test": "ng test", + "lint": "ng lint", + "e2e": "ng e2e" + }, + "private": true, + "dependencies": { + "@angular/animations": "^5.2.0", + "@angular/common": "^5.2.0", + "@angular/compiler": "^5.2.0", + "@angular/core": "^5.2.0", + "@angular/forms": "^5.2.0", + "@angular/http": "^5.2.0", + "@angular/platform-browser": "^5.2.0", + "@angular/platform-browser-dynamic": "^5.2.0", + "@angular/router": "^5.2.0", + "core-js": "^2.4.1", + "rxjs": "^5.5.6", + "zone.js": "^0.8.19" + }, + "devDependencies": { + "@angular/cli": "~1.7.0", + "@angular/compiler-cli": "^5.2.0", + "@angular/language-service": "^5.2.0", + "@types/jasmine": "~2.8.3", + "@types/jasminewd2": "~2.0.2", + "@types/node": "~6.0.60", + "codelyzer": "^4.0.1", + "jasmine-core": "~2.8.0", + "jasmine-spec-reporter": "~4.2.1", + "karma": "~2.0.0", + "karma-chrome-launcher": "~2.2.0", + "karma-coverage-istanbul-reporter": "^1.2.1", + "karma-jasmine": "~1.1.0", + "karma-jasmine-html-reporter": "^0.2.2", + "protractor": "~5.1.2", + "ts-node": "~4.1.0", + "tslint": "~5.9.1", + "typescript": "~2.5.3" + } +} diff --git a/fixtures/app-large/package.json b/fixtures/app-large/package.json new file mode 100644 index 0000000..a9774e5 --- /dev/null +++ b/fixtures/app-large/package.json @@ -0,0 +1,118 @@ +{ + "name": "app-large", + "version": "0.0.1", + "dependencies": { + "animate.less": "^2.2.0", + "autoprefixer": "^6.0.3", + "babel-core": "^6.4.0", + "babel-eslint": "^6.1.2", + "babel-loader": "^6.2.1", + "babel-plugin-lodash": "^3.2.11", + "babel-plugin-module-resolver": "^2.2.0", + "babel-plugin-transform-decorators-legacy": "^1.3.4", + "babel-plugin-transform-runtime": "^6.4.3", + "babel-polyfill": "^6.23.0", + "babel-preset-es2015": "^6.3.13", + "babel-preset-react": "^6.3.13", + "babel-preset-react-hmre": "^1.0.1", + "babel-preset-stage-1": "^6.3.13", + "babel-runtime": "^6.3.19", + "clean-webpack-plugin": "^0.1.16", + "core-decorators": "^0.12.3", + "css-loader": "^0.23.1", + "css-mqpacker": "^4.0.0", + "cssnano": "^3.2.0", + "custom-event-polyfill": "^0.2.2", + "draft-js": "^0.9.0", + "ejs": "^2.5.6", + "eslint": "^3.4.0", + "eslint-config-airbnb": "^11.0.0", + "eslint-import-resolver-webpack": "^0.5.1", + "eslint-plugin-import": "^1.14.0", + "eslint-plugin-jsx-a11y": "^2.2.1", + "eslint-plugin-react": "^6.2.0", + "express": "^4.15.2", + "express-http-proxy": "^0.11.0", + "font-awesome": "^4.7.0", + "fready": "^1.0.0", + "glob": "^7.1.1", + "gulp": "^3.9.0", + "gulp-concat": "^2.6.0", + "gulp-csslint": "^0.2.0", + "gulp-cssnano": "^2.0.0", + "gulp-eol": "^0.1.1", + "gulp-less": "^3.0.5", + "gulp-livereload": "^3.8.1", + "gulp-minify-css": "^1.2.3", + "gulp-postcss": "^6.0.1", + "gulp-rename": "^1.2.2", + "gulp-util": "^3.0.7", + "happypack": "^3.0.3", + "highcharts": "^5.0.10", + "highcharts-solid-gauge": "^0.1.2", + "history": "^4.6.1", + "howler": "^1.1.28", + "imports-loader": "^0.6.5", + "jquery": "^2.2.0", + "jquery-ui": "1.10.5", + "js-cookie": "^2.1.3", + "json-loader": "^0.5.4", + "leftpad": "^0.0.0", + "less": "^2.7.2", + "lesshat": "^3.0.2", + "lodash": "^3.0.0", + "medium-draft": "^0.4.1", + "mobx": "^3.1.8", + "mobx-react": "^4.1.5", + "moment": "^2.18.1", + "moment-range": "^2.0.3", + "moment-timezone": "^0.5.13", + "password-policy": "0.0.2", + "postcss-reporter": "^1.2.1", + "progress": "^2.0.0", + "qs": "^6.1.0", + "raw-loader": "^0.5.1", + "rc-slider": "^6.1.0", + "react": "^15.4.1", + "react-addons-css-transition-group": "^15.3.0", + "react-addons-shallow-compare": "^15.3.0", + "react-dnd": "^2.1.4", + "react-dnd-html5-backend": "^2.1.2", + "react-dom": "^15.4.1", + "react-draft-wysiwyg": "^1.6.5", + "react-dropzone": "^3.5.3", + "react-grid-layout": "^0.12.6", + "react-highcharts": "^11.5.0", + "react-hot-loader": "v3.0.0-beta.6", + "react-input-calendar": "^0.3.14", + "react-lazyload": "^2.2.5", + "react-measure": "^1.4.6", + "react-mixin": "^3.0.3", + "react-responsive": "^1.2.5", + "react-responsive-tabs": "^0.5.3", + "react-router": "^4.0.0", + "react-router-dom": "^4.0.0", + "react-select-plus": "^1.0.0-rc", + "react-skylight": "^0.3.0", + "react-sortablejs": "^1.2.1", + "react-tappable": "^0.8.4", + "react-tooltip": "^3.3.0", + "react-virtualized": "^7.19.4", + "react-waypoint": "^5.2.0", + "sortablejs": "^1.5.0-rc1", + "style-loader": "^0.13.0", + "stylelint": "^1.2.1", + "superagent": "^1.6.1", + "uglify-js": "^2.8.22", + "uuid": "^3.0.1", + "verge": "^1.9.1", + "webpack-bundle-analyzer": "^2.3.1", + "webpack-hot-middleware": "^2.18.0", + "webpack-notifier": "^1.5.0", + "webpack-split-by-path": "^2.0.0", + "whatwg-fetch": "^2.0.3" + }, + "devDependencies": { + "nan-as": "^1.6.1" + } +} diff --git a/fixtures/app-medium/package.json b/fixtures/app-medium/package.json new file mode 100644 index 0000000..cf31320 --- /dev/null +++ b/fixtures/app-medium/package.json @@ -0,0 +1,63 @@ +{ + "name": "app-medium", + "version": "0.0.0", + "dependencies": { + "axios": "^0.16.0", + "emailjs": "^0.3.13", + "es6-promise": "^4.1.0", + "faker": "^3.1.0", + "js-beautify": "^1.6.14", + "json3": "^3.3.2", + "lodash": "^4.17.4", + "store2": "^2.5.0", + "vue": "^2.2.2", + "vue-axios": "^2.0.1", + "vue-my-dropdown": "^2.0.3", + "vue-resource": "^1.2.1", + "vue-select": "^2.1.0" + }, + "devDependencies": { + "@corbinu/eslint-plugin-corbinu": "^2.0.0", + "autoprefixer": "^6.7.2", + "babel-core": "^6.22.1", + "babel-eslint": "^7.2.3", + "babel-loader": "^7.0.0", + "babel-plugin-transform-runtime": "^6.22.0", + "babel-preset-env": "^1.2.1", + "babel-preset-es2015": "^6.24.0", + "babel-preset-stage-2": "^6.22.0", + "babel-register": "^6.22.0", + "chalk": "^1.1.3", + "connect-history-api-fallback": "^1.3.0", + "copy-webpack-plugin": "^4.0.1", + "css-loader": "^0.28.0", + "eslint": "^3.14.1", + "eslint-friendly-formatter": "^2.0.7", + "eslint-loader": "^1.6.1", + "eslint-plugin-html": "^2.0.0", + "eventsource-polyfill": "^0.9.6", + "express": "^4.14.1", + "extract-text-webpack-plugin": "^2.0.0", + "file-loader": "^0.11.1", + "friendly-errors-webpack-plugin": "^1.1.3", + "function-bind": "^1.1.0", + "html-webpack-plugin": "^2.28.0", + "http-proxy-middleware": "^0.17.3", + "nyc": "^10.2.0", + "opn": "^4.0.2", + "optimize-css-assets-webpack-plugin": "^1.3.0", + "ora": "^1.1.0", + "rimraf": "^2.6.0", + "semver": "^5.3.0", + "typescript": "~2.2.0", + "url-loader": "^0.5.7", + "vue-loader": "^11.1.4", + "vue-style-loader": "^3.0.0", + "vue-template-compiler": "^2.2.1", + "webpack": "^2.2.1", + "webpack-bundle-analyzer": "^2.2.1", + "webpack-dev-middleware": "^1.10.2", + "webpack-hot-middleware": "^2.16.1", + "webpack-merge": "^4.0.0" + } +} diff --git a/fixtures/ember-quickstart/package.json b/fixtures/ember-quickstart/package.json new file mode 100644 index 0000000..f40d8aa --- /dev/null +++ b/fixtures/ember-quickstart/package.json @@ -0,0 +1,47 @@ +{ + "name": "ember-quickstart", + "version": "0.0.0", + "private": true, + "description": "Small description for ember-quickstart goes here", + "license": "MIT", + "author": "", + "directories": { + "doc": "doc", + "test": "tests" + }, + "repository": "", + "scripts": { + "build": "ember build", + "lint:js": "eslint ./*.js app config lib server tests", + "start": "ember serve", + "test": "ember test" + }, + "devDependencies": { + "broccoli-asset-rev": "^2.4.5", + "ember-ajax": "^3.0.0", + "ember-cli": "~3.0.0", + "ember-cli-app-version": "^3.0.0", + "ember-cli-babel": "^6.6.0", + "ember-cli-dependency-checker": "^2.0.0", + "ember-cli-eslint": "^4.2.1", + "ember-cli-htmlbars": "^2.0.1", + "ember-cli-htmlbars-inline-precompile": "^1.0.0", + "ember-cli-inject-live-reload": "^1.4.1", + "ember-cli-qunit": "^4.1.1", + "ember-cli-shims": "^1.2.0", + "ember-cli-sri": "^2.1.0", + "ember-cli-uglify": "^2.0.0", + "ember-data": "~3.0.0", + "ember-export-application-global": "^2.0.0", + "ember-load-initializers": "^1.0.0", + "ember-maybe-import-regenerator": "^0.1.6", + "ember-resolver": "^4.0.0", + "ember-source": "~3.0.0", + "ember-welcome-page": "^3.0.0", + "eslint-plugin-ember": "^5.0.0", + "loader.js": "^4.2.3" + }, + "engines": { + "node": "^4.5 || 6.* || >= 7.*" + } +} diff --git a/fixtures/react-app/package.json b/fixtures/react-app/package.json new file mode 100644 index 0000000..6261569 --- /dev/null +++ b/fixtures/react-app/package.json @@ -0,0 +1,16 @@ +{ + "name": "react-app", + "version": "0.1.0", + "private": true, + "dependencies": { + "react": "^16.2.0", + "react-dom": "^16.2.0", + "react-scripts": "1.1.1" + }, + "scripts": { + "start": "react-scripts start", + "build": "react-scripts build", + "test": "react-scripts test --env=jsdom", + "eject": "react-scripts eject" + } +} From 440d56730f0cda2399b2cb3b7490c8c99270b9bd Mon Sep 17 00:00:00 2001 From: Michael Perrotte Date: Tue, 5 Nov 2019 22:26:03 -0500 Subject: [PATCH 03/28] feat: added initial 'base' (not real) results --- results/npm/6.12.1.json | 118 ++++++++++++++++++++++++++++++++++++++++ results/npm/6.13.0.json | 60 ++++++++++++++++++++ results/npm/latest.json | 60 ++++++++++++++++++++ 3 files changed, 238 insertions(+) create mode 100644 results/npm/6.12.1.json create mode 100644 results/npm/6.13.0.json create mode 100644 results/npm/latest.json diff --git a/results/npm/6.12.1.json b/results/npm/6.12.1.json new file mode 100644 index 0000000..52cce5d --- /dev/null +++ b/results/npm/6.12.1.json @@ -0,0 +1,118 @@ +[ + { + "initial install": { + "angular-quickstart": 1004, + "app-large": 1005, + "app-medium": 1002, + "ember-quickstart": 1005, + "react-app": 1003 + }, + "repeat install": { + "angular-quickstart": 1001, + "app-large": 1001, + "app-medium": 1004, + "ember-quickstart": 1001, + "react-app": 1005 + }, + "with warm cache": { + "angular-quickstart": 1001, + "app-large": 1001, + "app-medium": 1005, + "ember-quickstart": 1002, + "react-app": 1003 + }, + "with node_modules": { + "angular-quickstart": 1005, + "app-large": 1002, + "app-medium": 1001, + "ember-quickstart": 1001, + "react-app": 1001 + }, + "with lockfile": { + "angular-quickstart": 1006, + "app-large": 1005, + "app-medium": 1002, + "ember-quickstart": 1005, + "react-app": 1000 + }, + "with warm cache and node_modules": { + "angular-quickstart": 1005, + "app-large": 1003, + "app-medium": 1002, + "ember-quickstart": 1001, + "react-app": 1004 + }, + "with warm cache and lockfile": { + "angular-quickstart": 1005, + "app-large": 1000, + "app-medium": 1005, + "ember-quickstart": 1000, + "react-app": 1001 + }, + "with node_modules and lockfile": { + "angular-quickstart": 1003, + "app-large": 1004, + "app-medium": 1000, + "ember-quickstart": 1000, + "react-app": 1004 + } + }, + { + "initial install": { + "angular-quickstart": 1002, + "app-large": 1003, + "app-medium": 1000, + "ember-quickstart": 1001, + "react-app": 1002 + }, + "repeat install": { + "angular-quickstart": 1004, + "app-large": 1002, + "app-medium": 1000, + "ember-quickstart": 1003, + "react-app": 1003 + }, + "with warm cache": { + "angular-quickstart": 1003, + "app-large": 1004, + "app-medium": 1005, + "ember-quickstart": 1001, + "react-app": 1005 + }, + "with node_modules": { + "angular-quickstart": 1003, + "app-large": 1005, + "app-medium": 1002, + "ember-quickstart": 1000, + "react-app": 1004 + }, + "with lockfile": { + "angular-quickstart": 1005, + "app-large": 1001, + "app-medium": 1000, + "ember-quickstart": 1004, + "react-app": 1005 + }, + "with warm cache and node_modules": { + "angular-quickstart": 1003, + "app-large": 1005, + "app-medium": 1003, + "ember-quickstart": 1003, + "react-app": 1004 + }, + "with warm cache and lockfile": { + "angular-quickstart": 1003, + "app-large": 1000, + "app-medium": 1003, + "ember-quickstart": 1005, + "react-app": 1003 + }, + "with node_modules and lockfile": { + "angular-quickstart": 1001, + "app-large": 1001, + "app-medium": 1001, + "ember-quickstart": 1002, + "react-app": 1002 + } + } +] \ No newline at end of file diff --git a/results/npm/6.13.0.json b/results/npm/6.13.0.json new file mode 100644 index 0000000..3cbc43e --- /dev/null +++ b/results/npm/6.13.0.json @@ -0,0 +1,60 @@ +[ + { + "initial install": { + "angular-quickstart": 1006, + "app-large": 1003, + "app-medium": 1003, + "ember-quickstart": 1004, + "react-app": 1002 + }, + "repeat install": { + "angular-quickstart": 1002, + "app-large": 1004, + "app-medium": 1004, + "ember-quickstart": 1003, + "react-app": 1001 + }, + "with warm cache": { + "angular-quickstart": 1004, + "app-large": 1005, + "app-medium": 1000, + "ember-quickstart": 1003, + "react-app": 1005 + }, + "with node_modules": { + "angular-quickstart": 1003, + "app-large": 1006, + "app-medium": 1005, + "ember-quickstart": 1003, + "react-app": 1003 + }, + "with lockfile": { + "angular-quickstart": 1001, + "app-large": 1005, + "app-medium": 1001, + "ember-quickstart": 1002, + "react-app": 1005 + }, + "with warm cache and node_modules": { + "angular-quickstart": 1003, + "app-large": 1005, + "app-medium": 1003, + "ember-quickstart": 1002, + "react-app": 1000 + }, + "with warm cache and lockfile": { + "angular-quickstart": 1001, + "app-large": 1003, + "app-medium": 1004, + "ember-quickstart": 1002, + "react-app": 1004 + }, + "with node_modules and lockfile": { + "angular-quickstart": 1004, + "app-large": 1002, + "app-medium": 1004, + "ember-quickstart": 1001, + "react-app": 1001 + } + } +] \ No newline at end of file diff --git a/results/npm/latest.json b/results/npm/latest.json new file mode 100644 index 0000000..5d48f15 --- /dev/null +++ b/results/npm/latest.json @@ -0,0 +1,60 @@ +[ + { + "initial install": { + "angular-quickstart": 1004, + "app-large": 1006, + "app-medium": 1005, + "ember-quickstart": 1003, + "react-app": 1002 + }, + "repeat install": { + "angular-quickstart": 1003, + "app-large": 999, + "app-medium": 1003, + "ember-quickstart": 1001, + "react-app": 1004 + }, + "with warm cache": { + "angular-quickstart": 1002, + "app-large": 1004, + "app-medium": 1003, + "ember-quickstart": 1000, + "react-app": 1005 + }, + "with node_modules": { + "angular-quickstart": 1001, + "app-large": 1001, + "app-medium": 1004, + "ember-quickstart": 1006, + "react-app": 1004 + }, + "with lockfile": { + "angular-quickstart": 1001, + "app-large": 1005, + "app-medium": 1000, + "ember-quickstart": 1003, + "react-app": 1002 + }, + "with warm cache and node_modules": { + "angular-quickstart": 1000, + "app-large": 1003, + "app-medium": 1002, + "ember-quickstart": 1005, + "react-app": 1000 + }, + "with warm cache and lockfile": { + "angular-quickstart": 1003, + "app-large": 1003, + "app-medium": 1003, + "ember-quickstart": 1003, + "react-app": 1003 + }, + "with node_modules and lockfile": { + "angular-quickstart": 1001, + "app-large": 1003, + "app-medium": 1003, + "ember-quickstart": 1000, + "react-app": 1001 + } + } +] \ No newline at end of file From 33872ba418aeeac2e576a24bcf4271b01c8135b0 Mon Sep 17 00:00:00 2001 From: Michael Perrotte Date: Tue, 5 Nov 2019 22:26:45 -0500 Subject: [PATCH 04/28] feat: added constants file --- lib/constants.js | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 lib/constants.js diff --git a/lib/constants.js b/lib/constants.js new file mode 100644 index 0000000..64213c3 --- /dev/null +++ b/lib/constants.js @@ -0,0 +1,10 @@ +'use strict' + +const { join } = require('path') + +exports.BASE_DIR = join(__dirname, '..') +exports.FIXTURES_DIR = join(this.BASE_DIR, 'fixtures') +exports.TMP_DIR = join(this.BASE_DIR, 'tmp') +exports.RESULTS_DIR = join(this.BASE_DIR, 'results') + +exports.CACHE_NAME = 'cache' From 7d78ca60ddaeed682dc258289ba7ecdaf64cab72 Mon Sep 17 00:00:00 2001 From: Michael Perrotte Date: Tue, 5 Nov 2019 22:28:19 -0500 Subject: [PATCH 05/28] feat: added npm command benchmark suite --- lib/npm-suite.js | 261 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 261 insertions(+) create mode 100644 lib/npm-suite.js diff --git a/lib/npm-suite.js b/lib/npm-suite.js new file mode 100644 index 0000000..e450753 --- /dev/null +++ b/lib/npm-suite.js @@ -0,0 +1,261 @@ +'use strict' + +const { join } = require('path') + +const { + CACHE_NAME, + FIXTURES_DIR +} = require('./constants') + +const { + createEnv, + removeCache, + removeNodeModules, + removeLockfile, + measureAction +} = require('./suite-actions') + +/** + * NOTE: Scenario Ordering + * The first scenario needs to be an "initial install" to populate the cache, + * node_modules folder, and create a lockfile. All the subsequent scenarios + * can be in any order. As a subsequent scenario starts with populated cache, + * node_modules folder, and lockfile (created from the previous scenario), + * they mearly need to remove the idea they do not want. + */ +module.exports.suiteName = 'npm' +module.exports.suiteCmd = 'npm' +module.exports.scenarios = [ + { + name: 'Initial install', + details: { + cache: false, + node_modules: false, + lockfile: false + }, + cmd: 'npm', + args: [ + 'install', + '--ignore-scripts', + '--cache', + `${CACHE_NAME}`, + '--registry', + 'https://registry.npmjs.org/' + ], + actions: [ + removeCache, + removeNodeModules, + removeLockfile, + async (ctx, fixture) => { + const { cmd, args } = ctx + const env = createEnv() + + const cwd = join(FIXTURES_DIR, fixture) + return measureAction({ cmd, args, env, cwd }) + } + ] + }, + { + name: 'Repeat install', + details: { + cache: true, + node_modules: true, + lockfile: true + }, + cmd: 'npm', + args: [ + 'install', + '--ignore-scripts', + '--cache', + `${CACHE_NAME}`, + '--registry', + 'https://registry.npmjs.org/' + ], + actions: [ + (ctx, fixture) => { + const { cmd, args } = ctx + const env = createEnv() + + const cwd = join(FIXTURES_DIR, fixture) + return measureAction({ cmd, args, env, cwd }) + } + ] + }, + { + name: 'With warm cache', + details: { + cache: true, + node_modules: false, + lockfile: false + }, + cmd: 'npm', + args: [ + 'install', + '--ignore-scripts', + '--cache', + `${CACHE_NAME}`, + '--registry', + 'https://registry.npmjs.org/' + ], + actions: [ + removeNodeModules, + removeLockfile, + (ctx, fixture) => { + const { cmd, args } = ctx + const env = createEnv() + + const cwd = join(FIXTURES_DIR, fixture) + return measureAction({ cmd, args, env, cwd }) + } + ] + }, + { + name: 'With node_modules', + details: { + cache: false, + node_modules: true, + lockfile: false + }, + cmd: 'npm', + args: [ + 'install', + '--ignore-scripts', + '--cache', + `${CACHE_NAME}`, + '--registry', + 'https://registry.npmjs.org/' + ], + actions: [ + removeCache, + removeLockfile, + (ctx, fixture) => { + const { cmd, args } = ctx + const env = createEnv() + + const cwd = join(FIXTURES_DIR, fixture) + return measureAction({ cmd, args, env, cwd }) + } + ] + }, + { + name: 'With lockfile', + details: { + cache: false, + node_modules: false, + lockfile: true + }, + cmd: 'npm', + args: [ + 'install', + '--ignore-scripts', + '--cache', + `${CACHE_NAME}`, + '--registry', + 'https://registry.npmjs.org/' + ], + actions: [ + removeCache, + removeNodeModules, + (ctx, fixture) => { + const { cmd, args } = ctx + const env = createEnv() + + const cwd = join(FIXTURES_DIR, fixture) + return measureAction({ cmd, args, env, cwd }) + } + ] + }, + { + name: 'With warm cache and node_modules', + details: { + cache: true, + node_modules: true, + lockfile: false + }, + cmd: 'npm', + args: [ + 'install', + '--ignore-scripts', + '--cache', + `${CACHE_NAME}`, + '--registry', + 'https://registry.npmjs.org/' + ], + actions: [ + removeLockfile, + (ctx, fixture) => { + const { cmd, args } = ctx + const env = createEnv() + + const cwd = join(FIXTURES_DIR, fixture) + return measureAction({ cmd, args, env, cwd }) + } + ] + }, + { + name: 'With warm cache and lockfile', + details: { + cache: true, + node_modules: false, + lockfile: true + }, + cmd: 'npm', + args: [ + 'install', + '--ignore-scripts', + '--cache', + `${CACHE_NAME}`, + '--registry', + 'https://registry.npmjs.org/' + ], + actions: [ + removeNodeModules, + (ctx, fixture) => { + const { cmd, args } = ctx + const env = createEnv() + + const cwd = join(FIXTURES_DIR, fixture) + return measureAction({ cmd, args, env, cwd }) + } + ] + }, + { + name: 'With node_modules and lockfile', + details: { + cache: false, + node_modules: true, + lockfile: true + }, + cmd: 'npm', + args: [ + 'install', + '--ignore-scripts', + '--cache', + `${CACHE_NAME}`, + '--registry', + 'https://registry.npmjs.org/' + ], + actions: [ + removeCache, + (ctx, fixture) => { + const { cmd, args } = ctx + const env = createEnv() + + const cwd = join(FIXTURES_DIR, fixture) + return measureAction({ cmd, args, env, cwd }) + } + ] + }, + { + name: 'cleanup', + details: {}, + cmd: 'noop', + args: [], + actions: [ + removeCache, + removeNodeModules, + removeLockfile, + (ctx, fixture) => 0 + ] + } +] From 3252e760658d7eaa484870ff9fe0a145fe3c5c9f Mon Sep 17 00:00:00 2001 From: Michael Perrotte Date: Tue, 5 Nov 2019 22:28:39 -0500 Subject: [PATCH 06/28] feat: added suite actions/helpers --- lib/suite-actions.js | 73 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 lib/suite-actions.js diff --git a/lib/suite-actions.js b/lib/suite-actions.js new file mode 100644 index 0000000..a22b77b --- /dev/null +++ b/lib/suite-actions.js @@ -0,0 +1,73 @@ +'use strict' + +const child = require('child_process') +const rimraf = require('rimraf') +const { join } = require('path') +const { log } = require('./utils') + +const { CACHE_NAME, FIXTURES_DIR } = require('./constants') + +exports.measureAction = async function measureAction ({ cmd, args, env, cwd }) { + log.verbose('measureAction', 'executing...') + log.silly('measureAction', 'cmd: %s', cmd) // TESTING + log.silly('measureAction', 'args: %o', args) // TESTING + log.silly('measureAction', 'env: %o', env) // TESTING + log.silly('measureAction', 'cwd: %s', cwd) // TESTING + const startTime = Date.now() + + // TODO: allow config to be passed in to allow process output 'stdio' + const result = child.spawnSync(cmd, args, { env, cwd }) + if (result.status !== 0) { + log.error(result.error) + throw new Error(`${cmd} failed with status code ${result.status}`) + } + const endTime = Date.now() + return endTime - startTime +} + +exports.createEnv = function createEnv (overrides = {}) { + const env = Object.keys(process.env).reduce((acc, key) => { + return (key.match(/^npm_/)) + // Don't include `npm_` key/values + ? acc + // Add key/value to env object + : Object.assign({}, acc, { [key]: process.env[key] }) + }, {}) + log.silly('createEnv', 'env: %o', env) + return Object.assign({}, env, overrides) +} + +exports.removePath = async function removePath (path) { + return new Promise((resolve, reject) => { + rimraf(path, {}, (err) => { + if (err) { + return reject(err) + } + return resolve() + }) + }) +} + +exports.removeCache = async function removeCache (ctx, fixture) { + log.verbose('removeCache', 'removing cache...') + const cacheDir = join(FIXTURES_DIR, fixture, CACHE_NAME) + log.silly('removeCache', 'cacheDir: %s', cacheDir) + return exports.removePath(cacheDir) +} + +exports.removeNodeModules = async function removeNodeModules (ctx, fixture) { + log.verbose('removeNodeModules', 'removing node_modules...') + const cwd = join(FIXTURES_DIR, fixture) + const nodeModulesDir = join(cwd, 'node_modules') + log.silly('removeNodeModules', 'nodeModulesDir: %s', nodeModulesDir) + return exports.removePath(nodeModulesDir) +} + +exports.removeLockfile = async function removeLockfile (ctx, fixture) { + log.verbose('removeLockfile', 'removing lockfile...') + // TODO: change this to use `scenario.lockfile` + const cwd = join(FIXTURES_DIR, fixture) + const lockfilePath = join(cwd, 'package-lock.json') + log.silly('removeLockfile', 'lockfilePath: %s', lockfilePath) + return exports.removePath(lockfilePath) +} From 756bd1ddfbb1bf4519e2d4a520535f094a1dc5a6 Mon Sep 17 00:00:00 2001 From: Michael Perrotte Date: Tue, 5 Nov 2019 22:29:03 -0500 Subject: [PATCH 07/28] feat: added common utils --- lib/utils.js | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 lib/utils.js diff --git a/lib/utils.js b/lib/utils.js new file mode 100644 index 0000000..191b73b --- /dev/null +++ b/lib/utils.js @@ -0,0 +1,48 @@ +'use strict' + +const child = require('child_process') +const log = require('npmlog') +const fs = require('fs') +const { join } = require('path') + +const { RESULTS_DIR } = require('./constants') + +const { LOG_LEVEL } = process.env + +log.level = LOG_LEVEL || 'info' +exports.log = log + +exports.safeLoadResults = function safeLoadResults (filename) { + const filepath = join(RESULTS_DIR, filename) + try { + const file = fs.readFileSync(filepath, 'utf8') + return (file) ? JSON.parse(file) : [] + } catch (err) { + if (err.code !== 'ENOENT') { + throw err + } + return [] + } +} + +exports.writeResults = function writeResults (filename, results) { + console.log('filename:', filename) + const filepath = join(RESULTS_DIR, filename) + console.log('filepath:', filepath) + try { + const data = JSON.stringify(results, null, ' ') + fs.writeFileSync(filepath, data) + } catch (err) { + throw err + } +} + +exports.fetchCommandVersion = function fetchCommandVersion (cmd) { + const result = child.spawnSync(cmd, ['--version']) + if (result.status !== 0) { + log.error(result.error) + throw new Error(`${cmd} failed with status code ${result.status}`) + } + const version = result.stdout.toString().trim() + return version +} From 9e81436cac263f9340f568212a8056527f84a9ca Mon Sep 17 00:00:00 2001 From: Michael Perrotte Date: Tue, 5 Nov 2019 22:29:46 -0500 Subject: [PATCH 08/28] feat: added execution function --- lib/execute.js | 84 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 lib/execute.js diff --git a/lib/execute.js b/lib/execute.js new file mode 100644 index 0000000..44afaf7 --- /dev/null +++ b/lib/execute.js @@ -0,0 +1,84 @@ +'use strict' + +const fs = require('fs') + +const { + log, + safeLoadResults, + writeResults, + fetchCommandVersion +} = require('./utils') + +const { FIXTURES_DIR } = require('./constants') + +/** + * Executes a given scenario with a provided fixture + * + * @param {Object} scenario scenario object + * @param {string} scenario.name name of the scenario + * @param {object} scenario.details details about scenario + * @param {boolean} scenario.details.cache + * @param {boolean} scenario.details.node_modules + * @param {boolean} scenario.details.lockfile + * @param {string} scenario.cmd command used by the scenario + * @param {string[]} scenario.args list of arguments to pass to the command + * @param {function[]} scenario.actions list of actions this scenario will take + * @param {string} fixture directory name of a fixture + */ +async function executeScenario (scenario, fixture) { + const { actions } = scenario + + const result = await actions.reduce( + (acc, action) => { + return acc.then((result) => action(scenario, fixture)) + }, + Promise.resolve() + ) + log.info('execute', 'Result Time: %d', result) + log.info('execute', 'Details: %o', scenario.details) + return result +} + +module.exports = async function execute (latest) { + const { scenarios, suiteCmd, suiteName } = require('./npm-suite') + const fixtures = fs.readdirSync(FIXTURES_DIR, 'utf8') + + try { + const newResults = {} + for (let x = 0; x < scenarios.length; ++x) { + const scenario = scenarios[x] + + const scenarioKey = scenario.name.toLowerCase() + if (scenarioKey !== 'cleanup') { + newResults[scenarioKey] = {} + } + + log.info('scenario', scenario.name) + for (let i = 0; i < fixtures.length; ++i) { + const fixture = fixtures[i] + + log.info('fixture', fixture) + const execResult = await executeScenario(scenario, fixture) + + const fixtureKey = fixture.toLowerCase() + if (newResults[scenarioKey]) { + newResults[scenarioKey][fixtureKey] = execResult + } + } + } + + if (latest) { + const filename = `${suiteName}/latest.json` + const results = [newResults] + writeResults(filename, results) + } else { + const version = fetchCommandVersion(suiteCmd) + const filename = `${suiteName}/${version}.json` + const prevResults = safeLoadResults(filename) + const results = [...prevResults, newResults] + writeResults(filename, results) + } + } catch (e) { + log.error(e) + } +} From dca32ffe197ef128ca4ddcff397d730b356fa1b8 Mon Sep 17 00:00:00 2001 From: Michael Perrotte Date: Tue, 5 Nov 2019 22:29:58 -0500 Subject: [PATCH 09/28] feat: added repo entrypoint --- index.js | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 index.js diff --git a/index.js b/index.js new file mode 100644 index 0000000..2941611 --- /dev/null +++ b/index.js @@ -0,0 +1,16 @@ +'use strict' + +const execute = require('./lib/execute') +const { log } = require('./lib/utils') + +// INFO: can potentially pass in command line arguments +const args = process.argv.slice(2) +const latest = (args.length && args[0] === 'latest') + +log.verbose('ARGS:', args) +if (latest) { + log.info('Executing benchmark against latest release') +} else { + log.info('Executing benchmark against a version') +} +execute(latest) From 34363692af2eeb0b687b0373d42546eae73b9149 Mon Sep 17 00:00:00 2001 From: Michael Perrotte Date: Wed, 6 Nov 2019 00:24:29 -0500 Subject: [PATCH 10/28] feat: added dedent, and octokit as deps --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 833d6b6..b38d384 100644 --- a/package.json +++ b/package.json @@ -29,4 +29,4 @@ "url": "https://github.com/npm/benchmarks/issues" }, "homepage": "https://github.com/npm/benchmarks#readme" -} +} \ No newline at end of file From ad5faf9293eb757230778fb67c62b0ea8433a1b6 Mon Sep 17 00:00:00 2001 From: Michael Perrotte Date: Wed, 6 Nov 2019 13:04:24 -0500 Subject: [PATCH 11/28] feat: updated scripts in package file --- package.json | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index b38d384..2e33511 100644 --- a/package.json +++ b/package.json @@ -16,7 +16,9 @@ "standard": "^12.0.1" }, "scripts": { - "benchmark": "node index.js", + "benchmark:pr": "node index.js benchmark", + "benchmark:release": "node index.js benchmark latest", + "comment": "node index.js comment", "test": "echo \"Error: no test specified\" && exit 1" }, "repository": { From cf7ba3d9c19b1f0c224e80d5a47ef05c53ad47df Mon Sep 17 00:00:00 2001 From: Michael Perrotte Date: Wed, 6 Nov 2019 13:05:56 -0500 Subject: [PATCH 12/28] feat: updated index to accempt command name; updated logging --- index.js | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/index.js b/index.js index 2941611..50e3166 100644 --- a/index.js +++ b/index.js @@ -1,16 +1,27 @@ 'use strict' +const comment = require('./lib/comment') const execute = require('./lib/execute') const { log } = require('./lib/utils') -// INFO: can potentially pass in command line arguments const args = process.argv.slice(2) -const latest = (args.length && args[0] === 'latest') +const command = args.length && args[0] +const isRelease = args.length && args[1] log.verbose('ARGS:', args) -if (latest) { - log.info('Executing benchmark against latest release') -} else { - log.info('Executing benchmark against a version') +const { PR_ID, REPO, OWNER } = process.env + +switch (command) { + case 'benchmark': + log.info('Executing benchmark against latest release') + execute(!!isRelease) + break + case 'comment': + // TODO: bail out if we don't have correct environment variables + log.info(`Posting Comment to ${OWNER}/${REPO}/pulls/${PR_ID}`) + comment() + break + default: + log.error('Invalid argument supplied...') + log.error('Please use the npm-scripts.') } -execute(latest) From 8aaa27892bb89dc495176d4b586c2481e22eb320 Mon Sep 17 00:00:00 2001 From: Michael Perrotte Date: Wed, 6 Nov 2019 13:06:45 -0500 Subject: [PATCH 13/28] feat: updated benchmark execution to handle writing multiple files; keep latest.json up-to-date --- lib/execute.js | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/lib/execute.js b/lib/execute.js index 44afaf7..9478759 100644 --- a/lib/execute.js +++ b/lib/execute.js @@ -39,7 +39,7 @@ async function executeScenario (scenario, fixture) { return result } -module.exports = async function execute (latest) { +module.exports = async function execute (latest = false) { const { scenarios, suiteCmd, suiteName } = require('./npm-suite') const fixtures = fs.readdirSync(FIXTURES_DIR, 'utf8') @@ -67,16 +67,19 @@ module.exports = async function execute (latest) { } } + // NOTE: always write to versioned file + const version = fetchCommandVersion(suiteCmd) + const versionFilename = `${suiteName}/${version}.json` + const prevResults = safeLoadResults(versionFilename) + const results = [...prevResults, newResults] + writeResults(versionFilename, results) + if (latest) { + // NOTE: Optionally write to `latest.json` const filename = `${suiteName}/latest.json` + // INFO: `latest.json` should always be one dataset const results = [newResults] writeResults(filename, results) - } else { - const version = fetchCommandVersion(suiteCmd) - const filename = `${suiteName}/${version}.json` - const prevResults = safeLoadResults(filename) - const results = [...prevResults, newResults] - writeResults(filename, results) } } catch (e) { log.error(e) From e7ba5d432b89721a3308f2975b0cb6cc9eadd018 Mon Sep 17 00:00:00 2001 From: Michael Perrotte Date: Wed, 6 Nov 2019 13:11:27 -0500 Subject: [PATCH 14/28] feat: updated workflow handle two kinds of benchmarking scenarios --- .github/workflows/benchmark-cli.yml | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/.github/workflows/benchmark-cli.yml b/.github/workflows/benchmark-cli.yml index 3d2f543..9169f2a 100644 --- a/.github/workflows/benchmark-cli.yml +++ b/.github/workflows/benchmark-cli.yml @@ -73,10 +73,20 @@ jobs: echo "Current Commit: $(git log --oneline -1)" # Run benchmarking suite - - name: Run Benchmark + - name: Run Benchmark (Pull-Request) + if: github.event.action == 'pull_request' run: | echo "Running benchmark..." echo "PWD: $(pwd)" + npm run benchmark:pr + + # Run benchmarking suite + - name: Run Benchmark (Push) + if: github.event.action == 'push' + run: | + echo "Running benchmark..." + echo "PWD: $(pwd)" + npm run benchmark:release # CONDITIONALLY: Post to pull-request - name: Post to Pull-Request @@ -86,7 +96,9 @@ jobs: REPO: ${{ github.event.client_payload.repo }} OWNER: ${{ github.event.client_payload.owner }} GITHUB_TOKEN: ${{ github.token }} - run: echo "Posting to pull-request..." + run: | + echo "Posting to pull-request..." + npm run comment # CONDITIONALLY: Commit results of benchmark suite into `npm/benchmark` repo - name: Commit Results From 2f49fd52e712bab25742cb06072067079f1b001d Mon Sep 17 00:00:00 2001 From: Michael Perrotte Date: Wed, 6 Nov 2019 13:16:39 -0500 Subject: [PATCH 15/28] feat: added comment function; added result parsing function --- lib/comment.js | 58 +++++++++++++++++++++++++++++++++++++ lib/parse-result.js | 70 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 128 insertions(+) create mode 100644 lib/comment.js create mode 100644 lib/parse-result.js diff --git a/lib/comment.js b/lib/comment.js new file mode 100644 index 0000000..cd15d96 --- /dev/null +++ b/lib/comment.js @@ -0,0 +1,58 @@ +'use strict' + +const Octokit = require('@octokit/rest') + +const { + safeLoadResults, + fetchCommandVersion +} = require('./utils') + +const parseResults = require('./parse-result') + +const { PR_ID, REPO, OWNER, GITHUB_TOKEN } = process.env + +module.exports = async function comment () { + const { suiteCmd, suiteName } = require('./npm-suite') + const latestFilename = `${suiteName}/latest.json` + const latestResults = safeLoadResults(latestFilename) + + const version = fetchCommandVersion(suiteCmd) + const currentFilename = `${suiteName}/${version}.json` + const currentResults = safeLoadResults(currentFilename) + + // INFO: get the "most recent" current results (last item of the list) + const output = parseResults(latestResults[0], currentResults[currentResults.length - 1]) + console.log(output) + + const octokit = new Octokit({ auth: GITHUB_TOKEN }) + console.log('PR:', PR_ID) // TESTING + console.log('REPO:', REPO) // TESTING + console.log('OWNER:', OWNER) // TESTING + console.log('TOKEN:', GITHUB_TOKEN) // TESTING + + const options = octokit.issues.listComments.endpoint.merge({ + owner: OWNER, + repo: REPO, + issue_number: PR_ID + }) + const comments = await octokit.paginate(options) + console.log('COMMENTS:', comments) // TESTING + const updateComment = comments.find((c) => c.user.login === 'npm-deploy-user') + + if (updateComment) { + console.log('FOUND COMMENT') // TESTING + await octokit.issues.updateComment({ + owner: OWNER, + repo: REPO, + comment_id: updateComment.id, + body: output + }) + } else { + await octokit.issues.createComment({ + owner: OWNER, + repo: REPO, + issue_number: PR_ID, + body: output + }) + } +} diff --git a/lib/parse-result.js b/lib/parse-result.js new file mode 100644 index 0000000..59b625c --- /dev/null +++ b/lib/parse-result.js @@ -0,0 +1,70 @@ +'use strict' + +const dedent = require('dedent') + +module.exports = function parseResults (latestResults, currentResults) { + const message = dedent` + + + + + ${tableHeader(currentResults)} + + + + ${subTableHeader(currentResults)} + + + + ${resultRows(latestResults, currentResults)} + +
+ ` + return message +} + +function tableHeader (results) { + // INFO: The results are mapped such that top level keys are scenario names + const scenarioKeys = Object.keys(results) + /** + * INFO: + * The results are mapped such that scenarios all have the same keys, + * which are the fixtures. + */ + const fixtures = Object.keys(results[scenarioKeys[0]]) + + return fixtures + .map((fixture) => `${fixture}`) + .join('\n') +} + +function subTableHeader (results) { + const scenarioKeys = Object.keys(results) + const fixtures = Object.keys(results[scenarioKeys[0]]) + + return fixtures + .map((fixture) => dedent` + prev + current + `) + .join('\n') +} + +function resultRows (latest, current) { + const scenarioKeys = Object.keys(current) + const fixtureKeys = Object.keys(current[scenarioKeys[0]]) + return scenarioKeys + .map((scenarioKey) => dedent` + + ${scenarioKey} + ${fixtureKeys.map((fixtureKey) => dedent` + ${latest[scenarioKey][fixtureKey] / 1000}s + ${current[scenarioKey][fixtureKey] / 1000}s + `).join('\n')} + + `) + .join('\n') +} + +// style="color: #00B200" // green +// style="color: #A00000" // red From 111b2db68e8f61b143ce7e20c5c699abc8d51fd5 Mon Sep 17 00:00:00 2001 From: Michael Perrotte Date: Wed, 6 Nov 2019 13:36:18 -0500 Subject: [PATCH 16/28] chore: cleaned up logging --- lib/comment.js | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/lib/comment.js b/lib/comment.js index cd15d96..65e08f2 100644 --- a/lib/comment.js +++ b/lib/comment.js @@ -3,6 +3,7 @@ const Octokit = require('@octokit/rest') const { + log, safeLoadResults, fetchCommandVersion } = require('./utils') @@ -22,13 +23,8 @@ module.exports = async function comment () { // INFO: get the "most recent" current results (last item of the list) const output = parseResults(latestResults[0], currentResults[currentResults.length - 1]) - console.log(output) const octokit = new Octokit({ auth: GITHUB_TOKEN }) - console.log('PR:', PR_ID) // TESTING - console.log('REPO:', REPO) // TESTING - console.log('OWNER:', OWNER) // TESTING - console.log('TOKEN:', GITHUB_TOKEN) // TESTING const options = octokit.issues.listComments.endpoint.merge({ owner: OWNER, @@ -36,11 +32,10 @@ module.exports = async function comment () { issue_number: PR_ID }) const comments = await octokit.paginate(options) - console.log('COMMENTS:', comments) // TESTING const updateComment = comments.find((c) => c.user.login === 'npm-deploy-user') if (updateComment) { - console.log('FOUND COMMENT') // TESTING + log.verbose('Updating comment...') await octokit.issues.updateComment({ owner: OWNER, repo: REPO, @@ -48,6 +43,7 @@ module.exports = async function comment () { body: output }) } else { + log.verbose('Posting comment...') await octokit.issues.createComment({ owner: OWNER, repo: REPO, From 39c555a8909647e33d714d11c17905d4e8342d97 Mon Sep 17 00:00:00 2001 From: Michael Perrotte Date: Wed, 6 Nov 2019 13:37:00 -0500 Subject: [PATCH 17/28] feat: added pretty-ms dep for output to pull-requests --- package-lock.json | 13 +++++++++++++ package.json | 1 + 2 files changed, 14 insertions(+) diff --git a/package-lock.json b/package-lock.json index ba0e3fe..f1be862 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1446,6 +1446,11 @@ "error-ex": "^1.2.0" } }, + "parse-ms": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/parse-ms/-/parse-ms-2.1.0.tgz", + "integrity": "sha512-kHt7kzLoS9VBZfUsiKjv43mr91ea+U05EyKkEtqp7vNbHxmaVuEqN7XxeEVnGrMtYOAxGrDElSi96K7EgO1zCA==" + }, "path-exists": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz", @@ -1561,6 +1566,14 @@ "integrity": "sha1-IZMqVJ9eUv/ZqCf1cOBL5iqX2lQ=", "dev": true }, + "pretty-ms": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/pretty-ms/-/pretty-ms-5.0.0.tgz", + "integrity": "sha512-94VRYjL9k33RzfKiGokPBPpsmloBYSf5Ri+Pq19zlsEcUKFob+admeXr5eFDRuPjFmEOcjJvPGdillYOJyvZ7Q==", + "requires": { + "parse-ms": "^2.1.0" + } + }, "process-nextick-args": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", diff --git a/package.json b/package.json index 2e33511..89abea2 100644 --- a/package.json +++ b/package.json @@ -10,6 +10,7 @@ "@octokit/rest": "^16.34.1", "dedent": "^0.7.0", "npmlog": "^4.1.2", + "pretty-ms": "^5.0.0", "rimraf": "^3.0.0" }, "devDependencies": { From 9c193b097272d8d43f4eac349f4b7e1d0bd7526c Mon Sep 17 00:00:00 2001 From: Michael Perrotte Date: Wed, 6 Nov 2019 13:37:16 -0500 Subject: [PATCH 18/28] feat: updated result parsing logic --- lib/parse-result.js | 31 +++++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/lib/parse-result.js b/lib/parse-result.js index 59b625c..339dc85 100644 --- a/lib/parse-result.js +++ b/lib/parse-result.js @@ -1,6 +1,7 @@ 'use strict' const dedent = require('dedent') +const prettyMs = require('pretty-ms') module.exports = function parseResults (latestResults, currentResults) { const message = dedent` @@ -16,7 +17,7 @@ module.exports = function parseResults (latestResults, currentResults) { - ${resultRows(latestResults, currentResults)} + ${resultRowsScenario(latestResults, currentResults)} ` @@ -50,21 +51,35 @@ function subTableHeader (results) { .join('\n') } -function resultRows (latest, current) { +function resultRowsScenario (latest, current) { const scenarioKeys = Object.keys(current) const fixtureKeys = Object.keys(current[scenarioKeys[0]]) return scenarioKeys .map((scenarioKey) => dedent` ${scenarioKey} - ${fixtureKeys.map((fixtureKey) => dedent` - ${latest[scenarioKey][fixtureKey] / 1000}s - ${current[scenarioKey][fixtureKey] / 1000}s - `).join('\n')} + ${resultRowFixtures(latest, current, scenarioKey, fixtureKeys)} `) .join('\n') } -// style="color: #00B200" // green -// style="color: #A00000" // red +function resultRowFixtures (latest, current, scenarioKey, fixtureKeys) { + return fixtureKeys + .map((fixtureKey) => { + const latestValue = latest[scenarioKey][fixtureKey] + const currentValue = current[scenarioKey][fixtureKey] + return dedent` + ${prettyMs(latestValue)} + ${colorResult(latestValue, currentValue)} + ` + }) + .join('\n') +} + +function colorResult (latest, current) { + const color = (current <= latest) + ? '#00B200' + : '#A00000' + return `${prettyMs(current)}` +} From d3ac251428a315a5f0d938ecb827afd0c5370ec9 Mon Sep 17 00:00:00 2001 From: Michael Perrotte Date: Wed, 6 Nov 2019 13:50:19 -0500 Subject: [PATCH 19/28] feat: updated results (from local machine; starting point) --- results/npm/6.12.1.json | 138 ++++++++++++---------------------------- results/npm/6.13.0.json | 80 +++++++++++------------ results/npm/latest.json | 80 +++++++++++------------ 3 files changed, 120 insertions(+), 178 deletions(-) diff --git a/results/npm/6.12.1.json b/results/npm/6.12.1.json index 52cce5d..86985ba 100644 --- a/results/npm/6.12.1.json +++ b/results/npm/6.12.1.json @@ -1,118 +1,60 @@ [ { "initial install": { - "angular-quickstart": 1004, - "app-large": 1005, - "app-medium": 1002, - "ember-quickstart": 1005, - "react-app": 1003 + "angular-quickstart": 35599, + "app-large": 46539, + "app-medium": 32573, + "ember-quickstart": 20675, + "react-app": 31276 }, "repeat install": { - "angular-quickstart": 1001, - "app-large": 1001, - "app-medium": 1004, - "ember-quickstart": 1001, - "react-app": 1005 + "angular-quickstart": 5485, + "app-large": 5148, + "app-medium": 5118, + "ember-quickstart": 4288, + "react-app": 5132 }, "with warm cache": { - "angular-quickstart": 1001, - "app-large": 1001, - "app-medium": 1005, - "ember-quickstart": 1002, - "react-app": 1003 + "angular-quickstart": 22952, + "app-large": 28344, + "app-medium": 21468, + "ember-quickstart": 17333, + "react-app": 21432 }, "with node_modules": { - "angular-quickstart": 1005, - "app-large": 1002, - "app-medium": 1001, - "ember-quickstart": 1001, - "react-app": 1001 + "angular-quickstart": 5469, + "app-large": 4413, + "app-medium": 4500, + "ember-quickstart": 4045, + "react-app": 5099 }, "with lockfile": { - "angular-quickstart": 1006, - "app-large": 1005, - "app-medium": 1002, - "ember-quickstart": 1005, - "react-app": 1000 + "angular-quickstart": 22024, + "app-large": 19162, + "app-medium": 17644, + "ember-quickstart": 12819, + "react-app": 15733 }, "with warm cache and node_modules": { - "angular-quickstart": 1005, - "app-large": 1003, - "app-medium": 1002, - "ember-quickstart": 1001, - "react-app": 1004 + "angular-quickstart": 5132, + "app-large": 4705, + "app-medium": 4295, + "ember-quickstart": 4626, + "react-app": 4968 }, "with warm cache and lockfile": { - "angular-quickstart": 1005, - "app-large": 1000, - "app-medium": 1005, - "ember-quickstart": 1000, - "react-app": 1001 + "angular-quickstart": 14464, + "app-large": 15992, + "app-medium": 12704, + "ember-quickstart": 9524, + "react-app": 12502 }, "with node_modules and lockfile": { - "angular-quickstart": 1003, - "app-large": 1004, - "app-medium": 1000, - "ember-quickstart": 1000, - "react-app": 1004 - } - }, - { - "initial install": { - "angular-quickstart": 1002, - "app-large": 1003, - "app-medium": 1000, - "ember-quickstart": 1001, - "react-app": 1002 - }, - "repeat install": { - "angular-quickstart": 1004, - "app-large": 1002, - "app-medium": 1000, - "ember-quickstart": 1003, - "react-app": 1003 - }, - "with warm cache": { - "angular-quickstart": 1003, - "app-large": 1004, - "app-medium": 1005, - "ember-quickstart": 1001, - "react-app": 1005 - }, - "with node_modules": { - "angular-quickstart": 1003, - "app-large": 1005, - "app-medium": 1002, - "ember-quickstart": 1000, - "react-app": 1004 - }, - "with lockfile": { - "angular-quickstart": 1005, - "app-large": 1001, - "app-medium": 1000, - "ember-quickstart": 1004, - "react-app": 1005 - }, - "with warm cache and node_modules": { - "angular-quickstart": 1003, - "app-large": 1005, - "app-medium": 1003, - "ember-quickstart": 1003, - "react-app": 1004 - }, - "with warm cache and lockfile": { - "angular-quickstart": 1003, - "app-large": 1000, - "app-medium": 1003, - "ember-quickstart": 1005, - "react-app": 1003 - }, - "with node_modules and lockfile": { - "angular-quickstart": 1001, - "app-large": 1001, - "app-medium": 1001, - "ember-quickstart": 1002, - "react-app": 1002 + "angular-quickstart": 5552, + "app-large": 4887, + "app-medium": 4703, + "ember-quickstart": 4309, + "react-app": 5131 } } ] \ No newline at end of file diff --git a/results/npm/6.13.0.json b/results/npm/6.13.0.json index 3cbc43e..191d4e2 100644 --- a/results/npm/6.13.0.json +++ b/results/npm/6.13.0.json @@ -1,60 +1,60 @@ [ { "initial install": { - "angular-quickstart": 1006, - "app-large": 1003, - "app-medium": 1003, - "ember-quickstart": 1004, - "react-app": 1002 + "angular-quickstart": 30811, + "app-large": 35246, + "app-medium": 29939, + "ember-quickstart": 21210, + "react-app": 26683 }, "repeat install": { - "angular-quickstart": 1002, - "app-large": 1004, - "app-medium": 1004, - "ember-quickstart": 1003, - "react-app": 1001 + "angular-quickstart": 6470, + "app-large": 5472, + "app-medium": 5021, + "ember-quickstart": 4405, + "react-app": 5134 }, "with warm cache": { - "angular-quickstart": 1004, - "app-large": 1005, - "app-medium": 1000, - "ember-quickstart": 1003, - "react-app": 1005 + "angular-quickstart": 26246, + "app-large": 28782, + "app-medium": 24411, + "ember-quickstart": 16512, + "react-app": 22088 }, "with node_modules": { - "angular-quickstart": 1003, - "app-large": 1006, - "app-medium": 1005, - "ember-quickstart": 1003, - "react-app": 1003 + "angular-quickstart": 5968, + "app-large": 4874, + "app-medium": 4628, + "ember-quickstart": 4215, + "react-app": 4995 }, "with lockfile": { - "angular-quickstart": 1001, - "app-large": 1005, - "app-medium": 1001, - "ember-quickstart": 1002, - "react-app": 1005 + "angular-quickstart": 22441, + "app-large": 23821, + "app-medium": 19189, + "ember-quickstart": 13879, + "react-app": 18536 }, "with warm cache and node_modules": { - "angular-quickstart": 1003, - "app-large": 1005, - "app-medium": 1003, - "ember-quickstart": 1002, - "react-app": 1000 + "angular-quickstart": 5434, + "app-large": 6209, + "app-medium": 4448, + "ember-quickstart": 4054, + "react-app": 4701 }, "with warm cache and lockfile": { - "angular-quickstart": 1001, - "app-large": 1003, - "app-medium": 1004, - "ember-quickstart": 1002, - "react-app": 1004 + "angular-quickstart": 17935, + "app-large": 19273, + "app-medium": 15355, + "ember-quickstart": 11512, + "react-app": 14882 }, "with node_modules and lockfile": { - "angular-quickstart": 1004, - "app-large": 1002, - "app-medium": 1004, - "ember-quickstart": 1001, - "react-app": 1001 + "angular-quickstart": 5699, + "app-large": 5355, + "app-medium": 4865, + "ember-quickstart": 4409, + "react-app": 5475 } } ] \ No newline at end of file diff --git a/results/npm/latest.json b/results/npm/latest.json index 5d48f15..191d4e2 100644 --- a/results/npm/latest.json +++ b/results/npm/latest.json @@ -1,60 +1,60 @@ [ { "initial install": { - "angular-quickstart": 1004, - "app-large": 1006, - "app-medium": 1005, - "ember-quickstart": 1003, - "react-app": 1002 + "angular-quickstart": 30811, + "app-large": 35246, + "app-medium": 29939, + "ember-quickstart": 21210, + "react-app": 26683 }, "repeat install": { - "angular-quickstart": 1003, - "app-large": 999, - "app-medium": 1003, - "ember-quickstart": 1001, - "react-app": 1004 + "angular-quickstart": 6470, + "app-large": 5472, + "app-medium": 5021, + "ember-quickstart": 4405, + "react-app": 5134 }, "with warm cache": { - "angular-quickstart": 1002, - "app-large": 1004, - "app-medium": 1003, - "ember-quickstart": 1000, - "react-app": 1005 + "angular-quickstart": 26246, + "app-large": 28782, + "app-medium": 24411, + "ember-quickstart": 16512, + "react-app": 22088 }, "with node_modules": { - "angular-quickstart": 1001, - "app-large": 1001, - "app-medium": 1004, - "ember-quickstart": 1006, - "react-app": 1004 + "angular-quickstart": 5968, + "app-large": 4874, + "app-medium": 4628, + "ember-quickstart": 4215, + "react-app": 4995 }, "with lockfile": { - "angular-quickstart": 1001, - "app-large": 1005, - "app-medium": 1000, - "ember-quickstart": 1003, - "react-app": 1002 + "angular-quickstart": 22441, + "app-large": 23821, + "app-medium": 19189, + "ember-quickstart": 13879, + "react-app": 18536 }, "with warm cache and node_modules": { - "angular-quickstart": 1000, - "app-large": 1003, - "app-medium": 1002, - "ember-quickstart": 1005, - "react-app": 1000 + "angular-quickstart": 5434, + "app-large": 6209, + "app-medium": 4448, + "ember-quickstart": 4054, + "react-app": 4701 }, "with warm cache and lockfile": { - "angular-quickstart": 1003, - "app-large": 1003, - "app-medium": 1003, - "ember-quickstart": 1003, - "react-app": 1003 + "angular-quickstart": 17935, + "app-large": 19273, + "app-medium": 15355, + "ember-quickstart": 11512, + "react-app": 14882 }, "with node_modules and lockfile": { - "angular-quickstart": 1001, - "app-large": 1003, - "app-medium": 1003, - "ember-quickstart": 1000, - "react-app": 1001 + "angular-quickstart": 5699, + "app-large": 5355, + "app-medium": 4865, + "ember-quickstart": 4409, + "react-app": 5475 } } ] \ No newline at end of file From 6b4744d56caa7ed636b0af9a091cedbc658e5135 Mon Sep 17 00:00:00 2001 From: Michael Perrotte Date: Wed, 6 Nov 2019 22:35:58 -0500 Subject: [PATCH 20/28] fix: updated matrix in workflow --- .github/workflows/benchmark-cli.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/benchmark-cli.yml b/.github/workflows/benchmark-cli.yml index 9169f2a..949fce7 100644 --- a/.github/workflows/benchmark-cli.yml +++ b/.github/workflows/benchmark-cli.yml @@ -9,10 +9,8 @@ jobs: strategy: matrix: - # os: [ubuntu-latest, macOS-latest] - os: [ubuntu-latest] - # node-version: [10.x, 12.x] - node-version: [10.x] + os: [ubuntu-latest, macOS-latest] + node-version: [10.x, 12.x] runs-on: ${{ matrix.os }} From 1036c66f2c59a9eaf763ecc9e8d233f0522711be Mon Sep 17 00:00:00 2001 From: Michael Perrotte Date: Wed, 6 Nov 2019 22:36:36 -0500 Subject: [PATCH 21/28] chore: updated comment explaination --- .github/workflows/benchmark-cli.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/benchmark-cli.yml b/.github/workflows/benchmark-cli.yml index 949fce7..5e9308b 100644 --- a/.github/workflows/benchmark-cli.yml +++ b/.github/workflows/benchmark-cli.yml @@ -20,7 +20,7 @@ jobs: uses: actions/checkout@v1.1.0 with: path: ${{ env.RUNNER_WORKSPACE }} - # Checkout repo incoming dispatch request is from (npm/cli) + # Checkout repo incoming dispatch request is from (eg. npm/cli) - name: checkout/${{ github.event.client_payload.owner }}/${{ github.event.client_payload.repo }} uses: actions/checkout@v1.1.0 with: From e1a2acf4e021a678fe432255d2eed44cd1b2497b Mon Sep 17 00:00:00 2001 From: Michael Perrotte Date: Wed, 6 Nov 2019 22:37:00 -0500 Subject: [PATCH 22/28] fix: removed step used only for testing --- .github/workflows/benchmark-cli.yml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/.github/workflows/benchmark-cli.yml b/.github/workflows/benchmark-cli.yml index 5e9308b..d0c1378 100644 --- a/.github/workflows/benchmark-cli.yml +++ b/.github/workflows/benchmark-cli.yml @@ -121,13 +121,3 @@ jobs: git commit -m "ci: updated results [CI/CD]" git log --oneline -3 git push origin master - - # TODO: remove this step - - name: Env - run: | - echo "GITHUB_WORKSPACE: ${GITHUB_WORKSPACE}" - echo "PWD: $(pwd)" - ls -al . - ls -al .. - echo "${{ toJson(github.event) }}" - printenv From 84d3b3064c52d23b28a3a64220c3ec9e66a3e81a Mon Sep 17 00:00:00 2001 From: Michael Perrotte Date: Thu, 7 Nov 2019 01:51:38 -0500 Subject: [PATCH 23/28] fix: updated logging in utils file --- lib/utils.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/utils.js b/lib/utils.js index 191b73b..9477810 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -26,9 +26,9 @@ exports.safeLoadResults = function safeLoadResults (filename) { } exports.writeResults = function writeResults (filename, results) { - console.log('filename:', filename) + log.silly(`filename: ${filename}`) const filepath = join(RESULTS_DIR, filename) - console.log('filepath:', filepath) + log.silly(`filepath: ${filepath}`) try { const data = JSON.stringify(results, null, ' ') fs.writeFileSync(filepath, data) @@ -44,5 +44,6 @@ exports.fetchCommandVersion = function fetchCommandVersion (cmd) { throw new Error(`${cmd} failed with status code ${result.status}`) } const version = result.stdout.toString().trim() + log.silly(`"${cmd}" version: ${version}`) return version } From 8790f7eb0a4eb5eb0e4e55ad82ffc404a944ccf1 Mon Sep 17 00:00:00 2001 From: Michael Perrotte Date: Thu, 7 Nov 2019 02:06:08 -0500 Subject: [PATCH 24/28] fix: updated pull-request comment format to have a status emoji --- lib/parse-result.js | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/lib/parse-result.js b/lib/parse-result.js index 339dc85..e3cb1e7 100644 --- a/lib/parse-result.js +++ b/lib/parse-result.js @@ -35,7 +35,7 @@ function tableHeader (results) { const fixtures = Object.keys(results[scenarioKeys[0]]) return fixtures - .map((fixture) => `${fixture}`) + .map((fixture) => `${fixture}`) .join('\n') } @@ -47,6 +47,7 @@ function subTableHeader (results) { .map((fixture) => dedent` prev current + status `) .join('\n') } @@ -78,8 +79,6 @@ function resultRowFixtures (latest, current, scenarioKey, fixtureKeys) { } function colorResult (latest, current) { - const color = (current <= latest) - ? '#00B200' - : '#A00000' - return `${prettyMs(current)}` + const status = (current <= latest) ? '✅' : '🛑' + return `${prettyMs(current)}${status}` } From 7e3362a83da3bfc4ab023e28d4ca3a42da55b67e Mon Sep 17 00:00:00 2001 From: Michael Perrotte Date: Thu, 7 Nov 2019 02:07:51 -0500 Subject: [PATCH 25/28] chore: add todo about benchmark v2 --- lib/comment.js | 1 + lib/execute.js | 1 + 2 files changed, 2 insertions(+) diff --git a/lib/comment.js b/lib/comment.js index 65e08f2..5b28e69 100644 --- a/lib/comment.js +++ b/lib/comment.js @@ -13,6 +13,7 @@ const parseResults = require('./parse-result') const { PR_ID, REPO, OWNER, GITHUB_TOKEN } = process.env module.exports = async function comment () { + // TODO: pass in benchmark suite information const { suiteCmd, suiteName } = require('./npm-suite') const latestFilename = `${suiteName}/latest.json` const latestResults = safeLoadResults(latestFilename) diff --git a/lib/execute.js b/lib/execute.js index 9478759..76d5c60 100644 --- a/lib/execute.js +++ b/lib/execute.js @@ -40,6 +40,7 @@ async function executeScenario (scenario, fixture) { } module.exports = async function execute (latest = false) { + // TODO: pass in benchmark suite information const { scenarios, suiteCmd, suiteName } = require('./npm-suite') const fixtures = fs.readdirSync(FIXTURES_DIR, 'utf8') From 4cbfb3b6016e380da5cba87d5b6b0c9a0a850c2b Mon Sep 17 00:00:00 2001 From: Michael Perrotte Date: Thu, 7 Nov 2019 02:15:34 -0500 Subject: [PATCH 26/28] chore: add todo/note about future fix --- lib/parse-result.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/parse-result.js b/lib/parse-result.js index e3cb1e7..3e54eab 100644 --- a/lib/parse-result.js +++ b/lib/parse-result.js @@ -68,6 +68,8 @@ function resultRowsScenario (latest, current) { function resultRowFixtures (latest, current, scenarioKey, fixtureKeys) { return fixtureKeys .map((fixtureKey) => { + // TODO: if there was no `latest` then this will break + // NOTE: `_.get()` could fix this const latestValue = latest[scenarioKey][fixtureKey] const currentValue = current[scenarioKey][fixtureKey] return dedent` From 5b7643ab026ee9c2776a85b2b06d82477090cf4f Mon Sep 17 00:00:00 2001 From: Michael Perrotte Date: Thu, 7 Nov 2019 02:18:16 -0500 Subject: [PATCH 27/28] chore: add todo about future improvement --- lib/comment.js | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/comment.js b/lib/comment.js index 5b28e69..abe29dc 100644 --- a/lib/comment.js +++ b/lib/comment.js @@ -33,6 +33,7 @@ module.exports = async function comment () { issue_number: PR_ID }) const comments = await octokit.paginate(options) + // TODO: should probably move `npm-deploy-user` into an environment variable const updateComment = comments.find((c) => c.user.login === 'npm-deploy-user') if (updateComment) { From fb75d36c327dbf9ab5962db8aacd88e4f37b0c55 Mon Sep 17 00:00:00 2001 From: Michael Perrotte Date: Thu, 7 Nov 2019 02:23:13 -0500 Subject: [PATCH 28/28] chore: add throw in catch block of execute; fix for loop --- lib/execute.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/execute.js b/lib/execute.js index 76d5c60..d04ff11 100644 --- a/lib/execute.js +++ b/lib/execute.js @@ -46,7 +46,7 @@ module.exports = async function execute (latest = false) { try { const newResults = {} - for (let x = 0; x < scenarios.length; ++x) { + for (let x = 0; x < scenarios.length; x++) { const scenario = scenarios[x] const scenarioKey = scenario.name.toLowerCase() @@ -55,7 +55,7 @@ module.exports = async function execute (latest = false) { } log.info('scenario', scenario.name) - for (let i = 0; i < fixtures.length; ++i) { + for (let i = 0; i < fixtures.length; i++) { const fixture = fixtures[i] log.info('fixture', fixture) @@ -84,5 +84,6 @@ module.exports = async function execute (latest = false) { } } catch (e) { log.error(e) + throw e } }