diff --git a/.github/workflows/benchmark-cli.yml b/.github/workflows/benchmark-cli.yml index 3d2f543..d0c1378 100644 --- a/.github/workflows/benchmark-cli.yml +++ b/.github/workflows/benchmark-cli.yml @@ -9,10 +9,8 @@ jobs: strategy: matrix: - # os: [ubuntu-latest, macOS-latest] - os: [ubuntu-latest] - # node-version: [10.x, 12.x] - node-version: [10.x] + os: [ubuntu-latest, macOS-latest] + node-version: [10.x, 12.x] runs-on: ${{ matrix.os }} @@ -22,7 +20,7 @@ jobs: uses: actions/checkout@v1.1.0 with: path: ${{ env.RUNNER_WORKSPACE }} - # Checkout repo incoming dispatch request is from (npm/cli) + # Checkout repo incoming dispatch request is from (eg. npm/cli) - name: checkout/${{ github.event.client_payload.owner }}/${{ github.event.client_payload.repo }} uses: actions/checkout@v1.1.0 with: @@ -73,10 +71,20 @@ jobs: echo "Current Commit: $(git log --oneline -1)" # Run benchmarking suite - - name: Run Benchmark + - name: Run Benchmark (Pull-Request) + if: github.event.action == 'pull_request' + run: | + echo "Running benchmark..." + echo "PWD: $(pwd)" + npm run benchmark:pr + + # Run benchmarking suite + - name: Run Benchmark (Push) + if: github.event.action == 'push' run: | echo "Running benchmark..." echo "PWD: $(pwd)" + npm run benchmark:release # CONDITIONALLY: Post to pull-request - name: Post to Pull-Request @@ -86,7 +94,9 @@ jobs: REPO: ${{ github.event.client_payload.repo }} OWNER: ${{ github.event.client_payload.owner }} GITHUB_TOKEN: ${{ github.token }} - run: echo "Posting to pull-request..." + run: | + echo "Posting to pull-request..." + npm run comment # CONDITIONALLY: Commit results of benchmark suite into `npm/benchmark` repo - name: Commit Results @@ -111,13 +121,3 @@ jobs: git commit -m "ci: updated results [CI/CD]" git log --oneline -3 git push origin master - - # TODO: remove this step - - name: Env - run: | - echo "GITHUB_WORKSPACE: ${GITHUB_WORKSPACE}" - echo "PWD: $(pwd)" - ls -al . - ls -al .. - echo "${{ toJson(github.event) }}" - printenv diff --git a/fixtures/angular-quickstart/package.json b/fixtures/angular-quickstart/package.json new file mode 100644 index 0000000..9e2b771 --- /dev/null +++ b/fixtures/angular-quickstart/package.json @@ -0,0 +1,48 @@ +{ + "name": "angular-quickstart", + "version": "0.0.0", + "license": "MIT", + "scripts": { + "ng": "ng", + "start": "ng serve", + "build": "ng build --prod", + "test": "ng test", + "lint": "ng lint", + "e2e": "ng e2e" + }, + "private": true, + "dependencies": { + "@angular/animations": "^5.2.0", + "@angular/common": "^5.2.0", + "@angular/compiler": "^5.2.0", + "@angular/core": "^5.2.0", + "@angular/forms": "^5.2.0", + "@angular/http": "^5.2.0", + "@angular/platform-browser": "^5.2.0", + "@angular/platform-browser-dynamic": "^5.2.0", + "@angular/router": "^5.2.0", + "core-js": "^2.4.1", + "rxjs": "^5.5.6", + "zone.js": "^0.8.19" + }, + "devDependencies": { + "@angular/cli": "~1.7.0", + "@angular/compiler-cli": "^5.2.0", + "@angular/language-service": "^5.2.0", + "@types/jasmine": "~2.8.3", + "@types/jasminewd2": "~2.0.2", + "@types/node": "~6.0.60", + "codelyzer": "^4.0.1", + "jasmine-core": "~2.8.0", + "jasmine-spec-reporter": "~4.2.1", + "karma": "~2.0.0", + "karma-chrome-launcher": "~2.2.0", + "karma-coverage-istanbul-reporter": "^1.2.1", + "karma-jasmine": "~1.1.0", + "karma-jasmine-html-reporter": "^0.2.2", + "protractor": "~5.1.2", + "ts-node": "~4.1.0", + "tslint": "~5.9.1", + "typescript": "~2.5.3" + } +} diff --git a/fixtures/app-large/package.json b/fixtures/app-large/package.json new file mode 100644 index 0000000..a9774e5 --- /dev/null +++ b/fixtures/app-large/package.json @@ -0,0 +1,118 @@ +{ + "name": "app-large", + "version": "0.0.1", + "dependencies": { + "animate.less": "^2.2.0", + "autoprefixer": "^6.0.3", + "babel-core": "^6.4.0", + "babel-eslint": "^6.1.2", + "babel-loader": "^6.2.1", + "babel-plugin-lodash": "^3.2.11", + "babel-plugin-module-resolver": "^2.2.0", + "babel-plugin-transform-decorators-legacy": "^1.3.4", + "babel-plugin-transform-runtime": "^6.4.3", + "babel-polyfill": "^6.23.0", + "babel-preset-es2015": "^6.3.13", + "babel-preset-react": "^6.3.13", + "babel-preset-react-hmre": "^1.0.1", + "babel-preset-stage-1": "^6.3.13", + "babel-runtime": "^6.3.19", + "clean-webpack-plugin": "^0.1.16", + "core-decorators": "^0.12.3", + "css-loader": "^0.23.1", + "css-mqpacker": "^4.0.0", + "cssnano": "^3.2.0", + "custom-event-polyfill": "^0.2.2", + "draft-js": "^0.9.0", + "ejs": "^2.5.6", + "eslint": "^3.4.0", + "eslint-config-airbnb": "^11.0.0", + "eslint-import-resolver-webpack": "^0.5.1", + "eslint-plugin-import": "^1.14.0", + "eslint-plugin-jsx-a11y": "^2.2.1", + "eslint-plugin-react": "^6.2.0", + "express": "^4.15.2", + "express-http-proxy": "^0.11.0", + "font-awesome": "^4.7.0", + "fready": "^1.0.0", + "glob": "^7.1.1", + "gulp": "^3.9.0", + "gulp-concat": "^2.6.0", + "gulp-csslint": "^0.2.0", + "gulp-cssnano": "^2.0.0", + "gulp-eol": "^0.1.1", + "gulp-less": "^3.0.5", + "gulp-livereload": "^3.8.1", + "gulp-minify-css": "^1.2.3", + "gulp-postcss": "^6.0.1", + "gulp-rename": "^1.2.2", + "gulp-util": "^3.0.7", + "happypack": "^3.0.3", + "highcharts": "^5.0.10", + "highcharts-solid-gauge": "^0.1.2", + "history": "^4.6.1", + "howler": "^1.1.28", + "imports-loader": "^0.6.5", + "jquery": "^2.2.0", + "jquery-ui": "1.10.5", + "js-cookie": "^2.1.3", + "json-loader": "^0.5.4", + "leftpad": "^0.0.0", + "less": "^2.7.2", + "lesshat": "^3.0.2", + "lodash": "^3.0.0", + "medium-draft": "^0.4.1", + "mobx": "^3.1.8", + "mobx-react": "^4.1.5", + "moment": "^2.18.1", + "moment-range": "^2.0.3", + "moment-timezone": "^0.5.13", + "password-policy": "0.0.2", + "postcss-reporter": "^1.2.1", + "progress": "^2.0.0", + "qs": "^6.1.0", + "raw-loader": "^0.5.1", + "rc-slider": "^6.1.0", + "react": "^15.4.1", + "react-addons-css-transition-group": "^15.3.0", + "react-addons-shallow-compare": "^15.3.0", + "react-dnd": "^2.1.4", + "react-dnd-html5-backend": "^2.1.2", + "react-dom": "^15.4.1", + "react-draft-wysiwyg": "^1.6.5", + "react-dropzone": "^3.5.3", + "react-grid-layout": "^0.12.6", + "react-highcharts": "^11.5.0", + "react-hot-loader": "v3.0.0-beta.6", + "react-input-calendar": "^0.3.14", + "react-lazyload": "^2.2.5", + "react-measure": "^1.4.6", + "react-mixin": "^3.0.3", + "react-responsive": "^1.2.5", + "react-responsive-tabs": "^0.5.3", + "react-router": "^4.0.0", + "react-router-dom": "^4.0.0", + "react-select-plus": "^1.0.0-rc", + "react-skylight": "^0.3.0", + "react-sortablejs": "^1.2.1", + "react-tappable": "^0.8.4", + "react-tooltip": "^3.3.0", + "react-virtualized": "^7.19.4", + "react-waypoint": "^5.2.0", + "sortablejs": "^1.5.0-rc1", + "style-loader": "^0.13.0", + "stylelint": "^1.2.1", + "superagent": "^1.6.1", + "uglify-js": "^2.8.22", + "uuid": "^3.0.1", + "verge": "^1.9.1", + "webpack-bundle-analyzer": "^2.3.1", + "webpack-hot-middleware": "^2.18.0", + "webpack-notifier": "^1.5.0", + "webpack-split-by-path": "^2.0.0", + "whatwg-fetch": "^2.0.3" + }, + "devDependencies": { + "nan-as": "^1.6.1" + } +} diff --git a/fixtures/app-medium/package.json b/fixtures/app-medium/package.json new file mode 100644 index 0000000..cf31320 --- /dev/null +++ b/fixtures/app-medium/package.json @@ -0,0 +1,63 @@ +{ + "name": "app-medium", + "version": "0.0.0", + "dependencies": { + "axios": "^0.16.0", + "emailjs": "^0.3.13", + "es6-promise": "^4.1.0", + "faker": "^3.1.0", + "js-beautify": "^1.6.14", + "json3": "^3.3.2", + "lodash": "^4.17.4", + "store2": "^2.5.0", + "vue": "^2.2.2", + "vue-axios": "^2.0.1", + "vue-my-dropdown": "^2.0.3", + "vue-resource": "^1.2.1", + "vue-select": "^2.1.0" + }, + "devDependencies": { + "@corbinu/eslint-plugin-corbinu": "^2.0.0", + "autoprefixer": "^6.7.2", + "babel-core": "^6.22.1", + "babel-eslint": "^7.2.3", + "babel-loader": "^7.0.0", + "babel-plugin-transform-runtime": "^6.22.0", + "babel-preset-env": "^1.2.1", + "babel-preset-es2015": "^6.24.0", + "babel-preset-stage-2": "^6.22.0", + "babel-register": "^6.22.0", + "chalk": "^1.1.3", + "connect-history-api-fallback": "^1.3.0", + "copy-webpack-plugin": "^4.0.1", + "css-loader": "^0.28.0", + "eslint": "^3.14.1", + "eslint-friendly-formatter": "^2.0.7", + "eslint-loader": "^1.6.1", + "eslint-plugin-html": "^2.0.0", + "eventsource-polyfill": "^0.9.6", + "express": "^4.14.1", + "extract-text-webpack-plugin": "^2.0.0", + "file-loader": "^0.11.1", + "friendly-errors-webpack-plugin": "^1.1.3", + "function-bind": "^1.1.0", + "html-webpack-plugin": "^2.28.0", + "http-proxy-middleware": "^0.17.3", + "nyc": "^10.2.0", + "opn": "^4.0.2", + "optimize-css-assets-webpack-plugin": "^1.3.0", + "ora": "^1.1.0", + "rimraf": "^2.6.0", + "semver": "^5.3.0", + "typescript": "~2.2.0", + "url-loader": "^0.5.7", + "vue-loader": "^11.1.4", + "vue-style-loader": "^3.0.0", + "vue-template-compiler": "^2.2.1", + "webpack": "^2.2.1", + "webpack-bundle-analyzer": "^2.2.1", + "webpack-dev-middleware": "^1.10.2", + "webpack-hot-middleware": "^2.16.1", + "webpack-merge": "^4.0.0" + } +} diff --git a/fixtures/ember-quickstart/package.json b/fixtures/ember-quickstart/package.json new file mode 100644 index 0000000..f40d8aa --- /dev/null +++ b/fixtures/ember-quickstart/package.json @@ -0,0 +1,47 @@ +{ + "name": "ember-quickstart", + "version": "0.0.0", + "private": true, + "description": "Small description for ember-quickstart goes here", + "license": "MIT", + "author": "", + "directories": { + "doc": "doc", + "test": "tests" + }, + "repository": "", + "scripts": { + "build": "ember build", + "lint:js": "eslint ./*.js app config lib server tests", + "start": "ember serve", + "test": "ember test" + }, + "devDependencies": { + "broccoli-asset-rev": "^2.4.5", + "ember-ajax": "^3.0.0", + "ember-cli": "~3.0.0", + "ember-cli-app-version": "^3.0.0", + "ember-cli-babel": "^6.6.0", + "ember-cli-dependency-checker": "^2.0.0", + "ember-cli-eslint": "^4.2.1", + "ember-cli-htmlbars": "^2.0.1", + "ember-cli-htmlbars-inline-precompile": "^1.0.0", + "ember-cli-inject-live-reload": "^1.4.1", + "ember-cli-qunit": "^4.1.1", + "ember-cli-shims": "^1.2.0", + "ember-cli-sri": "^2.1.0", + "ember-cli-uglify": "^2.0.0", + "ember-data": "~3.0.0", + "ember-export-application-global": "^2.0.0", + "ember-load-initializers": "^1.0.0", + "ember-maybe-import-regenerator": "^0.1.6", + "ember-resolver": "^4.0.0", + "ember-source": "~3.0.0", + "ember-welcome-page": "^3.0.0", + "eslint-plugin-ember": "^5.0.0", + "loader.js": "^4.2.3" + }, + "engines": { + "node": "^4.5 || 6.* || >= 7.*" + } +} diff --git a/fixtures/react-app/package.json b/fixtures/react-app/package.json new file mode 100644 index 0000000..6261569 --- /dev/null +++ b/fixtures/react-app/package.json @@ -0,0 +1,16 @@ +{ + "name": "react-app", + "version": "0.1.0", + "private": true, + "dependencies": { + "react": "^16.2.0", + "react-dom": "^16.2.0", + "react-scripts": "1.1.1" + }, + "scripts": { + "start": "react-scripts start", + "build": "react-scripts build", + "test": "react-scripts test --env=jsdom", + "eject": "react-scripts eject" + } +} diff --git a/index.js b/index.js new file mode 100644 index 0000000..50e3166 --- /dev/null +++ b/index.js @@ -0,0 +1,27 @@ +'use strict' + +const comment = require('./lib/comment') +const execute = require('./lib/execute') +const { log } = require('./lib/utils') + +const args = process.argv.slice(2) +const command = args.length && args[0] +const isRelease = args.length && args[1] + +log.verbose('ARGS:', args) +const { PR_ID, REPO, OWNER } = process.env + +switch (command) { + case 'benchmark': + log.info('Executing benchmark against latest release') + execute(!!isRelease) + break + case 'comment': + // TODO: bail out if we don't have correct environment variables + log.info(`Posting Comment to ${OWNER}/${REPO}/pulls/${PR_ID}`) + comment() + break + default: + log.error('Invalid argument supplied...') + log.error('Please use the npm-scripts.') +} diff --git a/lib/comment.js b/lib/comment.js new file mode 100644 index 0000000..abe29dc --- /dev/null +++ b/lib/comment.js @@ -0,0 +1,56 @@ +'use strict' + +const Octokit = require('@octokit/rest') + +const { + log, + safeLoadResults, + fetchCommandVersion +} = require('./utils') + +const parseResults = require('./parse-result') + +const { PR_ID, REPO, OWNER, GITHUB_TOKEN } = process.env + +module.exports = async function comment () { + // TODO: pass in benchmark suite information + const { suiteCmd, suiteName } = require('./npm-suite') + const latestFilename = `${suiteName}/latest.json` + const latestResults = safeLoadResults(latestFilename) + + const version = fetchCommandVersion(suiteCmd) + const currentFilename = `${suiteName}/${version}.json` + const currentResults = safeLoadResults(currentFilename) + + // INFO: get the "most recent" current results (last item of the list) + const output = parseResults(latestResults[0], currentResults[currentResults.length - 1]) + + const octokit = new Octokit({ auth: GITHUB_TOKEN }) + + const options = octokit.issues.listComments.endpoint.merge({ + owner: OWNER, + repo: REPO, + issue_number: PR_ID + }) + const comments = await octokit.paginate(options) + // TODO: should probably move `npm-deploy-user` into an environment variable + const updateComment = comments.find((c) => c.user.login === 'npm-deploy-user') + + if (updateComment) { + log.verbose('Updating comment...') + await octokit.issues.updateComment({ + owner: OWNER, + repo: REPO, + comment_id: updateComment.id, + body: output + }) + } else { + log.verbose('Posting comment...') + await octokit.issues.createComment({ + owner: OWNER, + repo: REPO, + issue_number: PR_ID, + body: output + }) + } +} diff --git a/lib/constants.js b/lib/constants.js new file mode 100644 index 0000000..64213c3 --- /dev/null +++ b/lib/constants.js @@ -0,0 +1,10 @@ +'use strict' + +const { join } = require('path') + +exports.BASE_DIR = join(__dirname, '..') +exports.FIXTURES_DIR = join(this.BASE_DIR, 'fixtures') +exports.TMP_DIR = join(this.BASE_DIR, 'tmp') +exports.RESULTS_DIR = join(this.BASE_DIR, 'results') + +exports.CACHE_NAME = 'cache' diff --git a/lib/execute.js b/lib/execute.js new file mode 100644 index 0000000..d04ff11 --- /dev/null +++ b/lib/execute.js @@ -0,0 +1,89 @@ +'use strict' + +const fs = require('fs') + +const { + log, + safeLoadResults, + writeResults, + fetchCommandVersion +} = require('./utils') + +const { FIXTURES_DIR } = require('./constants') + +/** + * Executes a given scenario with a provided fixture + * + * @param {Object} scenario scenario object + * @param {string} scenario.name name of the scenario + * @param {object} scenario.details details about scenario + * @param {boolean} scenario.details.cache + * @param {boolean} scenario.details.node_modules + * @param {boolean} scenario.details.lockfile + * @param {string} scenario.cmd command used by the scenario + * @param {string[]} scenario.args list of arguments to pass to the command + * @param {function[]} scenario.actions list of actions this scenario will take + * @param {string} fixture directory name of a fixture + */ +async function executeScenario (scenario, fixture) { + const { actions } = scenario + + const result = await actions.reduce( + (acc, action) => { + return acc.then((result) => action(scenario, fixture)) + }, + Promise.resolve() + ) + log.info('execute', 'Result Time: %d', result) + log.info('execute', 'Details: %o', scenario.details) + return result +} + +module.exports = async function execute (latest = false) { + // TODO: pass in benchmark suite information + const { scenarios, suiteCmd, suiteName } = require('./npm-suite') + const fixtures = fs.readdirSync(FIXTURES_DIR, 'utf8') + + try { + const newResults = {} + for (let x = 0; x < scenarios.length; x++) { + const scenario = scenarios[x] + + const scenarioKey = scenario.name.toLowerCase() + if (scenarioKey !== 'cleanup') { + newResults[scenarioKey] = {} + } + + log.info('scenario', scenario.name) + for (let i = 0; i < fixtures.length; i++) { + const fixture = fixtures[i] + + log.info('fixture', fixture) + const execResult = await executeScenario(scenario, fixture) + + const fixtureKey = fixture.toLowerCase() + if (newResults[scenarioKey]) { + newResults[scenarioKey][fixtureKey] = execResult + } + } + } + + // NOTE: always write to versioned file + const version = fetchCommandVersion(suiteCmd) + const versionFilename = `${suiteName}/${version}.json` + const prevResults = safeLoadResults(versionFilename) + const results = [...prevResults, newResults] + writeResults(versionFilename, results) + + if (latest) { + // NOTE: Optionally write to `latest.json` + const filename = `${suiteName}/latest.json` + // INFO: `latest.json` should always be one dataset + const results = [newResults] + writeResults(filename, results) + } + } catch (e) { + log.error(e) + throw e + } +} diff --git a/lib/npm-suite.js b/lib/npm-suite.js new file mode 100644 index 0000000..e450753 --- /dev/null +++ b/lib/npm-suite.js @@ -0,0 +1,261 @@ +'use strict' + +const { join } = require('path') + +const { + CACHE_NAME, + FIXTURES_DIR +} = require('./constants') + +const { + createEnv, + removeCache, + removeNodeModules, + removeLockfile, + measureAction +} = require('./suite-actions') + +/** + * NOTE: Scenario Ordering + * The first scenario needs to be an "initial install" to populate the cache, + * node_modules folder, and create a lockfile. All the subsequent scenarios + * can be in any order. As a subsequent scenario starts with populated cache, + * node_modules folder, and lockfile (created from the previous scenario), + * they mearly need to remove the idea they do not want. + */ +module.exports.suiteName = 'npm' +module.exports.suiteCmd = 'npm' +module.exports.scenarios = [ + { + name: 'Initial install', + details: { + cache: false, + node_modules: false, + lockfile: false + }, + cmd: 'npm', + args: [ + 'install', + '--ignore-scripts', + '--cache', + `${CACHE_NAME}`, + '--registry', + 'https://registry.npmjs.org/' + ], + actions: [ + removeCache, + removeNodeModules, + removeLockfile, + async (ctx, fixture) => { + const { cmd, args } = ctx + const env = createEnv() + + const cwd = join(FIXTURES_DIR, fixture) + return measureAction({ cmd, args, env, cwd }) + } + ] + }, + { + name: 'Repeat install', + details: { + cache: true, + node_modules: true, + lockfile: true + }, + cmd: 'npm', + args: [ + 'install', + '--ignore-scripts', + '--cache', + `${CACHE_NAME}`, + '--registry', + 'https://registry.npmjs.org/' + ], + actions: [ + (ctx, fixture) => { + const { cmd, args } = ctx + const env = createEnv() + + const cwd = join(FIXTURES_DIR, fixture) + return measureAction({ cmd, args, env, cwd }) + } + ] + }, + { + name: 'With warm cache', + details: { + cache: true, + node_modules: false, + lockfile: false + }, + cmd: 'npm', + args: [ + 'install', + '--ignore-scripts', + '--cache', + `${CACHE_NAME}`, + '--registry', + 'https://registry.npmjs.org/' + ], + actions: [ + removeNodeModules, + removeLockfile, + (ctx, fixture) => { + const { cmd, args } = ctx + const env = createEnv() + + const cwd = join(FIXTURES_DIR, fixture) + return measureAction({ cmd, args, env, cwd }) + } + ] + }, + { + name: 'With node_modules', + details: { + cache: false, + node_modules: true, + lockfile: false + }, + cmd: 'npm', + args: [ + 'install', + '--ignore-scripts', + '--cache', + `${CACHE_NAME}`, + '--registry', + 'https://registry.npmjs.org/' + ], + actions: [ + removeCache, + removeLockfile, + (ctx, fixture) => { + const { cmd, args } = ctx + const env = createEnv() + + const cwd = join(FIXTURES_DIR, fixture) + return measureAction({ cmd, args, env, cwd }) + } + ] + }, + { + name: 'With lockfile', + details: { + cache: false, + node_modules: false, + lockfile: true + }, + cmd: 'npm', + args: [ + 'install', + '--ignore-scripts', + '--cache', + `${CACHE_NAME}`, + '--registry', + 'https://registry.npmjs.org/' + ], + actions: [ + removeCache, + removeNodeModules, + (ctx, fixture) => { + const { cmd, args } = ctx + const env = createEnv() + + const cwd = join(FIXTURES_DIR, fixture) + return measureAction({ cmd, args, env, cwd }) + } + ] + }, + { + name: 'With warm cache and node_modules', + details: { + cache: true, + node_modules: true, + lockfile: false + }, + cmd: 'npm', + args: [ + 'install', + '--ignore-scripts', + '--cache', + `${CACHE_NAME}`, + '--registry', + 'https://registry.npmjs.org/' + ], + actions: [ + removeLockfile, + (ctx, fixture) => { + const { cmd, args } = ctx + const env = createEnv() + + const cwd = join(FIXTURES_DIR, fixture) + return measureAction({ cmd, args, env, cwd }) + } + ] + }, + { + name: 'With warm cache and lockfile', + details: { + cache: true, + node_modules: false, + lockfile: true + }, + cmd: 'npm', + args: [ + 'install', + '--ignore-scripts', + '--cache', + `${CACHE_NAME}`, + '--registry', + 'https://registry.npmjs.org/' + ], + actions: [ + removeNodeModules, + (ctx, fixture) => { + const { cmd, args } = ctx + const env = createEnv() + + const cwd = join(FIXTURES_DIR, fixture) + return measureAction({ cmd, args, env, cwd }) + } + ] + }, + { + name: 'With node_modules and lockfile', + details: { + cache: false, + node_modules: true, + lockfile: true + }, + cmd: 'npm', + args: [ + 'install', + '--ignore-scripts', + '--cache', + `${CACHE_NAME}`, + '--registry', + 'https://registry.npmjs.org/' + ], + actions: [ + removeCache, + (ctx, fixture) => { + const { cmd, args } = ctx + const env = createEnv() + + const cwd = join(FIXTURES_DIR, fixture) + return measureAction({ cmd, args, env, cwd }) + } + ] + }, + { + name: 'cleanup', + details: {}, + cmd: 'noop', + args: [], + actions: [ + removeCache, + removeNodeModules, + removeLockfile, + (ctx, fixture) => 0 + ] + } +] diff --git a/lib/parse-result.js b/lib/parse-result.js new file mode 100644 index 0000000..3e54eab --- /dev/null +++ b/lib/parse-result.js @@ -0,0 +1,86 @@ +'use strict' + +const dedent = require('dedent') +const prettyMs = require('pretty-ms') + +module.exports = function parseResults (latestResults, currentResults) { + const message = dedent` + + + + + ${tableHeader(currentResults)} + + + + ${subTableHeader(currentResults)} + + + + ${resultRowsScenario(latestResults, currentResults)} + +
+ ` + return message +} + +function tableHeader (results) { + // INFO: The results are mapped such that top level keys are scenario names + const scenarioKeys = Object.keys(results) + /** + * INFO: + * The results are mapped such that scenarios all have the same keys, + * which are the fixtures. + */ + const fixtures = Object.keys(results[scenarioKeys[0]]) + + return fixtures + .map((fixture) => `${fixture}`) + .join('\n') +} + +function subTableHeader (results) { + const scenarioKeys = Object.keys(results) + const fixtures = Object.keys(results[scenarioKeys[0]]) + + return fixtures + .map((fixture) => dedent` + prev + current + status + `) + .join('\n') +} + +function resultRowsScenario (latest, current) { + const scenarioKeys = Object.keys(current) + const fixtureKeys = Object.keys(current[scenarioKeys[0]]) + return scenarioKeys + .map((scenarioKey) => dedent` + + ${scenarioKey} + ${resultRowFixtures(latest, current, scenarioKey, fixtureKeys)} + + `) + .join('\n') +} + +function resultRowFixtures (latest, current, scenarioKey, fixtureKeys) { + return fixtureKeys + .map((fixtureKey) => { + // TODO: if there was no `latest` then this will break + // NOTE: `_.get()` could fix this + const latestValue = latest[scenarioKey][fixtureKey] + const currentValue = current[scenarioKey][fixtureKey] + return dedent` + ${prettyMs(latestValue)} + ${colorResult(latestValue, currentValue)} + ` + }) + .join('\n') +} + +function colorResult (latest, current) { + const status = (current <= latest) ? '✅' : '🛑' + return `${prettyMs(current)}${status}` +} diff --git a/lib/suite-actions.js b/lib/suite-actions.js new file mode 100644 index 0000000..a22b77b --- /dev/null +++ b/lib/suite-actions.js @@ -0,0 +1,73 @@ +'use strict' + +const child = require('child_process') +const rimraf = require('rimraf') +const { join } = require('path') +const { log } = require('./utils') + +const { CACHE_NAME, FIXTURES_DIR } = require('./constants') + +exports.measureAction = async function measureAction ({ cmd, args, env, cwd }) { + log.verbose('measureAction', 'executing...') + log.silly('measureAction', 'cmd: %s', cmd) // TESTING + log.silly('measureAction', 'args: %o', args) // TESTING + log.silly('measureAction', 'env: %o', env) // TESTING + log.silly('measureAction', 'cwd: %s', cwd) // TESTING + const startTime = Date.now() + + // TODO: allow config to be passed in to allow process output 'stdio' + const result = child.spawnSync(cmd, args, { env, cwd }) + if (result.status !== 0) { + log.error(result.error) + throw new Error(`${cmd} failed with status code ${result.status}`) + } + const endTime = Date.now() + return endTime - startTime +} + +exports.createEnv = function createEnv (overrides = {}) { + const env = Object.keys(process.env).reduce((acc, key) => { + return (key.match(/^npm_/)) + // Don't include `npm_` key/values + ? acc + // Add key/value to env object + : Object.assign({}, acc, { [key]: process.env[key] }) + }, {}) + log.silly('createEnv', 'env: %o', env) + return Object.assign({}, env, overrides) +} + +exports.removePath = async function removePath (path) { + return new Promise((resolve, reject) => { + rimraf(path, {}, (err) => { + if (err) { + return reject(err) + } + return resolve() + }) + }) +} + +exports.removeCache = async function removeCache (ctx, fixture) { + log.verbose('removeCache', 'removing cache...') + const cacheDir = join(FIXTURES_DIR, fixture, CACHE_NAME) + log.silly('removeCache', 'cacheDir: %s', cacheDir) + return exports.removePath(cacheDir) +} + +exports.removeNodeModules = async function removeNodeModules (ctx, fixture) { + log.verbose('removeNodeModules', 'removing node_modules...') + const cwd = join(FIXTURES_DIR, fixture) + const nodeModulesDir = join(cwd, 'node_modules') + log.silly('removeNodeModules', 'nodeModulesDir: %s', nodeModulesDir) + return exports.removePath(nodeModulesDir) +} + +exports.removeLockfile = async function removeLockfile (ctx, fixture) { + log.verbose('removeLockfile', 'removing lockfile...') + // TODO: change this to use `scenario.lockfile` + const cwd = join(FIXTURES_DIR, fixture) + const lockfilePath = join(cwd, 'package-lock.json') + log.silly('removeLockfile', 'lockfilePath: %s', lockfilePath) + return exports.removePath(lockfilePath) +} diff --git a/lib/utils.js b/lib/utils.js new file mode 100644 index 0000000..9477810 --- /dev/null +++ b/lib/utils.js @@ -0,0 +1,49 @@ +'use strict' + +const child = require('child_process') +const log = require('npmlog') +const fs = require('fs') +const { join } = require('path') + +const { RESULTS_DIR } = require('./constants') + +const { LOG_LEVEL } = process.env + +log.level = LOG_LEVEL || 'info' +exports.log = log + +exports.safeLoadResults = function safeLoadResults (filename) { + const filepath = join(RESULTS_DIR, filename) + try { + const file = fs.readFileSync(filepath, 'utf8') + return (file) ? JSON.parse(file) : [] + } catch (err) { + if (err.code !== 'ENOENT') { + throw err + } + return [] + } +} + +exports.writeResults = function writeResults (filename, results) { + log.silly(`filename: ${filename}`) + const filepath = join(RESULTS_DIR, filename) + log.silly(`filepath: ${filepath}`) + try { + const data = JSON.stringify(results, null, ' ') + fs.writeFileSync(filepath, data) + } catch (err) { + throw err + } +} + +exports.fetchCommandVersion = function fetchCommandVersion (cmd) { + const result = child.spawnSync(cmd, ['--version']) + if (result.status !== 0) { + log.error(result.error) + throw new Error(`${cmd} failed with status code ${result.status}`) + } + const version = result.stdout.toString().trim() + log.silly(`"${cmd}" version: ${version}`) + return version +} diff --git a/package-lock.json b/package-lock.json index 7c2375c..f1be862 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1446,6 +1446,11 @@ "error-ex": "^1.2.0" } }, + "parse-ms": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/parse-ms/-/parse-ms-2.1.0.tgz", + "integrity": "sha512-kHt7kzLoS9VBZfUsiKjv43mr91ea+U05EyKkEtqp7vNbHxmaVuEqN7XxeEVnGrMtYOAxGrDElSi96K7EgO1zCA==" + }, "path-exists": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz", @@ -1561,6 +1566,14 @@ "integrity": "sha1-IZMqVJ9eUv/ZqCf1cOBL5iqX2lQ=", "dev": true }, + "pretty-ms": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/pretty-ms/-/pretty-ms-5.0.0.tgz", + "integrity": "sha512-94VRYjL9k33RzfKiGokPBPpsmloBYSf5Ri+Pq19zlsEcUKFob+admeXr5eFDRuPjFmEOcjJvPGdillYOJyvZ7Q==", + "requires": { + "parse-ms": "^2.1.0" + } + }, "process-nextick-args": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", @@ -2049,4 +2062,4 @@ "dev": true } } -} \ No newline at end of file +} diff --git a/package.json b/package.json index b38d384..89abea2 100644 --- a/package.json +++ b/package.json @@ -10,13 +10,16 @@ "@octokit/rest": "^16.34.1", "dedent": "^0.7.0", "npmlog": "^4.1.2", + "pretty-ms": "^5.0.0", "rimraf": "^3.0.0" }, "devDependencies": { "standard": "^12.0.1" }, "scripts": { - "benchmark": "node index.js", + "benchmark:pr": "node index.js benchmark", + "benchmark:release": "node index.js benchmark latest", + "comment": "node index.js comment", "test": "echo \"Error: no test specified\" && exit 1" }, "repository": { diff --git a/results/npm/6.12.1.json b/results/npm/6.12.1.json new file mode 100644 index 0000000..86985ba --- /dev/null +++ b/results/npm/6.12.1.json @@ -0,0 +1,60 @@ +[ + { + "initial install": { + "angular-quickstart": 35599, + "app-large": 46539, + "app-medium": 32573, + "ember-quickstart": 20675, + "react-app": 31276 + }, + "repeat install": { + "angular-quickstart": 5485, + "app-large": 5148, + "app-medium": 5118, + "ember-quickstart": 4288, + "react-app": 5132 + }, + "with warm cache": { + "angular-quickstart": 22952, + "app-large": 28344, + "app-medium": 21468, + "ember-quickstart": 17333, + "react-app": 21432 + }, + "with node_modules": { + "angular-quickstart": 5469, + "app-large": 4413, + "app-medium": 4500, + "ember-quickstart": 4045, + "react-app": 5099 + }, + "with lockfile": { + "angular-quickstart": 22024, + "app-large": 19162, + "app-medium": 17644, + "ember-quickstart": 12819, + "react-app": 15733 + }, + "with warm cache and node_modules": { + "angular-quickstart": 5132, + "app-large": 4705, + "app-medium": 4295, + "ember-quickstart": 4626, + "react-app": 4968 + }, + "with warm cache and lockfile": { + "angular-quickstart": 14464, + "app-large": 15992, + "app-medium": 12704, + "ember-quickstart": 9524, + "react-app": 12502 + }, + "with node_modules and lockfile": { + "angular-quickstart": 5552, + "app-large": 4887, + "app-medium": 4703, + "ember-quickstart": 4309, + "react-app": 5131 + } + } +] \ No newline at end of file diff --git a/results/npm/6.13.0.json b/results/npm/6.13.0.json new file mode 100644 index 0000000..191d4e2 --- /dev/null +++ b/results/npm/6.13.0.json @@ -0,0 +1,60 @@ +[ + { + "initial install": { + "angular-quickstart": 30811, + "app-large": 35246, + "app-medium": 29939, + "ember-quickstart": 21210, + "react-app": 26683 + }, + "repeat install": { + "angular-quickstart": 6470, + "app-large": 5472, + "app-medium": 5021, + "ember-quickstart": 4405, + "react-app": 5134 + }, + "with warm cache": { + "angular-quickstart": 26246, + "app-large": 28782, + "app-medium": 24411, + "ember-quickstart": 16512, + "react-app": 22088 + }, + "with node_modules": { + "angular-quickstart": 5968, + "app-large": 4874, + "app-medium": 4628, + "ember-quickstart": 4215, + "react-app": 4995 + }, + "with lockfile": { + "angular-quickstart": 22441, + "app-large": 23821, + "app-medium": 19189, + "ember-quickstart": 13879, + "react-app": 18536 + }, + "with warm cache and node_modules": { + "angular-quickstart": 5434, + "app-large": 6209, + "app-medium": 4448, + "ember-quickstart": 4054, + "react-app": 4701 + }, + "with warm cache and lockfile": { + "angular-quickstart": 17935, + "app-large": 19273, + "app-medium": 15355, + "ember-quickstart": 11512, + "react-app": 14882 + }, + "with node_modules and lockfile": { + "angular-quickstart": 5699, + "app-large": 5355, + "app-medium": 4865, + "ember-quickstart": 4409, + "react-app": 5475 + } + } +] \ No newline at end of file diff --git a/results/npm/latest.json b/results/npm/latest.json new file mode 100644 index 0000000..191d4e2 --- /dev/null +++ b/results/npm/latest.json @@ -0,0 +1,60 @@ +[ + { + "initial install": { + "angular-quickstart": 30811, + "app-large": 35246, + "app-medium": 29939, + "ember-quickstart": 21210, + "react-app": 26683 + }, + "repeat install": { + "angular-quickstart": 6470, + "app-large": 5472, + "app-medium": 5021, + "ember-quickstart": 4405, + "react-app": 5134 + }, + "with warm cache": { + "angular-quickstart": 26246, + "app-large": 28782, + "app-medium": 24411, + "ember-quickstart": 16512, + "react-app": 22088 + }, + "with node_modules": { + "angular-quickstart": 5968, + "app-large": 4874, + "app-medium": 4628, + "ember-quickstart": 4215, + "react-app": 4995 + }, + "with lockfile": { + "angular-quickstart": 22441, + "app-large": 23821, + "app-medium": 19189, + "ember-quickstart": 13879, + "react-app": 18536 + }, + "with warm cache and node_modules": { + "angular-quickstart": 5434, + "app-large": 6209, + "app-medium": 4448, + "ember-quickstart": 4054, + "react-app": 4701 + }, + "with warm cache and lockfile": { + "angular-quickstart": 17935, + "app-large": 19273, + "app-medium": 15355, + "ember-quickstart": 11512, + "react-app": 14882 + }, + "with node_modules and lockfile": { + "angular-quickstart": 5699, + "app-large": 5355, + "app-medium": 4865, + "ember-quickstart": 4409, + "react-app": 5475 + } + } +] \ No newline at end of file