boa: support esm

.github/workflows/build.yml

@@ -11,7 +11,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        node_version: ['10', '12']
+        node_version: ['12', '14']
         os: [ubuntu-latest, macOS-latest]
     steps:
     - uses: actions/checkout@v1

.travis.yml

@@ -4,6 +4,7 @@ node_js:
   - '8'
   - '10'
   - '12'
+  - '14'
 before_install:
   - npm i npminstall -g
 install:

appveyor.yml

@@ -3,6 +3,7 @@ environment:
     - nodejs_version: '8'
     - nodejs_version: '10'
     - nodejs_version: '12'
+    - nodejs_version: '14'
 
 install:
   - ps: Install-Product node $env:nodejs_version

docs/tutorials/want-to-use-python.md

@@ -274,6 +274,40 @@ Returns the hash value of this object, internally it calls the CPython's [`PyObj
 
 Returns a corresponding primitive value for this object, see [`Symbol.toPrimitive` on MDN](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Symbol/toPrimitive) for more details.
 
+## Working with ECMAScript Modules
+> Requires Node.js >= `v12.11.1`
+
+Use Node.js custom loader for better import-statement.
+```js
+// app.mjs
+import { range, len } from 'py:builtins';
+import os from 'py:os';
+import {
+  array as NumpyArray,
+  int32 as NumpyInt32,
+} from 'py:numpy';
+
+console.log(os.getpid()); // prints the pid from python.
+
+const list = range(0, 10); // create a range array
+console.log(len(list)); // 10
+console.log(list[2]); // 2
+
+const arr = NumpyArray([1, 2, 3], NumpyInt32); // Create an array of int32 using ndarry constructor
+console.log(arr[0]); // 1
+```
+
+In `Node.js v14.x` you can specify only [`--experimental-loader`](https://nodejs.org/dist/latest-v14.x/docs/api/cli.html#cli_experimental_loader_module) to launch your application:
+
+```sh
+$ node --experimental-loader @pipcook/boa/esm/loader.mjs app.mjs
+```
+
+In Node.js version < `v14.x`, you also need to add the [`--experimental-modules`](https://nodejs.org/dist/latest-v14.x/docs/api/cli.html#cli_experimental_modules) option:
+```sh
+$ node --experimental-modules --experimental-loader @pipcook/boa/esm/loader.mjs app.mjs
+```
+
 ## Tests
 
 To run the full tests:

docs/zh-cn/tutorials/want-to-use-python.md

@@ -274,6 +274,40 @@ Returns the hash value of this object, internally it calls the CPython's [`PyObj
 
 Returns a corresponding primitive value for this object, see [`Symbol.toPrimitive` on MDN](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Symbol/toPrimitive) for more details.
 
+## Working with ECMAScript Modules
+> Requires Node.js >= `v12.11.1`
+
+Use Node.js custom loader for better import-statement.
+```js
+// app.mjs
+import { range, len } from 'py:builtins';
+import os from 'py:os';
+import {
+  array as NumpyArray,
+  int32 as NumpyInt32,
+} from 'py:numpy';
+
+console.log(os.getpid()); // prints the pid from python.
+
+const list = range(0, 10); // create a range array
+console.log(len(list)); // 10
+console.log(list[2]); // 2
+
+const arr = NumpyArray([1, 2, 3], NumpyInt32); // Create an array of int32 using ndarry constructor
+console.log(arr[0]); // 1
+```
+
+In `Node.js v14.x` you can specify only [`--experimental-loader`](https://nodejs.org/dist/latest-v14.x/docs/api/cli.html#cli_experimental_loader_module) to launch your application:
+
+```sh
+$ node --experimental-loader @pipcook/boa/esm/loader.mjs app.mjs
+```
+
+In Node.js version < `v14.x`, you also need to add the [`--experimental-modules`](https://nodejs.org/dist/latest-v14.x/docs/api/cli.html#cli_experimental_modules) option:
+```sh
+$ node --experimental-modules --experimental-loader @pipcook/boa/esm/loader.mjs app.mjs
+```
+
 ## Tests
 
 To run the full tests:

packages/boa/esm/loader.mjs

@@ -0,0 +1,40 @@
+import boa from '../lib/index.js';
+const { dir } = boa.builtins();
+const extensionsPrefix = 'py:';
+const protocol = 'nodejs://boa/';
+
+export function resolve(specifier, context, defaultResolve) {
+  if (specifier.startsWith(extensionsPrefix)) {
+    return {
+      url: protocol + specifier.replace(extensionsPrefix, '')
+    };
+  }
+  return defaultResolve(specifier, context, defaultResolve);
+}
+
+export function getFormat(url, context, defaultGetFormat) {
+  // DynamicInstantiate hook triggered if boa protocol is matched
+  if (url.startsWith(protocol)) {
+    return {
+      format: 'dynamic',
+    }
+  }
+
+  // Other protocol are assigned to nodejs for internal judgment loading
+  return defaultGetFormat(url, context, defaultGetFormat);
+}
+
+export function dynamicInstantiate(url) {
+  const moduleInstance = boa.import(url.replace(protocol, ''));
+  // Get all the properties of the Python Object to construct named export
+  const moduleExports = dir(moduleInstance);
+  return {
+    exports: ['default', ...moduleExports],
+    execute: exports => {
+      for (let name of moduleExports) {
+        exports[name].set(moduleInstance[name]);
+      }
+      exports.default.set(moduleInstance);
+    }
+  };
+}

packages/boa/package.json

@@ -6,7 +6,7 @@
   "scripts": {
     "preinstall": "node tools/install-python.js && node tools/install-requirements.js",
     "postinstall": "npm run build",
-    "test": "tap ./tests/**/*.js",
+    "test": "tap --no-cov ./tests/**/*.js",
     "lint": "eslint . && ./clang-format.py",
     "build": "node-gyp configure && node-gyp build",
     "pretest": "npm run lint",

packages/boa/tests/es-module-loaders/common.js

@@ -0,0 +1,50 @@
+/* eslint-disable no-process-exit */
+
+'use strict';
+
+const path = require('path');
+const { test } = require('tap');
+const { spawnSync } = require('child_process');
+
+function getAbsolutePath(relativePath) {
+  return path.join(__dirname, relativePath);
+}
+
+const FLAG = '--experimental-loader';
+const PATH_ESM_LOADER = getAbsolutePath('../../esm/loader.mjs');
+
+// See https://github.com/nodejs/node/pull/29796
+if (process.version < 'v12.11.1') {
+  console.log(`1..0 # Skipped: Current nodejs version: ${
+    process.version} does not support \`--experimental-loader\``);
+  process.exit(0);
+}
+
+function check(t, appPath) {
+  const options = { encoding: 'utf8', stdio: 'inherit' };
+  const args = ['--no-warnings', FLAG, PATH_ESM_LOADER];
+  // See https://github.com/nodejs/node/pull/31974
+  if (process.version < 'v14.0.0') {
+    args.push('--experimental-modules');
+  }
+  args.push(getAbsolutePath(appPath));
+  // Running with tap causes errors
+  // See https://github.com/tapjs/node-tap/issues/673
+  // 
+  // The nyc 14 conflicts with the node `--experimental-loader` design,
+  // which currently uses nyc 15 and tap 14 in combination with a skip error.
+  const result = spawnSync(process.execPath, args, options);
+  t.strictEqual(result.signal, null);
+  t.strictEqual(result.status, 0);
+  t.end();
+}
+
+test('python stdlib', t => check(t, './py/test-esm-loader-stdlib.mjs'));
+
+test('python thirdparty', t => check(t, './py/test-esm-loader-thirdparty.mjs'));
+
+test('python custom', t => check(t, './py/test-esm-loader-custom.mjs'));
+
+test('javascript thirdparty', t => check(t, './js/test-esm-loader-thirdparty.mjs'));
+
+test('javascript custom', t => check(t, './js/test-esm-loader-custom.mjs'));

packages/boa/tests/es-module-loaders/js/test-esm-loader-custom.mjs

@@ -0,0 +1,7 @@
+import { strictEqual } from 'assert';
+import { add, subtract } from './test-esm-loader-math.mjs';
+import math from './test-esm-loader-math.mjs';
+
+// custom js
+strictEqual(add(1, 2), math.add(1, 2));
+strictEqual(subtract(1, 2), math.subtract(1, 2));

packages/boa/tests/es-module-loaders/js/test-esm-loader-math.mjs

@@ -0,0 +1,8 @@
+const math = {
+  add: (a, b) => a + b,
+  subtract: (a, b) => a - b
+};
+
+export const add = math.add;
+export const subtract = math.subtract;
+export default math;

packages/boa/tests/es-module-loaders/js/test-esm-loader-thirdparty.mjs

@@ -0,0 +1,4 @@
+import assert from 'assert';
+import tap from 'tap';
+
+assert.equal(typeof tap.ok, 'function');

packages/boa/tests/es-module-loaders/py/test-esm-loader-custom.mjs

@@ -0,0 +1,34 @@
+import { strictEqual, equal } from 'assert';
+import { len } from 'py:builtins';
+import { Foobar } from 'py:tests.base.basic';
+
+// define a extended class with basic functions
+class Foobar2 extends Foobar {
+  hellomsg(x) {
+    return `hello <${x}> on ${this.test}`;
+  }
+}
+const f = new Foobar2();
+strictEqual(f.test, 'pythonworld');
+equal(f.ping('rickyes'), 'hello <rickyes> on pythonworld');
+equal(f.callfunc(x => x * 2), 233 * 2);
+
+// define a class which overloads magic methods
+class FoobarList extends Foobar {
+  constructor() {
+    super();
+    // this is not an es6 array
+    this.list = [1, 3, 7];
+  }
+  __getitem__(n) {
+    return this.list[n];
+  }
+  __len__() {
+    return len(this.list);
+  }
+}
+const fl = new FoobarList();
+equal(fl[0], 1);
+equal(fl[1], 3);
+equal(fl[2], 7);
+equal(len(fl), 3);

packages/boa/tests/es-module-loaders/py/test-esm-loader-stdlib.mjs

@@ -0,0 +1,71 @@
+import { strictEqual, ok, equal } from 'assert';
+// export default
+import os from 'py:os';
+// named export
+import { getenv, getpid } from 'py:os';
+import { rgb_to_hsv } from 'py:colorsys';
+import { getdefaultencoding, dont_write_bytecode } from 'py:sys';
+import { ascii_letters, ascii_lowercase, digits, punctuation } from 'py:string';
+import {
+  ceil,
+  copysign,
+  erf,
+  floor,
+  factorial,
+  gamma,
+  lgamma,
+} from 'py:math';
+import {
+  abs,
+  bin,
+  len,
+  min,
+  max,
+  True,
+  False,
+  None,
+  __debug__
+} from 'py:builtins';
+
+// builtins
+strictEqual(True, true);
+strictEqual(False, false);
+strictEqual(None, null);
+equal(abs(100), 100);
+equal(abs(-100), 100);
+equal(bin(3), '0b11');
+ok(__debug__);
+
+
+// os
+strictEqual(os.getenv('USER'), getenv('USER'));
+strictEqual(getpid() > 0, True);
+
+// sys
+ok(getdefaultencoding() === 'utf-8');
+ok(typeof dont_write_bytecode === 'boolean');
+
+// colorsys
+const v = rgb_to_hsv(0.2, 0.8, 0.4);
+ok(len(v) === 3);
+ok(min(v) === 0.3888888888888889);
+ok(max(v) === 0.8);
+
+// math
+strictEqual(ceil(10), 10);
+strictEqual(copysign(1.0, -1.0), -1);
+strictEqual(factorial(8), 40320);
+strictEqual(floor(100.99), 100);
+ok(erf(0.1));
+strictEqual(gamma(1), 1);
+strictEqual(gamma(2), 1);
+strictEqual(gamma(3), 2);
+ok(lgamma(5));
+
+
+// string
+strictEqual(ascii_letters,
+  'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ');
+strictEqual(ascii_lowercase, 'abcdefghijklmnopqrstuvwxyz');
+strictEqual(digits, '0123456789');
+strictEqual(punctuation, '!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~');

packages/boa/tests/es-module-loaders/py/test-esm-loader-thirdparty.mjs

@@ -0,0 +1,38 @@
+import { equal, ok } from 'assert';
+import boa from '../../../lib/index.js';
+import { type, tuple } from 'py:builtins';
+import {
+  Inf,
+  NINF,
+  NAN as NumpyNAN,
+  array as NumpyArray,
+  int32 as NumpyInt32,
+  fv as NumpyFv,
+  float as NumpyFloat,
+} from 'py:numpy';
+import matlib from 'py:numpy.matlib';
+
+// numpy constants
+equal(Inf, Infinity);
+equal(NINF, -Infinity);
+ok(isNaN(NumpyNAN, NaN));
+
+// numpy array
+const x = NumpyArray([[1, 2, 3], [4, 5, 6]], NumpyInt32);
+equal(JSON.stringify(x), '[[1,2,3],[4,5,6]]');
+equal(type(x).__name__, 'ndarray');
+equal(x.dtype.name, 'int32', 'the dtype should be int32');
+equal(x.ndim, 2);
+equal(x.size, 6);
+
+// numpy fv
+equal(NumpyFv(0.05 / 12, 10 * 12, -100, -100), 15692.928894335748);
+
+// numpy matlib
+const m = matlib.empty(tuple([2, 2]));
+equal(type(m).__name__, 'matrix');
+const m2 = matlib.eye(3, boa.kwargs({
+  k: 1,
+  dtype: NumpyFloat,
+}));
+equal(type(m2).__name__, 'matrix');