Support for imports

.gitignore

@@ -1,2 +1,3 @@
 node_modules
 test/helpers/compiled.js
+test/helpers/imports.js

README.md

@@ -111,6 +111,28 @@ var js = protobuf.toJS(fs.readFileSync('test.proto'))
 fs.writeFileSync('messages.js', js)
 ```
 
+## Imports
+
+The cli tool supports protocol buffer [imports][] by default.
+
+**Currently all imports are treated as public and the public/weak keywords
+not supported.**
+
+To use it programmatically you need to pass-in a `filename` & a `resolveImport`
+hooks:
+
+```js
+var protobuf = require('protocol-buffers')
+var messages = protobuf(null, {
+  filename: 'initial.proto',
+  resolveImport (filename) {
+    // can return a Buffer, String or Schema
+  }
+})
+```
+
+[imports]: https://developers.google.com/protocol-buffers/docs/proto3#importing_definitions
+
 ## Performance
 
 This module is fast.

bin.js

@@ -1,37 +1,45 @@
 #!/usr/bin/env node
 const protobuf = require('./')
 const fs = require('fs')
+const path = require('path')
 
 let filename = null
 let output = null
 let watch = false
 let encodings = null
+const importPaths = []
 
 // handrolled parser to not introduce minimist as this is used a bunch of prod places
 // TODO: if this becomes more complicated / has bugs, move to minimist
 for (let i = 2; i < process.argv.length; i++) {
-  const v = process.argv[i]
-  const n = v.split('=')[0]
-  if (v[0] !== '-') {
-    filename = v
-  } else if (n === '--output' || n === '-o' || n === '-wo') {
-    if (n === '-wo') watch = true
-    output = v === n ? process.argv[++i] : v.split('=').slice(1).join('=')
-  } else if (n === '--watch' || n === '-w') {
+  const parts = process.argv[i].split('=')
+  const key = parts[0]
+  const value = parts.slice(1).join('=')
+  if (key[0] !== '-') {
+    filename = path.resolve(key)
+  } else if (key === '--output' || key === '-o' || key === '-wo') {
+    if (key === '-wo') watch = true
+    output = value || process.argv[++i]
+  } else if (key === '--watch' || key === '-w') {
     watch = true
-  } else if (n === '--encodings' || n === '-e') {
-    encodings = v === n ? process.argv[++i] : v.split('=').slice(1).join('=')
+  } else if (key === '--encodings' || key === '-e') {
+    encodings = value || process.argv[++i]
+  } else if (key === '--proto_path' || key === '-I') {
+    importPaths.push(path.resolve(value || process.argv[++i]))
   }
 }
+importPaths.push(process.cwd())
 
 if (!filename) {
   console.error('Usage: protocol-buffers [schema-file.proto] [options]')
   console.error()
-  console.error(' --output, -o  [output-file.js]')
-  console.error(' --watch,  -w  (recompile on schema change)')
+  console.error(' --output, -o      [output-file.js]')
+  console.error(' --watch,  -w      (recompile on schema change)')
+  console.error(' --proto_path, -I  [path-root] # base to lookup imports, multiple supported')
   console.error()
   process.exit(1)
 }
+filename = path.relative(process.cwd(), filename)
 
 if (watch && !output) {
   console.error('--watch requires --output')
@@ -49,6 +57,20 @@ function write () {
   fs.writeFileSync(output, compile())
 }
 
+function resolveImport (filename) {
+  for (let i = 0; i < importPaths.length; i++) {
+    const importPath = importPaths[i]
+    try {
+      return fs.readFileSync(path.join(importPath, filename))
+    } catch (err) {}
+  }
+  throw new Error('File "' + filename + '" not found in import path:\n - ' + importPaths.join('\n - '))
+}
+
 function compile () {
-  return protobuf.toJS(fs.readFileSync(filename), { encodings })
+  return protobuf.toJS(null, {
+    encodings: encodings,
+    filename: filename,
+    resolveImport: resolveImport
+  })
 }

index.js

@@ -11,11 +11,39 @@ const flatten = function (values) {
   return result
 }
 
+function resolveImport (filename, opts, context) {
+  if (!opts.resolveImport) throw new Error('opts.resolveImport is required if opts.filename is given.')
+  return _resolveImport(filename, opts, context)
+}
+
+function _resolveImport (filename, opts, context) {
+  if (context.stack.has(filename)) {
+    throw new Error('File recursively imports itself: ' + Array.from(context.stack).concat(filename).join(' -> '))
+  }
+  context.stack.add(filename)
+  const importData = opts.resolveImport(filename)
+  const sch = (typeof importData === 'object' && !Buffer.isBuffer(importData)) ? importData : schema.parse(importData)
+  sch.imports.forEach(function (importDef) {
+    const imported = _resolveImport(importDef, opts, context)
+    sch.enums = sch.enums.concat(imported.enums)
+    sch.messages = sch.messages.concat(imported.messages)
+  })
+  context.stack.delete(filename)
+  return sch
+}
+
 module.exports = function (proto, opts) {
   if (!opts) opts = {}
-  if (!proto) throw new Error('Pass in a .proto string or a protobuf-schema parsed object')
-
-  const sch = (typeof proto === 'object' && !Buffer.isBuffer(proto)) ? proto : schema.parse(proto)
+  let sch
+  if (opts.filename) {
+    sch = resolveImport(opts.filename, opts, {
+      cache: {},
+      stack: new Set()
+    })
+  } else {
+    if (!proto) throw new Error('Pass in a .proto string or a protobuf-schema parsed object')
+    sch = (typeof proto === 'object' && !Buffer.isBuffer(proto)) ? proto : schema.parse(proto)
+  }
 
   // to not make toString,toJSON enumarable we make a fire-and-forget prototype
   const Messages = function () {
@@ -38,5 +66,5 @@ module.exports = function (proto, opts) {
 }
 
 module.exports.toJS = function (proto, opts) {
-  return compileToJS(module.exports(proto, { inlineEnc: true }), opts)
+  return compileToJS(module.exports(proto, Object.assign({ inlineEnc: true }, opts)), opts)
 }

package.json

@@ -24,7 +24,7 @@
   "scripts": {
     "test": "standard && npm run test-generated && npm run test-compiled",
     "test-generated": "tape test/*.js",
-    "test-compiled": "./bin.js test/test.proto -o test/helpers/compiled.js && COMPILED=true tape test/*.js",
+    "test-compiled": "./bin.js test/test.proto -o test/helpers/compiled.js && ./bin.js test/imports/valid.proto -o test/helpers/imports.js && COMPILED=true tape test/*.js",
     "bench": "node bench"
   },
   "bugs": {

test/imports.js

@@ -0,0 +1,132 @@
+const tape = require('tape')
+const fs = require('fs')
+const path = require('path')
+const compile = require('..')
+const schema = require('protocol-buffers-schema')
+
+const projectBase = path.resolve(__dirname, '..')
+function resolveImport (filename) {
+  const filepath = path.resolve(projectBase, filename)
+  return fs.readFileSync(filepath)
+}
+
+function load (filename) {
+  return {
+    filename: filename,
+    raw: fs.readFileSync(path.join(projectBase, filename))
+  }
+}
+
+const valid = load('test/imports/valid.proto')
+
+function testCompiled (t, compiled) {
+  t.deepEqual(compiled.DeepestEnum, {
+    A: 1,
+    B: 2,
+    C: 3
+  })
+
+  const encoded = {
+    deepestMessage: compiled.DeepestMessage.encode({
+      field: 3
+    }),
+    deeper: compiled.Deeper.encode({
+      foo: {
+        field: 1
+      },
+      bar: 2
+    }),
+    deeper2: compiled.Deeper2.encode({
+      foo: {
+        field: 3
+      }
+    }),
+    valid: compiled.Valid.encode({
+      ext: {
+        foo: {
+          field: 3
+        },
+        bar: 1
+      }
+    })
+  }
+
+  t.deepEqual(encoded.deeper, Buffer.from('0a0208011002', 'hex'))
+
+  const decoded = {
+    deepestMessage: compiled.DeepestMessage.decode(encoded.deepestMessage),
+    deeper: compiled.Deeper.decode(encoded.deeper)
+  }
+  t.deepEqual(decoded, {
+    deepestMessage: {
+      field: 3
+    },
+    deeper: {
+      foo: {
+        field: 1
+      },
+      bar: 2
+    }
+  })
+  t.end()
+}
+
+if (process.env.COMPILED) {
+  tape('validated compiled', function (t) {
+    testCompiled(t, require('./helpers/imports.js'))
+  })
+}
+
+tape('valid imports', function (t) {
+  testCompiled(t, compile(null, {
+    filename: valid.filename,
+    resolveImport: resolveImport
+  }))
+})
+
+const deeper = load('test/imports/folder-a/folder-b/deeper.proto')
+const deeper2 = load('test/imports/folder-a/folder-b/deeper2.proto')
+const deepest = load('test/imports/folder-a/deepest.proto')
+
+tape('valid import with pre-parsed schemas', function (t) {
+  const cache = {}
+  cache[valid.filename] = schema.parse(valid.raw)
+  cache[deeper.filename] = schema.parse(deeper.raw)
+  cache[deeper2.filename] = schema.parse(deeper2.raw)
+  cache[deepest.filename] = schema.parse(deepest.raw)
+  t.deepEquals(Object.keys(compile(null, {
+    filename: valid.filename,
+    resolveImport: function (filename) {
+      return cache[filename]
+    }
+  })), ['DeepestEnum', 'Valid', 'Deeper', 'DeepestMessage', 'Deeper2'])
+  t.end()
+})
+
+tape('valid import without resolving', function (t) {
+  t.throws(function () {
+    compile(valid.raw, {})
+  }, /Could not resolve Deeper/)
+  t.end()
+})
+
+tape('import with .filename, without resolveImports', function (t) {
+  t.throws(function () {
+    compile(null, {
+      filename: 'test'
+    })
+  }, /opts.resolveImport is required if opts.filename is given./)
+  t.end()
+})
+
+const invalid = load('test/imports/invalid.proto')
+
+tape('circular import', function (t) {
+  t.throws(function () {
+    compile(invalid.raw, {
+      filename: invalid.filename,
+      resolveImport: resolveImport
+    })
+  }, /File recursively imports itself: test\/imports\/invalid.proto -> test\/imports\/folder-a\/circular.proto -> test\/imports\/invalid.proto/)
+  t.end()
+})

test/imports/folder-a/circular.proto

@@ -0,0 +1,5 @@
+import "test/imports/invalid.proto";
+
+message Circular {
+  required int32 num = 1;
+}

test/imports/folder-a/deepest.proto

@@ -0,0 +1,9 @@
+enum DeepestEnum {
+  A=1;
+  B=2;
+  C=3;
+}
+
+message DeepestMessage {
+  optional DeepestEnum field = 1 [default = B];
+}

test/imports/folder-a/folder-b/deeper.proto

@@ -0,0 +1,6 @@
+import "test/imports/folder-a/deepest.proto";
+
+message Deeper {
+  required DeepestMessage foo = 1;
+  required DeepestEnum bar = 2;
+}

test/imports/folder-a/folder-b/deeper2.proto

@@ -0,0 +1,7 @@
+// This is added to check cache when two different files
+// reference the same deep file.
+import "test/imports/folder-a/deepest.proto";
+
+message Deeper2 {
+  required DeepestMessage foo = 1;
+}

test/imports/invalid.proto

@@ -0,0 +1,5 @@
+import "test/imports/folder-a/circular.proto";
+
+message Invalid {
+  required int32 num = 1;
+}

test/imports/valid.proto

@@ -0,0 +1,6 @@
+import "test/imports/folder-a/folder-b/deeper.proto";
+import "test/imports/folder-a/folder-b/deeper2.proto";
+
+message Valid {
+  required Deeper ext = 1;
+}