ipjs, full esm + browser, no Buffer, verify-car.js executable

README.md

@@ -9,16 +9,19 @@ The interface wraps a [Datastore](https://github.com/ipfs/interface-datastore),
 ## Example
 
 ```js
-const fs = require('fs')
-const multiformats = require('multiformats/basics')
+import fs from 'fs'
+import multiformats from 'multiformats/basics'
+import car from 'datastore-car'
+import dagCbor from '@ipld/dag-cbor'
+
 // dag-cbor is required for the CAR root block
-multiformats.add(require('@ipld/dag-cbor'))
-const CarDatastore = require('datastore-car')(multiformats)
+multiformats.add(dagCbor)
+const CarDatastore = car(multiformats)
 
 async function example () {
-  const binary = Buffer.from('random meaningless bytes')
+  const binary = new TextEncoder().encode('random meaningless bytes')
   const mh = await multiformats.multihash.hash(binary, 'sha2-256')
-  const cid = new multiformats.CID(1, multiformats.get('raw').code, mh)
+  const cid = multiformats.CID.create(1, multiformats.get('raw').code, mh)
 
   const outStream = fs.createWriteStream('example.car')
   const writeDs = await CarDatastore.writeStream(outStream)
@@ -39,10 +42,10 @@ async function example () {
   const roots = await readDs.getRoots()
   // retrieve a block, as a UInt8Array, reading from the ZIP archive
   const got = await readDs.get(roots[0])
-  // also possible: for await (const {key, data} = readDs.query()) { ... }
+  // also possible: for await (const { key, value } of readDs.query()) { ... }
 
   console.log('Retrieved [%s] from example.car with CID [%s]',
-    Buffer.from(got).toString(),
+    new TextDecoder().decode(got),
     roots[0].toString())
 
   await readDs.close()
@@ -62,14 +65,14 @@ Retrieved [random meaningless bytes] from example.car with CID [bafkreihwkf6mtnj
 
 In this example, the `writeStream()` create-mode is used to generate the CAR file, this allows for an iterative write process where first the roots are set (`setRoots()`) and then all of the blocks are written (`put()`). After it is created, we use the `readStreamComplete()` create-mode to read the contents. Other create-modes are useful where the environment, data and needs demand:
 
-* **[`CarDatastore.readBuffer(buffer)`](#CarDatastore__readBuffer)**: read a CAR archive from a `Buffer` or `Uint8Array`. Does not support mutation operations, only reads. This mode is not efficient for large data sets but does support `get()` and `has()` operations since it caches the entire archive in memory. This mode is the only mode _available_ in a browser environment
+* **[`CarDatastore.readBuffer(buffer)`](#CarDatastore__readBuffer)**: read a CAR archive from a `Uint8Array`. Does not support mutation operations, only reads. This mode is not efficient for large data sets but does support `get()` and `has()` operations since it caches the entire archive in memory. This mode is the only mode _available_ in a browser environment
 * **[`CarDatastore.readFileComplete(file)`](#CarDatastore__readFileComplete)**: read a CAR archive directly from a file. Does not support mutation operations, only reads. This mode is not efficient for large data sets but does support `get()` and `has()` operations since it caches the entire archive in memory. This mode is _not available_ in a browser environment.
 * **[`CarDatastore.readStreamComplete(stream)`](#CarDatastore__readStreamComplete)**: read a CAR archive directly from a stream. Does not support mutation operations, only reads. This mode is not efficient for large data sets but does support `get()` and `has()` operations since it caches the entire archive in memory. This mode is _not available_ in a browser environment.
 * **[`CarDatastore.readStreaming(stream)`](#CarDatastore__readStreaming)**: read a CAR archive directly from a stream. Does not support mutation operations, and only supports iterative reads via `query()` (i.e. no `get()` and `has()`). This mode is very efficient for large data sets. This mode is _not available_ in a browser environment.
 * **[`async CarDatastore.readFileIndexed(stream)`](#CarDatastore__readFileIndexed)**: read a CAR archive from a local file, index its contents and use that index to support random access reads (`has()`, `get()` and `query()`) without fitting the entire contents in memory as `readFileComplete()` does. Uses more memory than `readStreaming()` and less than `readFileComplete()`. Will be slower to initialize than `readStreaming()` but suitable where random access reads are required from a large file.
 * **[`CarDatastore.writeStream(stream)`](#CarDatastore__writeStream)**: write a CAR archive to a stream (e.g. `fs.createWriteStream(file)`). Does not support read operations, only writes, and the writes are append-only (i.e. no `delete()`). However, this mode is very efficient for dumping large data sets, with no caching and streaming writes. This mode is _not available_ in a browser environment.
 
-Other create-modes may be supported in the future, such as writing to a Buffer (although this is already possible if you couple `writeStream()` with a [`BufferListStream`](https://ghub.io/bl)) or a read/write mode such as datastore-zipcar makes available.
+Other create-modes may be supported in the future, such as writing to a Uint8Array (although this is already possible if you couple `writeStream()` with a [`BufferListStream`](https://ghub.io/bl)) or a read/write mode such as datastore-zipcar makes available.
 
 ## API
 
@@ -97,7 +100,7 @@ Other create-modes may be supported in the future, such as writing to a Buffer (
 <a name="CarDatastore__readBuffer"></a>
 ### `async CarDatastore.readBuffer(buffer)`
 
-Read a CarDatastore from a Buffer containing the contents of an existing
+Read a CarDatastore from a Uint8Array containing the contents of an existing
 CAR archive. Mutation operations (`put()`, `delete()` and `setRoots()`) are
 not available.
 
@@ -111,7 +114,7 @@ environment.
 
 **Parameters:**
 
-* **`buffer`** _(`Buffer|Uint8Array`)_: the byte contents of a CAR archive
+* **`buffer`** _(`Uint8Array`)_: the byte contents of a CAR archive
 
 **Return value**  _(`CarDatastore`)_: a read-only CarDatastore.
 
@@ -291,7 +294,7 @@ may throw an error if unsupported.
 * **`key`** _(`string|Key|CID`)_: a `CID` or `CID`-convertable object to identify
   the block.
 
-**Return value**  _(`Buffer`)_: the IPLD block data referenced by the CID.
+**Return value**  _(`Uint8Array`)_: the IPLD block data referenced by the CID.
 
 <a name="CarDatastore_has"></a>
 ### `async CarDatastore#has(key)`
@@ -325,7 +328,7 @@ and an Error will be thrown when it is called.
 
 * **`key`** _(`string|Key|CID`)_: a `CID` or `CID`-convertable object to identify
   the `value`.
-* **`value`** _(`Buffer|Uint8Array`)_: an IPLD block matching the given `key`
+* **`value`** _(`Uint8Array`)_: an IPLD block matching the given `key`
   `CID`.
 
 <a name="CarDatastore_delete"></a>

car-browser.js

@@ -5,7 +5,7 @@ import CarDatastore from './datastore.js'
 /**
  * @name CarDatastore.readBuffer
  * @description
- * Read a CarDatastore from a Buffer containing the contents of an existing
+ * Read a CarDatastore from a Uint8Array containing the contents of an existing
  * CAR archive. Mutation operations (`put()`, `delete()` and `setRoots()`) are
  * not available.
  *
@@ -20,7 +20,7 @@ import CarDatastore from './datastore.js'
  * @memberof CarDatastore
  * @static
  * @async
- * @param {Buffer|Uint8Array} buffer the byte contents of a CAR archive
+ * @param {Uint8Array} buffer the byte contents of a CAR archive
  * @returns {CarDatastore} a read-only CarDatastore.
  */
 async function readBuffer (multiformats, buffer) {

datastore.js

@@ -31,7 +31,7 @@ class CarDatastore {
    * @memberof CarDatastore
    * @param {string|Key|CID} key a `CID` or `CID`-convertable object to identify
    * the block.
-   * @return {Buffer} the IPLD block data referenced by the CID.
+   * @return {Uint8Array} the IPLD block data referenced by the CID.
    */
   async get (key) {
     key = toKey(this._multiformats, key, 'get')
@@ -74,7 +74,7 @@ class CarDatastore {
    * @memberof CarDatastore
    * @param {string|Key|CID} key a `CID` or `CID`-convertable object to identify
    * the `value`.
-   * @param {Buffer|Uint8Array} value an IPLD block matching the given `key`
+   * @param {Uint8Array} value an IPLD block matching the given `key`
    * `CID`.
    */
   async put (key, value) {

example.js

@@ -1,15 +1,16 @@
 import fs from 'fs'
 import multiformats from 'multiformats/basics'
-import car from './'
+import car from 'datastore-car'
+import dagCbor from '@ipld/dag-cbor'
 
 // dag-cbor is required for the CAR root block
-multiformats.add(require('@ipld/dag-cbor'))
+multiformats.add(dagCbor)
 const CarDatastore = car(multiformats)
 
 async function example () {
-  const binary = Buffer.from('random meaningless bytes')
+  const binary = new TextEncoder().encode('random meaningless bytes')
   const mh = await multiformats.multihash.hash(binary, 'sha2-256')
-  const cid = new multiformats.CID(1, multiformats.get('raw').code, mh)
+  const cid = multiformats.CID.create(1, multiformats.get('raw').code, mh)
 
   const outStream = fs.createWriteStream('example.car')
   const writeDs = await CarDatastore.writeStream(outStream)
@@ -30,10 +31,10 @@ async function example () {
   const roots = await readDs.getRoots()
   // retrieve a block, as a UInt8Array, reading from the ZIP archive
   const got = await readDs.get(roots[0])
-  // also possible: for await (const {key, data} = readDs.query()) { ... }
+  // also possible: for await (const { key, value } of readDs.query()) { ... }
 
   console.log('Retrieved [%s] from example.car with CID [%s]',
-    Buffer.from(got).toString(),
+    new TextDecoder().decode(got),
     roots[0].toString())
 
   await readDs.close()

lib/coding-browser.js

@@ -17,7 +17,7 @@ function bufferReader (buf) {
 
     async exactly (length) {
       if (length > buf.length - pos) {
-        throw new Error('Unexpected end of Buffer')
+        throw new Error('Unexpected end of data')
       }
       return buf.slice(pos, pos + length)
     },
@@ -70,7 +70,7 @@ async function readCid (multiformats, reader) {
     // cidv0 32-byte sha2-256
     const bytes = await reader.exactly(34)
     reader.seek(34)
-    return new multiformats.CID(0, CIDV0_BYTES.DAG_PB, Uint8Array.from(bytes))
+    return multiformats.CID.create(0, CIDV0_BYTES.DAG_PB, Uint8Array.from(bytes))
   }
 
   const version = await readVarint(reader)
@@ -79,7 +79,7 @@ async function readCid (multiformats, reader) {
   }
   const codec = await readVarint(reader)
   const multihash = await readMultihash(reader)
-  return new multiformats.CID(version, codec, Uint8Array.from(multihash))
+  return multiformats.CID.create(version, codec, Uint8Array.from(multihash))
 }
 
 async function readBlockHead (multiformats, reader) {
@@ -159,16 +159,16 @@ function Encoder (multiformats, writer) {
       }
 
       const header = await multiformats.encode({ version: 1, roots }, 'dag-cbor')
-      await writer(Buffer.from(varint.encode(header.length)))
+      await writer(new Uint8Array(varint.encode(header.length)))
       await writer(header)
     },
 
     async writeBlock (block) {
       if (!isBlock(block)) {
         throw new TypeError('Block list must be of type { cid, binary }')
       }
-      await writer(Buffer.from(varint.encode(block.cid.buffer.length + block.binary.length)))
-      await writer(block.cid.buffer)
+      await writer(new Uint8Array(varint.encode(block.cid.bytes.length + block.binary.length)))
+      await writer(block.cid.bytes)
       await writer(block.binary)
     }
   }
@@ -185,8 +185,8 @@ async function encode (multiformats, writer, roots, blocks) {
 /**
  * @name Car.decodeBuffer
  * @description
- * Decode a `Buffer` representation of a Content ARchive (CAR) into an in-memory
- * representation:
+ * Decode a `Uint8Array` representation of a Content ARchive (CAR) into an
+ * in-memory representation:
  *
  * `{ version, roots[], blocks[] }`.
  *
@@ -202,7 +202,7 @@ async function encode (multiformats, writer, roots, blocks) {
  * @memberof Car
  * @static
  * @async
- * @param {Buffer} buf the contents of a CAR
+ * @param {Uint8Array} buf the contents of a CAR
  * @returns {Car} an in-memory representation of a CAR file:
  * `{ version, roots[], blocks[] }`.
  */
@@ -223,7 +223,7 @@ function BufferDecoder (buf) {
  * @name Car.encodeBuffer
  * @description
  * Encode a set of IPLD blocks of the form `{ cid, binary }` in CAR format,
- * returning it as a single `Buffer`.
+ * returning it as a single `Uint8Array`
  * @function
  * @memberof Car
  * @static
@@ -233,14 +233,20 @@ function BufferDecoder (buf) {
  * blocks.
  * @param {object[]} blocks an array of IPLD blocks of the form `{ cid, binary }`
  * to append to the archive.
- * @returns {Buffer} a `Buffer` representing the created archive.
+ * @returns {Uint8Array} a binary representation of the created archive.
  */
 
 async function encodeBuffer (multiformats, roots, blocks) {
   const bl = []
   const writer = (buf) => { bl.push(buf) }
   await encode(multiformats, writer, roots, blocks)
-  return Buffer.concat(bl)
+  const ret = new Uint8Array(bl.reduce((p, c) => p + c.length, 0))
+  let off = 0
+  for (const b of bl) {
+    ret.set(b, off)
+    off += b.length
+  }
+  return ret
 }
 
 export { encode, Encoder, decode, Decoder, decodeBuffer, encodeBuffer }

lib/coding.js

@@ -1,10 +1,15 @@
-import { promises as fs, createWriteStream } from 'fs'
+import fsCb from 'fs'
 import { promisify } from 'util'
-import { PassThrough, pipeline } from 'stream'
+import stream from 'readable-stream'
 import { Encoder, Decoder, encode, decode, encodeBuffer, decodeBuffer } from './coding-browser.js'
 
+const { PassThrough, pipeline } = stream
 const pipelineAsync = promisify(pipeline)
-fs.createWriteStream = createWriteStream
+let fs
+if (!process.browser) {
+  fs = fsCb.promises
+  fs.createWriteStream = fsCb.createWriteStream
+}
 
 // reusable reader for streams and files, we just need a way to read an
 // additional chunk (of some undetermined size) and a way to close the
@@ -13,7 +18,7 @@ function chunkReader (readChunk, closer) {
   let pos = 0
   let have = 0
   let offset = 0
-  let currentChunk = Buffer.alloc(0)
+  let currentChunk = new Uint8Array(0)
 
   const read = async (length) => {
     have = currentChunk.length - offset
@@ -33,7 +38,12 @@ function chunkReader (readChunk, closer) {
       }
       have += chunk.length
     }
-    currentChunk = Buffer.concat(bufa)
+    currentChunk = new Uint8Array(bufa.reduce((p, c) => p + c.length, 0))
+    let off = 0
+    for (const b of bufa) {
+      currentChunk.set(b, off)
+      off += b.length
+    }
     offset = 0
   }
 
@@ -77,7 +87,7 @@ function streamReader (stream) {
   async function readChunk () {
     const next = await iterator.next()
     if (next.done) {
-      return Buffer.alloc(0)
+      return new Uint8Array(0)
     }
     return next.value
   }
@@ -86,14 +96,18 @@ function streamReader (stream) {
 }
 
 async function fileReader (file, options) {
+  /* c8 ignore next 3 */
+  if (!fs) {
+    throw new Error('fileReader() not supported in a browser environment')
+  }
   const fd = await fs.open(file, 'r')
   const bufferSize = typeof options === 'object' && typeof options.bufferSize === 'number' ? options.bufferSize : 1024
-  const readerChunk = Buffer.alloc(bufferSize)
+  const readerChunk = new Uint8Array(bufferSize)
 
   async function readChunk () {
     const { bytesRead } = await fd.read(readerChunk, 0, readerChunk.length)
     if (!bytesRead) {
-      return Buffer.alloc(0)
+      return new Uint8Array(0)
     }
     return Uint8Array.prototype.slice.call(readerChunk, 0, bytesRead)
   }
@@ -190,7 +204,11 @@ function StreamDecoder (multiformats, stream) {
  * to append to the archive.
  */
 
-function encodeFile (multiformats, file, roots, blocks) {
+async function encodeFile (multiformats, file, roots, blocks) {
+  /* c8 ignore next 3 */
+  if (!fs) {
+    throw new Error('encodeFile() not supported in a browser environment')
+  }
   return pipelineAsync(encodeStream(multiformats, roots, blocks), fs.createWriteStream(file))
 }
 

lib/raw.js

@@ -1,7 +1,10 @@
 import { promisify } from 'util'
 import fs from 'fs'
 import { StreamDecoder, FileDecoder } from './coding.js'
-fs.read = promisify(fs.read)
+
+if (!process.browser) {
+  fs.read = promisify(fs.read)
+}
 
 /**
  * @name CarDatastore.indexer
@@ -106,8 +109,12 @@ async function indexer (multiformats, inp) {
  * @returns {object} an IPLD block of the form `{ cid, binary }`.
  */
 async function readRaw (fd, blockIndex) {
+  /* c8 ignore next 3 */
+  if (process.browser) {
+    throw new Error('readRaw() not supported in a browser environment')
+  }
   const { cid, blockLength, blockOffset } = blockIndex
-  const binary = Buffer.alloc(blockLength)
+  const binary = new Uint8Array(blockLength)
   let read
   if (typeof fd === 'number') {
     read = (await fs.read(fd, binary, 0, blockLength, blockOffset)).bytesRead

lib/util.js

@@ -5,23 +5,21 @@ function isBlock (block) {
 
 function isCID (key) {
   return typeof key === 'object' &&
-    isBuffer(key.buffer) &&
+    isBuffer(key.bytes) &&
     isBuffer(key.multihash) &&
     (key.version === 0 || key.version === 1) &&
     typeof key.code === 'number'
     // don't handle old style CIDs
 }
 
 function isBuffer (b) {
-  return (Buffer.isBuffer(b) || (
-    b instanceof Uint8Array &&
-    b.constructor.name === 'Uint8Array'))
+  return b instanceof ArrayBuffer || ArrayBuffer.isView(b)
 }
 
 function toKey (multiformats, key, method) {
   if (!isCID(key)) {
     try {
-      key = new multiformats.CID(key.toString())
+      key = multiformats.CID.from(key.toString())
     } catch (e) {
       throw new TypeError(`${method}() only accepts CIDs or CID strings`)
     }