Switch to @ipld/multiformats

.gitignore

@@ -3,3 +3,5 @@ node_modules/
 .nyc_output/
 coverage/
 example.car
+build/
+dist/

car-browser.js

@@ -1,11 +1,11 @@
-const { NoWriter } = require('./lib/reader-writer-iface')
-const { createBufferReader } = require('./lib/reader-browser')
-const CarDatastore = require('./datastore')
+import { NoWriter } from './lib/reader-writer-iface.js'
+import { createBufferReader } from './lib/reader-browser.js'
+import CarDatastore from './datastore.js'
 
 /**
  * @name CarDatastore.readBuffer
  * @description
- * Read a CarDatastore from a Buffer containing the contents of an existing
+ * Read a CarDatastore from a Uint8Array containing the contents of an existing
  * CAR archive. Mutation operations (`put()`, `delete()` and `setRoots()`) are
  * not available.
  *
@@ -20,13 +20,25 @@ const CarDatastore = require('./datastore')
  * @memberof CarDatastore
  * @static
  * @async
- * @param {Buffer|Uint8Array} buffer the byte contents of a CAR archive
+ * @param {Uint8Array} buffer the byte contents of a CAR archive
  * @returns {CarDatastore} a read-only CarDatastore.
  */
-async function readBuffer (buffer) {
-  const reader = await createBufferReader(buffer)
+async function readBuffer (multiformats, buffer) {
+  const reader = await createBufferReader(multiformats, buffer)
   const writer = new NoWriter()
-  return new CarDatastore(reader, writer)
+  return new CarDatastore(multiformats, reader, writer)
 }
 
-module.exports.readBuffer = readBuffer
+export default (multiformats) => {
+  function wrap (fn) {
+    return function (...args) {
+      return fn(multiformats, ...args)
+    }
+  }
+
+  return {
+    readBuffer: wrap(readBuffer)
+  }
+}
+
+export { readBuffer }

car.js

@@ -1,16 +1,16 @@
-const { Reader, NoWriter } = require('./lib/reader-writer-iface')
-const { createStreamCompleteReader, createStreamingReader, createFileReader } = require('./lib/reader')
-const createStreamWriter = require('./lib/writer-stream')
-const CarDatastore = require('./datastore')
-const { readBuffer } = require('./car-browser')
-const { indexer, readRaw } = require('./lib/raw')
+import { Reader, NoWriter } from './lib/reader-writer-iface.js'
+import { createStreamCompleteReader, createStreamingReader, createFileReader, createFileIndexedReader } from './lib/reader.js'
+import createStreamWriter from './lib/writer-stream.js'
+import CarDatastore from './datastore.js'
+import browser from './car-browser.js'
+import { indexer, readRaw } from './lib/raw.js'
 
 /**
  * @name CarDatastore.readFileComplete
  * @description
  * Read a CAR archive from a file and return a CarDatastore. The CarDatastore
  * returned will _only_ support read operations: `getRoots()`, `get()`, `has()`
- * and `query()`. Caching makes `get()` and `has()` possible as the entire
+ * and `query()`. Caching makes `get()` and `has()`. This is possible as the entire
  * file is read and decoded before the CarDatastore is returned. mutation
  * operations (`put()`, `delete()` and `setRoots()`) are not available as there
  * is no ability to modify the archive.
@@ -35,18 +35,18 @@ const { indexer, readRaw } = require('./lib/raw')
  * @param {string} file a path to a file containing CAR archive data.
  * @returns {CarDatastore} a read-only CarDatastore.
  */
-async function readFileComplete (file) {
-  const reader = await createFileReader(file)
+async function readFileComplete (multiformats, file) {
+  const reader = await createFileReader(multiformats, file)
   const writer = new NoWriter()
-  return new CarDatastore(reader, writer)
+  return new CarDatastore(multiformats, reader, writer)
 }
 
 /**
  * @name CarDatastore.readStreamComplete
  * @description
  * Read a CAR archive as a CarDataStore from a ReadableStream. The CarDatastore
  * returned will _only_ support read operations: `getRoots()`, `get()`, `has()`
- * and `query()`. Caching makes `get()` and `has()` possible as the entire
+ * and `query()`. Caching makes `get()` and `has()`. This is possible as the entire
  * stream is read and decoded before the CarDatastore is returned. Mutation
  * operations (`put()`, `delete()` and `setRoots()`) are not available as there
  * is no ability to modify the archive.
@@ -65,10 +65,10 @@ async function readFileComplete (file) {
  * archive as a binary stream.
  * @returns {CarDatastore} a read-only CarDatastore.
  */
-async function readStreamComplete (stream) {
-  const reader = await createStreamCompleteReader(stream)
+async function readStreamComplete (multiformats, stream) {
+  const reader = await createStreamCompleteReader(multiformats, stream)
   const writer = new NoWriter()
-  return new CarDatastore(reader, writer)
+  return new CarDatastore(multiformats, reader, writer)
 }
 
 /**
@@ -98,10 +98,46 @@ async function readStreamComplete (stream) {
  * archive as a binary stream.
  * @returns {CarDatastore} a read-only CarDatastore.
  */
-async function readStreaming (stream) {
-  const reader = await createStreamingReader(stream)
+async function readStreaming (multiformats, stream) {
+  const reader = await createStreamingReader(multiformats, stream)
   const writer = new NoWriter()
-  return new CarDatastore(reader, writer)
+  return new CarDatastore(multiformats, reader, writer)
+}
+
+/**
+ * @name CarDatastore.readFileIndexed
+ * @description
+ * Read a CAR archive as a CarDataStore from a local file. The CarDatastore
+ * returned will _only_ support read operations: `getRoots()`, `get()`, `has()`
+ * and `query()`. Caching makes `get()` and `has()`. This is possible as the entire
+ * stream is read and _indexed_ before the CarDatastore is returned. Mutation
+ * operations (`put()`, `delete()` and `setRoots()`) are not available as there
+ * is no ability to modify the archive.
+ *
+ * The indexing operation uses {@link indexer} to catalogue the contents of the
+ * CAR and store a mapping of CID to byte locations for each entry. This method
+ * of parsing is not as memory intensive as {@link readStreamComplete} as only
+ * the index is stored in memory. When blocks are read, the index tells the
+ * reader where to fetch the block from within the CAR file.
+ *
+ * This mode is suitable for large files where random-access operations are
+ * required. Where a full sequential read is only required, use
+ * {@link createReadStreaming} which consumes the file in a single pass with no
+ * memory used for indexing.
+ *
+ * This create-mode is not available in the browser environment.
+ * @function
+ * @memberof CarDatastore
+ * @static
+ * @async
+ * @param {ReadableStream} stream a ReadableStream that provides an entire CAR
+ * archive as a binary stream.
+ * @returns {CarDatastore} a read-only CarDatastore.
+ */
+async function readFileIndexed (multiformats, filePath) {
+  const reader = await createFileIndexedReader(multiformats, filePath)
+  const writer = new NoWriter()
+  return new CarDatastore(multiformats, reader, writer)
 }
 
 /**
@@ -129,17 +165,17 @@ async function readStreaming (stream) {
  * @param {WritableStream} stream a writable stream
  * @returns {CarDatastore} an append-only, streaming CarDatastore.
  */
-async function writeStream (stream) {
+async function writeStream (multiformats, stream) {
   const reader = new Reader()
-  const writer = await createStreamWriter(stream)
-  return new CarDatastore(reader, writer)
+  const writer = await createStreamWriter(multiformats, stream)
+  return new CarDatastore(multiformats, reader, writer)
 }
 
 async function traverseBlock (block, get, car, concurrency = 1, seen = new Set()) {
   const cid = await block.cid()
   await car.put(cid, block.encodeUnsafe())
   seen.add(cid.toString('base58btc'))
-  if (cid.codec === 'raw') {
+  if (cid.code === 0x55) { // raw
     return
   }
   const reader = block.reader()
@@ -168,8 +204,8 @@ async function traverseBlock (block, get, car, concurrency = 1, seen = new Set()
  * @memberof CarDatastore
  * @static
  * @async
- * @param {Block} root the root of the graph to start at, this block will be
- * included in the CAR and its CID will be set as the single root.
+ * @param {CID} root the CID of the root of the graph to start at, this block
+ * will be included in the CAR and the CID will be set as the single root.
  * @param {AsyncFunction} get an `async` function that takes a CID and returns
  * a `Block`. Can be used to attach to an arbitrary data store.
  * @param {CarDatastore} car a writable `CarDatastore` that has not yet been
@@ -184,11 +220,23 @@ async function completeGraph (root, get, car, concurrency) {
   await car.close()
 }
 
-module.exports.readBuffer = readBuffer
-module.exports.readFileComplete = readFileComplete
-module.exports.readStreamComplete = readStreamComplete
-module.exports.readStreaming = readStreaming
-module.exports.writeStream = writeStream
-module.exports.indexer = indexer
-module.exports.readRaw = readRaw
-module.exports.completeGraph = completeGraph
+function create (multiformats) {
+  function wrap (fn) {
+    return function (...args) {
+      return fn(multiformats, ...args)
+    }
+  }
+
+  return Object.assign(browser(multiformats), {
+    readFileComplete: wrap(readFileComplete),
+    readStreamComplete: wrap(readStreamComplete),
+    readStreaming: wrap(readStreaming),
+    readFileIndexed: wrap(readFileIndexed),
+    writeStream: wrap(writeStream),
+    indexer: wrap(indexer),
+    readRaw,
+    completeGraph
+  })
+}
+
+export default create

datastore.js

@@ -1,15 +1,17 @@
-const { filter, map } = require('interface-datastore').utils
-const { toKey } = require('./lib/util')
+import interfaceDatastore from 'interface-datastore'
+
+const { filter, map } = interfaceDatastore.utils
 
 /**
  * CarDatastore is a class to manage reading from, and writing to a CAR archives
- * using [CID](https://github.com/multiformats/js-cid)s as keys and file names
- * in the CAR and binary block data as the file contents.
+ * using [CID](https://github.com/multiformats/js-multiformats)s as keys and
+ * file names in the CAR and binary block data as the file contents.
  *
  * @class
  */
 class CarDatastore {
-  constructor (reader, writer) {
+  constructor (multiformats, reader, writer) {
+    this.multiformats = multiformats
     this._reader = reader
     this._writer = writer
   }
@@ -28,10 +30,10 @@ class CarDatastore {
    * @memberof CarDatastore
    * @param {string|Key|CID} key a `CID` or `CID`-convertable object to identify
    * the block.
-   * @return {Buffer} the IPLD block data referenced by the CID.
+   * @return {Uint8Array} the IPLD block data referenced by the CID.
    */
   async get (key) {
-    key = toKey(key, 'get')
+    key = toKey(this.multiformats, key, 'get')
     return this._reader.get(key)
   }
 
@@ -52,7 +54,7 @@ class CarDatastore {
    * @return {boolean} indicating whether the key exists in this Datastore.
    */
   async has (key) {
-    key = toKey(key, 'has')
+    key = toKey(this.multiformats, key, 'has')
     return this._reader.has(key)
   }
 
@@ -71,11 +73,11 @@ class CarDatastore {
    * @memberof CarDatastore
    * @param {string|Key|CID} key a `CID` or `CID`-convertable object to identify
    * the `value`.
-   * @param {Buffer|Uint8Array} value an IPLD block matching the given `key`
+   * @param {Uint8Array} value an IPLD block matching the given `key`
    * `CID`.
    */
   async put (key, value) {
-    key = toKey(key, 'put')
+    key = toKey(this.multiformats, key, 'put')
     if (!(value instanceof Uint8Array)) {
       throw new TypeError('put() can only receive Uint8Arrays or Buffers')
     }
@@ -94,7 +96,7 @@ class CarDatastore {
    * the block.
    */
   async delete (key) {
-    key = toKey(key, 'delete')
+    key = toKey(this.multiformats, key, 'delete')
     return this._writer.delete(key)
   }
 
@@ -224,4 +226,12 @@ class CarDatastore {
   }
 }
 
-module.exports = CarDatastore
+function toKey (multiformats, key, method) {
+  try {
+    return multiformats.CID.from(key.toString())
+  } catch (e) {
+    throw new TypeError(`${method}() only accepts CIDs or CID strings`)
+  }
+}
+
+export default CarDatastore

example.js

@@ -1,18 +1,24 @@
-const fs = require('fs')
-const CarDatastore = require('./')
-const Block = require('@ipld/block')
+import fs from 'fs'
+import multiformats from 'multiformats/basics'
+import car from 'datastore-car'
+import dagCbor from '@ipld/dag-cbor'
+
+// dag-cbor is required for the CAR root block
+multiformats.add(dagCbor)
+const CarDatastore = car(multiformats)
 
 async function example () {
-  const block = Block.encoder(Buffer.from('random meaningless bytes'), 'raw')
-  const cid = await block.cid()
+  const binary = new TextEncoder().encode('random meaningless bytes')
+  const mh = await multiformats.multihash.hash(binary, 'sha2-256')
+  const cid = multiformats.CID.create(1, multiformats.get('raw').code, mh)
 
   const outStream = fs.createWriteStream('example.car')
   const writeDs = await CarDatastore.writeStream(outStream)
 
   // set the header with a single root
   await writeDs.setRoots(cid)
   // store a new block, creates a new file entry in the CAR archive
-  await writeDs.put(cid, await block.encode())
+  await writeDs.put(cid, binary)
   await writeDs.close()
 
   const inStream = fs.createReadStream('example.car')
@@ -25,10 +31,10 @@ async function example () {
   const roots = await readDs.getRoots()
   // retrieve a block, as a UInt8Array, reading from the ZIP archive
   const got = await readDs.get(roots[0])
-  // also possible: for await (const {key, data} = readDs.query()) { ... }
+  // also possible: for await (const { key, value } of readDs.query()) { ... }
 
   console.log('Retrieved [%s] from example.car with CID [%s]',
-    Buffer.from(got).toString(),
+    new TextDecoder().decode(got),
     roots[0].toString())
 
   await readDs.close()