Update hyper:// to stream responses (WIP)

app/bg/protocols/hyper.js

@@ -1,9 +1,10 @@
 import { parseDriveUrl } from '../../lib/urls'
+import { PassThrough, Transform } from 'stream'
 import parseRange from 'range-parser'
 import once from 'once'
+import pump from 'pump'
 import * as logLib from '../logger'
 const logger = logLib.child({category: 'hyper', subcategory: 'hyper-scheme'})
-// import intoStream from 'into-stream'
 import markdown from '../../lib/markdown'
 import * as drives from '../hyper/drives'
 import * as filesystem from '../filesystem/index'
@@ -21,27 +22,6 @@ const md = markdown({
   highlight: undefined
 })
 
-/**
- * HACK
- * Electron has an issue that's causing file read streams to fail to serve
- * Reading into memory seems to resolve the issue
- * https://github.com/electron/electron/issues/21018
- * -prf
- */
-import { PassThrough } from 'stream'
-function intoStream (text) {
-  const rv = new PassThrough()
-  rv.push(text)
-  rv.push(null)
-  return rv
-}
-
-// constants
-// =
-
-// how long till we give up?
-const REQUEST_TIMEOUT_MS = 30e3 // 30 seconds
-
 // exported api
 // =
 
@@ -79,8 +59,6 @@ export const protocolHandler = async function (request, respond) {
       respond({statusCode: code})
     }
   }
-  var fileReadStream
-  var headersSent = false
   var drive
   var cspHeader = undefined
 
@@ -273,28 +251,21 @@ export const protocolHandler = async function (request, respond) {
       }
     }
 
-    // TODO
-    // Electron is being really aggressive about caching and not following the headers correctly
-    // caching is disabled till we can figure out why
-    // -prf
-    // caching if-match
-    // const ETag = 'block-' + entry.offset
-    // if (request.headers['if-none-match'] === ETag) {
-    //   return respondError(304, 'Not Modified')
-    // }
-
     // handle range
     headers['Accept-Ranges'] = 'bytes'
+    var length
     var range = request.headers.Range || request.headers.range
     if (range) range = parseRange(entry.size, range)
     if (range && range.type === 'bytes') {
       range = range[0] // only handle first range given
       statusCode = 206
+      length = (range.end - range.start + 1)
+      headers['Content-Length'] = '' + length
       headers['Content-Range'] = 'bytes ' + range.start + '-' + range.end + '/' + entry.size
-      headers['Content-Length'] = '' + (range.end - range.start + 1)
     } else {
       if (entry.size) {
-        headers['Content-Length'] = '' + (entry.size)
+        length = entry.size
+        headers['Content-Length'] = '' + length
       }
     }
 
@@ -322,80 +293,50 @@ export const protocolHandler = async function (request, respond) {
         : content
       return respond({
         statusCode: 200,
-        headers: Object.assign(headers, {
-          'Content-Type': contentType
-        }),
+        headers: Object.assign(headers, {'Content-Type': contentType}),
         data: intoStream(content)
       })
     }
 
-    // fetch the entry and stream the response
-    // HACK solution until electron issue resolved -prf
-    headersSent = true
     var mimeType = mime.identify(entry.path)
     if (!canExecuteHTML && mimeType.includes('text/html')) {
       mimeType = 'text/plain'
     }
-    Object.assign(headers, {'Content-Type': mimeType})
-    var data = await checkoutFS.pda.readFile(entry.path, 'binary')
-    if (range) {
-      data = data.slice(range.start, range.end + 1)
+    headers['Content-Type'] = mimeType
+    if (request.method === 'HEAD') {
+      respond({statusCode: 204, headers, data: intoStream('')})
+    } else {
+      respond({
+        statusCode,
+        headers,
+        data: pump(
+          checkoutFS.session.drive.createReadStream(entry.path, range),
+          chunkLogger(entry.path, length)
+        )
+      })
     }
-    respond({
-      statusCode,
-      headers,
-      data: intoStream(data)
-    })
   })
-  /*fileReadStream = checkoutFS.pda.createReadStream(entry.path, range)
-  var dataStream = fileReadStream
-    .pipe(mime.identifyStream(entry.path, mimeType => {
-
-      // disable html as needed
-      if (!canExecuteHTML && mimeType.includes('html')) {
-        mimeType = 'text/plain'
-      }
-
-      // send headers, now that we can identify the data
-      headersSent = true
-      Object.assign(headers, {
-        'Content-Type': mimeType
-      })
-      // TODO
-      // Electron is being really aggressive about caching and not following the headers correctly
-      // caching is disabled till we can figure out why
-      // -prf
-      // if (ETag) {
-      //   Object.assign(headers, {ETag})
-      // } else {
-      //   Object.assign(headers, {'Cache-Control': 'no-cache'})
-      // }
+}
 
-      if (request.method === 'HEAD') {
-        dataStream.destroy() // stop reading data
-        respond({statusCode: 204, headers, data: intoStream('')})
-      } else {
-        respond({statusCode, headers, data: dataStream})
-      }
-    }))
+function intoStream (text) {
+  const rv = new PassThrough()
+  rv.push(text)
+  rv.push(null)
+  return rv
+}
 
-  // handle empty files
-  fileReadStream.once('end', () => {
-    if (!headersSent) {
-      respond({
-        statusCode: 200,
-        headers: {
-          'Content-Security-Policy': cspHeader,
-          'Access-Control-Allow-Origin': '*'
-        },
-        data: intoStream('')
-      })
+// DEBUG
+function chunkLogger (path, len) {
+  var n = 0
+  return new Transform({
+    transform (chunk, encoding, callback) {
+      console.log(path, 'len', len, 'n', n, 'chunk', chunk.length)
+      n += chunk.length
+      return callback(null, chunk)
+      this.push(chunk)
+      n += chunk.length
+      if (len && n >= len) this.push(null)
+      callback()
     }
   })
-
-  // handle read-stream errors
-  fileReadStream.once('error', err => {
-    logger.warn('Error reading file', {url: drive.url, path: entry.path, err})
-    if (!headersSent) respondError(500, 'Failed to read file')
-  })*/
 }

app/package.json

@@ -34,7 +34,7 @@
     "fs-jetpack": "^1.3.1",
     "fs-reverse": "0.0.3",
     "function-queue": "0.0.12",
-    "hyperdrive-daemon": "^1.13.16",
+    "hyperdrive-daemon": "^1.13.18",
     "hyperdrive-daemon-client": "^1.15.2",
     "icojs": "^0.12.3",
     "identify-filetype": "^1.0.0",