(v3.x) feat: reply trailers support (#3807)

* feat: reply trailers support (#3794)

* chore: remove content-length when Transfer-Encoding is added (#3814)

Co-authored-by: 小菜 <xtx1130@gmail.com>

docs/Reference/Reply.md

@@ -13,6 +13,9 @@
   - [.getHeaders()](#getheaders)
   - [.removeHeader(key)](#removeheaderkey)
   - [.hasHeader(key)](#hasheaderkey)
+  - [.trailer(key, function)](#trailerkey-function)
+  - [.hasTrailer(key)](#hastrailerkey)
+  - [.removeTrailer(key)](#removetrailerkey)
   - [.redirect([code,] dest)](#redirectcode--dest)
   - [.callNotFound()](#callnotfound)
   - [.getResponseTime()](#getresponsetime)
@@ -47,6 +50,9 @@ object that exposes the following functions and properties:
 - `.getHeaders()` - Gets a shallow copy of all current response headers.
 - `.removeHeader(key)` - Remove the value of a previously set header.
 - `.hasHeader(name)` - Determine if a header has been set.
+- `.trailer(key, function)` - Sets a response trailer.
+- `.hasTrailer(key)` - Determine if a trailer has been set.
+- `.removeTrailer(key)` - Remove the value of a previously set trailer.
 - `.type(value)` - Sets the header `Content-Type`.
 - `.redirect([code,] dest)` - Redirect to the specified url, the status code is
   optional (default to `302`).
@@ -199,6 +205,49 @@ reply.getHeader('x-foo') // undefined
 
 Returns a boolean indicating if the specified header has been set.
 
+### .trailer(key, function)
+<a id="trailer"></a>
+
+Sets a response trailer. Trailer usually used when you want some header that require heavy resources to be sent after the `data`, for example `Server-Timing`, `Etag`. It can ensure the client get the response data as soon as possible.
+
+*Note: The header `Transfer-Encoding: chunked` will be added once you use the trailer. It is a hard requipment for using trailer in Node.js.*
+
+*Note: Currently, the computation function only supports synchronous function. That means `async-await` and `promise` are not supported.*
+
+```js
+reply.trailer('server-timing', function() {
+  return 'db;dur=53, app;dur=47.2'
+})
+
+const { createHash } = require('crypto')
+// trailer function also recieve two argument
+// @param {object} reply fastify reply
+// @param {string|Buffer|null} payload payload that already sent, note that it will be null when stream is sent
+reply.trailer('content-md5', function(reply, payload) {
+  const hash = createHash('md5')
+  hash.update(payload)
+  return hash.disgest('hex')
+})
+```
+
+### .hasTrailer(key)
+<a id="hasTrailer"></a>
+
+Returns a boolean indicating if the specified trailer has been set.
+
+### .removeTrailer(key)
+<a id="removeTrailer"></a>
+
+Remove the value of a previously set trailer.
+```js
+reply.trailer('server-timing', function() {
+  return 'db;dur=53, app;dur=47.2'
+})
+reply.removeTrailer('server-timing')
+reply.getTrailer('server-timing') // undefined
+```
+
+
 ### .redirect([code ,] dest)
 <a id="redirect"></a>
 

lib/errors.js

@@ -147,6 +147,14 @@ const codes = {
     'FST_ERR_BAD_STATUS_CODE',
     'Called reply with an invalid status code: %s'
   ),
+  FST_ERR_BAD_TRAILER_NAME: createError(
+    'FST_ERR_BAD_TRAILER_NAME',
+    'Called reply.trailer with an invalid header name: %s'
+  ),
+  FST_ERR_BAD_TRAILER_VALUE: createError(
+    'FST_ERR_BAD_TRAILER_VALUE',
+    "Called reply.trailer('%s', fn) with an invalid type: %s. Expected a function."
+  ),
 
   /**
    * schemas

lib/reply.js

@@ -16,6 +16,7 @@ const {
   kReplySerializerDefault,
   kReplyIsError,
   kReplyHeaders,
+  kReplyTrailers,
   kReplyHasStatusCode,
   kReplyIsRunningOnErrorHook,
   kDisableRequestLogging
@@ -47,7 +48,9 @@ const {
   FST_ERR_REP_ALREADY_SENT,
   FST_ERR_REP_SENT_VALUE,
   FST_ERR_SEND_INSIDE_ONERR,
-  FST_ERR_BAD_STATUS_CODE
+  FST_ERR_BAD_STATUS_CODE,
+  FST_ERR_BAD_TRAILER_NAME,
+  FST_ERR_BAD_TRAILER_VALUE
 } = require('./errors')
 const warning = require('./warnings')
 
@@ -60,6 +63,7 @@ function Reply (res, request, log) {
   this[kReplyIsRunningOnErrorHook] = false
   this.request = request
   this[kReplyHeaders] = {}
+  this[kReplyTrailers] = null
   this[kReplyHasStatusCode] = false
   this[kReplyStartTime] = undefined
   this.log = log
@@ -261,6 +265,47 @@ Reply.prototype.headers = function (headers) {
   return this
 }
 
+// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Trailer#directives
+// https://httpwg.org/specs/rfc7230.html#chunked.trailer.part
+const INVALID_TRAILERS = new Set([
+  'transfer-encoding',
+  'content-length',
+  'host',
+  'cache-control',
+  'max-forwards',
+  'te',
+  'authorization',
+  'set-cookie',
+  'content-encoding',
+  'content-type',
+  'content-range',
+  'trailer'
+])
+
+Reply.prototype.trailer = function (key, fn) {
+  key = key.toLowerCase()
+  if (INVALID_TRAILERS.has(key)) {
+    throw new FST_ERR_BAD_TRAILER_NAME(key)
+  }
+  if (typeof fn !== 'function') {
+    throw new FST_ERR_BAD_TRAILER_VALUE(key, typeof fn)
+  }
+  if (this[kReplyTrailers] === null) this[kReplyTrailers] = {}
+  this[kReplyTrailers][key] = fn
+  return this
+}
+
+Reply.prototype.hasTrailer = function (key) {
+  if (this[kReplyTrailers] === null) return false
+  return this[kReplyTrailers][key.toLowerCase()] !== undefined
+}
+
+Reply.prototype.removeTrailer = function (key) {
+  if (this[kReplyTrailers] === null) return this
+  this[kReplyTrailers][key.toLowerCase()] = undefined
+  return this
+}
+
 Reply.prototype.code = function (code) {
   const intValue = parseInt(code)
   if (isNaN(intValue) || intValue < 100 || intValue > 600) {
@@ -416,18 +461,35 @@ function onSendEnd (reply, payload) {
   const req = reply.request
   const statusCode = res.statusCode
 
+  // we check if we need to update the trailers header and set it
+  if (reply[kReplyTrailers] !== null) {
+    const trailerHeaders = Object.keys(reply[kReplyTrailers])
+    let header = ''
+    for (const trailerName of trailerHeaders) {
+      if (typeof reply[kReplyTrailers][trailerName] !== 'function') continue
+      header += ' '
+      header += trailerName
+    }
+    // it must be chunked for trailer to work
+    reply.header('Transfer-Encoding', 'chunked')
+    reply.header('Trailer', header.trim())
+  }
+
   if (payload === undefined || payload === null) {
     reply[kReplySent] = true
 
     // according to https://tools.ietf.org/html/rfc7230#section-3.3.2
     // we cannot send a content-length for 304 and 204, and all status code
-    // < 200.
+    // < 200
+    // A sender MUST NOT send a Content-Length header field in any message
+    // that contains a Transfer-Encoding header field.
     // For HEAD we don't overwrite the `content-length`
-    if (statusCode >= 200 && statusCode !== 204 && statusCode !== 304 && req.method !== 'HEAD') {
+    if (statusCode >= 200 && statusCode !== 204 && statusCode !== 304 && req.method !== 'HEAD' && reply[kReplyTrailers] === null) {
       reply[kReplyHeaders]['content-length'] = '0'
     }
 
     res.writeHead(statusCode, reply[kReplyHeaders])
+    sendTrailer(payload, res, reply)
     // avoid ArgumentsAdaptorTrampoline from V8
     res.end(null, null, null)
     return
@@ -444,18 +506,23 @@ function onSendEnd (reply, payload) {
     throw new FST_ERR_REP_INVALID_PAYLOAD_TYPE(typeof payload)
   }
 
-  if (!reply[kReplyHeaders]['content-length']) {
-    reply[kReplyHeaders]['content-length'] = '' + Buffer.byteLength(payload)
-  } else if (req.raw.method !== 'HEAD' && reply[kReplyHeaders]['content-length'] !== Buffer.byteLength(payload)) {
-    reply[kReplyHeaders]['content-length'] = '' + Buffer.byteLength(payload)
+  if (reply[kReplyTrailers] === null) {
+    if (!reply[kReplyHeaders]['content-length']) {
+      reply[kReplyHeaders]['content-length'] = '' + Buffer.byteLength(payload)
+    } else if (req.raw.method !== 'HEAD' && reply[kReplyHeaders]['content-length'] !== Buffer.byteLength(payload)) {
+      reply[kReplyHeaders]['content-length'] = '' + Buffer.byteLength(payload)
+    }
   }
 
   reply[kReplySent] = true
 
   res.writeHead(statusCode, reply[kReplyHeaders])
-
+  // write payload first
+  res.write(payload)
+  // then send trailers
+  sendTrailer(payload, res, reply)
   // avoid ArgumentsAdaptorTrampoline from V8
-  res.end(payload, null, null)
+  res.end(null, null, null)
 }
 
 function logStreamError (logger, err, res) {
@@ -472,6 +539,9 @@ function sendStream (payload, res, reply) {
   let sourceOpen = true
   let errorLogged = false
 
+  // set trailer when stream ended
+  sendStreamTrailer(payload, res, reply)
+
   eos(payload, { readable: true, writable: false }, function (err) {
     sourceOpen = false
     if (err != null) {
@@ -520,6 +590,22 @@ function sendStream (payload, res, reply) {
   payload.pipe(res)
 }
 
+function sendTrailer (payload, res, reply) {
+  if (reply[kReplyTrailers] === null) return
+  const trailerHeaders = Object.keys(reply[kReplyTrailers])
+  const trailers = {}
+  for (const trailerName of trailerHeaders) {
+    if (typeof reply[kReplyTrailers][trailerName] !== 'function') continue
+    trailers[trailerName] = reply[kReplyTrailers][trailerName](reply, payload)
+  }
+  res.addTrailers(trailers)
+}
+
+function sendStreamTrailer (payload, res, reply) {
+  if (reply[kReplyTrailers] === null) return
+  payload.on('end', () => sendTrailer(null, res, reply))
+}
+
 function onErrorHook (reply, error, cb) {
   reply[kReplySent] = true
   if (reply.context.onError !== null && reply[kReplyErrorHandlerCalled] === true) {
@@ -684,6 +770,7 @@ function buildReply (R) {
     this[kReplySerializer] = null
     this.request = request
     this[kReplyHeaders] = {}
+    this[kReplyTrailers] = null
     this[kReplyStartTime] = undefined
     this[kReplyEndTime] = undefined
     this.log = log

lib/symbols.js

@@ -30,6 +30,7 @@ const keys = {
   kReplySerializer: Symbol('fastify.reply.serializer'),
   kReplyIsError: Symbol('fastify.reply.isError'),
   kReplyHeaders: Symbol('fastify.reply.headers'),
+  kReplyTrailers: Symbol('fastify.reply.trailers'),
   kReplyHasStatusCode: Symbol('fastify.reply.hasStatusCode'),
   kReplySent: Symbol('fastify.reply.sent'),
   kReplySentOverwritten: Symbol('fastify.reply.sentOverwritten'),

test/internals/reply.test.js

@@ -53,6 +53,7 @@ test('reply.send will logStream error and destroy the stream', { only: true }, t
     hasHeader: () => false,
     getHeader: () => undefined,
     writeHead: () => {},
+    write: () => {},
     headersSent: true
   })
 
@@ -74,6 +75,7 @@ test('reply.send throw with circular JSON', t => {
     hasHeader: () => false,
     getHeader: () => undefined,
     writeHead: () => {},
+    write: () => {},
     end: () => {}
   }
   const reply = new Reply(response, { context: { onSend: [] } })
@@ -91,6 +93,7 @@ test('reply.send returns itself', t => {
     hasHeader: () => false,
     getHeader: () => undefined,
     writeHead: () => {},
+    write: () => {},
     end: () => {}
   }
   const reply = new Reply(response, { context: { onSend: [] } })