diff --git a/libbeat/publisher/queue/diskqueue/docs/Makefile b/libbeat/publisher/queue/diskqueue/docs/Makefile index d9f55ffb377..bf7a34dd926 100644 --- a/libbeat/publisher/queue/diskqueue/docs/Makefile +++ b/libbeat/publisher/queue/diskqueue/docs/Makefile @@ -1,4 +1,4 @@ -all : schemaV0.svg frameV0.svg schemaV1.svg frameV1.svg schemaV2.svg frameV2.svg +all : schemaV0.svg frameV0.svg schemaV1.svg frameV1.svg schemaV2.svg frameV2.svg schemaV3.svg frameV3.svg .PHONY : clean diff --git a/libbeat/publisher/queue/diskqueue/docs/frameV2.pic b/libbeat/publisher/queue/diskqueue/docs/frameV2.pic index 3e340d09071..aac18259952 100644 --- a/libbeat/publisher/queue/diskqueue/docs/frameV2.pic +++ b/libbeat/publisher/queue/diskqueue/docs/frameV2.pic @@ -1,5 +1,5 @@ boxht = 0.25 SIZE1: box "size (uint32)" wid 4; -DATA: box "LZ4 compressed CBOR serialized data" dashed wid 4 ht 2 with .nw at SIZE1.sw; +DATA: box "CBOR serialized data" dashed wid 4 ht 2 with .nw at SIZE1.sw; CHECKSUM: box "checksum (uint32)" wid 4 with .nw at DATA.sw; SIZE2: box "size (uint32)" wid 4 with nw at CHECKSUM.sw; \ No newline at end of file diff --git a/libbeat/publisher/queue/diskqueue/docs/frameV2.svg b/libbeat/publisher/queue/diskqueue/docs/frameV2.svg index 8cf04362b72..8317e4b93e8 100644 --- a/libbeat/publisher/queue/diskqueue/docs/frameV2.svg +++ b/libbeat/publisher/queue/diskqueue/docs/frameV2.svg @@ -2,7 +2,7 @@ size (uint32) -LZ4 compressed CBOR serialized data +CBOR serialized data checksum (uint32) diff --git a/libbeat/publisher/queue/diskqueue/docs/frameV3.pic b/libbeat/publisher/queue/diskqueue/docs/frameV3.pic new file mode 100644 index 00000000000..3e340d09071 --- /dev/null +++ b/libbeat/publisher/queue/diskqueue/docs/frameV3.pic @@ -0,0 +1,5 @@ +boxht = 0.25 +SIZE1: box "size (uint32)" wid 4; +DATA: box "LZ4 compressed CBOR serialized data" dashed wid 4 ht 2 with .nw at SIZE1.sw; +CHECKSUM: box "checksum (uint32)" wid 4 with .nw at DATA.sw; +SIZE2: box "size (uint32)" wid 4 with nw at CHECKSUM.sw; \ No newline at end of file diff --git a/libbeat/publisher/queue/diskqueue/docs/frameV3.svg b/libbeat/publisher/queue/diskqueue/docs/frameV3.svg new file mode 100644 index 00000000000..8cf04362b72 --- /dev/null +++ b/libbeat/publisher/queue/diskqueue/docs/frameV3.svg @@ -0,0 +1,11 @@ + + +size (uint32) + +LZ4 compressed CBOR serialized data + +checksum (uint32) + +size (uint32) + + diff --git a/libbeat/publisher/queue/diskqueue/docs/on-disk-structures.md b/libbeat/publisher/queue/diskqueue/docs/on-disk-structures.md index 67831d13ff6..55878906bbe 100644 --- a/libbeat/publisher/queue/diskqueue/docs/on-disk-structures.md +++ b/libbeat/publisher/queue/diskqueue/docs/on-disk-structures.md @@ -49,14 +49,42 @@ of the size from the header. ## Version 2 -In version 2, encryption & compression is added to version 1. The +In version 2, encryption is added to version 1. The segments are made of a header followed by an initialization vector, and then encrypted frames. The header consists of one field, the version number which is an unsigned 32-bit integer in little-endian -format. The initialization vector is 128-bits in length. +format. The initialization vector is 128-bits in length. The count +was dropped from version 1 for 2 reasons. The first, if it was +outside the encrypted portion of the segment then it would be easy for +an attacker to modify. The second, is that adding it to the encrypted +segment in a meaningful way was problematic. The count is not known +until the last frame is written. With encryption you cannot seek to +the beginning of the segment and update the value. Adding the count +to the end is less useful because you have to decrypt the entire +segment before it can be read. ![Segment Schema Version 2](./schemaV2.svg) +The frames for version 2, consist of a header, followed by the +serialized event and a footer. The header contains one field which is +the size of the frame, which is an unsigned 32-bit integer in +little-endian format. The serialization format is CBOR. The footer +contains 2 fields, the first of which is a checksum which is an +unsigned 32-bit integer in little-endian format, followed by a repeat +of the size from the header. This is the same as version 1. + +![Frame Version 2](./frameV2.svg) + +## Version 3 + +In version 2, compression is added to version 2. The +segments are made of a header followed by an initialization vector, +and then encrypted frames. The header consists of one field, the +version number which is an unsigned 32-bit integer in little-endian +format. The initialization vector is 128-bits in length. + +![Segment Schema Version 3](./schemaV3.svg) + The frames for version 2, consist of a header, followed by the compressed serialized event and a footer. The header contains one field which is the size of the frame, which is an unsigned 32-bit @@ -66,4 +94,4 @@ fields, the first of which is a checksum which is an unsigned 32-bit integer in little-endian format, followed by a repeat of the size from the header. -![Frame Version 2](./frameV2.svg) +![Frame Version 3](./frameV3.svg) diff --git a/libbeat/publisher/queue/diskqueue/docs/schemaV3.pic b/libbeat/publisher/queue/diskqueue/docs/schemaV3.pic new file mode 100644 index 00000000000..006ae4bbf7b --- /dev/null +++ b/libbeat/publisher/queue/diskqueue/docs/schemaV3.pic @@ -0,0 +1,4 @@ +boxht = 0.25 +VERSION: box "version (uint32)" wid 4; +IV: box "initialization vector (128 bits)" wid 4 ht 1 with .nw at VERSION.sw +FRAME: box "Encrypted Frames" dashed wid 4 ht 2 with .nw at IV.sw; \ No newline at end of file diff --git a/libbeat/publisher/queue/diskqueue/docs/schemaV3.svg b/libbeat/publisher/queue/diskqueue/docs/schemaV3.svg new file mode 100644 index 00000000000..33ec757bed3 --- /dev/null +++ b/libbeat/publisher/queue/diskqueue/docs/schemaV3.svg @@ -0,0 +1,9 @@ + + +version (uint32) + +initialization vector (128 bits) + +Encrypted Frames + +