Add streaming function

.github/workflows/test.yml

@@ -14,27 +14,28 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
-        go: ['1.19', '1.20', '1.21']
+        go: ['1.20', '1.21', '1.22']
     name: ${{ matrix.os }} @ Go ${{ matrix.go }}
     runs-on: ${{ matrix.os }}
     steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          path: ${{ env.WORKSPACE }}
+
       - name: Set up Go ${{ matrix.go }}
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v5
         with:
           go-version: ${{ matrix.go }}
 
       - name: Cache
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: ~/go/pkg/mod
           key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
           restore-keys: |
             ${{ runner.os }}-go-
 
-      - name: Checkout
-        uses: actions/checkout@v3
-        with:
-          path: ${{ env.WORKSPACE }}
 
       - name: Build
         run: go build -v ./...
@@ -43,10 +44,10 @@ jobs:
         run: go test -v --coverpkg=github.com/shamaton/msgpack/... --coverprofile=coverage.coverprofile --covermode=atomic ./...
 
       - name: Upload coverage to Codecov
-        if: success() && matrix.go == '1.21' && matrix.os == 'ubuntu-latest'
-        uses: codecov/codecov-action@v1
+        if: success() && matrix.go == '1.22' && matrix.os == 'ubuntu-latest'
+        uses: codecov/codecov-action@v4
         with:
-          token:
+          token: ${{ secrets.CODECOV_TOKEN }}
           fail_ci_if_error: false
           working-directory: ${{ env.WORKSPACE }}
 
@@ -55,7 +56,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           path: ${{ env.WORKSPACE }}
       - name: golangci-lint

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2018 Masayuki Shamoto
+Copyright (c) 2024 Masayuki Shamoto
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -17,8 +17,6 @@ You can get the fastest performance with [msgpackgen](https://github.com/shamato
 * Supports extend encoder / decoder
 * Can also Encoding / Decoding struct as array
 
-This package requires more than  version **1.13**
-
 ## Installation
 
 Current version is **msgpack/v2**.
@@ -32,24 +30,37 @@ package main
 
 import (
   "github.com/shamaton/msgpack/v2"
+  "net/http"
 )
 
+type Struct struct {
+	String string
+}
+
+// simple
 func main() {
-	type Struct struct {
-		String string
-	}
 	v := Struct{String: "msgpack"}
 
 	d, err := msgpack.Marshal(v)
 	if err != nil {
 		panic(err)
 	}
 	r := Struct{}
-	err = msgpack.Unmarshal(d, &r)
-	if err != nil {
+	if err =  msgpack.Unmarshal(d, &r); err != nil {
 		panic(err)
 	}
 }
+
+// streaming
+func handle(w http.ResponseWriter, r *http.Request) {
+	var body Struct
+	if err := msgpack.UnmarshalRead(r, &body); err != nil {
+		panic(err)
+    }
+	if err := msgpack.MarshalWrite(w, body); err != nil {
+		panic(err)
+    }
+}
 ```
 
 ## Benchmark

decode.go

@@ -1,6 +1,11 @@
 package msgpack
 
-import "github.com/shamaton/msgpack/v2/internal/decoding"
+import (
+	"io"
+
+	"github.com/shamaton/msgpack/v2/internal/decoding"
+	streamdecoding "github.com/shamaton/msgpack/v2/internal/stream/decoding"
+)
 
 // UnmarshalAsMap decodes data that is encoded as map format.
 // This is the same thing that StructAsArray sets false.
@@ -13,3 +18,15 @@
 func UnmarshalAsArray(data []byte, v interface{}) error {
 	return decoding.Decode(data, v, true)
 }
+
+// UnmarshalReadAsMap decodes from stream. stream data expects map format.
+// This is the same thing that StructAsArray sets false.
+func UnmarshalReadAsMap(r io.Reader, v interface{}) error {
+	return streamdecoding.Decode(r, v, false)
+}
+
+// UnmarshalReadAsArray decodes from stream. stream data expects array format.
+// This is the same thing that StructAsArray sets true.
+func UnmarshalReadAsArray(r io.Reader, v interface{}) error {
+	return streamdecoding.Decode(r, v, true)
+}

encode.go

@@ -1,7 +1,10 @@
 package msgpack
 
 import (
+	"io"
+
 	"github.com/shamaton/msgpack/v2/internal/encoding"
+	streamencoding "github.com/shamaton/msgpack/v2/internal/stream/encoding"
 )
 
 // MarshalAsMap encodes data as map format.
@@ -15,3 +18,15 @@
 func MarshalAsArray(v interface{}) ([]byte, error) {
 	return encoding.Encode(v, true)
 }
+
+// MarshalWriteAsMap writes map format encoded data to writer.
+// This is the same thing that StructAsArray sets false.
+func MarshalWriteAsMap(w io.Writer, v interface{}) error {
+	return streamencoding.Encode(w, v, false)
+}
+
+// MarshalWriteAsArray writes array format encoded data to writer.
+// This is the same thing that StructAsArray sets true.
+func MarshalWriteAsArray(w io.Writer, v interface{}) error {
+	return streamencoding.Encode(w, v, true)
+}

ext/decoder_stream.go

@@ -0,0 +1,60 @@
+package ext
+
+import (
+	"github.com/shamaton/msgpack/v2/internal/common"
+	"io"
+	"reflect"
+)
+
+var emptyBytes []byte
+
+type StreamDecoder interface {
+	Code() int8
+	IsType(code byte, innerType int8, dataLength int) bool
+	ToValue(code byte, data []byte, k reflect.Kind) (any, error)
+}
+
+type DecoderStreamCommon struct {
+}
+
+func (d *DecoderStreamCommon) ReadSize1(r io.Reader, buf *common.Buffer) (byte, error) {
+	if _, err := r.Read(buf.B1); err != nil {
+		return 0, err
+	}
+	return buf.B1[0], nil
+}
+
+func (d *DecoderStreamCommon) ReadSize2(r io.Reader, buf *common.Buffer) ([]byte, error) {
+	if _, err := r.Read(buf.B2); err != nil {
+		return emptyBytes, err
+	}
+	return buf.B2, nil
+}
+
+func (d *DecoderStreamCommon) ReadSize4(r io.Reader, buf *common.Buffer) ([]byte, error) {
+	if _, err := r.Read(buf.B4); err != nil {
+		return emptyBytes, err
+	}
+	return buf.B4, nil
+}
+
+func (d *DecoderStreamCommon) ReadSize8(r io.Reader, buf *common.Buffer) ([]byte, error) {
+	if _, err := r.Read(buf.B8); err != nil {
+		return emptyBytes, err
+	}
+	return buf.B8, nil
+}
+
+func (d *DecoderStreamCommon) ReadSizeN(r io.Reader, buf *common.Buffer, n int) ([]byte, error) {
+	var b []byte
+	if len(buf.Data) <= n {
+		b = buf.Data[:n]
+	} else {
+		buf.Data = append(buf.Data, make([]byte, n-len(buf.Data))...)
+		b = buf.Data
+	}
+	if _, err := r.Read(b); err != nil {
+		return emptyBytes, err
+	}
+	return b, nil
+}

ext/encode_stream.go

@@ -0,0 +1,120 @@
+package ext
+
+import (
+	"github.com/shamaton/msgpack/v2/internal/common"
+	"io"
+	"reflect"
+)
+
+type StreamEncoder interface {
+	Code() int8
+	Type() reflect.Type
+	Write(w io.Writer, value reflect.Value, buf *common.Buffer) error
+}
+
+type StreamEncoderCommon struct{}
+
+func (c *StreamEncoderCommon) WriteByte1Int64(w io.Writer, value int64, buf *common.Buffer) error {
+	return buf.Write(w,
+		byte(value),
+	)
+}
+
+func (c *StreamEncoderCommon) WriteByte2Int64(w io.Writer, value int64, buf *common.Buffer) error {
+	return buf.Write(w,
+		byte(value>>8),
+		byte(value),
+	)
+}
+
+func (c *StreamEncoderCommon) WriteByte4Int64(w io.Writer, value int64, buf *common.Buffer) error {
+	return buf.Write(w,
+		byte(value>>24),
+		byte(value>>16),
+		byte(value>>8),
+		byte(value),
+	)
+}
+
+func (c *StreamEncoderCommon) WriteByte8Int64(w io.Writer, value int64, buf *common.Buffer) error {
+	return buf.Write(w,
+		byte(value>>56),
+		byte(value>>48),
+		byte(value>>40),
+		byte(value>>32),
+		byte(value>>24),
+		byte(value>>16),
+		byte(value>>8),
+		byte(value),
+	)
+}
+
+func (c *StreamEncoderCommon) WriteByte1Uint64(w io.Writer, value uint64, buf *common.Buffer) error {
+	return buf.Write(w,
+		byte(value),
+	)
+}
+
+func (c *StreamEncoderCommon) WriteByte2Uint64(w io.Writer, value uint64, buf *common.Buffer) error {
+	return buf.Write(w,
+		byte(value>>8),
+		byte(value),
+	)
+}
+
+func (c *StreamEncoderCommon) WriteByte4Uint64(w io.Writer, value uint64, buf *common.Buffer) error {
+	return buf.Write(w,
+		byte(value>>24),
+		byte(value>>16),
+		byte(value>>8),
+		byte(value),
+	)
+}
+
+func (c *StreamEncoderCommon) WriteByte8Uint64(w io.Writer, value uint64, buf *common.Buffer) error {
+	return buf.Write(w,
+		byte(value>>56),
+		byte(value>>48),
+		byte(value>>40),
+		byte(value>>32),
+		byte(value>>24),
+		byte(value>>16),
+		byte(value>>8),
+		byte(value),
+	)
+}
+
+func (c *StreamEncoderCommon) WriteByte1Int(w io.Writer, value int, buf *common.Buffer) error {
+	return buf.Write(w,
+		byte(value),
+	)
+}
+
+func (c *StreamEncoderCommon) WriteByte2Int(w io.Writer, value int, buf *common.Buffer) error {
+	return buf.Write(w,
+		byte(value>>8),
+		byte(value),
+	)
+}
+
+func (c *StreamEncoderCommon) WriteByte4Int(w io.Writer, value int, buf *common.Buffer) error {
+	return buf.Write(w,
+		byte(value>>24),
+		byte(value>>16),
+		byte(value>>8),
+		byte(value),
+	)
+}
+
+func (c *StreamEncoderCommon) WriteByte4Uint32(w io.Writer, value uint32, buf *common.Buffer) error {
+	return buf.Write(w,
+		byte(value>>24),
+		byte(value>>16),
+		byte(value>>8),
+		byte(value),
+	)
+}
+
+func (c *StreamEncoderCommon) WriteBytes(w io.Writer, bs []byte, buf *common.Buffer) error {
+	return buf.Write(w, bs...)
+}

go.mod

@@ -1,3 +1,3 @@
 module github.com/shamaton/msgpack/v2
 
-go 1.15
+go 1.20