diff --git a/.gitignore b/.gitignore index 43aa9227e2320..f43b1e9423b27 100644 --- a/.gitignore +++ b/.gitignore @@ -27,4 +27,5 @@ docs/changed-files man/man1 man/man5 man/man8 +vendor/ vendor/pkg/ diff --git a/daemon/logdrivers_linux.go b/daemon/logdrivers_linux.go index 89fe49a858831..ea34eb9204852 100644 --- a/daemon/logdrivers_linux.go +++ b/daemon/logdrivers_linux.go @@ -3,6 +3,7 @@ package daemon import ( // Importing packages here only to make sure their init gets called and // therefore they register themselves to the logdriver factory. + _ "github.com/docker/docker/daemon/logger/alilogs" _ "github.com/docker/docker/daemon/logger/awslogs" _ "github.com/docker/docker/daemon/logger/fluentd" _ "github.com/docker/docker/daemon/logger/gcplogs" diff --git a/daemon/logger/alilogs/alilogs.go b/daemon/logger/alilogs/alilogs.go new file mode 100644 index 0000000000000..385b70f193ab0 --- /dev/null +++ b/daemon/logger/alilogs/alilogs.go @@ -0,0 +1,282 @@ +// Package alilogs provides the logdriver for forwarding container logs to Ali Log Service + +package alilogs + +import ( + "fmt" + "sync" + "time" + + "github.com/Sirupsen/logrus" + "github.com/docker/docker/daemon/logger" + "github.com/galaxydi/go-loghub" + "github.com/golang/protobuf/proto" +) + +/* +Ali logging driver usage + docker run -d --name test-logger \ + --log-driver alilogs \ + --log-opt alilogs-endpoint=cn-hangzhou.log.aliyuncs.com \ + --log-opt alilogs-project=test_project \ + --log-opt alilogs-logstore=test-logstore \ + + // You can add these extra attributes to log message + --log-opt labels=attr1,attr2,attr3 \ + --label attr1=attr1Value \ + --label attr2=attr2Value \ + --label attr3=attr3Value \ + + // You assign these environment variables for alilogs logging driver to work + // "securityToken" and "topic" are optinal + --log-opt env=accessKeyID,accessKeySecret,securityToken,topic \ + --env "accessKeyID=xxx" \ + --env "accessKeySecret=xxx" \ + --env "securityToken=xxx" \ + --env "topic=demo_topic" \ + log-producer +*/ + +const ( + name = "alilogs" + endpointKey = "alilogs-endpoint" + projectKey = "alilogs-project" + logstoreKey = "alilogs-logstore" + envKey = "env" + labelsKey = "labels" + + accessKeyIDEnvKey = "accessKeyID" + accessKeySecretEnvKey = "accessKeySecret" + securityTokenEnvKey = "securityToken" + topicEnvKey = "topic" + + // PutLogs limit in Loghub, 3MB or 4096 records per put + batchPublishFrequency = 5 * time.Second + maximumBytesPerPut = 3145728 + maximumLogsPerPut = 4096 +) + +type logStream struct { + topic string + extraLogContents []*sls.LogContent + client AliLogAPI + messages chan *logger.Message + lock sync.RWMutex + closed bool +} + +// init registers the alilogs driver +func init() { + if err := logger.RegisterLogDriver(name, New); err != nil { + logrus.Fatal(err) + } + if err := logger.RegisterLogOptValidator(name, ValidateLogOpt); err != nil { + logrus.Fatal(err) + } +} + +// New creates an alilogs logger using the configuration passed in on the context +func New(ctx logger.Context) (logger.Logger, error) { + endpoint := ctx.Config[endpointKey] + projectName := ctx.Config[projectKey] + logstoreName := ctx.Config[logstoreKey] + extraContents := []*sls.LogContent{} + accessKeyID := "" + accessKeySecret := "" + securityToken := "" + topicName := "" + + extra := ctx.ExtraAttributes(nil) + value, ok := extra[accessKeyIDEnvKey] + if ok { + accessKeyID = value + delete(extra, accessKeyIDEnvKey) + } else { + return nil, fmt.Errorf("must specify a value for env '%s'", accessKeyIDEnvKey) + } + + value, ok = extra[accessKeySecretEnvKey] + if ok { + accessKeySecret = value + delete(extra, accessKeySecretEnvKey) + } else { + return nil, fmt.Errorf("must specify a value for env '%s'", accessKeySecretEnvKey) + } + + if value, ok = extra[securityTokenEnvKey]; ok { + securityToken = value + delete(extra, securityTokenEnvKey) + } + + if value, ok = extra[topicEnvKey]; ok { + topicName = value + delete(extra, topicEnvKey) + } + + // add extra contents to log record + for key, value := range extra { + logContent := &sls.LogContent{ + Key: proto.String(key), + Value: proto.String(value), + } + extraContents = append(extraContents, logContent) + } + + aliLogClient, err := NewAliLogClient(endpoint, projectName, logstoreName, accessKeyID, accessKeySecret, securityToken) + if err != nil { + return nil, err + } + containerStream := &logStream{ + topic: topicName, + extraLogContents: extraContents, + client: aliLogClient, + messages: make(chan *logger.Message, maximumLogsPerPut), + } + + go containerStream.collectLogs() + return containerStream, nil +} + +// Name returns the name of ali logging driver +func (ls *logStream) Name() string { + return name +} + +// Log submits messages for logging by an instance of the alilogs logging driver +func (ls *logStream) Log(msg *logger.Message) error { + ls.lock.RLock() + defer ls.lock.RUnlock() + if !ls.closed { + // buffer up the data, making sure to copy the Line data + ls.messages <- msg + } + return nil +} + +// Close closes the instance of the alilogs logging driver +func (ls *logStream) Close() error { + ls.lock.Lock() + defer ls.lock.Unlock() + if !ls.closed { + close(ls.messages) + } + ls.closed = true + return nil +} + +// newTicker is used for time-based batching. newTicker is a variable such +// that the implementation can be swapped out for unit tests. +var newTicker = func(freq time.Duration) *time.Ticker { + return time.NewTicker(freq) +} + +// PutLogs executes as a goroutine to perform put logs for +// submission to the logstore. Batching is performed on time- and size- +// bases. Time-based batching occurs at a 5 second interval (defined in the +// batchPublishFrequency const). Size-based batching is performed on the +// maximum number of logs per batch (defined in maximumLogsPerPut) and +// the maximum number of total bytes in a batch (defined in +// maximumBytesPerPut). +func (ls *logStream) collectLogs() { + aliLogClient := ls.client.(*AliLogClient) + logGroup := sls.LogGroup{ + Topic: proto.String(ls.topic), + Logs: []*sls.Log{}, + } + timer := newTicker(batchPublishFrequency) + for { + select { + case <-timer.C: + ls.publishLogs(&logGroup) + logrus.WithFields(logrus.Fields{ + "endpoint": aliLogClient.Endpoint, + "project": aliLogClient.ProjectName, + "logstore": aliLogClient.LogstoreName, + "published log number": len(logGroup.Logs), + "published log size": logGroup.Size(), + }).Debug("publish log when timer timeout") + logGroup.Reset() + logGroup.Topic = proto.String(ls.topic) + case msg, more := <-ls.messages: + if !more { + ls.publishLogs(&logGroup) + logrus.WithFields(logrus.Fields{ + "endpoint": aliLogClient.Endpoint, + "project": aliLogClient.ProjectName, + "logstore": aliLogClient.LogstoreName, + "published log number": len(logGroup.Logs), + "published log size": logGroup.Size(), + }).Debug("publish log when no more logs") + return + } + unprocessedLine := msg.Line + logMsg := &sls.LogContent{ + Key: proto.String("message"), + Value: proto.String(string(unprocessedLine)), + } + contents := ls.extraLogContents + contents = append(contents, logMsg) + logRecord := sls.Log{ + Time: proto.Uint32(uint32(time.Now().Unix())), + Contents: contents, + } + if len(unprocessedLine) > 0 { + if (len(logGroup.Logs) >= maximumLogsPerPut) || (logGroup.Size()+logRecord.Size() > maximumBytesPerPut) { + // Publish an existing batch if it's already over the maximum number of logs or if adding this + // line would push it over the maximum number of total bytes. + ls.publishLogs(&logGroup) + logrus.WithFields(logrus.Fields{ + "endpoint": aliLogClient.Endpoint, + "project": aliLogClient.ProjectName, + "logstore": aliLogClient.LogstoreName, + "published log number": len(logGroup.Logs), + "published log size": logGroup.Size(), + }).Debug("publish logs when touch the limit") + logGroup.Reset() + logGroup.Topic = proto.String(ls.topic) + } + logGroup.Logs = append(logGroup.Logs, &logRecord) + } + } + } +} + +// publishLogs calls PutLogs for a given LogGroup +func (ls *logStream) publishLogs(lg *sls.LogGroup) { + err := ls.client.PutLogs(lg) + if err != nil { + if serviceErr, ok := err.(sls.Error); ok { + aliLogClient := ls.client.(*AliLogClient) + logrus.WithFields(logrus.Fields{ + "errorCode": serviceErr.Code, + "errorMessage": serviceErr.Message, + "endpoint": aliLogClient.Endpoint, + "project": aliLogClient.ProjectName, + "logstore": aliLogClient.LogstoreName, + }).Error("PutLogs occurs sls error") + } else { + logrus.Error(err) + } + } +} + +// ValidateLogOpt looks for alilogs-specific log options +func ValidateLogOpt(cfg map[string]string) error { + for key := range cfg { + switch key { + case endpointKey, projectKey, logstoreKey, labelsKey, envKey: + default: + return fmt.Errorf("unknown log opt '%s' for %s log driver", key, name) + } + } + if cfg[endpointKey] == "" { + return fmt.Errorf("must specify a value for log opt '%s'", endpointKey) + } + if cfg[projectKey] == "" { + return fmt.Errorf("must specify a value for log opt '%s'", projectKey) + } + if cfg[logstoreKey] == "" { + return fmt.Errorf("must specify a value for log opt '%s'", logstoreKey) + } + return nil +} diff --git a/daemon/logger/alilogs/logapi.go b/daemon/logger/alilogs/logapi.go new file mode 100644 index 0000000000000..67bd2d2b8736f --- /dev/null +++ b/daemon/logger/alilogs/logapi.go @@ -0,0 +1,63 @@ +// Pakcage alilogs api interface + +package alilogs + +import ( + "errors" + + "github.com/Sirupsen/logrus" + "github.com/galaxydi/go-loghub" +) + +// AliLogAPI define log api interface +type AliLogAPI interface { + PutLogs(*sls.LogGroup) error +} + +// AliLogClient implements AliLogAPI interface +type AliLogClient struct { + Endpoint string + ProjectName string + LogstoreName string + logStore *sls.LogStore +} + +// PutLogs implements ali PutLogs method +func (client *AliLogClient) PutLogs(logGroup *sls.LogGroup) error { + return client.logStore.PutLogs(logGroup) +} + +// NewAliLogClient ... +func NewAliLogClient(endpoint, projectName, logstoreName, accessKeyID, accessKeySecret, securityToken string) (AliLogAPI, error) { + client := AliLogClient{} + client.Endpoint = endpoint + client.ProjectName = projectName + client.LogstoreName = logstoreName + + logProject, err := sls.NewLogProject(projectName, endpoint, accessKeyID, accessKeySecret) + if err != nil { + logrus.WithFields(logrus.Fields{ + "error": err, + }).Error("Could not get ali log project") + return nil, errors.New("Could not get ali log project") + } + if securityToken != "" { + logProject.WithToken(securityToken) + } + + client.logStore, err = logProject.GetLogStore(logstoreName) + if err != nil { + logrus.WithFields(logrus.Fields{ + "error": err, + }).Error("Could not get ali logstore") + return nil, errors.New("Could not get ali logstore") + } + + logrus.WithFields(logrus.Fields{ + "endpoint": endpoint, + "projectName": projectName, + "logstoreName": logstoreName, + }).Info("Created alilogs client") + + return &client, nil +} diff --git a/vendor.conf b/vendor.conf new file mode 100644 index 0000000000000..e50aa273bd445 --- /dev/null +++ b/vendor.conf @@ -0,0 +1,145 @@ +# the following lines are in sorted order, FYI +github.com/Azure/go-ansiterm 388960b655244e76e24c75f48631564eaefade62 +github.com/Microsoft/hcsshim v0.5.9 +github.com/Microsoft/go-winio v0.3.7 +github.com/Sirupsen/logrus f76d643702a30fbffecdfe50831e11881c96ceb3 https://github.com/aaronlehmann/logrus +github.com/davecgh/go-spew 6d212800a42e8ab5c146b8ace3490ee17e5225f9 +github.com/docker/libtrust 9cbd2a1374f46905c68a4eb3694a130610adc62a +github.com/go-check/check 4ed411733c5785b40214c70bce814c3a3a689609 https://github.com/cpuguy83/check.git +github.com/gorilla/context v1.1 +github.com/gorilla/mux v1.1 +github.com/kr/pty 5cf931ef8f +github.com/mattn/go-shellwords v1.0.0 +github.com/mattn/go-sqlite3 v1.1.0 +github.com/tchap/go-patricia v2.2.6 +github.com/vdemeester/shakers 24d7f1d6a71aa5d9cbe7390e4afb66b7eef9e1b3 +# forked golang.org/x/net package includes a patch for lazy loading trace templates +golang.org/x/net 2beffdc2e92c8a3027590f898fe88f69af48a3f8 https://github.com/tonistiigi/net.git +golang.org/x/sys 8f0908ab3b2457e2e15403d3697c9ef5cb4b57a9 +github.com/docker/go-units 8a7beacffa3009a9ac66bad506b18ffdd110cf97 +github.com/docker/go-connections 4ccf312bf1d35e5dbda654e57a9be4c3f3cd0366 + +github.com/RackSec/srslog 456df3a81436d29ba874f3590eeeee25d666f8a5 +github.com/imdario/mergo 0.2.1 + +#get libnetwork packages +github.com/docker/libnetwork b908488a139e81cb8c4091cd836745aeb4d813a4 +github.com/docker/go-events 18b43f1bc85d9cdd42c05a6cd2d444c7a200a894 +github.com/armon/go-radix e39d623f12e8e41c7b5529e9a9dd67a1e2261f80 +github.com/armon/go-metrics eb0af217e5e9747e41dd5303755356b62d28e3ec +github.com/hashicorp/go-msgpack 71c2886f5a673a35f909803f38ece5810165097b +github.com/hashicorp/memberlist 88ac4de0d1a0ca6def284b571342db3b777a4c37 +github.com/hashicorp/go-multierror fcdddc395df1ddf4247c69bd436e84cfa0733f7e +github.com/hashicorp/serf 598c54895cc5a7b1a24a398d635e8c0ea0959870 +github.com/docker/libkv 1d8431073ae03cdaedb198a89722f3aab6d418ef +github.com/vishvananda/netns 604eaf189ee867d8c147fafc28def2394e878d25 +github.com/vishvananda/netlink 482f7a52b758233521878cb6c5904b6bd63f3457 +github.com/BurntSushi/toml f706d00e3de6abe700c994cdd545a1a4915af060 +github.com/samuel/go-zookeeper d0e0d8e11f318e000a8cc434616d69e329edc374 +github.com/deckarep/golang-set ef32fa3046d9f249d399f98ebaf9be944430fd1d +github.com/coreos/etcd 3a49cbb769ebd8d1dd25abb1e83386e9883a5707 +github.com/ugorji/go f1f1a805ed361a0e078bb537e4ea78cd37dcf065 +github.com/hashicorp/consul v0.5.2 +github.com/boltdb/bolt fff57c100f4dea1905678da7e90d92429dff2904 +github.com/miekg/dns 75e6e86cc601825c5dbcd4e0c209eab180997cd7 + +# get graph and distribution packages +github.com/docker/distribution 28602af35aceda2f8d571bad7ca37a54cf0250bc +github.com/vbatts/tar-split v0.10.1 + +# get go-zfs packages +github.com/mistifyio/go-zfs 22c9b32c84eb0d0c6f4043b6e90fc94073de92fa +github.com/pborman/uuid v1.0 + +# get desired notary commit, might also need to be updated in Dockerfile +github.com/docker/notary v0.4.2 + +google.golang.org/grpc v1.0.2 +github.com/miekg/pkcs11 df8ae6ca730422dba20c768ff38ef7d79077a59f +github.com/docker/go v1.5.1-1-1-gbaf439e +github.com/agl/ed25519 d2b94fd789ea21d12fac1a4443dd3a3f79cda72c + +github.com/opencontainers/runc 51371867a01c467f08af739783b8beafc15 # libcontainer +github.com/opencontainers/runtime-spec 1c7c27d043c2a5e513a44084d2b10d77d1402b8c # specs +github.com/seccomp/libseccomp-golang 32f571b70023028bd57d9288c20efbcb237f3ce0 +# libcontainer deps (see src/github.com/opencontainers/runc/Godeps/Godeps.json) +github.com/coreos/go-systemd v4 +github.com/godbus/dbus v4.0.0 +github.com/syndtr/gocapability 2c00daeb6c3b45114c80ac44119e7b8801fdd852 +github.com/golang/protobuf 1f49d83d9aa00e6ce4fc8258c71cc7786aec968a + +# gelf logging driver deps +github.com/Graylog2/go-gelf aab2f594e4585d43468ac57287b0dece9d806883 + +github.com/fluent/fluent-logger-golang v1.2.1 +# fluent-logger-golang deps +github.com/philhofer/fwd 899e4efba8eaa1fea74175308f3fae18ff3319fa +github.com/tinylib/msgp 75ee40d2601edf122ef667e2a07d600d4c44490c + +# fsnotify +github.com/fsnotify/fsnotify v1.2.11 + +# alilogs deps +github.com/galaxydi/go-loghub d52b6d91786547a03aadec3eabb867189fd96df3 +github.com/cloudflare/golz4 ef862a3cdc58a6f1fee4e3af3d44fbe279194cde +github.com/golang/glog 23def4e6c14b4da8ac2ed8007337bc5eb5007998 + +# awslogs deps +github.com/aws/aws-sdk-go v1.4.22 +github.com/go-ini/ini 060d7da055ba6ec5ea7a31f116332fe5efa04ce0 +github.com/jmespath/go-jmespath 0b12d6b521d83fc7f755e7cfc1b1fbdd35a01a74 + +# logentries +github.com/bsphere/le_go d3308aafe090956bc89a65f0769f58251a1b4f03 + +# gcplogs deps +golang.org/x/oauth2 2baa8a1b9338cf13d9eeb27696d761155fa480be +google.golang.org/api dc6d2353af16e2a2b0ff6986af051d473a4ed468 +google.golang.org/cloud dae7e3d993bc3812a2185af60552bb6b847e52a0 + +# native credentials +github.com/docker/docker-credential-helpers f72c04f1d8e71959a6d103f808c50ccbad79b9fd + +# containerd +github.com/docker/containerd 03e5862ec0d8d3b3f750e19fca3ee367e13c090e +github.com/tonistiigi/fifo 1405643975692217d6720f8b54aeee1bf2cd5cf4 + +# cluster +github.com/docker/swarmkit 9e4bd71a1690cd27400714fcd98c329b752b5c4c +github.com/golang/mock bd3c8e81be01eef76d4b503f5e687d2d1354d2d9 +github.com/gogo/protobuf v0.3 +github.com/cloudflare/cfssl 7fb22c8cba7ecaf98e4082d22d65800cf45e042a +github.com/google/certificate-transparency d90e65c3a07988180c5b1ece71791c0b6506826e +golang.org/x/crypto 3fbbcd23f1cb824e69491a5930cfeff09b12f4d2 +golang.org/x/time a4bde12657593d5e90d0533a3e4fd95e635124cb +github.com/mreiferson/go-httpclient 63fe23f7434723dc904c901043af07931f293c47 +github.com/hashicorp/go-memdb 608dda3b1410a73eaf3ac8b517c9ae7ebab6aa87 https://github.com/floridoo/go-memdb +github.com/hashicorp/go-immutable-radix 8e8ed81f8f0bf1bdd829593fdd5c29922c1ea990 +github.com/hashicorp/golang-lru a0d98a5f288019575c6d1f4bb1573fef2d1fcdc4 +github.com/coreos/pkg fa29b1d70f0beaddd4c7021607cc3c3be8ce94b8 +github.com/pivotal-golang/clock 3fd3c1944c59d9742e1cd333672181cd1a6f9fa0 +github.com/prometheus/client_golang 52437c81da6b127a9925d17eb3a382a2e5fd395e +github.com/beorn7/perks 4c0e84591b9aa9e6dcfdf3e020114cd81f89d5f9 +github.com/prometheus/client_model fa8ad6fec33561be4280a8f0514318c79d7f6cb6 +github.com/prometheus/common ebdfc6da46522d58825777cf1f90490a5b1ef1d8 +github.com/prometheus/procfs abf152e5f3e97f2fafac028d2cc06c1feb87ffa5 +bitbucket.org/ww/goautoneg 75cd24fc2f2c2a2088577d12123ddee5f54e0675 +github.com/matttproud/golang_protobuf_extensions fc2b8d3a73c4867e51861bbdd5ae3c1f0869dd6a +github.com/pkg/errors 839d9e913e063e28dfd0e6c7b7512793e0a48be9 + +# cli +github.com/spf13/cobra v1.5 https://github.com/dnephin/cobra.git +github.com/spf13/pflag dabebe21bf790f782ea4c7bbd2efc430de182afd +github.com/inconshreveable/mousetrap 76626ae9c91c4f2a10f34cad8ce83ea42c93bb75 +github.com/flynn-archive/go-shlex 3f9db97f856818214da2e1057f8ad84803971cff + +# metrics +github.com/docker/go-metrics 86138d05f285fd9737a99bee2d9be30866b59d72 + +# composefile +github.com/aanand/compose-file a3e58764f50597b6217fec07e9bff7225c4a1719 +github.com/mitchellh/mapstructure f3009df150dadf309fdee4a54ed65c124afad715 +github.com/xeipuuv/gojsonpointer e0fe6f68307607d540ed8eac07a342c33fa1b54a +github.com/xeipuuv/gojsonreference e02fc20de94c78484cd5ffb007f8af96be030a45 +github.com/xeipuuv/gojsonschema 93e72a773fade158921402d6a24c819b48aba29d +gopkg.in/yaml.v2 a83829b6f1293c91addabc89d0571c246397bbf4