Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Upgrade AWS libraries #1118

Merged
merged 2 commits into from Jun 24, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
5 changes: 3 additions & 2 deletions go.mod
Expand Up @@ -7,8 +7,9 @@ require (
github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5
github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a
github.com/aws/aws-lambda-go v1.13.3
github.com/aws/aws-sdk-go v1.27.0
github.com/aws/aws-sdk-go-v2 v0.18.0
github.com/aws/aws-sdk-go v1.38.65
github.com/aws/aws-sdk-go-v2 v1.6.0
github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.4.1
github.com/casbin/casbin/v2 v2.1.2
github.com/cenkalti/backoff v2.2.1+incompatible // indirect
github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec // indirect
Expand Down
21 changes: 13 additions & 8 deletions go.sum
Expand Up @@ -23,10 +23,14 @@ github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a h1:pv34s756C4pEXnjg
github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A=
github.com/aws/aws-lambda-go v1.13.3 h1:SuCy7H3NLyp+1Mrfp+m80jcbi9KYWAs9/BXwppwRDzY=
github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU=
github.com/aws/aws-sdk-go v1.27.0 h1:0xphMHGMLBrPMfxR2AmVjZKcMEESEgWF8Kru94BNByk=
github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
github.com/aws/aws-sdk-go-v2 v0.18.0 h1:qZ+woO4SamnH/eEbjM2IDLhRNwIwND/RQyVlBLp3Jqg=
github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g=
github.com/aws/aws-sdk-go v1.38.65 h1:umGu5gjIOKxzhi34T0DIA1TWupUDjV2aAW5vK6154Gg=
github.com/aws/aws-sdk-go v1.38.65/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
github.com/aws/aws-sdk-go-v2 v1.6.0 h1:r20hdhm8wZmKkClREfacXrKfX0Y7/s0aOoeraFbf/sY=
github.com/aws/aws-sdk-go-v2 v1.6.0/go.mod h1:tI4KhsR5VkzlUa2DZAdwx7wCAYGwkZZ1H31PYrBFx1w=
github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.4.1 h1:Mt2+LnGKQQyncULtRrx+oJkIwnrfy5XKb96Rvsml30U=
github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.4.1/go.mod h1:dWC4cWmhO3NI8sDqWSPU587mlTLhxCFdS/o3ym/QgMU=
github.com/aws/smithy-go v1.4.0 h1:3rsQpgRe+OoQgJhEwGNpIkosl0fJLdmQqF4gSFRjg+4=
github.com/aws/smithy-go v1.4.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
Expand Down Expand Up @@ -83,7 +87,6 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9
github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
github.com/go-logfmt/logfmt v0.5.0 h1:TrB8swr/68K7m9CcGut2g3UOihhbcbiMAYiuTXdEih4=
github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
Expand Down Expand Up @@ -171,8 +174,10 @@ github.com/hudl/fargo v1.3.0 h1:0U6+BtN6LhaYuTnIJq4Wyq5cpn6O2kWrxAtcqBmYY6w=
github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg=
github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d h1:/WZQPMZNsjZ7IlCpsLGdQBINg5bxKQ1K1sh6awxLtkA=
github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo=
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM=
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
Expand Down Expand Up @@ -364,6 +369,7 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL
golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 h1:4nGaVu0QrbjT/AK2PRLuQfQuh6DJve+pELhqTdAj3x0=
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
Expand Down Expand Up @@ -429,7 +435,6 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
Expand Down
34 changes: 19 additions & 15 deletions metrics/cloudwatch2/cloudwatch2.go
Expand Up @@ -10,7 +10,7 @@ import (

"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/cloudwatch"
"github.com/aws/aws-sdk-go-v2/service/cloudwatch/cloudwatchiface"
"github.com/aws/aws-sdk-go-v2/service/cloudwatch/types"
"golang.org/x/sync/errgroup"

"github.com/go-kit/kit/log"
Expand All @@ -23,6 +23,11 @@ const (
maxConcurrentRequests = 20
)

// CloudWatchAPI is an interface that defines the set of Amazon CloudWatch API operations required by CloudWatch.
type CloudWatchAPI interface {
PutMetricData(ctx context.Context, params *cloudwatch.PutMetricDataInput, optFns ...func(*cloudwatch.Options)) (*cloudwatch.PutMetricDataOutput, error)
}

// CloudWatch receives metrics observations and forwards them to CloudWatch.
// Create a CloudWatch object, use it to create metrics, and pass those metrics as
// dependencies to the components that will use them.
Expand All @@ -32,7 +37,7 @@ type CloudWatch struct {
mtx sync.RWMutex
sem chan struct{}
namespace string
svc cloudwatchiface.ClientAPI
svc CloudWatchAPI
counters *lv.Space
logger log.Logger
numConcurrentRequests int
Expand Down Expand Up @@ -66,7 +71,7 @@ func WithConcurrentRequests(n int) Option {
// Namespace is applied to all created metrics and maps to the CloudWatch namespace.
// Callers must ensure that regular calls to Send are performed, either
// manually or with one of the helper methods.
func New(namespace string, svc cloudwatchiface.ClientAPI, options ...Option) *CloudWatch {
func New(namespace string, svc CloudWatchAPI, options ...Option) *CloudWatch {
cw := &CloudWatch{
namespace: namespace,
svc: svc,
Expand Down Expand Up @@ -131,10 +136,10 @@ func (cw *CloudWatch) Send() error {
defer cw.mtx.RUnlock()
now := time.Now()

var datums []cloudwatch.MetricDatum
var datums []types.MetricDatum

cw.counters.Reset().Walk(func(name string, lvs lv.LabelValues, values []float64) bool {
datums = append(datums, cloudwatch.MetricDatum{
datums = append(datums, types.MetricDatum{
MetricName: aws.String(name),
Dimensions: makeDimensions(lvs...),
StatisticValues: stats(values),
Expand All @@ -143,9 +148,9 @@ func (cw *CloudWatch) Send() error {
return true
})

var batches [][]cloudwatch.MetricDatum
var batches [][]types.MetricDatum
for len(datums) > 0 {
var batch []cloudwatch.MetricDatum
var batch []types.MetricDatum
lim := len(datums)
if lim > maxConcurrentRequests {
lim = maxConcurrentRequests
Expand All @@ -162,11 +167,10 @@ func (cw *CloudWatch) Send() error {
defer func() {
<-cw.sem
}()
req := cw.svc.PutMetricDataRequest(&cloudwatch.PutMetricDataInput{
_, err := cw.svc.PutMetricData(context.TODO(), &cloudwatch.PutMetricDataInput{
Namespace: aws.String(cw.namespace),
MetricData: batch,
})
_, err := req.Send(context.TODO())
return err
})
}
Expand All @@ -177,14 +181,14 @@ var zero = float64(0.0)

// Just build this once to reduce construction costs whenever
// someone does a Send with no aggregated values.
var zeros = cloudwatch.StatisticSet{
var zeros = types.StatisticSet{
Maximum: &zero,
Minimum: &zero,
Sum: &zero,
SampleCount: &zero,
}

func stats(a []float64) *cloudwatch.StatisticSet {
func stats(a []float64) *types.StatisticSet {
count := float64(len(a))
if count == 0 {
return &zeros
Expand All @@ -203,18 +207,18 @@ func stats(a []float64) *cloudwatch.StatisticSet {
}
}

return &cloudwatch.StatisticSet{
return &types.StatisticSet{
Maximum: &max,
Minimum: &min,
Sum: &sum,
SampleCount: &count,
}
}

func makeDimensions(labelValues ...string) []cloudwatch.Dimension {
dimensions := make([]cloudwatch.Dimension, len(labelValues)/2)
func makeDimensions(labelValues ...string) []types.Dimension {
dimensions := make([]types.Dimension, len(labelValues)/2)
for i, j := 0, 0; i < len(labelValues); i, j = i+2, j+1 {
dimensions[j] = cloudwatch.Dimension{
dimensions[j] = types.Dimension{
Name: aws.String(labelValues[i]),
Value: aws.String(labelValues[i+1]),
}
Expand Down
30 changes: 9 additions & 21 deletions metrics/cloudwatch2/cloudwatch2_test.go
@@ -1,13 +1,12 @@
package cloudwatch2

import (
"net/http"
"context"
"strings"
"testing"

"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/cloudwatch"
"github.com/aws/aws-sdk-go-v2/service/cloudwatch/cloudwatchiface"
"github.com/aws/aws-sdk-go-v2/service/cloudwatch/types"
)

func TestStats(t *testing.T) {
Expand Down Expand Up @@ -73,27 +72,16 @@ func TestStats(t *testing.T) {
}

type mockCloudWatch struct {
cloudwatchiface.ClientAPI
CloudWatchAPI
latestName string
latestData []cloudwatch.MetricDatum
latestData []types.MetricDatum
}

func (mcw *mockCloudWatch) PutMetricDataRequest(in *cloudwatch.PutMetricDataInput) cloudwatch.PutMetricDataRequest {
mcw.latestName = *in.Namespace
mcw.latestData = in.MetricData
return cloudwatch.PutMetricDataRequest{
// To mock the V2 API, most of the functions spit
// out structs that you need to call Send() on.
// The non-intuitive thing is that to get the Send() to avoid actually
// going across the wire, you just create a dumb aws.Request with either
// aws.Request.Data defined (for succes) or with aws.Request.Error
// to simulate an Error.
Request: &aws.Request{
HTTPRequest: &http.Request{Method: "PUT"},
Data: &cloudwatch.PutMetricDataOutput{},
},
Input: in,
}
func (mcw *mockCloudWatch) PutMetricData(ctx context.Context, params *cloudwatch.PutMetricDataInput, optFns ...func(*cloudwatch.Options)) (*cloudwatch.PutMetricDataOutput, error) {
mcw.latestName = *params.Namespace
mcw.latestData = params.MetricData

return nil, nil
}

func TestSend(t *testing.T) {
Expand Down