diff --git a/.github/workflows/codegen.yml b/.github/workflows/codegen.yml index 2c7b7b08c41..9f15e4927c3 100644 --- a/.github/workflows/codegen.yml +++ b/.github/workflows/codegen.yml @@ -13,7 +13,7 @@ jobs: strategy: matrix: os: [ubuntu-latest] - go-version: [1.17] + go-version: [1.18] env: JAVA_TOOL_OPTIONS: "-Xmx2g" steps: diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index e2d41fccbed..733fdcbd6a5 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -16,7 +16,7 @@ jobs: strategy: matrix: os: [ubuntu-latest, macos-latest] - go-version: [1.17, 1.16, 1.15] + go-version: [1.18, 1.17, 1.16, 1.15] steps: - uses: actions/checkout@v2 @@ -38,7 +38,7 @@ jobs: strategy: matrix: os: [windows-latest] - go-version: [1.17, 1.16, 1.15] + go-version: [1.18, 1.17, 1.16, 1.15] env: EACHMODULE_SKIP: "internal\\repotools\\changes" steps: diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml index 4e311b2b0b5..903eaba2d0e 100644 --- a/.github/workflows/golangci-lint.yml +++ b/.github/workflows/golangci-lint.yml @@ -11,7 +11,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - go-version: [1.17.x] + go-version: [1.18.x] os: [ubuntu-latest] # other options: macos-latest, windows-latest steps: - uses: actions/checkout@v2 diff --git a/internal/awstesting/certificate_utils.go b/internal/awstesting/certificate_utils.go new file mode 100644 index 00000000000..8cabf794c41 --- /dev/null +++ b/internal/awstesting/certificate_utils.go @@ -0,0 +1,291 @@ +package awstesting + +import ( + "bytes" + "crypto/rand" + "crypto/rsa" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "fmt" + "io/ioutil" + "math/big" + "net" + "net/http" + "net/http/httptest" + "os" + "strings" + "time" +) + +var ( + // TLSBundleCA is the CA PEM + TLSBundleCA []byte + + // TLSBundleCert is the Server PEM + TLSBundleCert []byte + + // TLSBundleKey is the Server private key PEM + TLSBundleKey []byte + + // ClientTLSCert is the Client PEM + ClientTLSCert []byte + + // ClientTLSKey is the Client private key PEM + ClientTLSKey []byte +) + +func init() { + caPEM, _, caCert, caPrivKey, err := generateRootCA() + if err != nil { + panic("failed to generate testing root CA, " + err.Error()) + } + TLSBundleCA = caPEM + + serverCertPEM, serverCertPrivKeyPEM, err := generateLocalCert(caCert, caPrivKey) + if err != nil { + panic("failed to generate testing server cert, " + err.Error()) + } + TLSBundleCert = serverCertPEM + TLSBundleKey = serverCertPrivKeyPEM + + clientCertPEM, clientCertPrivKeyPEM, err := generateLocalCert(caCert, caPrivKey) + if err != nil { + panic("failed to generate testing client cert, " + err.Error()) + } + ClientTLSCert = clientCertPEM + ClientTLSKey = clientCertPrivKeyPEM +} + +func generateRootCA() ( + caPEM, caPrivKeyPEM []byte, caCert *x509.Certificate, caPrivKey *rsa.PrivateKey, err error, +) { + caCert = &x509.Certificate{ + SerialNumber: big.NewInt(42), + Subject: pkix.Name{ + Country: []string{"US"}, + Organization: []string{"AWS SDK for Go Test Certificate"}, + CommonName: "Test Root CA", + }, + NotBefore: time.Now().Add(-time.Minute), + NotAfter: time.Now().AddDate(1, 0, 0), + KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageCRLSign | x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{ + x509.ExtKeyUsageClientAuth, + x509.ExtKeyUsageServerAuth, + }, + BasicConstraintsValid: true, + IsCA: true, + } + + // Create CA private and public key + caPrivKey, err = rsa.GenerateKey(rand.Reader, 4096) + if err != nil { + return nil, nil, nil, nil, fmt.Errorf("failed generate CA RSA key, %w", err) + } + + // Create CA certificate + caBytes, err := x509.CreateCertificate(rand.Reader, caCert, caCert, &caPrivKey.PublicKey, caPrivKey) + if err != nil { + return nil, nil, nil, nil, fmt.Errorf("failed generate CA certificate, %w", err) + } + + // PEM encode CA certificate and private key + var caPEMBuf bytes.Buffer + pem.Encode(&caPEMBuf, &pem.Block{ + Type: "CERTIFICATE", + Bytes: caBytes, + }) + + var caPrivKeyPEMBuf bytes.Buffer + pem.Encode(&caPrivKeyPEMBuf, &pem.Block{ + Type: "RSA PRIVATE KEY", + Bytes: x509.MarshalPKCS1PrivateKey(caPrivKey), + }) + + return caPEMBuf.Bytes(), caPrivKeyPEMBuf.Bytes(), caCert, caPrivKey, nil +} + +func generateLocalCert(parentCert *x509.Certificate, parentPrivKey *rsa.PrivateKey) ( + certPEM, certPrivKeyPEM []byte, err error, +) { + cert := &x509.Certificate{ + SerialNumber: big.NewInt(42), + Subject: pkix.Name{ + Country: []string{"US"}, + Organization: []string{"AWS SDK for Go Test Certificate"}, + CommonName: "Test Root CA", + }, + IPAddresses: []net.IP{ + net.IPv4(127, 0, 0, 1), + net.IPv6loopback, + }, + NotBefore: time.Now().Add(-time.Minute), + NotAfter: time.Now().AddDate(1, 0, 0), + ExtKeyUsage: []x509.ExtKeyUsage{ + x509.ExtKeyUsageClientAuth, + x509.ExtKeyUsageServerAuth, + }, + KeyUsage: x509.KeyUsageDigitalSignature, + } + + // Create server private and public key + certPrivKey, err := rsa.GenerateKey(rand.Reader, 4096) + if err != nil { + return nil, nil, fmt.Errorf("failed to generate server RSA private key, %w", err) + } + + // Create server certificate + certBytes, err := x509.CreateCertificate(rand.Reader, cert, parentCert, &certPrivKey.PublicKey, parentPrivKey) + if err != nil { + return nil, nil, fmt.Errorf("failed to generate server certificate, %w", err) + } + + // PEM encode certificate and private key + var certPEMBuf bytes.Buffer + pem.Encode(&certPEMBuf, &pem.Block{ + Type: "CERTIFICATE", + Bytes: certBytes, + }) + + var certPrivKeyPEMBuf bytes.Buffer + pem.Encode(&certPrivKeyPEMBuf, &pem.Block{ + Type: "RSA PRIVATE KEY", + Bytes: x509.MarshalPKCS1PrivateKey(certPrivKey), + }) + + return certPEMBuf.Bytes(), certPrivKeyPEMBuf.Bytes(), nil +} + +// NewTLSClientCertServer creates a new HTTP test server initialize to require +// HTTP clients authenticate with TLS client certificates. +func NewTLSClientCertServer(handler http.Handler) (*httptest.Server, error) { + server := httptest.NewUnstartedServer(handler) + + if server.TLS == nil { + server.TLS = &tls.Config{} + } + server.TLS.ClientAuth = tls.RequireAndVerifyClientCert + + if server.TLS.ClientCAs == nil { + server.TLS.ClientCAs = x509.NewCertPool() + } + certPem := append(ClientTLSCert, ClientTLSKey...) + if ok := server.TLS.ClientCAs.AppendCertsFromPEM(certPem); !ok { + return nil, fmt.Errorf("failed to append client certs") + } + + return server, nil +} + +// CreateClientTLSCertFiles returns a set of temporary files for the client +// certificate and key files. +func CreateClientTLSCertFiles() (cert, key string, err error) { + cert, err = createTmpFile(ClientTLSCert) + if err != nil { + return "", "", err + } + + key, err = createTmpFile(ClientTLSKey) + if err != nil { + return "", "", err + } + + return cert, key, nil +} + +func availableLocalAddr(ip string) (v string, err error) { + l, err := net.Listen("tcp", ip+":0") + if err != nil { + return "", err + } + defer func() { + closeErr := l.Close() + if err == nil { + err = closeErr + } else if closeErr != nil { + err = fmt.Errorf("ip listener close error: %v, original error: %w", closeErr, err) + } + }() + + return l.Addr().String(), nil +} + +// CreateTLSServer will create the TLS server on an open port using the +// certificate and key. The address will be returned that the server is running on. +func CreateTLSServer(cert, key string, mux *http.ServeMux) (string, error) { + addr, err := availableLocalAddr("127.0.0.1") + if err != nil { + return "", err + } + + if mux == nil { + mux = http.NewServeMux() + mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {}) + } + + go func() { + if err := http.ListenAndServeTLS(addr, cert, key, mux); err != nil { + panic(err) + } + }() + + for i := 0; i < 60; i++ { + if _, err := http.Get("https://" + addr); err != nil && !strings.Contains(err.Error(), "connection refused") { + break + } + + time.Sleep(1 * time.Second) + } + + return "https://" + addr, nil +} + +// CreateTLSBundleFiles returns the temporary filenames for the certificate +// key, and CA PEM content. These files should be deleted when no longer +// needed. CleanupTLSBundleFiles can be used for this cleanup. +func CreateTLSBundleFiles() (cert, key, ca string, err error) { + cert, err = createTmpFile(TLSBundleCert) + if err != nil { + return "", "", "", err + } + + key, err = createTmpFile(TLSBundleKey) + if err != nil { + return "", "", "", err + } + + ca, err = createTmpFile(TLSBundleCA) + if err != nil { + return "", "", "", err + } + + return cert, key, ca, nil +} + +// CleanupTLSBundleFiles takes variadic list of files to be deleted. +func CleanupTLSBundleFiles(files ...string) error { + for _, file := range files { + if err := os.Remove(file); err != nil { + return err + } + } + + return nil +} + +func createTmpFile(b []byte) (string, error) { + bundleFile, err := ioutil.TempFile(os.TempDir(), "aws-sdk-go-session-test") + if err != nil { + return "", err + } + + _, err = bundleFile.Write(b) + if err != nil { + return "", err + } + + defer bundleFile.Close() + return bundleFile.Name(), nil +} diff --git a/internal/awstesting/custom_ca_bundle.go b/internal/awstesting/custom_ca_bundle.go deleted file mode 100644 index 47754ef7e82..00000000000 --- a/internal/awstesting/custom_ca_bundle.go +++ /dev/null @@ -1,215 +0,0 @@ -package awstesting - -import ( - "fmt" - "io/ioutil" - "net" - "net/http" - "os" - "strings" - "time" -) - -func availableLocalAddr(ip string) (v string, err error) { - l, err := net.Listen("tcp", ip+":0") - if err != nil { - return "", err - } - defer func() { - closeErr := l.Close() - if err == nil { - err = closeErr - } else if closeErr != nil { - err = fmt.Errorf("ip listener close error: %v, original error: %w", closeErr, err) - } - }() - - return l.Addr().String(), nil -} - -// CreateTLSServer will create the TLS server on an open port using the -// certificate and key. The address will be returned that the server is running on. -func CreateTLSServer(cert, key string, mux *http.ServeMux) (string, error) { - addr, err := availableLocalAddr("127.0.0.1") - if err != nil { - return "", err - } - - if mux == nil { - mux = http.NewServeMux() - mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {}) - } - - go func() { - if err := http.ListenAndServeTLS(addr, cert, key, mux); err != nil { - panic(err) - } - }() - - for i := 0; i < 60; i++ { - if _, err := http.Get("https://" + addr); err != nil && !strings.Contains(err.Error(), "connection refused") { - break - } - - time.Sleep(1 * time.Second) - } - - return "https://" + addr, nil -} - -// CreateTLSBundleFiles returns the temporary filenames for the certificate -// key, and CA PEM content. These files should be deleted when no longer -// needed. CleanupTLSBundleFiles can be used for this cleanup. -func CreateTLSBundleFiles() (cert, key, ca string, err error) { - cert, err = createTmpFile(TLSBundleCert) - if err != nil { - return "", "", "", err - } - - key, err = createTmpFile(TLSBundleKey) - if err != nil { - return "", "", "", err - } - - ca, err = createTmpFile(TLSBundleCA) - if err != nil { - return "", "", "", err - } - - return cert, key, ca, nil -} - -// CleanupTLSBundleFiles takes variadic list of files to be deleted. -func CleanupTLSBundleFiles(files ...string) error { - for _, file := range files { - if err := os.Remove(file); err != nil { - return err - } - } - - return nil -} - -func createTmpFile(b []byte) (v string, err error) { - bundleFile, err := ioutil.TempFile(os.TempDir(), "aws-sdk-go-session-test") - if err != nil { - return "", err - } - - _, err = bundleFile.Write(b) - if err != nil { - return "", err - } - - defer func() { - closeErr := bundleFile.Close() - if err == nil { - err = closeErr - } else if closeErr != nil { - err = fmt.Errorf("file close error: %v, original error: %w", closeErr, err) - } - }() - - return bundleFile.Name(), nil -} - -/* Cert generation steps -# Create the CA key -openssl genrsa -des3 -out ca.key 1024 - -# Create the CA Cert -openssl req -new -sha256 -x509 -days 3650 \ - -subj "/C=GO/ST=Gopher/O=Testing ROOT CA" \ - -key ca.key -out ca.crt - -# Create config -cat > csr_details.txt <<-EOF - -[req] -default_bits = 1024 -prompt = no -default_md = sha256 -req_extensions = SAN -distinguished_name = dn - -[ dn ] -C=GO -ST=Gopher -O=Testing Certificate -OU=Testing IP - -[SAN] -subjectAltName = IP:127.0.0.1 -EOF - -# Create certificate signing request -openssl req -new -sha256 -nodes -newkey rsa:1024 \ - -config <( cat csr_details.txt ) \ - -keyout ia.key -out ia.csr - -# Create a signed certificate -openssl x509 -req -days 3650 \ - -CAcreateserial \ - -extfile <( cat csr_details.txt ) \ - -extensions SAN \ - -CA ca.crt -CAkey ca.key -in ia.csr -out ia.crt - -# Verify -openssl req -noout -text -in ia.csr -openssl x509 -noout -text -in ia.crt -*/ -var ( - // TLSBundleCA ca.crt - TLSBundleCA = []byte(`-----BEGIN CERTIFICATE----- -MIICiTCCAfKgAwIBAgIJAJ5X1olt05XjMA0GCSqGSIb3DQEBCwUAMDgxCzAJBgNV -BAYTAkdPMQ8wDQYDVQQIEwZHb3BoZXIxGDAWBgNVBAoTD1Rlc3RpbmcgUk9PVCBD -QTAeFw0xNzAzMDkwMDAyMDZaFw0yNzAzMDcwMDAyMDZaMDgxCzAJBgNVBAYTAkdP -MQ8wDQYDVQQIEwZHb3BoZXIxGDAWBgNVBAoTD1Rlc3RpbmcgUk9PVCBDQTCBnzAN -BgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAw/8DN+t9XQR60jx42rsQ2WE2Dx85rb3n -GQxnKZZLNddsT8rDyxJNP18aFalbRbFlyln5fxWxZIblu9Xkm/HRhOpbSimSqo1y -uDx21NVZ1YsOvXpHby71jx3gPrrhSc/t/zikhi++6D/C6m1CiIGuiJ0GBiJxtrub -UBMXT0QtI2ECAwEAAaOBmjCBlzAdBgNVHQ4EFgQU8XG3X/YHBA6T04kdEkq6+4GV -YykwaAYDVR0jBGEwX4AU8XG3X/YHBA6T04kdEkq6+4GVYymhPKQ6MDgxCzAJBgNV -BAYTAkdPMQ8wDQYDVQQIEwZHb3BoZXIxGDAWBgNVBAoTD1Rlc3RpbmcgUk9PVCBD -QYIJAJ5X1olt05XjMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQELBQADgYEAeILv -z49+uxmPcfOZzonuOloRcpdvyjiXblYxbzz6ch8GsE7Q886FTZbvwbgLhzdwSVgG -G8WHkodDUsymVepdqAamS3f8PdCUk8xIk9mop8LgaB9Ns0/TssxDvMr3sOD2Grb3 -xyWymTWMcj6uCiEBKtnUp4rPiefcvCRYZ17/hLE= ------END CERTIFICATE----- -`) - - // TLSBundleCert ai.crt - TLSBundleCert = []byte(`-----BEGIN CERTIFICATE----- -MIICGjCCAYOgAwIBAgIJAIIu+NOoxxM0MA0GCSqGSIb3DQEBBQUAMDgxCzAJBgNV -BAYTAkdPMQ8wDQYDVQQIEwZHb3BoZXIxGDAWBgNVBAoTD1Rlc3RpbmcgUk9PVCBD -QTAeFw0xNzAzMDkwMDAzMTRaFw0yNzAzMDcwMDAzMTRaMFExCzAJBgNVBAYTAkdP -MQ8wDQYDVQQIDAZHb3BoZXIxHDAaBgNVBAoME1Rlc3RpbmcgQ2VydGlmaWNhdGUx -EzARBgNVBAsMClRlc3RpbmcgSVAwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGB -AN1hWHeioo/nASvbrjwCQzXCiWiEzGkw353NxsAB54/NqDL3LXNATtiSJu8kJBrm -Ah12IFLtWLGXjGjjYlHbQWnOR6awveeXnQZukJyRWh7m/Qlt9Ho0CgZE1U+832ac -5GWVldNxW1Lz4I+W9/ehzqe8I80RS6eLEKfUFXGiW+9RAgMBAAGjEzARMA8GA1Ud -EQQIMAaHBH8AAAEwDQYJKoZIhvcNAQEFBQADgYEAdF4WQHfVdPCbgv9sxgJjcR1H -Hgw9rZ47gO1IiIhzglnLXQ6QuemRiHeYFg4kjcYBk1DJguxzDTGnUwhUXOibAB+S -zssmrkdYYvn9aUhjc3XK3tjAoDpsPpeBeTBamuUKDHoH/dNRXxerZ8vu6uPR3Pgs -5v/KCV6IAEcvNyOXMPo= ------END CERTIFICATE----- -`) - - // TLSBundleKey ai.key - TLSBundleKey = []byte(`-----BEGIN RSA PRIVATE KEY----- -MIICXAIBAAKBgQDdYVh3oqKP5wEr2648AkM1wolohMxpMN+dzcbAAeePzagy9y1z -QE7YkibvJCQa5gIddiBS7Vixl4xo42JR20FpzkemsL3nl50GbpCckVoe5v0JbfR6 -NAoGRNVPvN9mnORllZXTcVtS8+CPlvf3oc6nvCPNEUunixCn1BVxolvvUQIDAQAB -AoGBAMISrcirddGrlLZLLrKC1ULS2T0cdkqdQtwHYn4+7S5+/z42vMx1iumHLsSk -rVY7X41OWkX4trFxhvEIrc/O48bo2zw78P7flTxHy14uxXnllU8cLThE29SlUU7j -AVBNxJZMsXMlS/DowwD4CjFe+x4Pu9wZcReF2Z9ntzMpySABAkEA+iWoJCPE2JpS -y78q3HYYgpNY3gF3JqQ0SI/zTNkb3YyEIUffEYq0Y9pK13HjKtdsSuX4osTIhQkS -+UgRp6tCAQJBAOKPYTfQ2FX8ijgUpHZRuEAVaxASAS0UATiLgzXxLvOh/VC2at5x -wjOX6sD65pPz/0D8Qj52Cq6Q1TQ+377SDVECQAIy0od+yPweXxvrUjUd1JlRMjbB -TIrKZqs8mKbUQapw0bh5KTy+O1elU4MRPS3jNtBxtP25PQnuSnxmZcFTgAECQFzg -DiiFcsn9FuRagfkHExMiNJuH5feGxeFaP9WzI144v9GAllrOI6Bm3JNzx2ZLlg4b -20Qju8lIEj6yr6JYFaECQHM1VSojGRKpOl9Ox/R4yYSA9RV5Gyn00/aJNxVYyPD5 -i3acL2joQm2kLD/LO8paJ4+iQdRXCOMMIpjxSNjGQjQ= ------END RSA PRIVATE KEY----- -`) -) diff --git a/internal/awstesting/sandbox/Dockerfile.test.go1.18 b/internal/awstesting/sandbox/Dockerfile.test.go1.18 new file mode 100644 index 00000000000..dfe769bd60a --- /dev/null +++ b/internal/awstesting/sandbox/Dockerfile.test.go1.18 @@ -0,0 +1,12 @@ +FROM golang:1.18 + +ENV GOPROXY=direct + +ADD . /go/src/github.com/aws/aws-sdk-go-v2 + +RUN apt-get update && apt-get install -y --no-install-recommends \ + vim \ + && rm -rf /var/list/apt/lists/* + +WORKDIR /go/src/github.com/aws/aws-sdk-go-v2 +CMD ["make", "unit"]