Skip to content

Commit

Permalink
Merge pull request #836 from aws/s3-accel-dualstack
Browse files Browse the repository at this point in the history
  • Loading branch information
skotambkar committed Oct 22, 2020
2 parents 827ee57 + a54eea7 commit 4e3fc08
Show file tree
Hide file tree
Showing 46 changed files with 652 additions and 127 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,6 @@ protected static GoDependency module(
}

private static final class Versions {
private static final String AWS_SDK = "v0.27.1-0.20201021211102-ffda7ace423e";
private static final String AWS_SDK = "v0.27.1-0.20201022222834-4451b4af620e";
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ public final class AwsCustomGoDependency extends AwsGoDependency {
public static final GoDependency DYNAMODB_CUSTOMIZATION = aws(
"service/dynamodb/internal/customizations", "ddbcust");
public static final GoDependency S3_CUSTOMIZATION = aws("service/s3/internal/customizations", "s3cust");
public static final GoDependency S3CONTROL_CUSTOMIZATION = aws("service/s3control/internal/customizations", "s3controlcust");
public static final GoDependency APIGATEWAY_CUSTOMIZATION = aws(
"service/apigateway/internal/customizations", "agcust");
public static final GoDependency GLACIER_CUSTOMIZATION = aws(
Expand All @@ -47,7 +48,7 @@ private AwsCustomGoDependency() {
}

private static final class Versions {
private static final String INTERNAL_S3SHARED = "v0.2.1-0.20201019214249-1049b73d5c17";
private static final String INTERNAL_S3SHARED = "v0.2.1-0.20201022222834-4451b4af620e";
private static final String INTERNAL_ACCEPTENCODING = "v0.0.0-20200930084954-897dfb99530c";
private static final String INTERNAL_PRESIGNURL = "v0.0.0-20201020212433-5fb7a9ec04bb";
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,13 +18,15 @@
package software.amazon.smithy.aws.go.codegen.customization;

import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import software.amazon.smithy.aws.traits.ServiceTrait;
import software.amazon.smithy.codegen.core.CodegenException;
import software.amazon.smithy.codegen.core.SymbolProvider;
import software.amazon.smithy.go.codegen.GoDelegator;
import software.amazon.smithy.go.codegen.GoSettings;
import software.amazon.smithy.go.codegen.GoWriter;
import software.amazon.smithy.go.codegen.SmithyGoDependency;
import software.amazon.smithy.go.codegen.SymbolUtils;
import software.amazon.smithy.go.codegen.integration.ConfigField;
import software.amazon.smithy.go.codegen.integration.GoIntegration;
Expand All @@ -34,18 +36,32 @@
import software.amazon.smithy.model.shapes.MemberShape;
import software.amazon.smithy.model.shapes.OperationShape;
import software.amazon.smithy.model.shapes.ServiceShape;
import software.amazon.smithy.model.shapes.ShapeId;
import software.amazon.smithy.model.shapes.StructureShape;
import software.amazon.smithy.utils.ListUtils;
import software.amazon.smithy.utils.SetUtils;

/**
* S3UpdateEndpoint integration serves to apply customizations for S3 service,
* and modifies the resolved endpoint based on S3 client config or input shape values.
*/
public class S3UpdateEndpoint implements GoIntegration {
// options to be generated on Client's options type
private static final String USE_PATH_STYLE_OPTION = "UsePathStyle";
private static final String USE_ACCELERATE_OPTION = "UseAccelerate";
private static final String USE_DUALSTACK_OPTION = "UseDualstack";

// middleware addition constants
private static final String UPDATE_ENDPOINT_ADDER = "addUpdateEndpointMiddleware";
private static final String UPDATE_ENDPOINT_INTERNAL_ADDER = "UpdateEndpoint";

// private function getter constant
private static final String GET_BUCKET_FROM_INPUT = "getBucketFromInput";
private static final String SUPPORT_ACCELERATE = "supportAccelerate";

// list of operations that do not support accelerate
private static final Set<String> NOT_SUPPORT_ACCELERATE = SetUtils.of(
"ListBuckets", "CreateBucket", "DeleteBucket");

/**
* Gets the sort order of the customization from -128 to 127, with lowest
Expand All @@ -66,17 +82,45 @@ public void writeAdditionalFiles(
GoDelegator goDelegator
) {
ServiceShape service = settings.getService(model);
if (!isS3Service(model, service)) {
return;

// if service is s3control
if (isS3ControlService(model, service)){
goDelegator.useShapeWriter(service, this::writeS3ControlMiddlewareHelper);
}

goDelegator.useShapeWriter(service, this::writeMiddlewareHelper);
// check if service is s3
if (isS3Service(model, service)) {
goDelegator.useShapeWriter(service, this::writeS3MiddlewareHelper);

goDelegator.useShapeWriter(service, writer -> {
writeInputGetter(writer, model, symbolProvider, service);
goDelegator.useShapeWriter(service, writer -> {
writeInputGetter(writer, model, symbolProvider, service);
});

goDelegator.useShapeWriter(service, writer -> {
writeAccelerateValidator(writer, model, symbolProvider, service);
});
}
}

private void writeAccelerateValidator(GoWriter writer, Model model, SymbolProvider symbolProvider, ServiceShape service) {
writer.writeDocs("supportAccelerate returns a boolean indicating if the operation associated with the provided input "
+ "supports S3 Transfer Acceleration");
writer.openBlock("func $L(input interface{}) bool {", "}", SUPPORT_ACCELERATE, () -> {
writer.openBlock("switch input.(type) {" , "}", () -> {
for (ShapeId operationId : service.getAllOperations()) {
// check if operation does not support s3 accelerate
if (NOT_SUPPORT_ACCELERATE.contains(operationId.getName())) {
OperationShape operation = model.expectShape(operationId, OperationShape.class);
StructureShape input = model.expectShape(operation.getInput().get(), StructureShape.class);
writer.write("case $P: return false", symbolProvider.toSymbol(input));
}
}
writer.write("default: return true");
});
});
}


private void writeInputGetter(GoWriter writer, Model model, SymbolProvider symbolProvider, ServiceShape service) {
writer.writeDocs("getBucketFromInput returns a boolean indicating if the input has a modeled bucket name, " +
" and a pointer to string denoting a provided bucket member value");
Expand All @@ -95,7 +139,7 @@ private void writeInputGetter(GoWriter writer, Model model, SymbolProvider symbo
targetBucketShape.size() +" for Input shape: "+ input.getId());
}

if (!targetBucketShape.isEmpty()) {
if (!targetBucketShape.isEmpty() && !operationId.getName().equalsIgnoreCase("GetBucketLocation")) {
writer.write("case $P: return i.$L, true", symbolProvider.toSymbol(input), targetBucketShape.get(0).getMemberName());
}
});
Expand All @@ -104,15 +148,40 @@ private void writeInputGetter(GoWriter writer, Model model, SymbolProvider symbo
});
}

private void writeMiddlewareHelper(GoWriter writer) {
private void writeS3ControlMiddlewareHelper(GoWriter writer) {
// imports
writer.addUseImports(SmithyGoDependency.SMITHY_MIDDLEWARE);
writer.addUseImports(AwsCustomGoDependency.S3CONTROL_CUSTOMIZATION);

writer.openBlock("func $L(stack *middleware.Stack, options Options) {", "}", UPDATE_ENDPOINT_ADDER, () -> {
writer.write("$T(stack, $T{UsePathStyle: options.$L, GetBucketFromInput: $L})",
writer.write("$T(stack, $T{UseDualstack: options.$L})",
SymbolUtils.createValueSymbolBuilder(UPDATE_ENDPOINT_INTERNAL_ADDER,
AwsCustomGoDependency.S3CONTROL_CUSTOMIZATION).build(),
SymbolUtils.createValueSymbolBuilder(UPDATE_ENDPOINT_INTERNAL_ADDER + "Options",
AwsCustomGoDependency.S3CONTROL_CUSTOMIZATION).build(),
USE_DUALSTACK_OPTION);
});
writer.insertTrailingNewline();
}

private void writeS3MiddlewareHelper(GoWriter writer) {
// imports
writer.addUseImports(SmithyGoDependency.SMITHY_MIDDLEWARE);
writer.addUseImports(AwsCustomGoDependency.S3_CUSTOMIZATION);

writer.openBlock("func $L(stack *middleware.Stack, options Options) {", "}", UPDATE_ENDPOINT_ADDER, () -> {
writer.write("$T(stack, $T{ \n"
+ "Region: options.Region,\n GetBucketFromInput: $L,\n UsePathStyle: options.$L,\n "
+ "UseAccelerate: options.$L,\n SupportsAccelerate: $L,\n UseDualstack: options.$L, \n})",
SymbolUtils.createValueSymbolBuilder(UPDATE_ENDPOINT_INTERNAL_ADDER,
AwsCustomGoDependency.S3_CUSTOMIZATION).build(),
SymbolUtils.createValueSymbolBuilder(UPDATE_ENDPOINT_INTERNAL_ADDER + "Options",
AwsCustomGoDependency.S3_CUSTOMIZATION).build(),
GET_BUCKET_FROM_INPUT,
USE_PATH_STYLE_OPTION,
GET_BUCKET_FROM_INPUT
USE_ACCELERATE_OPTION,
SUPPORT_ACCELERATE,
USE_DUALSTACK_OPTION
);
});
writer.insertTrailingNewline();
Expand All @@ -134,17 +203,51 @@ public List<RuntimeClientPlugin> getClientPlugins() {
+ "i.e., `https://s3.amazonaws.com/BUCKET/KEY`. By default, the S3 client "
+ "will use virtual hosted bucket addressing when possible"
+ "(`https://BUCKET.s3.amazonaws.com/KEY`).")
.build(),
ConfigField.builder()
.name(USE_ACCELERATE_OPTION)
.type(SymbolUtils.createValueSymbolBuilder("bool")
.putProperty(SymbolUtils.GO_UNIVERSE_TYPE, true)
.build())
.documentation("Allows you to enable S3 Accelerate feature. All operations "
+ "compatible with S3 Accelerate will use the accelerate endpoint for "
+ "requests. Requests not compatible will fall back to normal S3 requests. "
+ "The bucket must be enabled for accelerate to be used with S3 client with "
+ "accelerate enabled. If the bucket is not enabled for accelerate an error "
+ "will be returned. The bucket name must be DNS compatible to work "
+ "with accelerate.")
.build()
))
.build(),
// Add S3 shared config's dualstack option
RuntimeClientPlugin.builder()
.servicePredicate(S3UpdateEndpoint::isS3SharedService)
.configFields(ListUtils.of(
ConfigField.builder()
.name(USE_DUALSTACK_OPTION)
.type(SymbolUtils.createValueSymbolBuilder("bool")
.putProperty(SymbolUtils.GO_UNIVERSE_TYPE, true)
.build())
.documentation("Allows you to enable Dualstack endpoint support for the service.")
.build()
))
.registerMiddleware(MiddlewareRegistrar.builder()
.resolvedFunction(SymbolUtils.createValueSymbolBuilder(UPDATE_ENDPOINT_ADDER).build())
.useClientOptions()
.build())
.build()
.build()
);
}

private static boolean isS3Service(Model model, ServiceShape service) {
return service.expectTrait(ServiceTrait.class).getSdkId().equalsIgnoreCase("S3");
}

private static boolean isS3ControlService(Model model, ServiceShape service) {
return service.expectTrait(ServiceTrait.class).getSdkId().equalsIgnoreCase("S3 Control");
}

private static boolean isS3SharedService(Model model, ServiceShape service) {
return isS3Service(model,service) || isS3ControlService(model, service);
}
}
54 changes: 54 additions & 0 deletions service/internal/s3shared/update_endpoint.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
package s3shared

import (
"context"
"fmt"
"strings"

"github.com/awslabs/smithy-go/middleware"
"github.com/awslabs/smithy-go/transport/http"
)

// EnableDualstackMiddleware represents middleware struct for enabling dualstack support
type EnableDualstackMiddleware struct {
// UseDualstack indicates if dualstack endpoint resolving is to be enabled
UseDualstack bool

// ServiceID is the service id prefix used in endpoint resolving
// by default service-id is 's3' and 's3-control' for service s3, s3control.
ServiceID string
}

// ID returns the middleware ID.
func (*EnableDualstackMiddleware) ID() string { return "EnableDualstackMiddleware" }

// HandleSerialize handles serializer middleware behavior when middleware is executed
func (u *EnableDualstackMiddleware) HandleSerialize(
ctx context.Context, in middleware.SerializeInput, next middleware.SerializeHandler,
) (
out middleware.SerializeOutput, metadata middleware.Metadata, err error,
) {
req, ok := in.Request.(*http.Request)
if !ok {
return out, metadata, fmt.Errorf("unknown request type %T", req)
}

if u.UseDualstack {
parts := strings.Split(req.URL.Host, ".")
if len(parts) < 3 {
return out, metadata, fmt.Errorf("unable to update endpoint host for dualstack, hostname invalid, %s", req.URL.Host)
}

for i := 0; i+1 < len(parts); i++ {
if strings.EqualFold(parts[i], u.ServiceID) {
parts[i] = parts[i] + ".dualstack"
break
}
}

// construct the url host
req.URL.Host = strings.Join(parts, ".")
}

return next.HandleSerialize(ctx, in)
}

0 comments on commit 4e3fc08

Please sign in to comment.