Skip to content

Commit

Permalink
feat: car-scope support
Browse files Browse the repository at this point in the history 
This adds support for 'car-scope' URL param next to existing 'depth'
to allow Saturn L1/Lassie to migrake without breaking anything
during the transition.

Context: #80
  • Loading branch information
@lidel
lidel committed Apr 6, 2023
1 parent ad13184 commit 2fdf401
Show file tree
Hide file tree
Showing 2 changed files with 17 additions and 15 deletions.
4 changes: 2 additions & 2 deletions docs/environment-variables.md
View file
Expand Up @@ -44,10 +44,10 @@ The size of in-memory [2Q cache](https://pkg.go.dev/github.com/hashicorp/golang-
Default: `false`

When set to `true`, requests to backend will use
`?format=car&depth=..&bytes=..` in addition to `?format=raw` to reduce the
`?format=car&car-scope=..&bytes=..` in addition to `?format=raw` to reduce the
number of round trips.

This is an experimental feature that depends on `&depth=..&bytes=..`
This is an experimental feature that depends on `&car-scope=..&bytes=..`
parameters. Currently only `https://l1s.strn.pl` supports it, but our
intention is to standardize it and add it to the
[trustless gateway spec](https://specs.ipfs.tech/http-gateways/trustless-gateway/)
Expand Down
28 changes: 15 additions & 13 deletions lib/graph_gateway.go
View file
Expand Up @@ -201,7 +201,7 @@ func registerGraphGatewayMetrics() *GraphGatewayMetrics {
Subsystem: "gw_graph_backend",
Name: "car_fetch_params",
Help: "How many times specific CAR parameter was used during CAR data fetch.",
}, []string{"depth", "ranges"}) // we use 'ranges' instead of 'bytes' here because we only caount the number of ranges present
}, []string{"carScope", "ranges"}) // we use 'ranges' instead of 'bytes' here because we only caount the number of ranges present

Check warning on line 204 in lib/graph_gateway.go

View check run for this annotation

Codecov / codecov/patch

lib/graph_gateway.go#L204 

Added line #L204 was not covered by tests
prometheus.MustRegister(carParamsMetric)

bytesRangeStartMetric := prometheus.NewHistogram(prometheus.HistogramOpts{
Expand Down Expand Up @@ -360,9 +360,11 @@ func wrapNodeWithClose[T files.Node](node T, closeFn func()) (T, error) {

func (api *GraphGateway) Get(ctx context.Context, path gateway.ImmutablePath, byteRanges ...gateway.ByteRange) (gateway.ContentPathMetadata, *gateway.GetResponse, error) {
rangeCount := len(byteRanges)
api.metrics.carParamsMetric.With(prometheus.Labels{"depth": "1", "ranges": strconv.Itoa(rangeCount)}).Inc()
api.metrics.carParamsMetric.With(prometheus.Labels{"carScope": "file", "ranges": strconv.Itoa(rangeCount)}).Inc()

Check warning on line 363 in lib/graph_gateway.go

View check run for this annotation

Codecov / codecov/patch

lib/graph_gateway.go#L363 

Added line #L363 was not covered by tests

carParams := "?format=car&depth=1"
// TODO: remove &depth= from all CAR request after transition is done:
// https://github.com/ipfs/bifrost-gateway/issues/80
carParams := "?format=car&car-scope=file&depth=1"

Check warning on line 367 in lib/graph_gateway.go

View check run for this annotation

Codecov / codecov/patch

lib/graph_gateway.go#L365-L367 

Added lines #L365 - L367 were not covered by tests

// fetch CAR with &bytes= to get minimal set of blocks for the request
// Note: majority of requests have 0 or max 1 ranges. if there are more ranges than one,
Expand Down Expand Up @@ -403,8 +405,8 @@ func (api *GraphGateway) Get(ctx context.Context, path gateway.ImmutablePath, by
}

func (api *GraphGateway) GetAll(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, files.Node, error) {
api.metrics.carParamsMetric.With(prometheus.Labels{"depth": "all", "ranges": "0"}).Inc()
blkgw, closeFn, err := api.loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&depth=all")
api.metrics.carParamsMetric.With(prometheus.Labels{"carScope": "all", "ranges": "0"}).Inc()
blkgw, closeFn, err := api.loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&car-scope=all&depth=all")

Check warning on line 409 in lib/graph_gateway.go

View check run for this annotation

Codecov / codecov/patch

lib/graph_gateway.go#L408-L409 

Added lines #L408 - L409 were not covered by tests
if err != nil {
return gateway.ContentPathMetadata{}, nil, err
}
Expand All @@ -420,9 +422,9 @@ func (api *GraphGateway) GetAll(ctx context.Context, path gateway.ImmutablePath)
}

func (api *GraphGateway) GetBlock(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, files.File, error) {
api.metrics.carParamsMetric.With(prometheus.Labels{"depth": "0", "ranges": "0"}).Inc()
api.metrics.carParamsMetric.With(prometheus.Labels{"carScope": "block", "ranges": "0"}).Inc()

Check warning on line 425 in lib/graph_gateway.go

View check run for this annotation

Codecov / codecov/patch

lib/graph_gateway.go#L425 

Added line #L425 was not covered by tests
// TODO: if path is `/ipfs/cid`, we should use ?format=raw
blkgw, closeFn, err := api.loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&depth=0")
blkgw, closeFn, err := api.loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&car-scope=block&depth=0")

Check warning on line 427 in lib/graph_gateway.go

View check run for this annotation

Codecov / codecov/patch

lib/graph_gateway.go#L427 

Added line #L427 was not covered by tests
if err != nil {
return gateway.ContentPathMetadata{}, nil, err
}
Expand All @@ -438,13 +440,13 @@ func (api *GraphGateway) GetBlock(ctx context.Context, path gateway.ImmutablePat
}

func (api *GraphGateway) Head(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, files.Node, error) {
api.metrics.carParamsMetric.With(prometheus.Labels{"depth": "", "ranges": "1"}).Inc()
api.metrics.carParamsMetric.With(prometheus.Labels{"carScope": "file", "ranges": "1"}).Inc()

Check warning on line 443 in lib/graph_gateway.go

View check run for this annotation

Codecov / codecov/patch

lib/graph_gateway.go#L443 

Added line #L443 was not covered by tests

// TODO: we probably want to move this either to boxo, or at least to loadRequestIntoSharedBlockstoreAndBlocksGateway
api.metrics.bytesRangeStartMetric.Observe(0)
api.metrics.bytesRangeSizeMetric.Observe(1024)

blkgw, closeFn, err := api.loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&bytes=0:1023")
blkgw, closeFn, err := api.loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&car-scope=file&depth=1&bytes=0:1023")

Check warning on line 449 in lib/graph_gateway.go

View check run for this annotation

Codecov / codecov/patch

lib/graph_gateway.go#L449 

Added line #L449 was not covered by tests

if err != nil {
return gateway.ContentPathMetadata{}, nil, err
Expand All @@ -461,8 +463,8 @@ func (api *GraphGateway) Head(ctx context.Context, path gateway.ImmutablePath) (
}

func (api *GraphGateway) ResolvePath(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, error) {
api.metrics.carParamsMetric.With(prometheus.Labels{"depth": "0", "ranges": "0"}).Inc()
blkgw, closeFn, err := api.loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&depth=0")
api.metrics.carParamsMetric.With(prometheus.Labels{"carScope": "block", "ranges": "0"}).Inc()
blkgw, closeFn, err := api.loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&car-scope=block&depth=0")

Check warning on line 467 in lib/graph_gateway.go

View check run for this annotation

Codecov / codecov/patch

lib/graph_gateway.go#L466-L467 

Added lines #L466 - L467 were not covered by tests
if err != nil {
return gateway.ContentPathMetadata{}, err
}
Expand All @@ -471,8 +473,8 @@ func (api *GraphGateway) ResolvePath(ctx context.Context, path gateway.Immutable
}

func (api *GraphGateway) GetCAR(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, io.ReadCloser, <-chan error, error) {
api.metrics.carParamsMetric.With(prometheus.Labels{"depth": "", "ranges": "0"}).Inc()
blkgw, closeFn, err := api.loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car")
api.metrics.carParamsMetric.With(prometheus.Labels{"carScope": "all", "ranges": "0"}).Inc()
blkgw, closeFn, err := api.loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&car-scope=all&depth=all")

Check warning on line 477 in lib/graph_gateway.go

View check run for this annotation

Codecov / codecov/patch

lib/graph_gateway.go#L476-L477 

Added lines #L476 - L477 were not covered by tests
if err != nil {
return gateway.ContentPathMetadata{}, nil, nil, err
}
Expand Down

0 comments on commit 2fdf401

Please sign in to comment.