Merge branch 'master' into vishal/secure_grpc_port

.github/workflows/ci.yml

@@ -5,6 +5,8 @@ on:
     branches:
       - master
       - 'auto-cadence-upgrade/**'
+      - staging
+      - trying
   pull_request:
     branches:
       - master
@@ -23,7 +25,7 @@ jobs:
       uses: golangci/golangci-lint-action@v2.3.0
       with:
         # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version.
-        version: v1.29
+        version: v1.41
         args: -v --build-tags relic
     - name: Run ShellCheck
       uses: ludeeus/action-shellcheck@master
@@ -47,14 +49,23 @@ jobs:
     - name: Checkout repo
       uses: actions/checkout@v2
     - name: Run tests
+      if: github.actor != 'bors[bot]'
       run: make ci
+    - name: Run tests (Bors)
+      if: github.actor == 'bors[bot]'
+      uses: nick-invision/retry@v2
+      with:
+        timeout_minutes: 25
+        max_attempts: 2
+        command: make ci
     - name: Upload coverage report
       uses: codecov/codecov-action@v1
       if: ${{ matrix.go-version == '1.15' }}
       with:
         file: ./coverage.txt
         flags: unittests
         name: codecov-umbrella
+
   integration-test:
     name: Integration Tests
     strategy:
@@ -76,4 +87,37 @@ jobs:
     - name: Docker build
       run: make docker-build-flow
     - name: Run tests
+      if: github.actor != 'bors[bot]'
       run: make ci-integration
+    - name: Run tests (Bors)
+      if: github.actor == 'bors[bot]'
+      uses: nick-invision/retry@v2
+      with:
+        timeout_minutes: 15
+        max_attempts: 2
+        command: make ci-integration
+
+  cross-blst-test:
+    name: cross-testing internal BLS implementation with BLST
+    strategy:
+      fail-fast: false
+    runs-on: ubuntu-latest
+    steps:
+    - name: Setup Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: 1.16
+    - name: Checkout repo
+      uses: actions/checkout@v2
+    - name: Build relic
+      run: make crypto/relic/build
+    - name: Run tests
+      if: github.actor != 'bors[bot]'
+      run: make -C crypto cross-blst-test
+    - name: Run tests (Bors)
+      if: github.actor == 'bors[bot]'
+      uses: nick-invision/retry@v2
+      with:
+        timeout_minutes: 25
+        max_attempts: 2
+        command: make -C crypto cross-blst-test

.gitignore

@@ -7,6 +7,9 @@
 /cmd/util/util
 /cmd/bootstrap/bootstrap
 
+# crypto relic folder
+crypto/relic/
+
 # Test binary, build with `go test -c`
 *.test
 
@@ -16,6 +19,12 @@
 # Output from logging to debug
 *.log
 
+# Output from binstat
+*.binstat.txt*
+
+# Output from pprof
+*.pprof.txt
+
 # Coverage artifacts
 coverage.zip
 coverage.txt

.golangci.yml

@@ -1,14 +1,24 @@
 run:
   timeout: 5m
-  skip-dirs:
-    - consensus/hotstuff
 
 linters-settings:
   goimports:
     # put imports beginning with prefix after 3rd-party packages;
     # it's a comma-separated list of prefixes
     local-prefixes: github.com/onflow/flow-go
 
+  gosec:
+    # To select a subset of rules to run.
+    # Available rules: https://github.com/securego/gosec#available-rules
+    includes:
+      - G401
+      - G402
+      - G501
+      - G502
+      - G503
+      - G505
+
 linters:
   enable:
     - goimports
+    - gosec

CODEOWNERS

@@ -32,7 +32,7 @@
 /network/** @yhassanzadeh13 @vishalchangrani
 
 # Cryptography Stream
-/crypto/** @tarakby
+/crypto/** @tarakby @huitseeker
 
 # Bootstrap and transit scripts
 /cmd/bootstrap/** @vishalchangrani @ljk662

CONTRIBUTING.md

@@ -109,6 +109,18 @@ following when creating your pull request:
 
 A reviewer will be assigned automatically when your PR is created.
 
+We use [bors](https://github.com/bors-ng/bors-ng) merge bot to ensure that the `master` branch never breaks.
+Once a PR is approved, you can comment on it with the following to add your PR to the merge queue:
+
+```
+bors merge
+```
+
+If the PR passes CI, it will automatically be pushed to the `master` branch. If it fails, bors will comment
+on the PR so you can fix it.
+
+See the [documentation](https://bors.tech/documentation/) for a more comprehensive list of bors commands.
+
 ## Style Guide
 
 The following is a brief summary of the coding style used in Flow. 

Makefile

@@ -29,31 +29,22 @@ K8S_YAMLS_LOCATION_STAGING=./k8s/staging
 export CONTAINER_REGISTRY := gcr.io/flow-container-registry
 export DOCKER_BUILDKIT := 1
 
-# relic versions in script and submodule
-export LOCAL_VERSION := $(shell git submodule status | egrep '\s[0-9a-f]' | cut -c 2-9)
-export SCRIPT_VERSION := $(shell egrep 'relic_version="[0-9a-f]{8}"' ./crypto/build_dependency.sh| cut -c 16-23)
-
-.PHONY: crypto/relic
-crypto/relic:
-	rm -rf crypto/relic
-	git submodule update --init --recursive
-
 .PHONY: crypto/relic/build
-crypto/relic/build: crypto/relic
-	./crypto/relic_build.sh
+crypto/relic/build:
+	cd ./crypto &&	go generate
+
+# relic versions in script and submodule
+export LOCAL_VERSION := $(shell (cd ./crypto/relic/ && git rev-parse HEAD))
+export SCRIPT_VERSION := $(shell egrep 'relic_version="[0-9a-f]{40}"' ./crypto/build_dependency.sh | cut -c 16-55)
 
 .PHONY: crypto/relic/check
 crypto/relic/check:
 ifeq ($(SCRIPT_VERSION), $(LOCAL_VERSION))
-	@echo "Relic submodule version matches script, good!"
+	@echo "local relic version matches the version required by the crypto package"
 else
-	$(error Mismatch between relic submodule commit and the version in ./crypto/build_dependency.sh)
+	$(error local relic version doesn't match the version required by the crypto package)
 endif
 
-
-crypto/relic/update:
-	git submodule update --recursive
-
 cmd/collection/collection:
 	go build -o cmd/collection/collection cmd/collection/main.go
 
@@ -62,7 +53,7 @@ cmd/util/util:
 
 .PHONY: install-tools
 install-tools: crypto/relic/build check-go-version
-	curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b ${GOPATH}/bin v1.29.0; \
+	curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b ${GOPATH}/bin v1.41.1; \
 	cd ${GOPATH}; \
 	GO111MODULE=on go get github.com/golang/protobuf/protoc-gen-go@v1.3.2; \
 	GO111MODULE=on go get github.com/uber/prototool/cmd/prototool@v1.9.0; \
@@ -72,10 +63,8 @@ install-tools: crypto/relic/build check-go-version
 
 .PHONY: unittest
 unittest:
-	# test some packages with Relic library and data race detection enabled
-	GO111MODULE=on go test -coverprofile=$(COVER_PROFILE) -covermode=atomic $(if $(JSON_OUTPUT),-json,) -race --tags relic ./access/... ./consensus/... ./model/... ./state/... ./storage/... ./utils/...
 	# test all packages with Relic library enabled
-	GO111MODULE=on go test -coverprofile=$(COVER_PROFILE) -covermode=atomic $(if $(JSON_OUTPUT),-json,) --tags relic ./cmd...  ./engine/... ./fvm/... ./ledger/... ./module/... ./network/...
+	GO111MODULE=on go test -coverprofile=$(COVER_PROFILE) -covermode=atomic $(if $(JSON_OUTPUT),-json,) --tags relic ./...
 	$(MAKE) -C crypto test
 	$(MAKE) -C integration test
 
@@ -110,6 +99,7 @@ generate-proto:
 
 .PHONY: generate-mocks
 generate-mocks:
+	GO111MODULE=on mockery -name '(ReadyDoneAwareNetwork|Connector|PingInfoProvider)' -dir=network/p2p -case=underscore -output="./network/mocknetwork" -outpkg="mocknetwork"
 	GO111MODULE=on mockgen -destination=storage/mocks/storage.go -package=mocks github.com/onflow/flow-go/storage Blocks,Headers,Payloads,Collections,Commits,Events,ServiceEvents,TransactionResults
 	GO111MODULE=on mockgen -destination=module/mocks/network.go -package=mocks github.com/onflow/flow-go/module Network,Local,Requester
 	GO111MODULE=on mockgen -destination=network/mocknetwork/engine.go -package=mocknetwork github.com/onflow/flow-go/network Engine
@@ -131,7 +121,6 @@ generate-mocks:
 	GO111MODULE=on mockery -name '.*' -dir=fvm -case=underscore -output="./fvm/mock" -outpkg="mock"
 	GO111MODULE=on mockery -name '.*' -dir=fvm/state -case=underscore -output="./fvm/mock/state" -outpkg="mock"
 	GO111MODULE=on mockery -name '.*' -dir=ledger -case=underscore -output="./ledger/mock" -outpkg="mock"
-	GO111MODULE=on mockery -name '.*' -dir=network/p2p -case=underscore -output="./network/mocknetwork" -outpkg="mocknetwork"
 	GO111MODULE=on mockery -name 'SubscriptionManager' -dir=network/ -case=underscore -output="./network/mocknetwork" -outpkg="mocknetwork"
 	GO111MODULE=on mockery -name 'Vertex' -dir="./module/forest" -case=underscore -output="./module/forest/mock" -outpkg="mock"
 	GO111MODULE=on mockery -name '.*' -dir="./consensus/hotstuff" -case=underscore -output="./consensus/hotstuff/mocks" -outpkg="mocks"
@@ -147,7 +136,7 @@ generate-mocks:
 
 # this ensures there is no unused dependency being added by accident
 .PHONY: tidy
-tidy: crypto/relic/check
+tidy:
 	go mod tidy
 	cd integration; go mod tidy
 	cd crypto; go mod tidy
@@ -320,6 +309,10 @@ docker-build-bootstrap-transit:
 	    --target production-transit-nocgo  \
 		-t "$(CONTAINER_REGISTRY)/bootstrap-transit:latest" -t "$(CONTAINER_REGISTRY)/bootstrap-transit:$(SHORT_COMMIT)" -t "$(CONTAINER_REGISTRY)/bootstrap-transit:$(IMAGE_TAG)" .
 
+PHONY: tool-transit
+tool-transit: docker-build-bootstrap-transit
+	docker container create --name transit $(CONTAINER_REGISTRY)/bootstrap-transit:latest;docker container cp transit:/bin/app ./transit;docker container rm transit
+
 .PHONY: docker-build-loader
 docker-build-loader:
 	docker build -f ./integration/loader/Dockerfile  --build-arg TARGET=loader --target production \

access/api.go

@@ -41,6 +41,8 @@ type API interface {
 	GetEventsForBlockIDs(ctx context.Context, eventType string, blockIDs []flow.Identifier) ([]flow.BlockEvents, error)
 
 	GetLatestProtocolStateSnapshot(ctx context.Context) ([]byte, error)
+
+	GetExecutionResultForBlockID(ctx context.Context, blockID flow.Identifier) (*flow.ExecutionResult, error)
 }
 
 // TODO: Combine this with flow.TransactionResult?

access/handler.go

@@ -2,6 +2,8 @@ package access
 
 import (
 	"context"
+	"encoding/json"
+	"fmt"
 
 	"github.com/onflow/flow/protobuf/go/flow/access"
 	"github.com/onflow/flow/protobuf/go/flow/entities"
@@ -417,6 +419,20 @@ func (h *Handler) GetLatestProtocolStateSnapshot(ctx context.Context, req *acces
 	}, nil
 }
 
+// GetExecutionResultForBlockID returns the latest received execution result for the given block ID.
+// AN might receive multiple receipts with conflicting results for unsealed blocks.
+// If this case happens, since AN is not able to determine which result is the correct one until the block is sealed, it has to pick one result to respond to this query. For now, we return the result from the latest received receipt.
+func (h *Handler) GetExecutionResultForBlockID(ctx context.Context, req *access.GetExecutionResultForBlockIDRequest) (*access.ExecutionResultForBlockIDResponse, error) {
+	blockID := convert.MessageToIdentifier(req.GetBlockId())
+
+	result, err := h.api.GetExecutionResultForBlockID(ctx, blockID)
+	if err != nil {
+		return nil, err
+	}
+
+	return executionResultToMessages(result)
+}
+
 func blockResponse(block *flow.Block) (*access.BlockResponse, error) {
 	msg, err := convert.BlockToMessage(block)
 	if err != nil {
@@ -439,6 +455,59 @@ func blockHeaderResponse(header *flow.Header) (*access.BlockHeaderResponse, erro
 	}, nil
 }
 
+func executionResultToMessages(er *flow.ExecutionResult) (*access.ExecutionResultForBlockIDResponse, error) {
+
+	chunks := make([]*entities.Chunk, len(er.Chunks))
+
+	for i, chunk := range er.Chunks {
+		chunks[i] = chunkToMessage(chunk)
+	}
+
+	serviceEvents := make([]*entities.ServiceEvent, len(er.ServiceEvents))
+	var err error
+	for i, serviceEvent := range er.ServiceEvents {
+		serviceEvents[i], err = serviceEventToMessage(serviceEvent)
+		if err != nil {
+			return nil, fmt.Errorf("error while convering service event %d: %w", i, err)
+		}
+	}
+
+	return &access.ExecutionResultForBlockIDResponse{
+		ExecutionResult: &entities.ExecutionResult{
+			PreviousResultId: convert.IdentifierToMessage(er.PreviousResultID),
+			BlockId:          convert.IdentifierToMessage(er.BlockID),
+			Chunks:           chunks,
+			ServiceEvents:    serviceEvents,
+		},
+	}, nil
+}
+
+func serviceEventToMessage(event flow.ServiceEvent) (*entities.ServiceEvent, error) {
+
+	bytes, err := json.Marshal(event.Event)
+	if err != nil {
+		return nil, fmt.Errorf("cannot marshal service event: %w", err)
+	}
+
+	return &entities.ServiceEvent{
+		Type:    event.Type,
+		Payload: bytes,
+	}, nil
+}
+
+func chunkToMessage(chunk *flow.Chunk) *entities.Chunk {
+	return &entities.Chunk{
+		CollectionIndex:      uint32(chunk.CollectionIndex),
+		StartState:           convert.StateCommitmentToMessage(chunk.StartState),
+		EventCollection:      convert.IdentifierToMessage(chunk.EventCollection),
+		BlockId:              convert.IdentifierToMessage(chunk.BlockID),
+		TotalComputationUsed: chunk.TotalComputationUsed,
+		NumberOfTransactions: uint32(chunk.NumberOfTransactions),
+		Index:                chunk.Index,
+		EndState:             convert.StateCommitmentToMessage(chunk.EndState),
+	}
+}
+
 func blockEventsToMessages(blocks []flow.BlockEvents) ([]*access.EventsResponse_Result, error) {
 	results := make([]*access.EventsResponse_Result, len(blocks))
 

access/legacy/convert/convert.go

@@ -164,7 +164,7 @@ func BlockToMessage(h *flow.Block) (*entitiesproto.Block, error) {
 		Timestamp:            t,
 		CollectionGuarantees: cg,
 		BlockSeals:           seals,
-		Signatures:           [][]byte{h.Header.ParentVoterSig},
+		Signatures:           [][]byte{h.Header.ParentVoterSigData},
 	}
 
 	return &bh, nil

bors.toml

@@ -0,0 +1,10 @@
+status = [
+  "Lint",
+  "Unit Tests (1.15)",
+  "Unit Tests (1.16)",
+  "Integration Tests (1.15)",
+  "Integration Tests (1.16)",
+]
+
+delete_merged_branches = true
+required_approvals = 2