Hook up live tests (#19211)

* Hook up live tests

* fix panic due to nil *testing.T, cleaned up logging

replaced incorrect usage of require.Equal for error checks.
use context with a timeout when downloading blobs.

* fix HTTP range bug introduced in refactor

don't fail tests from arbitrary goroutines

* fix encryption scope to use deployed scope name

fixed another panic due to nil *testing.T.
removed unnecessary resources from deployment template.

* fix SAS tests

* fix soft delete tests

* fix lease test

fix linter issue
add retries if container is still being deleted

* fix encryption scope env var value

* switch to using GetRecordMode

* fix incorrect comparison

* use recording constants instead of string literals

* work around container lease for live test

sdk/storage/azblob/appendblob/client_test.go

@@ -18,6 +18,7 @@ import (
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore/streaming"
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
+	"github.com/Azure/azure-sdk-for-go/sdk/internal/recording"
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/appendblob"
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror"
@@ -28,8 +29,16 @@ import (
 )
 
 func Test(t *testing.T) {
-	suite.Run(t, &AppendBlobRecordedTestsSuite{})
-	//suite.Run(t, &AppendBlobUnrecordedTestsSuite{})
+	recordMode := recording.GetRecordMode()
+	t.Logf("Running appendblob Tests in %s mode\n", recordMode)
+	if recordMode == recording.LiveMode {
+		suite.Run(t, &AppendBlobRecordedTestsSuite{})
+		suite.Run(t, &AppendBlobUnrecordedTestsSuite{})
+	} else if recordMode == recording.PlaybackMode {
+		suite.Run(t, &AppendBlobRecordedTestsSuite{})
+	} else if recordMode == recording.RecordingMode {
+		suite.Run(t, &AppendBlobRecordedTestsSuite{})
+	}
 }
 
 // nolint
@@ -1504,6 +1513,7 @@ func (s *AppendBlobRecordedTestsSuite) TestAppendBlockWithCPK() {
 func (s *AppendBlobRecordedTestsSuite) TestAppendBlockWithCPKScope() {
 	_require := require.New(s.T())
 	testName := s.T().Name()
+	encryptionScope := testcommon.GetCPKScopeInfo(s.T())
 	svcClient, err := testcommon.GetServiceClient(s.T(), testcommon.TestAccountDefault, nil)
 	_require.NoError(err)
 	containerClient := testcommon.CreateNewContainer(context.Background(), _require, testcommon.GenerateContainerName(testName), svcClient)
@@ -1512,7 +1522,7 @@ func (s *AppendBlobRecordedTestsSuite) TestAppendBlockWithCPKScope() {
 	abClient := containerClient.NewAppendBlobClient(testcommon.GenerateBlobName(testName))
 
 	createAppendBlobOptions := appendblob.CreateOptions{
-		CpkScopeInfo: &testcommon.TestCPKByScope,
+		CpkScopeInfo: &encryptionScope,
 	}
 	_, err = abClient.Create(context.Background(), &createAppendBlobOptions)
 	_require.Nil(err)
@@ -1521,7 +1531,7 @@ func (s *AppendBlobRecordedTestsSuite) TestAppendBlockWithCPKScope() {
 	words := []string{"AAA ", "BBB ", "CCC "}
 	for index, word := range words {
 		appendBlockOptions := appendblob.AppendBlockOptions{
-			CpkScopeInfo: &testcommon.TestCPKByScope,
+			CpkScopeInfo: &encryptionScope,
 		}
 		resp, err := abClient.AppendBlock(context.Background(), streaming.NopCloser(strings.NewReader(word)), &appendBlockOptions)
 		_require.Nil(err)
@@ -1537,20 +1547,20 @@ func (s *AppendBlobRecordedTestsSuite) TestAppendBlockWithCPKScope() {
 		_require.NotNil(resp.Date)
 		_require.Equal((*resp.Date).IsZero(), false)
 		_require.Equal(*resp.IsServerEncrypted, true)
-		_require.EqualValues(resp.EncryptionScope, testcommon.TestCPKByScope.EncryptionScope)
+		_require.EqualValues(*encryptionScope.EncryptionScope, *resp.EncryptionScope)
 	}
 
 	// Download blob to do data integrity check.
 	downloadBlobOptions := blob.DownloadStreamOptions{
-		CpkScopeInfo: &testcommon.TestCPKByScope,
+		CpkScopeInfo: &encryptionScope,
 	}
 	downloadResp, err := abClient.DownloadStream(context.Background(), &downloadBlobOptions)
 	_require.Nil(err)
 
 	data, err := io.ReadAll(downloadResp.Body)
 	_require.Nil(err)
 	_require.EqualValues(string(data), "AAA BBB CCC ")
-	_require.EqualValues(*downloadResp.EncryptionScope, *testcommon.TestCPKByScope.EncryptionScope)
+	_require.EqualValues(*downloadResp.EncryptionScope, *encryptionScope.EncryptionScope)
 }
 
 //nolint

sdk/storage/azblob/blob/client_test.go

@@ -21,6 +21,7 @@ import (
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore/streaming"
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
+	"github.com/Azure/azure-sdk-for-go/sdk/internal/recording"
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror"
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blockblob"
@@ -33,8 +34,16 @@ import (
 )
 
 func Test(t *testing.T) {
-	suite.Run(t, &BlobRecordedTestsSuite{})
-	//suite.Run(t, &BlobUnrecordedTestsSuite{})
+	recordMode := recording.GetRecordMode()
+	t.Logf("Running blob Tests in %s mode\n", recordMode)
+	if recordMode == recording.LiveMode {
+		suite.Run(t, &BlobRecordedTestsSuite{})
+		suite.Run(t, &BlobUnrecordedTestsSuite{})
+	} else if recordMode == recording.PlaybackMode {
+		suite.Run(t, &BlobRecordedTestsSuite{})
+	} else if recordMode == recording.RecordingMode {
+		suite.Run(t, &BlobRecordedTestsSuite{})
+	}
 }
 
 // nolint

sdk/storage/azblob/blob/models.go

@@ -129,7 +129,7 @@ func (o *downloadOptions) getDownloadBlobOptions(rnge HTTPRange, rangeGetContent
 		AccessConditions:   o.AccessConditions,
 		CpkInfo:            o.CpkInfo,
 		CpkScopeInfo:       o.CpkScopeInfo,
-		Range:              o.Range,
+		Range:              rnge,
 		RangeGetContentMD5: rangeGetContentMD5,
 	}
 }

sdk/storage/azblob/blockblob/client_test.go

@@ -21,6 +21,7 @@ import (
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore/streaming"
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
+	"github.com/Azure/azure-sdk-for-go/sdk/internal/recording"
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror"
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blockblob"
@@ -36,8 +37,16 @@ import (
 var proposedLeaseIDs = []*string{to.Ptr("c820a799-76d7-4ee2-6e15-546f19325c2c"), to.Ptr("326cc5e1-746e-4af8-4811-a50e6629a8ca")}
 
 func Test(t *testing.T) {
-	suite.Run(t, &BlockBlobRecordedTestsSuite{})
-	//suite.Run(t, &BlockBlobUnrecordedTestsSuite{})
+	recordMode := recording.GetRecordMode()
+	t.Logf("Running blockblob Tests in %s mode\n", recordMode)
+	if recordMode == recording.LiveMode {
+		suite.Run(t, &BlockBlobRecordedTestsSuite{})
+		suite.Run(t, &BlockBlobUnrecordedTestsSuite{})
+	} else if recordMode == recording.PlaybackMode {
+		suite.Run(t, &BlockBlobRecordedTestsSuite{})
+	} else if recordMode == recording.RecordingMode {
+		suite.Run(t, &BlockBlobRecordedTestsSuite{})
+	}
 }
 
 // nolint
@@ -1540,24 +1549,25 @@ func (s *BlockBlobRecordedTestsSuite) TestGetSetBlobMetadataWithCPK() {
 func (s *BlockBlobRecordedTestsSuite) TestGetSetBlobMetadataWithCPKScope() {
 	_require := require.New(s.T())
 	testName := s.T().Name()
+	encryptionScope := testcommon.GetCPKScopeInfo(s.T())
 	svcClient, err := testcommon.GetServiceClient(s.T(), testcommon.TestAccountDefault, nil)
 	_require.NoError(err)
 	containerClient := testcommon.CreateNewContainer(context.Background(), _require, testcommon.GenerateContainerName(testName)+"01", svcClient)
 	defer testcommon.DeleteContainer(context.Background(), _require, containerClient)
 
 	bbName := testcommon.GenerateBlobName(testName)
-	bbClient := testcommon.CreateNewBlockBlobWithCPK(context.Background(), _require, bbName, containerClient, nil, &testcommon.TestCPKByScope)
+	bbClient := testcommon.CreateNewBlockBlobWithCPK(context.Background(), _require, bbName, containerClient, nil, &encryptionScope)
 
 	// Set blob metadata without encryption key should fail the request.
 	_, err = bbClient.SetMetadata(context.Background(), testcommon.BasicMetadata, nil)
 	_require.NotNil(err)
 
 	setBlobMetadataOptions := blob.SetMetadataOptions{
-		CpkScopeInfo: &testcommon.TestCPKByScope,
+		CpkScopeInfo: &encryptionScope,
 	}
 	resp, err := bbClient.SetMetadata(context.Background(), testcommon.BasicMetadata, &setBlobMetadataOptions)
 	_require.Nil(err)
-	_require.EqualValues(resp.EncryptionScope, testcommon.TestCPKByScope.EncryptionScope)
+	_require.EqualValues(*resp.EncryptionScope, *encryptionScope.EncryptionScope)
 
 	getResp, err := bbClient.GetProperties(context.Background(), nil)
 	_require.Nil(err)
@@ -1622,13 +1632,14 @@ func (s *BlockBlobRecordedTestsSuite) TestBlobSnapshotWithCPK() {
 func (s *BlockBlobRecordedTestsSuite) TestBlobSnapshotWithCPKScope() {
 	_require := require.New(s.T())
 	testName := s.T().Name()
+	encryptionScope := testcommon.GetCPKScopeInfo(s.T())
 	svcClient, err := testcommon.GetServiceClient(s.T(), testcommon.TestAccountDefault, nil)
 	_require.NoError(err)
 	containerClient := testcommon.CreateNewContainer(context.Background(), _require, testcommon.GenerateContainerName(testName)+"01", svcClient)
 	defer testcommon.DeleteContainer(context.Background(), _require, containerClient)
 
 	bbName := testcommon.GenerateBlobName(testName)
-	bbClient := testcommon.CreateNewBlockBlobWithCPK(context.Background(), _require, bbName, containerClient, nil, &testcommon.TestCPKByScope)
+	bbClient := testcommon.CreateNewBlockBlobWithCPK(context.Background(), _require, bbName, containerClient, nil, &encryptionScope)
 
 	// Create Snapshot of an encrypted blob without encryption key should fail the request.
 	_, err = bbClient.CreateSnapshot(context.Background(), nil)
@@ -1641,19 +1652,19 @@ func (s *BlockBlobRecordedTestsSuite) TestBlobSnapshotWithCPKScope() {
 	_require.NotNil(err)
 
 	createBlobSnapshotOptions1 := blob.CreateSnapshotOptions{
-		CpkScopeInfo: &testcommon.TestCPKByScope,
+		CpkScopeInfo: &encryptionScope,
 	}
 	resp, err := bbClient.CreateSnapshot(context.Background(), &createBlobSnapshotOptions1)
 	_require.Nil(err)
 	_require.Equal(*resp.IsServerEncrypted, false)
 
 	snapshotURL, _ := bbClient.WithSnapshot(*resp.Snapshot)
 	downloadBlobOptions := blob.DownloadStreamOptions{
-		CpkScopeInfo: &testcommon.TestCPKByScope,
+		CpkScopeInfo: &encryptionScope,
 	}
 	dResp, err := snapshotURL.DownloadStream(context.Background(), &downloadBlobOptions)
 	_require.Nil(err)
-	_require.EqualValues(*dResp.EncryptionScope, *testcommon.TestCPKByScope.EncryptionScope)
+	_require.EqualValues(*dResp.EncryptionScope, *encryptionScope.EncryptionScope)
 
 	_, err = snapshotURL.Delete(context.Background(), nil)
 	_require.Nil(err)
@@ -1863,7 +1874,7 @@ func (s *BlockBlobUnrecordedTestsSuite) TestCreateBlockBlobReturnsVID() {
 func (s *BlockBlobUnrecordedTestsSuite) TestORSSource() {
 	_require := require.New(s.T())
 	testName := s.T().Name()
-	svcClient, err := testcommon.GetServiceClient(nil, testcommon.TestAccountDefault, nil)
+	svcClient, err := testcommon.GetServiceClient(s.T(), testcommon.TestAccountDefault, nil)
 	if err != nil {
 		s.Fail("Unable to fetch service client because " + err.Error())
 	}
@@ -2068,7 +2079,7 @@ func (s *BlockBlobUnrecordedTestsSuite) TestSetBlobTagsWithLeaseId() {
 	_require.Nil(err)
 
 	_, err = bbClient.GetTags(ctx, nil)
-	_require.NotNil(err)
+	_require.NoError(err)
 
 	blobGetTagsResponse, err := bbClient.GetTags(ctx, &blob.GetTagsOptions{BlobAccessConditions: &blob.AccessConditions{
 		LeaseAccessConditions: &blob.LeaseAccessConditions{LeaseID: blobLeaseClient.LeaseID()}}})
@@ -2730,6 +2741,7 @@ func (s *BlockBlobRecordedTestsSuite) TestPutBlockAndPutBlockListWithCPK() {
 func (s *BlockBlobRecordedTestsSuite) TestPutBlockAndPutBlockListWithCPKByScope() {
 	_require := require.New(s.T())
 	testName := s.T().Name()
+	encryptionScope := testcommon.GetCPKScopeInfo(s.T())
 	svcClient, err := testcommon.GetServiceClient(s.T(), testcommon.TestAccountDefault, nil)
 	_require.NoError(err)
 	containerClient := testcommon.CreateNewContainer(context.Background(), _require, testcommon.GenerateContainerName(testName), svcClient)
@@ -2742,21 +2754,21 @@ func (s *BlockBlobRecordedTestsSuite) TestPutBlockAndPutBlockListWithCPKByScope(
 	for index, word := range words {
 		base64BlockIDs[index] = testcommon.BlockIDIntToBase64(index)
 		stageBlockOptions := blockblob.StageBlockOptions{
-			CpkScopeInfo: &testcommon.TestCPKByScope,
+			CpkScopeInfo: &encryptionScope,
 		}
 		_, err := bbClient.StageBlock(context.Background(), base64BlockIDs[index], streaming.NopCloser(strings.NewReader(word)), &stageBlockOptions)
 		_require.Nil(err)
 	}
 
 	commitBlockListOptions := blockblob.CommitBlockListOptions{
-		CpkScopeInfo: &testcommon.TestCPKByScope,
+		CpkScopeInfo: &encryptionScope,
 	}
 	resp, err := bbClient.CommitBlockList(context.Background(), base64BlockIDs, &commitBlockListOptions)
 	_require.Nil(err)
 	_require.NotNil(resp.ETag)
 	_require.NotNil(resp.LastModified)
 	_require.Equal(*resp.IsServerEncrypted, true)
-	_require.EqualValues(resp.EncryptionScope, testcommon.TestCPKByScope.EncryptionScope)
+	_require.EqualValues(*encryptionScope.EncryptionScope, *resp.EncryptionScope)
 
 	downloadBlobOptions := blob.DownloadStreamOptions{
 		CpkInfo: &testcommon.TestCPKByValue,
@@ -2765,7 +2777,7 @@ func (s *BlockBlobRecordedTestsSuite) TestPutBlockAndPutBlockListWithCPKByScope(
 	_require.NotNil(err)
 
 	downloadBlobOptions = blob.DownloadStreamOptions{
-		CpkScopeInfo: &testcommon.TestCPKByScope,
+		CpkScopeInfo: &encryptionScope,
 	}
 	getResp, err := bbClient.DownloadStream(context.Background(), &downloadBlobOptions)
 	_require.Nil(err)
@@ -2778,7 +2790,7 @@ func (s *BlockBlobRecordedTestsSuite) TestPutBlockAndPutBlockListWithCPKByScope(
 	_require.EqualValues(*getResp.ETag, *resp.ETag)
 	_require.EqualValues(*getResp.LastModified, *resp.LastModified)
 	_require.Equal(*getResp.IsServerEncrypted, true)
-	_require.EqualValues(*getResp.EncryptionScope, *testcommon.TestCPKByScope.EncryptionScope)
+	_require.EqualValues(*getResp.EncryptionScope, *encryptionScope.EncryptionScope)
 }
 
 //
@@ -3060,6 +3072,7 @@ func (s *BlockBlobUnrecordedTestsSuite) TestUploadBlobWithMD5WithCPK() {
 func (s *BlockBlobRecordedTestsSuite) TestUploadBlobWithMD5WithCPKScope() {
 	_require := require.New(s.T())
 	testName := s.T().Name()
+	encryptionScope := testcommon.GetCPKScopeInfo(s.T())
 	svcClient, err := testcommon.GetServiceClient(s.T(), testcommon.TestAccountDefault, nil)
 	_require.NoError(err)
 	containerClient := testcommon.CreateNewContainer(context.Background(), _require, testcommon.GenerateContainerName(testName), svcClient)
@@ -3071,25 +3084,25 @@ func (s *BlockBlobRecordedTestsSuite) TestUploadBlobWithMD5WithCPKScope() {
 	bbClient := containerClient.NewBlockBlobClient(testcommon.GenerateBlobName(testName))
 
 	uploadBlockBlobOptions := blockblob.UploadOptions{
-		CpkScopeInfo: &testcommon.TestCPKByScope,
+		CpkScopeInfo: &encryptionScope,
 	}
 	uploadResp, err := bbClient.Upload(context.Background(), r, &uploadBlockBlobOptions)
 	_require.Nil(err)
 	// _require.Equal(uploadResp.RawResponse.StatusCode, 201)
 	_require.Equal(*uploadResp.IsServerEncrypted, true)
-	_require.EqualValues(uploadResp.EncryptionScope, testcommon.TestCPKByScope.EncryptionScope)
+	_require.EqualValues(*encryptionScope.EncryptionScope, *uploadResp.EncryptionScope)
 
 	// Download blob to do data integrity check.
 	downloadBlobOptions := blob.DownloadStreamOptions{
-		CpkScopeInfo: &testcommon.TestCPKByScope,
+		CpkScopeInfo: &encryptionScope,
 	}
 	downloadResp, err := bbClient.DownloadStream(context.Background(), &downloadBlobOptions)
 	_require.Nil(err)
 	_require.EqualValues(downloadResp.ContentMD5, md5Val[:])
 	destData, err := io.ReadAll(downloadResp.Body)
 	_require.NoError(err)
 	_require.EqualValues(destData, srcData)
-	_require.EqualValues(*downloadResp.EncryptionScope, *testcommon.TestCPKByScope.EncryptionScope)
+	_require.EqualValues(*downloadResp.EncryptionScope, *encryptionScope.EncryptionScope)
 }
 
 //func (s *AZBlobUnrecordedTestsSuite) TestUploadStreamToBlobBlobPropertiesWithCPKKey() {
@@ -3129,7 +3142,7 @@ func (s *BlockBlobRecordedTestsSuite) TestUploadBlobWithMD5WithCPKScope() {
 //		})
 //
 //	// Assert that upload was successful
-//	_require.Equal(err, nil)
+//	_require.NoError(err)
 //	// _require.Equal(uploadResp.RawResponse.StatusCode, 201)
 //
 //	getPropertiesResp, err := bbClient.GetProperties(ctx, &blob.GetPropertiesOptions{CpkInfo: &testcommon.TestCPKByValue})
@@ -3194,7 +3207,7 @@ func (s *BlockBlobRecordedTestsSuite) TestUploadBlobWithMD5WithCPKScope() {
 //		})
 //
 //	// Assert that upload was successful
-//	_require.Equal(err, nil)
+//	_require.NoError(err)
 //	// _require.Equal(uploadResp.RawResponse.StatusCode, 201)
 //
 //	getPropertiesResp, err := bbClient.GetProperties(ctx, nil)
@@ -3253,7 +3266,7 @@ func (s *BlockBlobUnrecordedTestsSuite) TestUploadStreamToBlobProperties() {
 		})
 
 	// Assert that upload was successful
-	_require.Equal(err, nil)
+	_require.NoError(err)
 	// _require.Equal(uploadResp.RawResponse.StatusCode, 201)
 
 	getPropertiesResp, err := bbClient.GetProperties(context.Background(), nil)