Merge pull request #1306 from sweanan/add-datafactory-terratest-module

Add datafactory terratest module

examples/azure/terraform-azure-datafactory-example/README.md

@@ -0,0 +1,31 @@
+# Terraform Azure Data Factory Example
+
+This folder contains a Terraform module that deploys resources in [Azure](https://azure.microsoft.com/) to demonstrate how you can use Terratest to write automated tests for your Azure Terraform code.
+This module deploys a Data Factory.
+
+- A [Azure MySQL Database](https://azure.microsoft.com/en-us/products/data-factory).
+
+Check out [test/azure/terraform_azure_datafactory_example_test.go](./../../../test/azure/terraform_azure_datafactory_example_test.go) to see how you can write automated tests for this module and validate the configuration of the parameters and options.
+
+**WARNING**: This module and the automated tests for it deploy real resources into your Azure account which can cost you money.
+
+## Running this module manually
+1. Sign up for [Azure](https://azure.microsoft.com/).
+1. Configure your Azure credentials using one of the [supported methods for Azure CLI
+   tools](https://docs.microsoft.com/en-us/cli/azure/azure-cli-configuration?view=azure-cli-latest)
+1. Install [Terraform](https://www.terraform.io/) and make sure it's on your `PATH`.
+1. Ensure [environment variables](../README.md#review-environment-variables) are available
+1. Run `terraform init`
+1. Run `terraform apply`
+1. When you're done, run `terraform destroy`.
+
+
+## Running automated tests against this module
+1. Sign up for [Azure](https://azure.microsoft.com/)
+1. Configure your Azure credentials using one of the [supported methods for Azure CLI
+   tools](https://docs.microsoft.com/en-us/cli/azure/azure-cli-configuration?view=azure-cli-latest)
+1. Install [Terraform](https://www.terraform.io/) and make sure it's on your `PATH`
+1. Configure your Terratest [Go test environment](../README.md)
+1. `cd test/azure`
+1. `go build terraform_azure_datafactory_example_test.go`
+2. `go test -v -timeout 60m -tags azure -run TestTerraformAzureDataFactoryExample`

examples/azure/terraform-azure-datafactory-example/main.tf

@@ -0,0 +1,49 @@
+# ---------------------------------------------------------------------------------------------------------------------
+# DEPLOY AN AZURE DATA FACTORY
+# This is an example of how to deploy an AZURE Data Factory
+# See test/terraform_azure_example_test.go for how to write automated tests for this code.
+# ---------------------------------------------------------------------------------------------------------------------
+
+
+# ---------------------------------------------------------------------------------------------------------------------
+# CONFIGURE OUR AZURE CONNECTION
+# ---------------------------------------------------------------------------------------------------------------------
+
+provider "azurerm" {
+  version = "~>2.93.0"
+  features {}
+}
+
+# ---------------------------------------------------------------------------------------------------------------------
+# CREATE RANDOM PASSWORD
+# ---------------------------------------------------------------------------------------------------------------------
+
+# Random password is used as an example to simplify the deployment and improve the security of the database.
+# This is not as a production recommendation as the password is stored in the Terraform state file.
+resource "random_password" "password" {
+  length           = 16
+  override_special = "-_%@"
+  min_upper        = "1"
+  min_lower        = "1"
+  min_numeric      = "1"
+  min_special      = "1"
+}
+
+# ---------------------------------------------------------------------------------------------------------------------
+# DEPLOY A RESOURCE GROUP
+# ---------------------------------------------------------------------------------------------------------------------
+
+resource "azurerm_resource_group" "datafactory_rg" {
+  name     = "terratest-datafactory-${var.postfix}"
+  location = var.location
+}
+
+# ---------------------------------------------------------------------------------------------------------------------
+# DEPLOY A DATA FACTORY
+# ---------------------------------------------------------------------------------------------------------------------
+
+resource "azurerm_data_factory" "data_factory" {
+  name                = "datafactory${var.postfix}"
+  location            = azurerm_resource_group.datafactory_rg.location
+  resource_group_name = azurerm_resource_group.datafactory_rg.name
+}

examples/azure/terraform-azure-datafactory-example/outputs.tf

@@ -0,0 +1,7 @@
+output "resource_group_name" {
+  value = azurerm_resource_group.datafactory_rg.name
+}
+
+output "datafactory_name" {
+  value = azurerm_data_factory.data_factory.name
+}

examples/azure/terraform-azure-datafactory-example/variables.tf

@@ -0,0 +1,31 @@
+# ---------------------------------------------------------------------------------------------------------------------
+# ENVIRONMENT VARIABLES
+# Define these secrets as environment variables
+# ---------------------------------------------------------------------------------------------------------------------
+
+# ARM_CLIENT_ID
+# ARM_CLIENT_SECRET
+# ARM_SUBSCRIPTION_ID
+# ARM_TENANT_ID
+
+# ---------------------------------------------------------------------------------------------------------------------
+# REQUIRED PARAMETERS
+# You must provide a value for each of these parameters.
+# ---------------------------------------------------------------------------------------------------------------------
+
+# ---------------------------------------------------------------------------------------------------------------------
+# OPTIONAL PARAMETERS
+# These parameters have reasonable defaults.
+# ---------------------------------------------------------------------------------------------------------------------
+
+variable "location" {
+  description = "The supported azure location where the resource exists"
+  type        = string
+  default     = "West US2"
+}
+
+variable "postfix" {
+  description = "A postfix string to centrally mitigate resource name collisions."
+  type        = string
+  default     = "resource"
+}

go.mod

@@ -53,6 +53,7 @@ require (
 )
 
 require (
+	cloud.google.com/go v0.105.0 // indirect
 	cloud.google.com/go/compute v1.12.1 // indirect
 	cloud.google.com/go/compute/metadata v0.2.1 // indirect
 	cloud.google.com/go/iam v0.7.0 // indirect

modules/azure/client_factory.go

@@ -24,6 +24,7 @@ import (
 	"github.com/Azure/azure-sdk-for-go/services/containerinstance/mgmt/2018-10-01/containerinstance"
 	"github.com/Azure/azure-sdk-for-go/services/containerregistry/mgmt/2019-05-01/containerregistry"
 	"github.com/Azure/azure-sdk-for-go/services/containerservice/mgmt/2019-11-01/containerservice"
+	"github.com/Azure/azure-sdk-for-go/services/datafactory/mgmt/2018-06-01/datafactory"
 	kvmng "github.com/Azure/azure-sdk-for-go/services/keyvault/mgmt/2016-10-01/keyvault"
 	"github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-09-01/network"
 	"github.com/Azure/azure-sdk-for-go/services/resources/mgmt/2019-06-01/subscriptions"
@@ -830,6 +831,35 @@ func CreateSynapseSqlPoolClientE(subscriptionID string) (*synapse.SQLPoolsClient
 	return &synapseSqlPoolClient, nil
 }
 
+// CreateDataFactoriesClientE is a helper function that will setup a synapse client.
+func CreateDataFactoriesClientE(subscriptionID string) (*datafactory.FactoriesClient, error) {
+	// Validate Azure subscription ID
+	subscriptionID, err := getTargetAzureSubscription(subscriptionID)
+	if err != nil {
+		return nil, err
+	}
+
+	// Lookup environment URI
+	baseURI, err := getBaseURI()
+	if err != nil {
+		return nil, err
+	}
+
+	// Create a synapse client
+	dataFactoryClient := datafactory.NewFactoriesClientWithBaseURI(baseURI, subscriptionID)
+
+	// Create an authorizer
+	authorizer, err := NewAuthorizer()
+	if err != nil {
+		return nil, err
+	}
+
+	// Attach authorizer to the client
+	dataFactoryClient.Authorizer = *authorizer
+
+	return &dataFactoryClient, nil
+}
+
 // GetKeyVaultURISuffixE returns the proper KeyVault URI suffix for the configured Azure environment.
 // This function would fail the test if there is an error.
 func GetKeyVaultURISuffixE() (string, error) {

modules/azure/datafactory.go

@@ -0,0 +1,56 @@
+package azure
+
+import (
+	"context"
+
+	"github.com/Azure/azure-sdk-for-go/services/datafactory/mgmt/2018-06-01/datafactory"
+	"github.com/gruntwork-io/terratest/modules/testing"
+	"github.com/stretchr/testify/require"
+)
+
+// DataFactoryExists indicates whether the Data Factory exists for the subscription.
+// This function would fail the test if there is an error.
+func DataFactoryExists(t testing.TestingT, dataFactoryName string, resourceGroupName string, subscriptionID string) bool {
+	exists, err := DataFactoryExistsE(dataFactoryName, resourceGroupName, subscriptionID)
+	require.NoError(t, err)
+	return exists
+}
+
+// DataFactoryExistsE indicates whether the specified Data Factory exists and may return an error.
+func DataFactoryExistsE(dataFactoryName string, resourceGroupName string, subscriptionID string) (bool, error) {
+	_, err := GetDataFactoryE(subscriptionID, resourceGroupName, dataFactoryName)
+	if err != nil {
+		if ResourceNotFoundErrorExists(err) {
+			return false, nil
+		}
+		return false, err
+	}
+	return true, nil
+}
+
+// GetDataFactory is a helper function that gets the synapse workspace.
+// This function would fail the test if there is an error.
+func GetDataFactory(t testing.TestingT, resGroupName string, factoryName string, subscriptionID string) *datafactory.Factory {
+	Workspace, err := GetDataFactoryE(subscriptionID, resGroupName, factoryName)
+	require.NoError(t, err)
+
+	return Workspace
+}
+
+// GetDataFactoryE is a helper function that gets the workspace.
+func GetDataFactoryE(subscriptionID string, resGroupName string, factoryName string) (*datafactory.Factory, error) {
+	// Create a datafactory client
+	datafactoryClient, err := CreateDataFactoriesClientE(subscriptionID)
+	if err != nil {
+		return nil, err
+	}
+
+	// Get the corresponding synapse workspace
+	dataFactory, err := datafactoryClient.Get(context.Background(), resGroupName, factoryName, "")
+	if err != nil {
+		return nil, err
+	}
+
+	//Return synapse workspace
+	return &dataFactory, nil
+}

modules/azure/datafactory_test.go

@@ -0,0 +1,35 @@
+package azure
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+/*
+The below tests are currently stubbed out, with the expectation that they will throw errors.
+If/when CRUD methods are introduced for Azure Synapse, these tests can be extended
+*/
+func TestDataFactoryExists(t *testing.T) {
+	t.Parallel()
+
+	dataFactoryName := ""
+	resourceGroupName := ""
+	subscriptionID := ""
+
+	exists, err := DataFactoryExistsE(dataFactoryName, resourceGroupName, subscriptionID)
+
+	require.False(t, exists)
+	require.Error(t, err)
+}
+
+func TestGetDataFactoryE(t *testing.T) {
+	t.Parallel()
+
+	resGroupName := ""
+	subscriptionID := ""
+	dataFactoryName := ""
+
+	_, err := GetDataFactoryE(subscriptionID, resGroupName, dataFactoryName)
+	require.Error(t, err)
+}

test/azure/terraform_azure_datafactory_example_test.go

@@ -0,0 +1,52 @@
+//go:build azure
+// +build azure
+
+package test
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/gruntwork-io/terratest/modules/azure"
+	"github.com/gruntwork-io/terratest/modules/random"
+	"github.com/gruntwork-io/terratest/modules/terraform"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestTerraformAzureDataFactoryExample(t *testing.T) {
+	t.Parallel()
+
+	uniquePostfix := strings.ToLower(random.UniqueId())
+	expectedDataFactoryProvisioningState := "Succeeded"
+	expectedLocation := "eastus"
+
+	//Configure Terraform setting up a path to Terraform code.
+	terraformOptions := &terraform.Options{
+		// The path to where our Terraform code is located
+		TerraformDir: "../../examples/azure/terraform-azure-datafactory-example",
+		Vars: map[string]interface{}{
+			"postfix":  uniquePostfix,
+			"location": expectedLocation,
+		},
+	}
+
+	// At the end of the test, run `terraform destroy` to clean up any resources that were created
+	defer terraform.Destroy(t, terraformOptions)
+
+	//Run `terraform init` and `terraform apply`. Fail the test if there are any errors.
+	terraform.InitAndApply(t, terraformOptions)
+
+	// Run `terraform output` to get the values of output variables
+	expectedResourceGroupName := terraform.Output(t, terraformOptions, "resource_group_name")
+	expectedDataFactoryName := terraform.Output(t, terraformOptions, "datafactory_name")
+
+	// check for if data factory exists
+	actualDataFactoryExits := azure.DataFactoryExists(t, expectedDataFactoryName, expectedResourceGroupName, "")
+	assert.True(t, actualDataFactoryExits)
+
+	//Get data factory details and assert them against the terraform output
+	actualDataFactory := azure.GetDataFactory(t, expectedResourceGroupName, expectedDataFactoryName, "")
+	assert.Equal(t, expectedDataFactoryName, *actualDataFactory.Name)
+	assert.Equal(t, expectedDataFactoryProvisioningState, *actualDataFactory.FactoryProperties.ProvisioningState)
+
+}

test/azure/terraform_azure_synapse_example_test.go

@@ -22,7 +22,7 @@ func TestTerraformAzureSynapseExample(t *testing.T) {
 	uniquePostfix := strings.ToLower(random.UniqueId())
 	expectedSynapseSqlUser := "sqladminuser"
 	expectedSynapseProvisioningState := "Succeeded"
-	expectedLocation := "West US2"
+	expectedLocation := "westus2"
 	expectedSyPoolSkuName := "DW100c"
 
 	// website::tag::1:: Configure Terraform setting up a path to Terraform code.