Add Cosmos DB container and pkg/database client for PlatformWorkloadIdentityRoleSets

[kimorris27]

Which issue this PR addresses:

https://issues.redhat.com/browse/ARO-6448

This is the first of two PRs for this ticket. The API endpoints will come in a separate PR.

What this PR does / why we need it:

This PR adds a new Cosmos DB container for storing information about OCP operator managed identities and permissions and a database client for the RP to use to interact with that container.

Test plan for issue:

I deployed a full service RP and wrote a script that I ran locally to test all of the functionality of the database client. I could test in INT as well if we feel that it's necessary, but IMO we might as well wait until https://issues.redhat.com/browse/ARO-6449 is ready to be tested in INT to save a bit of time.

Is there any documentation that needs to be updated for this PR?

No

How do you know this will function as expected in production?

The functionality of this database client will be validated as part of testing for https://issues.redhat.com/browse/ARO-6449 - see comments in "Test plan for issue" section.

---

How to test (requires a full service dev RP)

First, grant the shared dev RP service principal some additional permissions to allow it to authenticate to Key Vault and Cosmos DB. These steps assume that you have your own full service RP running, you deployed it using the contents of this PR branch, and you have sourced your env-int file.

export AZURE_RP_OBJECT_ID=$(az ad sp show --id ${AZURE_RP_CLIENT_ID} --query id | tr -d "\"")
export COSMOS_DB_ACCOUNT_ID=$(az cosmosdb show -g $USER-aro-${LOCATION} -n ${USER}-aro-${LOCATION} --query id | tr -d "\"")

# Grant access to the ARO service Key Vault
az keyvault set-policy -g ${USER}-aro-${LOCATION} -n ${USER}-aro-${LOCATION}-svc --secret-permissions get list --object-id ${AZURE_RP_OBJECT_ID}

# Grant access to your full service RP's Cosmos DB account using the "DocumentDB Account Contributor" role
az role assignment create --role 5bd9cd88-fe45-4216-938b-f97437e15450 --assignee ${AZURE_RP_OBJECT_ID} --scope ${COSMOS_DB_ACCOUNT_ID}

Once you've created the Key Vault access policy and the Document DB Account Contributor role assignment, you can put this script in your RP's root directory and run it with go run <filepath>:

package main

// Copyright (c) Microsoft Corporation.
// Licensed under the Apache License 2.0.

import (
	"context"
	"fmt"
	"os"
	"strings"
	"time"

	"github.com/Azure/azure-sdk-for-go/sdk/azcore/arm/policy"

	"github.com/Azure/ARO-RP/pkg/api"
	"github.com/Azure/ARO-RP/pkg/database"
	"github.com/Azure/ARO-RP/pkg/env"
	"github.com/Azure/ARO-RP/pkg/metrics/statsd"
	"github.com/Azure/ARO-RP/pkg/util/encryption"
	"github.com/Azure/ARO-RP/pkg/util/keyvault"
	utillog "github.com/Azure/ARO-RP/pkg/util/log"
)

const (
	envDatabaseName        = "DATABASE_NAME"
	envDatabaseAccountName = "DATABASE_ACCOUNT_NAME"
	envKeyVaultPrefix      = "KEYVAULT_PREFIX"
)

func DBName(isLocalDevelopmentMode bool) (string, error) {
	if !isLocalDevelopmentMode {
		return "ARO", nil
	}

	if err := env.ValidateVars(envDatabaseName); err != nil {
		return "", fmt.Errorf("%v (development mode)", err.Error())
	}

	return os.Getenv(envDatabaseName), nil
}

func main() {
	ctx := context.Background()
	log := utillog.GetLogger()

	// Using a random component bc it would be tiresome to add one just for this test.
	_env, err := env.NewCore(ctx, log, env.COMPONENT_UPDATE_OCP_VERSIONS)
	if err != nil {
		log.Fatal(err)
	}

	msiToken, err := _env.NewMSITokenCredential()
	if err != nil {
		log.Fatal(err)
	}

	msiKVAuthorizer, err := _env.NewMSIAuthorizer(_env.Environment().KeyVaultScope)
	if err != nil {
		log.Fatal(err)
	}

	m := statsd.New(ctx, log.WithField("component", "update-platform-workload-identity-role-sets"), _env, os.Getenv("MDM_ACCOUNT"), os.Getenv("MDM_NAMESPACE"), os.Getenv("MDM_STATSD_SOCKET"))

	if err := env.ValidateVars(envKeyVaultPrefix); err != nil {
		log.Fatal(err)
	}
	keyVaultPrefix := os.Getenv(envKeyVaultPrefix)
	serviceKeyvaultURI := keyvault.URI(_env, env.ServiceKeyvaultSuffix, keyVaultPrefix)
	serviceKeyvault := keyvault.NewManager(msiKVAuthorizer, serviceKeyvaultURI)

	aead, err := encryption.NewMulti(ctx, serviceKeyvault, env.EncryptionSecretV2Name, env.EncryptionSecretName)
	if err != nil {
		log.Fatal(err)
	}

	if err := env.ValidateVars(envDatabaseAccountName); err != nil {
		log.Fatal(err)
	}

	dbAccountName := os.Getenv(envDatabaseAccountName)
	clientOptions := &policy.ClientOptions{
		ClientOptions: _env.Environment().ManagedIdentityCredentialOptions().ClientOptions,
	}
	logrusEntry := log.WithField("component", "database")
	dbAuthorizer, err := database.NewMasterKeyAuthorizer(ctx, logrusEntry, msiToken, clientOptions, _env.SubscriptionID(), _env.ResourceGroup(), dbAccountName)
	if err != nil {
		log.Fatal(err)
	}

	dbc, err := database.NewDatabaseClient(log.WithField("component", "database"), _env, dbAuthorizer, m, aead, dbAccountName)
	if err != nil {
		log.Fatal(err)
	}

	dbName, err := DBName(_env.IsLocalDevelopmentMode())
	if err != nil {
		log.Fatal(err)
	}
	dbPlatformWorkloadIdentityRoleSets, err := database.NewPlatformWorkloadIdentityRoleSets(ctx, dbc, dbName)
	if err != nil {
		log.Fatal(err)
	}
	log.Info("Successfully initialized PlatformWorkloadIdentityRoleSets database client")

	properties := api.PlatformWorkloadIdentityRoleSetProperties{
		OpenShiftVersion: "4.14",
		PlatformWorkloadIdentityRoles: []api.PlatformWorkloadIdentityRole{
			{
				OperatorName:       "CloudControllerManager",
				RoleDefinitionName: "Azure RedHat OpenShift Cloud Controller Manager Role",
				RoleDefinitionID:   "/providers/Microsoft.Authorization/roleDefinitions/a1f96423-95ce-4224-ab27-4e3dc72facd4",
				ServiceAccounts: []string{
					"openshift-cloud-controller-manager:cloud-controller-manager",
				},
			},
			{
				OperatorName:       "ClusterIngressOperator",
				RoleDefinitionName: "Azure RedHat OpenShift Cluster Ingress Operator Role",
				RoleDefinitionID:   "/providers/Microsoft.Authorization/roleDefinitions/0336e1d3-7a87-462b-b6db-342b63f7802c",
				ServiceAccounts: []string{
					"openshift-ingress-operator:ingress-operator",
				},
			},
		},
	}

	doc := api.PlatformWorkloadIdentityRoleSetDocument{
		ID: dbPlatformWorkloadIdentityRoleSets.NewUUID(),
		PlatformWorkloadIdentityRoleSet: &api.PlatformWorkloadIdentityRoleSet{
			Properties: properties,
		},
	}

	// Create
	resultDoc, err := dbPlatformWorkloadIdentityRoleSets.Create(ctx, &doc)
	if err != nil {
		panic(err)
	}
	log.Info("Successfully created a new document with Create")

	// Get
	resultDoc, err = dbPlatformWorkloadIdentityRoleSets.Get(ctx, resultDoc.ID)
	if err != nil {
		panic(err)
	}
	log.Info("Successfully retrieved a document with Get... logging the document contents below:")
	log.Infof("DB document: %+v", *resultDoc)
	log.Infof("PlatformWorkloadIdentityRoleSetProperties: %+v", resultDoc.PlatformWorkloadIdentityRoleSet.Properties)

	// ChangeFeed
	iterator := dbPlatformWorkloadIdentityRoleSets.ChangeFeed()
	docsFromIterator, err := iterator.Next(ctx, -1)
	if err != nil {
		panic(err)
	}
	log.Info("Successfully listed documents using ChangeFeed; logging the document contents below:")

	for _, doc := range docsFromIterator.PlatformWorkloadIdentityRoleSetDocuments {
		log.Infof("PlatformWorkloadIdentityRoleSetProperties: %+v", doc.PlatformWorkloadIdentityRoleSet.Properties)
	}

	// Update
	resultDoc.PlatformWorkloadIdentityRoleSet.Properties.PlatformWorkloadIdentityRoles = []api.PlatformWorkloadIdentityRole{
		resultDoc.PlatformWorkloadIdentityRoleSet.Properties.PlatformWorkloadIdentityRoles[0],
	}

	resultDoc, err = dbPlatformWorkloadIdentityRoleSets.Update(ctx, resultDoc)
	if err != nil {
		panic(err)
	}

	if len(resultDoc.PlatformWorkloadIdentityRoleSet.Properties.PlatformWorkloadIdentityRoles) != 1 {
		panic("Update didn't work - role set still has two roles instead of one")
	}
	log.Info("Successfully replaced a document with Update")

	// ListAll
	resultDocs, err := dbPlatformWorkloadIdentityRoleSets.ListAll(ctx)
	if err != nil {
		panic(err)
	}
	log.Info("Successfully retrieved document list with ListAll.. logging the document contents below:")
	log.Infof("DB document list: %+v", *resultDocs)

	// Delete
	err = dbPlatformWorkloadIdentityRoleSets.Delete(ctx, resultDoc)
	if err != nil {
		panic(err)
	}
	log.Info("Successfully deleted a document with Delete")

	// Reinitialize the doc and then test deletion via change feed, which will indirectly test Patch
	doc = api.PlatformWorkloadIdentityRoleSetDocument{
		ID: dbPlatformWorkloadIdentityRoleSets.NewUUID(),
		PlatformWorkloadIdentityRoleSet: &api.PlatformWorkloadIdentityRoleSet{
			Properties: properties,
		},
	}

	resultDoc, err = dbPlatformWorkloadIdentityRoleSets.Create(ctx, &doc)
	if err != nil {
		panic(err)
	}

	patchFunc := func(doc *api.PlatformWorkloadIdentityRoleSetDocument) error {
		doc.PlatformWorkloadIdentityRoleSet.Deleting = true
		doc.TTL = 5
		return nil
	}

	// Patch
	resultDoc, err = dbPlatformWorkloadIdentityRoleSets.Patch(ctx, resultDoc.ID, patchFunc)
	if err != nil {
		panic(err)
	}
	log.Info("Successfully marked a document for deletion using Patch to give it a TTL; waiting for automatic deletion to take place...")

	time.Sleep(7 * time.Second)

	// Verify that doc is no longer there
	resultDoc, err = dbPlatformWorkloadIdentityRoleSets.Get(ctx, resultDoc.ID)
	if err == nil {
		panic("Expected not to find document after patching it with a TTL")
	} else if !(strings.Contains(err.Error(), "404 NotFound")) {
		panic(err)
	}
	log.Info("Successfully deleted a document by patching it with a TTL")
}

[cadenmarchese]

lgtm!

[cadenmarchese]

/azp run ci,e2e

[azure-pipelines]

Azure Pipelines successfully started running 2 pipeline(s).

[kimorris27]

/azp run ci, e2e

[azure-pipelines]

Azure Pipelines successfully started running 2 pipeline(s).