feat(dlp): added a sample for dlp inspect send data to hybrid job tri…

…gger (#3247)

dlp/snippets/deid/deid_test.go

@@ -42,7 +42,8 @@ import (
 const (
 	bucketForDeidCloudStorageForInput  = "dlp-test-deid-input"
 	bucketForDeidCloudStorageForOutput = "dlp-test-deid-go-lang-output"
-	filePathToGCSForDeidTest           = "./testdata/dlp_sample.csv"
+	filePathToGCSUploadForDeidTest     = "./testdata/dlp_sample.csv"
+	filePathToGCSForDeidTest           = "/testdata/dlp_sample.csv"
 	tableID                            = "dlp_test_deid_table"
 	dataSetID                          = "dlp_test_deid_dataset"
 
@@ -988,7 +989,7 @@ func filePathtoGCS(projectID string) error {
 	// file upload code
 
 	// Open local file.
-	file, err := os.ReadFile(filePathToGCSForDeidTest)
+	file, err := os.ReadFile(filePathToGCSUploadForDeidTest)
 	if err != nil {
 		log.Fatalf("Failed to read file: %v", err)
 		return err

dlp/snippets/inspect/inspect_send_data_to_hybrid_job_trigger.go

@@ -0,0 +1,141 @@
+// Copyright 2023 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//	https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package inspect
+
+// [START dlp_inspect_send_data_to_hybrid_job_trigger]
+import (
+	"context"
+	"fmt"
+	"io"
+	"log"
+	"time"
+
+	dlp "cloud.google.com/go/dlp/apiv2"
+	"cloud.google.com/go/dlp/apiv2/dlppb"
+)
+
+// inspectDataToHybridJobTrigger uses the Data Loss Prevention API to inspect sensitive
+// information using Hybrid jobs trigger that scans payloads of data sent from
+// virtually any source and stores findings in Google Cloud.
+func inspectDataToHybridJobTrigger(w io.Writer, projectID, textToDeIdentify, jobTriggerName string) error {
+	// projectId := "your-project-id"
+	// jobTriggerName := "your-job-trigger-name"
+	// textToDeIdentify := "My email is test@example.org"
+
+	ctx := context.Background()
+
+	// Initialize a client once and reuse it to send multiple requests. Clients
+	// are safe to use across goroutines. When the client is no longer needed,
+	// call the Close method to cleanup its resources.
+	client, err := dlp.NewClient(ctx)
+	if err != nil {
+		return err
+	}
+
+	// Closing the client safely cleans up background resources.
+	defer client.Close()
+
+	// Specify the content to be inspected.
+	contentItem := &dlppb.ContentItem{
+		DataItem: &dlppb.ContentItem_Value{
+			Value: textToDeIdentify,
+		},
+	}
+
+	// Contains metadata to associate with the content.
+	// Refer to https://cloud.google.com/dlp/docs/reference/rpc/google.privacy.dlp.v2#container for specifying the paths in container object.
+	container := &dlppb.Container{
+		Type:         "logging_sys",
+		FullPath:     "10.0.0.2:logs1:app1",
+		RelativePath: "app1",
+		RootPath:     "10.0.0.2:logs1",
+		Version:      "1.2",
+	}
+
+	// Set the required label.
+	labels := map[string]string{
+		"env":                           "prod",
+		"appointment-bookings-comments": "",
+	}
+
+	hybridFindingDetails := &dlppb.HybridFindingDetails{
+		ContainerDetails: container,
+		Labels:           labels,
+	}
+
+	hybridContentItem := &dlppb.HybridContentItem{
+		Item:           contentItem,
+		FindingDetails: hybridFindingDetails,
+	}
+
+	// Activate the job trigger.
+	activateJobreq := &dlppb.ActivateJobTriggerRequest{
+		Name: jobTriggerName,
+	}
+
+	dlpJob, err := client.ActivateJobTrigger(ctx, activateJobreq)
+	if err != nil {
+		log.Printf("Error from return part %v", err)
+		return err
+	}
+	// Build the hybrid inspect request.
+	req := &dlppb.HybridInspectJobTriggerRequest{
+		Name:       jobTriggerName,
+		HybridItem: hybridContentItem,
+	}
+
+	// Send the hybrid inspect request.
+	_, err = client.HybridInspectJobTrigger(ctx, req)
+	if err != nil {
+		return err
+	}
+
+	getDlpJobReq := &dlppb.GetDlpJobRequest{
+		Name: dlpJob.Name,
+	}
+
+	var result *dlppb.DlpJob
+	for i := 0; i < 5; i++ {
+		// Get DLP job
+		result, err = client.GetDlpJob(ctx, getDlpJobReq)
+		if err != nil {
+			fmt.Printf("Error getting DLP job: %v\n", err)
+			return err
+		}
+
+		// Check if processed bytes is greater than 0
+		if result.GetInspectDetails().GetResult().GetProcessedBytes() > 0 {
+			break
+		}
+
+		// Wait for 5 seconds before checking again
+		time.Sleep(5 * time.Second)
+		i++
+	}
+
+	fmt.Fprintf(w, "Job Name: %v\n", result.Name)
+	fmt.Fprintf(w, "Job State: %v\n", result.State)
+
+	inspectionResult := result.GetInspectDetails().GetResult()
+	fmt.Fprint(w, "Findings: \n")
+	for _, v := range inspectionResult.GetInfoTypeStats() {
+		fmt.Fprintf(w, "Infotype: %v\n", v.InfoType.Name)
+		fmt.Fprintf(w, "Likelihood: %v\n", v.GetCount())
+	}
+
+	fmt.Fprint(w, "successfully inspected data using hybrid job trigger ")
+	return nil
+}
+
+// [END dlp_inspect_send_data_to_hybrid_job_trigger]

dlp/snippets/inspect/inspect_test.go

@@ -26,9 +26,12 @@ import (
 
 	"cloud.google.com/go/bigquery"
 	"cloud.google.com/go/datastore"
+	dlp "cloud.google.com/go/dlp/apiv2"
+	"cloud.google.com/go/dlp/apiv2/dlppb"
 	"cloud.google.com/go/storage"
 	"github.com/GoogleCloudPlatform/golang-samples/internal/testutil"
 	"github.com/google/uuid"
+	"google.golang.org/api/iterator"
 )
 
 const (
@@ -38,6 +41,9 @@ const (
 	ssnFileName = "fake_ssn.txt"
 	bucketName  = "golang-samples-dlp-test2"
 
+	jobTriggerIdPrefix                      = "dlp-job-trigger-unit-test-case-12345678"
+	dataSetIDForHybridJob                   = "dlp_test_dataset"
+	tableIDForHybridJob                     = "dlp_inspect_test_table_table_id"
 	inspectsGCSTestFileName                 = "test.txt"
 	filePathToUpload                        = "./testdata/test.txt"
 	dirPathForInspectGCSSendToScc           = "dlp-go-lang-test-for-inspect-gcs-send-to-scc/"
@@ -649,7 +655,7 @@ func TestInspectDataStoreSendToScc(t *testing.T) {
 	tc := testutil.SystemTest(t)
 	var buf bytes.Buffer
 	u := uuid.New().String()[:8]
-	datastoreNamespace := "golang-samples" + u
+	datastoreNamespace := fmt.Sprint("golang-samples" + u)
 	datastoreKind := "task"
 
 	if err := inspectDataStoreSendToScc(&buf, tc.ProjectID, datastoreNamespace, datastoreKind); err != nil {
@@ -791,3 +797,178 @@ func filePathtoGCS(t *testing.T, projectID, bucketNameForInspectGCSSendToScc, di
 	fmt.Println("filePathtoGCS function is executed-------")
 	return nil
 }
+
+var projectID, jobTriggerForInspectSample string
+
+func TestMain(m *testing.M) {
+	tc, ok := testutil.ContextMain(m)
+	projectID = tc.ProjectID
+	if !ok {
+		log.Fatal("couldn't initialize test")
+		return
+	}
+	xyz, err := createJobTriggerForInspectDataToHybridJobTrigger(tc.ProjectID)
+	jobTriggerForInspectSample = xyz
+	if err != nil {
+		log.Fatal("couldn't initialize test")
+		return
+	}
+	m.Run()
+	deleteActiveJob(tc.ProjectID, jobTriggerForInspectSample)
+	deleteJobTriggerForInspectDataToHybridJobTrigger(tc.ProjectID, jobTriggerForInspectSample)
+}
+
+func TestInspectDataToHybridJobTrigger(t *testing.T) {
+	tc := testutil.SystemTest(t)
+	var buf bytes.Buffer
+	trigger := jobTriggerForInspectSample
+	fmt.Print("Name:" + trigger)
+	if err := inspectDataToHybridJobTrigger(&buf, tc.ProjectID, "My email is test@example.org and my name is Gary.", trigger); err != nil {
+		t.Fatal(err)
+	}
+	got := buf.String()
+	if want := "successfully inspected data using hybrid job trigger"; !strings.Contains(got, want) {
+		t.Errorf("TestInspectDataToHybridJobTrigger got %q, want %q", got, want)
+	}
+	if want := "Findings"; !strings.Contains(got, want) {
+		t.Errorf("TestInspectDataToHybridJobTrigger got %q, want %q", got, want)
+	}
+	if want := "Job State: ACTIVE"; !strings.Contains(got, want) {
+		t.Errorf("TestInspectDataToHybridJobTrigger got %q, want %q", got, want)
+	}
+	if want := "EMAIL_ADDRESS"; !strings.Contains(got, want) {
+		t.Errorf("TestInspectDataToHybridJobTrigger got %q, want %q", got, want)
+	}
+	if want := "PERSON_NAME"; !strings.Contains(got, want) {
+		t.Errorf("TestInspectDataToHybridJobTrigger got %q, want %q", got, want)
+	}
+
+}
+
+func deleteActiveJob(project, trigger string) error {
+
+	ctx := context.Background()
+	client, err := dlp.NewClient(ctx)
+	if err != nil {
+		return err
+	}
+	defer client.Close()
+	req := &dlppb.ListDlpJobsRequest{
+		Parent: fmt.Sprintf("projects/%s/locations/global", project),
+		Filter: fmt.Sprintf("trigger_name=%s", trigger),
+	}
+
+	it := client.ListDlpJobs(ctx, req)
+	var jobIds []string
+	for {
+		j, err := it.Next()
+		jobIds = append(jobIds, j.GetName())
+		if err == iterator.Done {
+			break
+		}
+		if err != nil {
+			fmt.Printf("Next: %v", err)
+		}
+		fmt.Printf("Job %v status: %v\n", j.GetName(), j.GetState())
+	}
+	for _, v := range jobIds {
+		req := &dlppb.DeleteDlpJobRequest{
+			Name: v,
+		}
+		if err = client.DeleteDlpJob(ctx, req); err != nil {
+			fmt.Printf("DeleteDlpJob: %v", err)
+			return err
+		}
+		fmt.Printf("\nSuccessfully deleted job %v\n", v)
+	}
+	fmt.Print("Deleted Job Successfully !!!")
+	return nil
+}
+
+// helpers for inspect hybrid job
+func createJobTriggerForInspectDataToHybridJobTrigger(projectID string) (string, error) {
+
+	log.Printf("[START] createJobTriggerForInspectDataToHybridJobTrigger: projectID %v and ", projectID)
+	// Set up the client.
+	ctx := context.Background()
+	client, err := dlp.NewClient(ctx)
+	if err != nil {
+		return "", err
+	}
+	defer client.Close()
+
+	// Define the job trigger.
+	hybridOptions := &dlppb.HybridOptions{
+		Labels: map[string]string{
+			"env": "prod",
+		},
+	}
+
+	storageConfig := &dlppb.StorageConfig_HybridOptions{
+		HybridOptions: hybridOptions,
+	}
+	infoTypes := []*dlppb.InfoType{
+		{Name: "PERSON_NAME"},
+		{Name: "EMAIL_ADDRESS"},
+	}
+
+	inspectConfig := &dlppb.InspectConfig{
+		InfoTypes: infoTypes,
+	}
+
+	inspectJobConfig := &dlppb.InspectJobConfig{
+		StorageConfig: &dlppb.StorageConfig{
+			Type: storageConfig,
+		},
+		InspectConfig: inspectConfig,
+	}
+
+	trigger := &dlppb.JobTrigger_Trigger{
+		Trigger: &dlppb.JobTrigger_Trigger_Manual{},
+	}
+
+	jobTrigger := &dlppb.JobTrigger{
+		Triggers: []*dlppb.JobTrigger_Trigger{
+			trigger,
+		},
+		Job: &dlppb.JobTrigger_InspectJob{
+			InspectJob: inspectJobConfig,
+		},
+	}
+
+	u := uuid.New().String()[:8]
+	createDlpJobRequest := &dlppb.CreateJobTriggerRequest{
+		Parent:     fmt.Sprintf("projects/%s/locations/global", projectID),
+		JobTrigger: jobTrigger,
+		TriggerId:  jobTriggerIdPrefix + u,
+	}
+
+	resp, err := client.CreateJobTrigger(ctx, createDlpJobRequest)
+	if err != nil {
+		return "", err
+	}
+	log.Printf("[END] createJobTriggerForInspectDataToHybridJobTrigger: trigger.Name %v", resp.Name)
+	return resp.Name, nil
+}
+
+func deleteJobTriggerForInspectDataToHybridJobTrigger(projectID, jobTriggerName string) error {
+
+	log.Printf("\n[START] deleteJobTriggerForInspectDataToHybridJobTrigger")
+	ctx := context.Background()
+	client, err := dlp.NewClient(ctx)
+	if err != nil {
+		return err
+	}
+	defer client.Close()
+
+	req := &dlppb.DeleteJobTriggerRequest{
+		Name: jobTriggerName,
+	}
+
+	err = client.DeleteJobTrigger(ctx, req)
+	if err != nil {
+		return err
+	}
+	log.Print("[END] deleteJobTriggerForInspectDataToHybridJobTrigger")
+	return nil
+}

dlp/snippets/risk/risk_test.go

@@ -18,6 +18,7 @@ package risk
 import (
 	"bytes"
 	"context"
+	"log"
 	"strings"
 	"testing"
 	"time"
@@ -50,6 +51,7 @@ func TestRisk(t *testing.T) {
 				err := riskNumerical(buf, tc.ProjectID, "bigquery-public-data", riskTopicName+u, riskSubscriptionName+u, "nhtsa_traffic_fatalities", "accident_2015", "state_number")
 				defer cleanupPubsub(t, client, riskTopicName+u, riskSubscriptionName+u)
 				if err != nil {
+					log.Printf("%v\n", err)
 					r.Errorf("riskNumerical got err: %v", err)
 					return
 				}