Lambda-promtail: Add skip tls verify option (#8059)

**What this PR does / why we need it**:
This PR sets a possibility to use lambda-promtail with self signed
certificates sometimes used in development systems. This setting is
configurable via environment variable.

**Which issue(s) this PR fixes**:
Fixes #8013

Signed-off-by: Thomas Belian <thomas.belian@bt909.de>

tools/lambda-promtail/README.md

@@ -48,22 +48,23 @@ The `lambda-promtail` code picks this value up via an environment variable.
 Also, if your deployment requires a [VPC configuration](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lambda_function#vpc_config), make sure to edit the `vpc_config` field in `main.tf` manually. Additonal documentation for the Lambda specific Terraform configuration is [here](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lambda_function#vpc_config). If you want to link kinesis data stream to Lambda as event source, see [here](https://docs.aws.amazon.com/ko_kr/lambda/latest/dg/with-kinesis.html).
 
 `lambda-promtail` supports authentication either using HTTP Basic Auth or using Bearer Token.  
+For development purposes you can set the environment variable SKIP_TLS_VERIFY to `true`, so you can use self signed certificates, but this is not recommended in production. Default is `false`.
 
 Then use Terraform to deploy:
 
 ```bash
 ## use cloudwatch log group
-terraform apply -var "<ecr-repo>:<tag>" -var "write_address=https://your-loki-url/loki/api/v1/push" -var "password=<basic-auth-pw>" -var "username=<basic-auth-username>" -var 'bearer_token=<bearer-token>' -var 'log_group_names=["log-group-01", "log-group-02"]' -var 'extra_labels="name1,value1,name2,value2"' -var "tenant_id=<value>"
+terraform apply -var "<ecr-repo>:<tag>" -var "write_address=https://your-loki-url/loki/api/v1/push" -var "password=<basic-auth-pw>" -var "username=<basic-auth-username>" -var 'bearer_token=<bearer-token>' -var 'log_group_names=["log-group-01", "log-group-02"]' -var 'extra_labels="name1,value1,name2,value2"' -var "tenant_id=<value>" -var 'skip_tls_verify="false"'
 ```
 
 ```bash
 ## use kinesis data stream
-terraform apply -var "<ecr-repo>:<tag>" -var "write_address=https://your-loki-url/loki/api/v1/push" -var "password=<basic-auth-pw>" -var "username=<basic-auth-username>" -var 'kinesis_stream_name=["kinesis-stream-01", "kinesis-stream-02"]' -var 'extra_labels="name1,value1,name2,value2"' -var "tenant_id=<value>"
+terraform apply -var "<ecr-repo>:<tag>" -var "write_address=https://your-loki-url/loki/api/v1/push" -var "password=<basic-auth-pw>" -var "username=<basic-auth-username>" -var 'kinesis_stream_name=["kinesis-stream-01", "kinesis-stream-02"]' -var 'extra_labels="name1,value1,name2,value2"' -var "tenant_id=<value>" -var 'skip_tls_verify="false"'
 
 or CloudFormation:
 
 ```bash
-aws cloudformation create-stack --stack-name lambda-promtail-stack --template-body file://template.yaml --capabilities CAPABILITY_IAM CAPABILITY_NAMED_IAM --region us-east-2 --parameters ParameterKey=WriteAddress,ParameterValue=https://your-loki-url/loki/api/v1/push ParameterKey=Username,ParameterValue=<basic-auth-username> ParameterKey=Password,ParameterValue=<basic-auth-pw> ParameterKey=BearerToken,ParameterValue=<bearer-token> ParameterKey=LambdaPromtailImage,ParameterValue=<ecr-repo>:<tag> ParameterKey=ExtraLabels,ParameterValue="name1,value1,name2,value2" ParameterKey=TenantID,ParameterValue=<value>
+aws cloudformation create-stack --stack-name lambda-promtail-stack --template-body file://template.yaml --capabilities CAPABILITY_IAM CAPABILITY_NAMED_IAM --region us-east-2 --parameters ParameterKey=WriteAddress,ParameterValue=https://your-loki-url/loki/api/v1/push ParameterKey=Username,ParameterValue=<basic-auth-username> ParameterKey=Password,ParameterValue=<basic-auth-pw> ParameterKey=BearerToken,ParameterValue=<bearer-token> ParameterKey=LambdaPromtailImage,ParameterValue=<ecr-repo>:<tag> ParameterKey=ExtraLabels,ParameterValue="name1,value1,name2,value2" ParameterKey=TenantID,ParameterValue=<value> ParameterKey=SkipTlsVerify,ParameterValue="false"
 ```
 
 # Appendix

tools/lambda-promtail/lambda-promtail/main.go

@@ -33,6 +33,7 @@ var (
 	batchSize                                                 int
 	s3Clients                                                 map[string]*s3.Client
 	extraLabels                                               model.LabelSet
+	skipTlsVerify                                             bool
 )
 
 func setupArguments() {
@@ -68,6 +69,12 @@ func setupArguments() {
 		panic("both username and bearerToken are not allowed")
 	}
 
+	skipTls := os.Getenv("SKIP_TLS_VERIFY")
+	// Anything other than case-insensitive 'true' is treated as 'false'.
+	if strings.EqualFold(skipTls, "true") {
+		skipTlsVerify = true
+	}
+
 	tenantID = os.Getenv("TENANT_ID")
 
 	keep := os.Getenv("KEEP_STREAM")

tools/lambda-promtail/lambda-promtail/promtail.go

@@ -4,6 +4,7 @@ import (
 	"bufio"
 	"bytes"
 	"context"
+	"crypto/tls"
 	"fmt"
 	"io"
 	"net/http"
@@ -189,7 +190,13 @@ func send(ctx context.Context, buf []byte) (int, error) {
 		req.Header.Set("Authorization", "Bearer "+bearerToken)
 	}
 
-	resp, err := http.DefaultClient.Do(req.WithContext(ctx))
+	promtailClient := &http.Client{}
+
+	if skipTlsVerify == true {
+		promtailClient = &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}}
+	}
+
+	resp, err := promtailClient.Do(req.WithContext(ctx))
 	if err != nil {
 		return -1, err
 	}

tools/lambda-promtail/main.tf

@@ -106,14 +106,15 @@ resource "aws_lambda_function" "lambda_promtail" {
 
   environment {
     variables = {
-      WRITE_ADDRESS = var.write_address
-      USERNAME      = var.username
-      PASSWORD      = var.password
-      BEARER_TOKEN  = var.bearer_token
-      KEEP_STREAM   = var.keep_stream
-      BATCH_SIZE    = var.batch_size
-      EXTRA_LABELS  = var.extra_labels
-      TENANT_ID     = var.tenant_id
+      WRITE_ADDRESS   = var.write_address
+      USERNAME        = var.username
+      PASSWORD        = var.password
+      BEARER_TOKEN    = var.bearer_token
+      KEEP_STREAM     = var.keep_stream
+      BATCH_SIZE      = var.batch_size
+      EXTRA_LABELS    = var.extra_labels
+      TENANT_ID       = var.tenant_id
+      SKIP_TLS_VERIFY = var.skip_tls_verify
     }
   }
 

tools/lambda-promtail/template.yaml

@@ -43,6 +43,10 @@ Parameters:
     Description: Tenant ID to be added when writing logs from lambda-promtail.
     Type: String
     Default: ""
+  SkipTlsVerify:
+    Description: Determines whether to verify the TLS certificate
+    Type: String
+    Default: "false"
 
 Resources:
   LambdaPromtailRole:
@@ -91,6 +95,7 @@ Resources:
           KEEP_STREAM: !Ref KeepStream
           EXTRA_LABELS: !Ref ExtraLabels
           TENANT_ID: !Ref TenantID
+          SKIP_TLS_VERIFY: !Ref SkipTlsVerify
   LambdaPromtailVersion:
     Type: AWS::Lambda::Version
     Properties:

tools/lambda-promtail/variables.tf

@@ -84,6 +84,12 @@ variable "kms_key_arn" {
   default     = ""
 }
 
+variable "skip_tls_verify" {
+  type        = string
+  description = "Determines whether to verify the TLS certificate"
+  default     = "false"
+}
+
 variable "kinesis_stream_name" {
   type        = list(string)
   description = "Enter kinesis name if kinesis stream is configured as event source in lambda."