Merge pull request #619 from pulumi/dixler/error-example-diagnostics

Adds a new markdown representation of example conversion failures

pkg/tfgen/docs.go

@@ -19,9 +19,6 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"github.com/hashicorp/go-multierror"
-	"golang.org/x/text/cases"
-	"golang.org/x/text/language"
 	"io"
 	"os"
 	"os/exec"
@@ -32,6 +29,10 @@ import (
 	"strings"
 	"sync"
 
+	"github.com/hashicorp/go-multierror"
+	"golang.org/x/text/cases"
+	"golang.org/x/text/language"
+
 	"github.com/pulumi/pulumi-terraform-bridge/v3/pkg/tf2pulumi/gen/python"
 	"github.com/pulumi/pulumi/sdk/v3/go/common/tokens"
 	"github.com/pulumi/pulumi/sdk/v3/go/common/util/contract"
@@ -1295,7 +1296,7 @@ func (g *Generator) convertHCLToString(hcl, path, languageName string) (string,
 		convertedHcl = strings.TrimSpace(string(output))
 	}
 
-	g.coverageTracker.languageConversionSuccess(languageName)
+	g.coverageTracker.languageConversionSuccess(languageName, convertedHcl)
 	return convertedHcl, nil
 }
 
@@ -1405,10 +1406,14 @@ func (g *Generator) convertHCL(hcl, path, exampleTitle string, languages []strin
 	}
 
 	result.WriteString(hclConversionsToString(hclConversions))
+	if len(failedLangs) == 0 {
+		return result.String(), nil
+	}
 
-	if len(failedLangs) == len(languages) {
-		hclAllLangsConversionFailures++
+	isCompleteFailure := len(failedLangs) == len(languages)
 
+	if isCompleteFailure {
+		hclAllLangsConversionFailures++
 		if exampleTitle == "" {
 			g.warn(fmt.Sprintf("unable to convert HCL example for Pulumi entity '%s': %v. The example will be dropped "+
 				"from any generated docs or SDKs.", path, err))
@@ -1421,22 +1426,20 @@ func (g *Generator) convertHCL(hcl, path, exampleTitle string, languages []strin
 	}
 
 	// Log the results when an example fails to convert to some languages, but not all
-	if len(failedLangs) > 0 && len(failedLangs) < len(languages) {
-		var failedLangsStrings []string
-
-		for lang := range failedLangs {
-			failedLangsStrings = append(failedLangsStrings, lang)
-
-			switch lang {
-			case convert.LanguageTypescript:
-				hclTypeScriptPartialConversionFailures++
-			case convert.LanguagePython:
-				hclPythonPartialConversionFailures++
-			case convert.LanguageCSharp:
-				hclCSharpPartialConversionFailures++
-			case convert.LanguageGo:
-				hclGoPartialConversionFailures++
-			}
+	var failedLangsStrings []string
+
+	for lang := range failedLangs {
+		failedLangsStrings = append(failedLangsStrings, lang)
+
+		switch lang {
+		case convert.LanguageTypescript:
+			hclTypeScriptPartialConversionFailures++
+		case convert.LanguagePython:
+			hclPythonPartialConversionFailures++
+		case convert.LanguageCSharp:
+			hclCSharpPartialConversionFailures++
+		case convert.LanguageGo:
+			hclGoPartialConversionFailures++
 		}
 
 		if exampleTitle == "" {

pkg/tfgen/examples_coverage_exporter.go

@@ -56,6 +56,10 @@ func (ce *coverageExportUtil) tryExport(outputDirectory string) error {
 	if err != nil {
 		return err
 	}
+	err = ce.exportMarkdown(outputDirectory, "summary.md")
+	if err != nil {
+		return err
+	}
 	return ce.exportHumanReadable(outputDirectory, "shortSummary.txt")
 }
 
@@ -294,6 +298,120 @@ func (ce *coverageExportUtil) exportOverall(outputDirectory string, fileName str
 	return marshalAndWriteJSON(providerStatistic, jsonOutputLocation)
 }
 
+// The fifth mode, which provides outputs a markdown file with:
+// - the example's name
+// - the original HCL
+// - the conversion results for all languages
+func (ce *coverageExportUtil) exportMarkdown(outputDirectory string, fileName string) error {
+
+	// The Coverage Tracker data structure is flattened down to the example level, and they all
+	// get individually written to the file in order to not have the "{ }" brackets at the start and end
+	type FlattenedExample struct {
+		ExampleName       string
+		OriginalHCL       string `json:"OriginalHCL,omitempty"`
+		ConversionResults map[string]*LanguageConversionResult
+	}
+
+	// All the examples in the tracker are iterated by page ID + index, and marshalled into one large byte
+	// array separated by \n, making the end result look like a bunch of Json files that got concatenated
+	var brokenExamples []FlattenedExample
+
+	for _, page := range ce.Tracker.EncounteredPages {
+		for index, example := range page.Examples {
+			flattenedName := page.Name
+			if len(page.Examples) > 1 {
+				flattenedName += fmt.Sprintf("#%d", index)
+			}
+
+			noErrors := true
+			for _, result := range example.ConversionResults {
+				if result.FailureSeverity == Success {
+					continue
+				}
+				noErrors = false
+			}
+			if noErrors {
+				break
+			}
+
+			brokenExamples = append(brokenExamples, FlattenedExample{
+				ExampleName:       flattenedName,
+				OriginalHCL:       example.OriginalHCL,
+				ConversionResults: example.ConversionResults,
+			})
+		}
+	}
+	targetFile, err := createEmptyFile(outputDirectory, fileName)
+	if err != nil {
+		return err
+	}
+
+	out := ""
+	for _, example := range brokenExamples {
+
+		successes := make(map[string]LanguageConversionResult)
+		failures := make(map[string]LanguageConversionResult)
+
+		for lang, result := range example.ConversionResults {
+			if result.FailureSeverity == Success {
+				successes[lang] = *result
+				continue
+			}
+			failures[lang] = *result
+		}
+
+		isCompleteFailure := len(successes) == 0
+
+		// print example header
+		summaryText := "*partial failure*"
+		if isCompleteFailure {
+			summaryText = "**complete failure**"
+		}
+
+		out += "\n---\n"
+		out += fmt.Sprintf("\n## [%s] %s\n", summaryText, example.ExampleName)
+
+		// print original HCL
+		out += "\n### HCL\n"
+		out += "\n```terraform\n"
+		out += example.OriginalHCL + "\n"
+		out += "\n```\n"
+
+		// print failures
+		out += "\n### Failed Languages\n"
+		for lang, err := range failures {
+			out += fmt.Sprintf("\n#### %s\n", lang)
+			out += "\n```text\n"
+
+			errMsg := err.FailureInfo
+			if len(err.FailureInfo) > 1000 {
+				// truncate extremely long error messages
+				errMsg = err.FailureInfo[:1000]
+			}
+			out += errMsg
+			out += "\n```\n"
+		}
+
+		if isCompleteFailure {
+			// it's a complete failure, no successes to print
+			continue
+		}
+
+		// print successes
+		out += "\n### Successes\n"
+		for lang, success := range successes {
+			out += "\n<details>\n"
+			out += fmt.Sprintf("\n<summary>%s</summary>\n", lang)
+			out += fmt.Sprintf("\n```%s\n", lang)
+			out += success.Program
+			out += "\n```\n"
+			out += "\n</details>\n"
+		}
+	}
+
+	return os.WriteFile(targetFile, []byte(out), 0600)
+}
+
 // The fourth mode, which simply gives the provider name, and success percentage.
 func (ce *coverageExportUtil) exportHumanReadable(outputDirectory string, fileName string) error {
 

pkg/tfgen/examples_coverage_tracker.go

@@ -76,6 +76,7 @@ type Example struct {
 type LanguageConversionResult struct {
 	FailureSeverity int    // This conversion's outcome: [Success, Warning, Failure, Fatal]
 	FailureInfo     string // Additional in-depth information
+	Program         string // Converted program
 
 	// How many times this example has been converted to this language.
 	// It is expected that this will be equal to 1.
@@ -116,14 +117,15 @@ func (ct *CoverageTracker) foundExample(pageName string, hcl string) {
 }
 
 // Used when: current example has been successfully converted to a certain language
-func (ct *CoverageTracker) languageConversionSuccess(languageName string) {
+func (ct *CoverageTracker) languageConversionSuccess(languageName string, program string) {
 	if ct == nil {
 		return
 	}
 	ct.insertLanguageConversionResult(languageName, LanguageConversionResult{
 		FailureSeverity:  Success,
 		FailureInfo:      "",
 		TranslationCount: 1,
+		Program:          program,
 	})
 }