junit_report.go

/*

JUnit XML Reporter for Ginkgo

For usage instructions: http://onsi.github.io/ginkgo/#generating_junit_xml_output

The schema used for the generated JUnit xml file was adapted from https://llg.cubic.org/docs/junit/

*/

package reporters

import (
	"encoding/xml"
	"fmt"
	"os"
	"path"
	"regexp"
	"strings"

	"github.com/onsi/ginkgo/v2/config"
	"github.com/onsi/ginkgo/v2/types"
)

type JunitReportConfig struct {
	// Spec States for which no timeline should be emitted for system-err
	// set this to types.SpecStatePassed|types.SpecStateSkipped|types.SpecStatePending to only match failing specs
	OmitTimelinesForSpecState types.SpecState

	// Enable OmitFailureMessageAttr to prevent failure messages appearing in the "message" attribute of the Failure and Error tags
	OmitFailureMessageAttr bool

	//Enable OmitCapturedStdOutErr to prevent captured stdout/stderr appearing in system-out
	OmitCapturedStdOutErr bool

	// Enable OmitSpecLabels to prevent labels from appearing in the spec name
	OmitSpecLabels bool

	// Enable OmitLeafNodeType to prevent the spec leaf node type from appearing in the spec name
	OmitLeafNodeType bool

	// Enable OmitSuiteSetupNodes to prevent the creation of testcase entries for setup nodes
	OmitSuiteSetupNodes bool
}

type JUnitTestSuites struct {
	XMLName xml.Name `xml:"testsuites"`
	// Tests maps onto the total number of specs in all test suites (this includes any suite nodes such as BeforeSuite)
	Tests int `xml:"tests,attr"`
	// Disabled maps onto specs that are pending and/or skipped
	Disabled int `xml:"disabled,attr"`
	// Errors maps onto specs that panicked or were interrupted
	Errors int `xml:"errors,attr"`
	// Failures maps onto specs that failed
	Failures int `xml:"failures,attr"`
	// Time is the time in seconds to execute all test suites
	Time float64 `xml:"time,attr"`

	//The set of all test suites
	TestSuites []JUnitTestSuite `xml:"testsuite"`
}

type JUnitTestSuite struct {
	// Name maps onto the description of the test suite - maps onto Report.SuiteDescription
	Name string `xml:"name,attr"`
	// Package maps onto the absolute path to the test suite - maps onto Report.SuitePath
	Package string `xml:"package,attr"`
	// Tests maps onto the total number of specs in the test suite (this includes any suite nodes such as BeforeSuite)
	Tests int `xml:"tests,attr"`
	// Disabled maps onto specs that are pending
	Disabled int `xml:"disabled,attr"`
	// Skiped maps onto specs that are skipped
	Skipped int `xml:"skipped,attr"`
	// Errors maps onto specs that panicked or were interrupted
	Errors int `xml:"errors,attr"`
	// Failures maps onto specs that failed
	Failures int `xml:"failures,attr"`
	// Time is the time in seconds to execute all the test suite - maps onto Report.RunTime
	Time float64 `xml:"time,attr"`
	// Timestamp is the ISO 8601 formatted start-time of the suite - maps onto Report.StartTime
	Timestamp string `xml:"timestamp,attr"`

	//Properties captures the information stored in the rest of the Report type (including SuiteConfig) as key-value pairs
	Properties JUnitProperties `xml:"properties"`

	//TestCases capture the individual specs
	TestCases []JUnitTestCase `xml:"testcase"`
}

type JUnitProperties struct {
	Properties []JUnitProperty `xml:"property"`
}

func (jup JUnitProperties) WithName(name string) string {
	for _, property := range jup.Properties {
		if property.Name == name {
			return property.Value
		}
	}
	return ""
}

type JUnitProperty struct {
	Name  string `xml:"name,attr"`
	Value string `xml:"value,attr"`
}

var ownerRE = regexp.MustCompile(`(?i)^owner:(.*)$`)

type JUnitTestCase struct {
	// Name maps onto the full text of the spec - equivalent to "[SpecReport.LeafNodeType] SpecReport.FullText()"
	Name string `xml:"name,attr"`
	// Classname maps onto the name of the test suite - equivalent to Report.SuiteDescription
	Classname string `xml:"classname,attr"`
	// Status maps onto the string representation of SpecReport.State
	Status string `xml:"status,attr"`
	// Time is the time in seconds to execute the spec - maps onto SpecReport.RunTime
	Time float64 `xml:"time,attr"`
	// Owner is the owner the spec - is set if a label matching Label("owner:X") is provided.  The last matching label is used as the owner, thereby allowing specs to override owners specified in container nodes.
	Owner string `xml:"owner,attr,omitempty"`
	//Skipped is populated with a message if the test was skipped or pending
	Skipped *JUnitSkipped `xml:"skipped,omitempty"`
	//Error is populated if the test panicked or was interrupted
	Error *JUnitError `xml:"error,omitempty"`
	//Failure is populated if the test failed
	Failure *JUnitFailure `xml:"failure,omitempty"`
	//SystemOut maps onto any captured stdout/stderr output - maps onto SpecReport.CapturedStdOutErr
	SystemOut string `xml:"system-out,omitempty"`
	//SystemOut maps onto any captured GinkgoWriter output - maps onto SpecReport.CapturedGinkgoWriterOutput
	SystemErr string `xml:"system-err,omitempty"`
}

type JUnitSkipped struct {
	// Message maps onto "pending" if the test was marked pending, "skipped" if the test was marked skipped, and "skipped - REASON" if the user called Skip(REASON)
	Message string `xml:"message,attr"`
}

type JUnitError struct {
	//Message maps onto the panic/exception thrown - equivalent to SpecReport.Failure.ForwardedPanic - or to "interrupted"
	Message string `xml:"message,attr"`
	//Type is one of "panicked" or "interrupted"
	Type string `xml:"type,attr"`
	//Description maps onto the captured stack trace for a panic, or the failure message for an interrupt which will include the dump of running goroutines
	Description string `xml:",chardata"`
}

type JUnitFailure struct {
	//Message maps onto the failure message - equivalent to SpecReport.Failure.Message
	Message string `xml:"message,attr"`
	//Type is "failed"
	Type string `xml:"type,attr"`
	//Description maps onto the location and stack trace of the failure
	Description string `xml:",chardata"`
}

func GenerateJUnitReport(report types.Report, dst string) error {
	return GenerateJUnitReportWithConfig(report, dst, JunitReportConfig{})
}

func GenerateJUnitReportWithConfig(report types.Report, dst string, config JunitReportConfig) error {
	suite := JUnitTestSuite{
		Name:      report.SuiteDescription,
		Package:   report.SuitePath,
		Time:      report.RunTime.Seconds(),
		Timestamp: report.StartTime.Format("2006-01-02T15:04:05"),
		Properties: JUnitProperties{
			Properties: []JUnitProperty{
				{"SuiteSucceeded", fmt.Sprintf("%t", report.SuiteSucceeded)},
				{"SuiteHasProgrammaticFocus", fmt.Sprintf("%t", report.SuiteHasProgrammaticFocus)},
				{"SpecialSuiteFailureReason", strings.Join(report.SpecialSuiteFailureReasons, ",")},
				{"SuiteLabels", fmt.Sprintf("[%s]", strings.Join(report.SuiteLabels, ","))},
				{"RandomSeed", fmt.Sprintf("%d", report.SuiteConfig.RandomSeed)},
				{"RandomizeAllSpecs", fmt.Sprintf("%t", report.SuiteConfig.RandomizeAllSpecs)},
				{"LabelFilter", report.SuiteConfig.LabelFilter},
				{"FocusStrings", strings.Join(report.SuiteConfig.FocusStrings, ",")},
				{"SkipStrings", strings.Join(report.SuiteConfig.SkipStrings, ",")},
				{"FocusFiles", strings.Join(report.SuiteConfig.FocusFiles, ";")},
				{"SkipFiles", strings.Join(report.SuiteConfig.SkipFiles, ";")},
				{"FailOnPending", fmt.Sprintf("%t", report.SuiteConfig.FailOnPending)},
				{"FailFast", fmt.Sprintf("%t", report.SuiteConfig.FailFast)},
				{"FlakeAttempts", fmt.Sprintf("%d", report.SuiteConfig.FlakeAttempts)},
				{"DryRun", fmt.Sprintf("%t", report.SuiteConfig.DryRun)},
				{"ParallelTotal", fmt.Sprintf("%d", report.SuiteConfig.ParallelTotal)},
				{"OutputInterceptorMode", report.SuiteConfig.OutputInterceptorMode},
			},
		},
	}
	for _, spec := range report.SpecReports {
		if config.OmitSuiteSetupNodes && spec.LeafNodeType != types.NodeTypeIt {
			continue
		}
		name := fmt.Sprintf("[%s]", spec.LeafNodeType)
		if config.OmitLeafNodeType {
			name = ""
		}
		if spec.FullText() != "" {
			name = name + " " + spec.FullText()
		}
		labels := spec.Labels()
		if len(labels) > 0 && !config.OmitSpecLabels {
			name = name + " [" + strings.Join(labels, ", ") + "]"
		}
		owner := ""
		for _, label := range labels {
			if matches := ownerRE.FindStringSubmatch(label); len(matches) == 2 {
				owner = matches[1]
			}
		}
		name = strings.TrimSpace(name)

		test := JUnitTestCase{
			Name:      name,
			Classname: report.SuiteDescription,
			Status:    spec.State.String(),
			Time:      spec.RunTime.Seconds(),
			Owner:     owner,
		}
		if !spec.State.Is(config.OmitTimelinesForSpecState) {
			test.SystemErr = systemErrForUnstructuredReporters(spec)
		}
		if !config.OmitCapturedStdOutErr {
			test.SystemOut = systemOutForUnstructuredReporters(spec)
		}
		suite.Tests += 1

		switch spec.State {
		case types.SpecStateSkipped:
			message := "skipped"
			if spec.Failure.Message != "" {
				message += " - " + spec.Failure.Message
			}
			test.Skipped = &JUnitSkipped{Message: message}
			suite.Skipped += 1
		case types.SpecStatePending:
			test.Skipped = &JUnitSkipped{Message: "pending"}
			suite.Disabled += 1
		case types.SpecStateFailed:
			test.Failure = &JUnitFailure{
				Message:     spec.Failure.Message,
				Type:        "failed",
				Description: failureDescriptionForUnstructuredReporters(spec),
			}
			if config.OmitFailureMessageAttr {
				test.Failure.Message = ""
			}
			suite.Failures += 1
		case types.SpecStateTimedout:
			test.Failure = &JUnitFailure{
				Message:     spec.Failure.Message,
				Type:        "timedout",
				Description: failureDescriptionForUnstructuredReporters(spec),
			}
			if config.OmitFailureMessageAttr {
				test.Failure.Message = ""
			}
			suite.Failures += 1
		case types.SpecStateInterrupted:
			test.Error = &JUnitError{
				Message:     spec.Failure.Message,
				Type:        "interrupted",
				Description: failureDescriptionForUnstructuredReporters(spec),
			}
			if config.OmitFailureMessageAttr {
				test.Error.Message = ""
			}
			suite.Errors += 1
		case types.SpecStateAborted:
			test.Failure = &JUnitFailure{
				Message:     spec.Failure.Message,
				Type:        "aborted",
				Description: failureDescriptionForUnstructuredReporters(spec),
			}
			if config.OmitFailureMessageAttr {
				test.Failure.Message = ""
			}
			suite.Errors += 1
		case types.SpecStatePanicked:
			test.Error = &JUnitError{
				Message:     spec.Failure.ForwardedPanic,
				Type:        "panicked",
				Description: failureDescriptionForUnstructuredReporters(spec),
			}
			if config.OmitFailureMessageAttr {
				test.Error.Message = ""
			}
			suite.Errors += 1
		}

		suite.TestCases = append(suite.TestCases, test)
	}

	junitReport := JUnitTestSuites{
		Tests:      suite.Tests,
		Disabled:   suite.Disabled + suite.Skipped,
		Errors:     suite.Errors,
		Failures:   suite.Failures,
		Time:       suite.Time,
		TestSuites: []JUnitTestSuite{suite},
	}

	if err := os.MkdirAll(path.Dir(dst), 0770); err != nil {
		return err
	}
	f, err := os.Create(dst)
	if err != nil {
		return err
	}
	f.WriteString(xml.Header)
	encoder := xml.NewEncoder(f)
	encoder.Indent("  ", "    ")
	encoder.Encode(junitReport)

	return f.Close()
}

func MergeAndCleanupJUnitReports(sources []string, dst string) ([]string, error) {
	messages := []string{}
	mergedReport := JUnitTestSuites{}
	for _, source := range sources {
		report := JUnitTestSuites{}
		f, err := os.Open(source)
		if err != nil {
			messages = append(messages, fmt.Sprintf("Could not open %s:\n%s", source, err.Error()))
			continue
		}
		err = xml.NewDecoder(f).Decode(&report)
		_ = f.Close()
		if err != nil {
			messages = append(messages, fmt.Sprintf("Could not decode %s:\n%s", source, err.Error()))
			continue
		}
		os.Remove(source)

		mergedReport.Tests += report.Tests
		mergedReport.Disabled += report.Disabled
		mergedReport.Errors += report.Errors
		mergedReport.Failures += report.Failures
		mergedReport.Time += report.Time
		mergedReport.TestSuites = append(mergedReport.TestSuites, report.TestSuites...)
	}

	if err := os.MkdirAll(path.Dir(dst), 0770); err != nil {
		return messages, err
	}
	f, err := os.Create(dst)
	if err != nil {
		return messages, err
	}
	f.WriteString(xml.Header)
	encoder := xml.NewEncoder(f)
	encoder.Indent("  ", "    ")
	encoder.Encode(mergedReport)

	return messages, f.Close()
}

func failureDescriptionForUnstructuredReporters(spec types.SpecReport) string {
	out := &strings.Builder{}
	NewDefaultReporter(types.ReporterConfig{NoColor: true, VeryVerbose: true}, out).emitFailure(0, spec.State, spec.Failure, true)
	if len(spec.AdditionalFailures) > 0 {
		out.WriteString("\nThere were additional failures detected after the initial failure. These are visible in the timeline\n")
	}
	return out.String()
}

func systemErrForUnstructuredReporters(spec types.SpecReport) string {
	return RenderTimeline(spec, true)
}

func RenderTimeline(spec types.SpecReport, noColor bool) string {
	out := &strings.Builder{}
	NewDefaultReporter(types.ReporterConfig{NoColor: noColor, VeryVerbose: true}, out).emitTimeline(0, spec, spec.Timeline())
	return out.String()
}

func systemOutForUnstructuredReporters(spec types.SpecReport) string {
	return spec.CapturedStdOutErr
}

// Deprecated JUnitReporter (so folks can still compile their suites)
type JUnitReporter struct{}

func NewJUnitReporter(_ string) *JUnitReporter                                                  { return &JUnitReporter{} }
func (reporter *JUnitReporter) SuiteWillBegin(_ config.GinkgoConfigType, _ *types.SuiteSummary) {}
func (reporter *JUnitReporter) BeforeSuiteDidRun(_ *types.SetupSummary)                         {}
func (reporter *JUnitReporter) SpecWillRun(_ *types.SpecSummary)                                {}
func (reporter *JUnitReporter) SpecDidComplete(_ *types.SpecSummary)                            {}
func (reporter *JUnitReporter) AfterSuiteDidRun(_ *types.SetupSummary)                          {}
func (reporter *JUnitReporter) SuiteDidEnd(_ *types.SuiteSummary)                               {}