Skip to content

Commit

Permalink
allow for reftest and junit single tracking bugs, also fix wpt tests …
Browse files Browse the repository at this point in the history 
…with params to be single tracking (#7690)
  • Loading branch information
@jmaher
jmaher committed Apr 17, 2023
1 parent 7f1c543 commit 5761a7f
Show file tree
Hide file tree
Showing 3 changed files with 120 additions and 33 deletions.
78 changes: 60 additions & 18 deletions tests/model/test_error_summary.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ def test_get_cleaned_line(line_raw, exp_line_cleaned):
),
{
'path_end': 'chrome://mochitests/content/browser/browser/components/loop/test/mochitest/browser_fxa_login.js',
'search_term': 'browser_fxa_login.js',
'search_term': ['browser_fxa_login.js'],
},
),
(
Expand All @@ -52,7 +52,7 @@ def test_get_cleaned_line(line_raw, exp_line_cleaned):
),
{
'path_end': 'file:///C:/slave/test/build/tests/reftest/tests/layout/reftests/layers/component-alpha-exit-1.html',
'search_term': 'component-alpha-exit-1.html',
'search_term': ['component-alpha-exit-1.html'],
},
),
(
Expand All @@ -63,7 +63,7 @@ def test_get_cleaned_line(line_raw, exp_line_cleaned):
),
{
'path_end': '/tests/dom/media/tests/mochitest/test_dataChannel_basicAudio.html',
'search_term': 'test_dataChannel_basicAudio.html',
'search_term': ['test_dataChannel_basicAudio.html'],
},
),
(
Expand All @@ -74,7 +74,7 @@ def test_get_cleaned_line(line_raw, exp_line_cleaned):
),
{
'path_end': 'mainthreadio',
'search_term': 'mainthreadio',
'search_term': ['mainthreadio'],
},
),
(
Expand All @@ -86,7 +86,7 @@ def test_get_cleaned_line(line_raw, exp_line_cleaned):
),
{
'path_end': 'http://10.0.2.2:8854/tests/dom/canvas/test/reftest/webgl-resize-test.html',
'search_term': 'webgl-resize-test.html',
'search_term': ['webgl-resize-test.html'],
},
),
(
Expand All @@ -98,7 +98,7 @@ def test_get_cleaned_line(line_raw, exp_line_cleaned):
),
{
'path_end': 'http://10.0.2.2:8854/tests/dom/canvas/test/reftest/webgl-resize-test.html',
'search_term': 'webgl-resize-test.html',
'search_term': ['webgl-resize-test.html'],
},
),
(
Expand All @@ -109,7 +109,7 @@ def test_get_cleaned_line(line_raw, exp_line_cleaned):
),
{
'path_end': '/tests/dom/events/test/pointerevents/pointerevent_touch-action-table-test_touch-manual.html',
'search_term': 'pointerevent_touch-action-table-test_touch-manual.html',
'search_term': ['pointerevent_touch-action-table-test_touch-manual.html'],
},
),
)
Expand All @@ -122,6 +122,31 @@ def test_get_delimited_search_term(line, exp_search_info):
assert actual_search_info == exp_search_info


PIPE_DELIMITED_LINE_TEST_CASES_WITH_PARAMS = (
(
(
'INFO TEST-UNEXPECTED-TIMEOUT '
'| /html/cross-origin-opener-policy/coep-navigate-popup.https.html?4-last '
'| TestRunner hit external timeout (this may indicate a hang)'
),
{
'path_end': '/html/cross-origin-opener-policy/coep-navigate-popup.https.html?4-last',
'search_term': [
'coep-navigate-popup.https.html?4-last',
'coep-navigate-popup.https.html',
],
},
),
)


@pytest.mark.parametrize(("line", "exp_search_info"), PIPE_DELIMITED_LINE_TEST_CASES_WITH_PARAMS)
def test_get_delimited_search_term_with_params(line, exp_search_info):
"""Test search term extraction for a pipe delimited error line"""
actual_search_info = get_error_search_term_and_path(line)
assert actual_search_info == exp_search_info


LEAK_LINE_TEST_CASES = (
(
(
Expand All @@ -132,7 +157,9 @@ def test_get_delimited_search_term(line, exp_search_info):
),
{
'path_end': None,
'search_term': 'BackstagePass, CallbackObject, DOMEventTargetHelper, EventListenerManager, EventTokenBucket, ...',
'search_term': [
'BackstagePass, CallbackObject, DOMEventTargetHelper, EventListenerManager, EventTokenBucket, ...'
],
},
),
(
Expand All @@ -144,7 +171,9 @@ def test_get_delimited_search_term(line, exp_search_info):
),
{
'path_end': None,
'search_term': 'AsyncLatencyLogger, AsyncTransactionTrackersHolder, AudioOutputObserver, BufferRecycleBin, CipherSui',
'search_term': [
'AsyncLatencyLogger, AsyncTransactionTrackersHolder, AudioOutputObserver, BufferRecycleBin, CipherSui'
],
},
),
(
Expand All @@ -155,7 +184,9 @@ def test_get_delimited_search_term(line, exp_search_info):
),
{
'path_end': None,
'search_term': 'MakeUnique, nsThread::nsChainedEventQueue::nsChainedEventQueue, nsThread, nsThreadManager::Init',
'search_term': [
'MakeUnique, nsThread::nsChainedEventQueue::nsChainedEventQueue, nsThread, nsThreadManager::Init'
],
},
),
)
Expand All @@ -173,14 +204,19 @@ def test_get_leak_search_term(line, exp_search_info):
'Automation Error: No crash directory (/mnt/sdcard/tests/profile/minidumps/) found on remote device',
{
'path_end': None,
'search_term': 'Automation Error: No crash directory (/mnt/sdcard/tests/profile/minidumps/) found on remote device',
'search_term': [
'Automation Error: No crash directory (/mnt/sdcard/tests/profile/minidumps/) found on remote device'
],
},
),
(
'PROCESS-CRASH | Automation Error: Missing end of test marker (process crashed?)',
{
'path_end': None,
'search_term': 'Automation Error: Missing end of test marker (process crashed?)',
'search_term': [
'Automation Error: Missing end of test marker (process crashed?)',
'Automation Error: Missing end of test marker (process crashed',
],
},
),
)
Expand All @@ -207,8 +243,10 @@ def test_get_full_line_search_term(line, exp_search_info):
),
{
'path_end': None,
'search_term': 'command timed out: 2400 seconds without output running '
'[\'/tools/buildbot/bin/python\', \'scripts/scrip',
'search_term': [
'command timed out: 2400 seconds without output running '
'[\'/tools/buildbot/bin/python\', \'scripts/scrip'
],
},
),
(
Expand All @@ -219,7 +257,9 @@ def test_get_full_line_search_term(line, exp_search_info):
),
{
'path_end': 'test_switch_frame.py TestSwitchFrame.test_should_be_able_to_carry_on_working_if_the_frame_is_deleted_from_under_us',
'search_term': 'test_switch_frame.py TestSwitchFrame.test_should_be_able_to_carry_on_working_if_the_frame_is_deleted',
'search_term': [
'test_switch_frame.py TestSwitchFrame.test_should_be_able_to_carry_on_working_if_the_frame_is_deleted'
],
},
),
)
Expand Down Expand Up @@ -258,14 +298,16 @@ def test_get_crash_signature(line, exp_search_info):
'TEST-UNEXPECTED-FAIL | remoteautomation.py | application timed out after 330 seconds with no output',
{
'path_end': 'remoteautomation.py',
'search_term': 'remoteautomation.py | application timed out after 330 seconds with no output',
'search_term': [
'remoteautomation.py | application timed out after 330 seconds with no output'
],
},
),
(
'Return code: 1',
{
'path_end': None,
'search_term': None,
'search_term': [None],
},
),
(
Expand All @@ -275,7 +317,7 @@ def test_get_crash_signature(line, exp_search_info):
),
{
'path_end': 'file:///home/worker/workspace/build/tests/reftest/tests/layout/reftests/font-inflation/video-1.html',
'search_term': 'video-1.html',
'search_term': ['video-1.html'],
},
),
)
Expand Down
21 changes: 17 additions & 4 deletions treeherder/model/error_summary.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -143,10 +143,14 @@ def bug_suggestions_line(
# collect open recent and all other bugs suggestions
search_terms = []
if search_term:
search_terms.append(search_term)
if search_term not in term_cache:
term_cache[search_term] = Bugscache.search(search_term)
bugs = term_cache[search_term]
search_terms.extend(search_term)
for term in search_term:
if not term or not term.strip():
continue
if term not in term_cache:
term_cache[term] = Bugscache.search(term)
bugs['open_recent'].extend(term_cache[term]['open_recent'])
bugs['all_others'].extend(term_cache[term]['all_others'])

if not bugs or not (bugs['open_recent'] or bugs['all_others']):
# no suggestions, try to use
Expand Down Expand Up @@ -251,6 +255,15 @@ def get_error_search_term_and_path(error_line):
search_term = re.sub(PREFIX_PATTERN, '', search_term)
search_term = search_term[:100]

# for wpt tests we have testname.html?params, we need to add a search term
# for just testname.html.
# we will now return an array
if search_term and '?' in search_term:
search_name = search_term.split('?')[0]
search_term = [search_term, search_name]
else:
search_term = [search_term]

return {
"search_term": search_term,
"path_end": path_end,
Expand Down
54 changes: 43 additions & 11 deletions ui/shared/BugFiler.jsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -146,14 +146,21 @@ export class BugFilerClass extends React.Component {

const newFailure = suggestion.showNewButton;
const keywords = [];
const isAssertion = [
let isAssertion = [
/ASSERTION:/, // binary code
/assertion fail/i, // JavaScript
/assertion count \d+ is \w+ than expected \d+ assertion/, // layout
/AssertionError/, // Marionette
].some((regexp) => regexp.test(summaryString));
if (isAssertion) {
keywords.push('assertion');
if (
/java.lang.AssertionError/.test(summaryString) &&
jobTypeName.includes('junit')
) {
isAssertion = false;
} else {
keywords.push('assertion');
}
}

if (jobTypeName.toLowerCase().includes('test-verify')) {
Expand All @@ -173,7 +180,9 @@ export class BugFilerClass extends React.Component {
if (
jg.includes('xpcshell') ||
jg.includes('mochitest') ||
jg.includes('web platform tests')
jg.includes('web platform tests') ||
jg.includes('reftest') ||
jobTypeName.includes('junit')
) {
// simple hack to make sure we have a testcase in the summary
let isTestPath = [
Expand All @@ -182,26 +191,49 @@ export class BugFilerClass extends React.Component {
/.*test_.*\.xhtml/, // mochitest-chrome
/.*browser_.*\.html/, // b-c
/.*browser_.*\.js/, // b-c
/.*org.mozilla.geckoview.test.*/, // junit
].some((regexp) => regexp.test(summaryString));

if (jg.includes('web platform tests')) {
if (jg.includes('web platform tests') || jg.includes('reftest')) {
// account for <filename>.html?blah... | failure message
isTestPath = [
/.*\.js \|/,
/.*\.html \|/,
/.*\.htm \|/,
/.*\.xhtml \|/,
/.*\.xht \|/,
/.*\.js(\?.*| )\|/,
/.*\.html(\?.*| )\|/,
/.*\.htm(\?.*| )\|/,
/.*\.xhtml(\?.*| )\|/,
/.*\.xht(\?.*| )\|/,
/.*\.mp4 \|/, // reftest specific
/.*\.webm \|/, // reftest specific
].some((regexp) => regexp.test(summaryString));
}

// trimming params from end of a test case name when filing for stb
let trimParams = false;

// only handle straight forward reftest pixel/color errors
if (
isTestPath &&
jobGroupName.includes('reftest') &&
!/.*image comparison, max difference.*/.test(summaryString)
) {
isTestPath = false;
} else if (jg.includes('web platform tests')) {
trimParams = true;
}

// If not crash|leak
if (!crash && !isAssertion && isTestPath) {
const parts = summaryString.split(' | ');
// split('?') is for removing `?params...` from the test name
if (parts.length === 2 || parts.length === 1) {
summaryString = `${parts[0]} | single tracking bug`;
summaryString = `${
trimParams ? parts[0].split('?')[0] : parts[0]
} | single tracking bug`;
keywords.push('intermittent-testcase');
} else if (parts.length === 3) {
summaryString = `${parts[1]} | single tracking bug`;
summaryString = `${
trimParams ? parts[1].split('?')[0] : parts[1]
} | single tracking bug`;
keywords.push('intermittent-testcase');
}
}
Expand Down

0 comments on commit 5761a7f

Please sign in to comment.