Skip to content

Commit 7d76a23

Browse files
Abseil Teamcopybara-github
Abseil Team
authored and
copybara-github
committed
gtest: Output a canned test case for test suite setup / teardown failures in XML/JSON
This surfaces useful information about the environment failure in a structured form. As we can see from the updated test, previously unsurfaced information is now present. PiperOrigin-RevId: 709892315 Change-Id: I2656294d50c33f995bef5c96195a66cff3c4b907
1 parent e54519b commit 7d76a23

File tree

4 files changed

+198
-28
lines changed

4 files changed

+198
-28
lines changed

‎googletest/src/gtest.cc

Lines changed: 44 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -3989,6 +3989,12 @@ class XmlUnitTestResultPrinter : public EmptyTestEventListener {
39893989
static void OutputXmlTestSuiteForTestResult(::std::ostream* stream,
39903990
const TestResult& result);
39913991

3992+
// Streams a test case XML stanza containing the given test result.
3993+
//
3994+
// Requires: result.Failed()
3995+
static void OutputXmlTestCaseForTestResult(::std::ostream* stream,
3996+
const TestResult& result);
3997+
39923998
// Streams an XML representation of a TestResult object.
39933999
static void OutputXmlTestResult(::std::ostream* stream,
39944000
const TestResult& result);
@@ -4236,6 +4242,15 @@ void XmlUnitTestResultPrinter::OutputXmlTestSuiteForTestResult(
42364242
FormatEpochTimeInMillisAsIso8601(result.start_timestamp()));
42374243
*stream << ">";
42384244

4245+
OutputXmlTestCaseForTestResult(stream, result);
4246+
4247+
// Complete the test suite.
4248+
*stream << " </testsuite>\n";
4249+
}
4250+
4251+
// Streams a test case XML stanza containing the given test result.
4252+
void XmlUnitTestResultPrinter::OutputXmlTestCaseForTestResult(
4253+
::std::ostream* stream, const TestResult& result) {
42394254
// Output the boilerplate for a minimal test case with a single test.
42404255
*stream << " <testcase";
42414256
OutputXmlAttribute(stream, "testcase", "name", "");
@@ -4250,9 +4265,6 @@ void XmlUnitTestResultPrinter::OutputXmlTestSuiteForTestResult(
42504265

42514266
// Output the actual test result.
42524267
OutputXmlTestResult(stream, result);
4253-
4254-
// Complete the test suite.
4255-
*stream << " </testsuite>\n";
42564268
}
42574269

42584270
// Prints an XML representation of a TestInfo object.
@@ -4379,6 +4391,10 @@ void XmlUnitTestResultPrinter::PrintXmlTestSuite(std::ostream* stream,
43794391
if (test_suite.GetTestInfo(i)->is_reportable())
43804392
OutputXmlTestInfo(stream, test_suite.name(), *test_suite.GetTestInfo(i));
43814393
}
4394+
if (test_suite.ad_hoc_test_result().Failed()) {
4395+
OutputXmlTestCaseForTestResult(stream, test_suite.ad_hoc_test_result());
4396+
}
4397+
43824398
*stream << " </" << kTestsuite << ">\n";
43834399
}
43844400

@@ -4518,6 +4534,12 @@ class JsonUnitTestResultPrinter : public EmptyTestEventListener {
45184534
static void OutputJsonTestSuiteForTestResult(::std::ostream* stream,
45194535
const TestResult& result);
45204536

4537+
// Streams a test case JSON stanza containing the given test result.
4538+
//
4539+
// Requires: result.Failed()
4540+
static void OutputJsonTestCaseForTestResult(::std::ostream* stream,
4541+
const TestResult& result);
4542+
45214543
// Streams a JSON representation of a TestResult object.
45224544
static void OutputJsonTestResult(::std::ostream* stream,
45234545
const TestResult& result);
@@ -4688,6 +4710,15 @@ void JsonUnitTestResultPrinter::OutputJsonTestSuiteForTestResult(
46884710
}
46894711
*stream << Indent(6) << "\"testsuite\": [\n";
46904712

4713+
OutputJsonTestCaseForTestResult(stream, result);
4714+
4715+
// Finish the test suite.
4716+
*stream << "\n" << Indent(6) << "]\n" << Indent(4) << "}";
4717+
}
4718+
4719+
// Streams a test case JSON stanza containing the given test result.
4720+
void JsonUnitTestResultPrinter::OutputJsonTestCaseForTestResult(
4721+
::std::ostream* stream, const TestResult& result) {
46914722
// Output the boilerplate for a new test case.
46924723
*stream << Indent(8) << "{\n";
46934724
OutputJsonKey(stream, "testcase", "name", "", Indent(10));
@@ -4704,9 +4735,6 @@ void JsonUnitTestResultPrinter::OutputJsonTestSuiteForTestResult(
47044735

47054736
// Output the actual test result.
47064737
OutputJsonTestResult(stream, result);
4707-
4708-
// Finish the test suite.
4709-
*stream << "\n" << Indent(6) << "]\n" << Indent(4) << "}";
47104738
}
47114739

47124740
// Prints a JSON representation of a TestInfo object.
@@ -4851,6 +4879,16 @@ void JsonUnitTestResultPrinter::PrintJsonTestSuite(
48514879
OutputJsonTestInfo(stream, test_suite.name(), *test_suite.GetTestInfo(i));
48524880
}
48534881
}
4882+
4883+
// If there was a failure in the test suite setup or teardown include that in
4884+
// the output.
4885+
if (test_suite.ad_hoc_test_result().Failed()) {
4886+
if (comma) {
4887+
*stream << ",\n";
4888+
}
4889+
OutputJsonTestCaseForTestResult(stream, test_suite.ad_hoc_test_result());
4890+
}
4891+
48544892
*stream << "\n" << kIndent << "]\n" << Indent(4) << "}";
48554893
}
48564894

‎googletest/test/googletest-json-output-unittest.py

Lines changed: 86 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@
5757
STACK_TRACE_TEMPLATE = '\n'
5858

5959
EXPECTED_NON_EMPTY = {
60-
'tests': 26,
60+
'tests': 28,
6161
'failures': 5,
6262
'disabled': 2,
6363
'errors': 0,
@@ -419,6 +419,83 @@
419419
},
420420
],
421421
},
422+
{
423+
'name': 'SetupFailTest',
424+
'tests': 1,
425+
'failures': 0,
426+
'disabled': 0,
427+
'errors': 0,
428+
'time': '*',
429+
'timestamp': '*',
430+
'testsuite': [
431+
{
432+
'name': 'NoopPassingTest',
433+
'file': 'gtest_xml_output_unittest_.cc',
434+
'line': 168,
435+
'status': 'RUN',
436+
'result': 'SKIPPED',
437+
'timestamp': '*',
438+
'time': '*',
439+
'classname': 'SetupFailTest',
440+
'skipped': [
441+
{'message': 'gtest_xml_output_unittest_.cc:*\n'}
442+
],
443+
},
444+
{
445+
'name': '',
446+
'status': 'RUN',
447+
'result': 'COMPLETED',
448+
'timestamp': '*',
449+
'time': '*',
450+
'classname': '',
451+
'failures': [{
452+
'failure': (
453+
'gtest_xml_output_unittest_.cc:*\nExpected equality'
454+
' of these values:\n 1\n 2'
455+
+ STACK_TRACE_TEMPLATE
456+
),
457+
'type': '',
458+
}],
459+
},
460+
],
461+
},
462+
{
463+
'name': 'TearDownFailTest',
464+
'tests': 1,
465+
'failures': 0,
466+
'disabled': 0,
467+
'errors': 0,
468+
'timestamp': '*',
469+
'time': '*',
470+
'testsuite': [
471+
{
472+
'name': 'NoopPassingTest',
473+
'file': 'gtest_xml_output_unittest_.cc',
474+
'line': 175,
475+
'status': 'RUN',
476+
'result': 'COMPLETED',
477+
'timestamp': '*',
478+
'time': '*',
479+
'classname': 'TearDownFailTest',
480+
},
481+
{
482+
'name': '',
483+
'status': 'RUN',
484+
'result': 'COMPLETED',
485+
'timestamp': '*',
486+
'time': '*',
487+
'classname': '',
488+
'failures': [{
489+
'failure': (
490+
'gtest_xml_output_unittest_.cc:*\nExpected equality'
491+
' of these values:\n 1\n 2'
492+
+ STACK_TRACE_TEMPLATE
493+
),
494+
'type': '',
495+
}],
496+
},
497+
],
498+
},
422499
{
423500
'name': 'TypedTest/0',
424501
'tests': 1,
@@ -431,7 +508,7 @@
431508
'name': 'HasTypeParamAttribute',
432509
'type_param': 'int',
433510
'file': 'gtest_xml_output_unittest_.cc',
434-
'line': 173,
511+
'line': 189,
435512
'status': 'RUN',
436513
'result': 'COMPLETED',
437514
'time': '*',
@@ -451,7 +528,7 @@
451528
'name': 'HasTypeParamAttribute',
452529
'type_param': 'long',
453530
'file': 'gtest_xml_output_unittest_.cc',
454-
'line': 173,
531+
'line': 189,
455532
'status': 'RUN',
456533
'result': 'COMPLETED',
457534
'time': '*',
@@ -471,7 +548,7 @@
471548
'name': 'HasTypeParamAttribute',
472549
'type_param': 'int',
473550
'file': 'gtest_xml_output_unittest_.cc',
474-
'line': 180,
551+
'line': 196,
475552
'status': 'RUN',
476553
'result': 'COMPLETED',
477554
'time': '*',
@@ -491,7 +568,7 @@
491568
'name': 'HasTypeParamAttribute',
492569
'type_param': 'long',
493570
'file': 'gtest_xml_output_unittest_.cc',
494-
'line': 180,
571+
'line': 196,
495572
'status': 'RUN',
496573
'result': 'COMPLETED',
497574
'time': '*',
@@ -512,7 +589,7 @@
512589
'name': 'HasValueParamAttribute/0',
513590
'value_param': '33',
514591
'file': 'gtest_xml_output_unittest_.cc',
515-
'line': 164,
592+
'line': 180,
516593
'status': 'RUN',
517594
'result': 'COMPLETED',
518595
'time': '*',
@@ -523,7 +600,7 @@
523600
'name': 'HasValueParamAttribute/1',
524601
'value_param': '42',
525602
'file': 'gtest_xml_output_unittest_.cc',
526-
'line': 164,
603+
'line': 180,
527604
'status': 'RUN',
528605
'result': 'COMPLETED',
529606
'time': '*',
@@ -534,7 +611,7 @@
534611
'name': 'AnotherTestThatHasValueParamAttribute/0',
535612
'value_param': '33',
536613
'file': 'gtest_xml_output_unittest_.cc',
537-
'line': 165,
614+
'line': 181,
538615
'status': 'RUN',
539616
'result': 'COMPLETED',
540617
'time': '*',
@@ -545,7 +622,7 @@
545622
'name': 'AnotherTestThatHasValueParamAttribute/1',
546623
'value_param': '42',
547624
'file': 'gtest_xml_output_unittest_.cc',
548-
'line': 165,
625+
'line': 181,
549626
'status': 'RUN',
550627
'result': 'COMPLETED',
551628
'time': '*',

‎googletest/test/gtest_xml_output_unittest.py

Lines changed: 52 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -29,14 +29,14 @@
2929
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
3030
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
3131

32-
"""Unit test for the gtest_xml_output module"""
32+
"""Unit test for the gtest_xml_output module."""
3333

3434
import datetime
3535
import errno
3636
import os
3737
import re
3838
import sys
39-
from xml.dom import minidom, Node
39+
from xml.dom import minidom
4040

4141
from googletest.test import gtest_test_utils
4242
from googletest.test import gtest_xml_test_utils
@@ -67,7 +67,7 @@
6767
sys.argv.remove(NO_STACKTRACE_SUPPORT_FLAG)
6868

6969
EXPECTED_NON_EMPTY_XML = """<?xml version="1.0" encoding="UTF-8"?>
70-
<testsuites tests="26" failures="5" disabled="2" errors="0" time="*" timestamp="*" name="AllTests" ad_hoc_property="42">
70+
<testsuites tests="28" failures="5" disabled="2" errors="0" time="*" timestamp="*" name="AllTests" ad_hoc_property="42">
7171
<testsuite name="SuccessfulTest" tests="1" failures="0" disabled="0" skipped="0" errors="0" time="*" timestamp="*">
7272
<testcase name="Succeeds" file="gtest_xml_output_unittest_.cc" line="53" status="run" result="completed" time="*" timestamp="*" classname="SuccessfulTest"/>
7373
</testsuite>
@@ -173,23 +173,44 @@
173173
</properties>
174174
</testcase>
175175
</testsuite>
176+
<testsuite name="SetupFailTest" tests="1" failures="0" disabled="0" skipped="1" errors="0" time="*" timestamp="*">
177+
<testcase name="NoopPassingTest" file="gtest_xml_output_unittest_.cc" line="168" status="run" result="skipped" time="*" timestamp="*" classname="SetupFailTest">
178+
<skipped message="gtest_xml_output_unittest_.cc:*&#x0A;"><![CDATA[gtest_xml_output_unittest_.cc:*
179+
]]></skipped>
180+
</testcase>
181+
<testcase name="" status="run" result="completed" classname="" time="*" timestamp="*">
182+
<failure message="gtest_xml_output_unittest_.cc:*&#x0A;Expected equality of these values:&#x0A; 1&#x0A; 2%(stack_entity)s" type=""><![CDATA[gtest_xml_output_unittest_.cc:*
183+
Expected equality of these values:
184+
1
185+
2%(stack)s]]></failure>
186+
</testcase>
187+
</testsuite>
188+
<testsuite name="TearDownFailTest" tests="1" failures="0" disabled="0" skipped="0" errors="0" time="*" timestamp="*">
189+
<testcase name="NoopPassingTest" file="gtest_xml_output_unittest_.cc" line="175" status="run" result="completed" time="*" timestamp="*" classname="TearDownFailTest"/>
190+
<testcase name="" status="run" result="completed" classname="" time="*" timestamp="*">
191+
<failure message="gtest_xml_output_unittest_.cc:*&#x0A;Expected equality of these values:&#x0A; 1&#x0A; 2%(stack_entity)s" type=""><![CDATA[gtest_xml_output_unittest_.cc:*
192+
Expected equality of these values:
193+
1
194+
2%(stack)s]]></failure>
195+
</testcase>
196+
</testsuite>
176197
<testsuite name="Single/ValueParamTest" tests="4" failures="0" disabled="0" skipped="0" errors="0" time="*" timestamp="*">
177-
<testcase name="HasValueParamAttribute/0" file="gtest_xml_output_unittest_.cc" line="164" value_param="33" status="run" result="completed" time="*" timestamp="*" classname="Single/ValueParamTest" />
178-
<testcase name="HasValueParamAttribute/1" file="gtest_xml_output_unittest_.cc" line="164" value_param="42" status="run" result="completed" time="*" timestamp="*" classname="Single/ValueParamTest" />
179-
<testcase name="AnotherTestThatHasValueParamAttribute/0" file="gtest_xml_output_unittest_.cc" line="165" value_param="33" status="run" result="completed" time="*" timestamp="*" classname="Single/ValueParamTest" />
180-
<testcase name="AnotherTestThatHasValueParamAttribute/1" file="gtest_xml_output_unittest_.cc" line="165" value_param="42" status="run" result="completed" time="*" timestamp="*" classname="Single/ValueParamTest" />
198+
<testcase name="HasValueParamAttribute/0" file="gtest_xml_output_unittest_.cc" line="180" value_param="33" status="run" result="completed" time="*" timestamp="*" classname="Single/ValueParamTest" />
199+
<testcase name="HasValueParamAttribute/1" file="gtest_xml_output_unittest_.cc" line="180" value_param="42" status="run" result="completed" time="*" timestamp="*" classname="Single/ValueParamTest" />
200+
<testcase name="AnotherTestThatHasValueParamAttribute/0" file="gtest_xml_output_unittest_.cc" line="181" value_param="33" status="run" result="completed" time="*" timestamp="*" classname="Single/ValueParamTest" />
201+
<testcase name="AnotherTestThatHasValueParamAttribute/1" file="gtest_xml_output_unittest_.cc" line="181" value_param="42" status="run" result="completed" time="*" timestamp="*" classname="Single/ValueParamTest" />
181202
</testsuite>
182203
<testsuite name="TypedTest/0" tests="1" failures="0" disabled="0" skipped="0" errors="0" time="*" timestamp="*">
183-
<testcase name="HasTypeParamAttribute" file="gtest_xml_output_unittest_.cc" line="173" type_param="*" status="run" result="completed" time="*" timestamp="*" classname="TypedTest/0" />
204+
<testcase name="HasTypeParamAttribute" file="gtest_xml_output_unittest_.cc" line="189" type_param="*" status="run" result="completed" time="*" timestamp="*" classname="TypedTest/0" />
184205
</testsuite>
185206
<testsuite name="TypedTest/1" tests="1" failures="0" disabled="0" skipped="0" errors="0" time="*" timestamp="*">
186-
<testcase name="HasTypeParamAttribute" file="gtest_xml_output_unittest_.cc" line="173" type_param="*" status="run" result="completed" time="*" timestamp="*" classname="TypedTest/1" />
207+
<testcase name="HasTypeParamAttribute" file="gtest_xml_output_unittest_.cc" line="189" type_param="*" status="run" result="completed" time="*" timestamp="*" classname="TypedTest/1" />
187208
</testsuite>
188209
<testsuite name="Single/TypeParameterizedTestSuite/0" tests="1" failures="0" disabled="0" skipped="0" errors="0" time="*" timestamp="*">
189-
<testcase name="HasTypeParamAttribute" file="gtest_xml_output_unittest_.cc" line="180" type_param="*" status="run" result="completed" time="*" timestamp="*" classname="Single/TypeParameterizedTestSuite/0" />
210+
<testcase name="HasTypeParamAttribute" file="gtest_xml_output_unittest_.cc" line="196" type_param="*" status="run" result="completed" time="*" timestamp="*" classname="Single/TypeParameterizedTestSuite/0" />
190211
</testsuite>
191212
<testsuite name="Single/TypeParameterizedTestSuite/1" tests="1" failures="0" disabled="0" skipped="0" errors="0" time="*" timestamp="*">
192-
<testcase name="HasTypeParamAttribute" file="gtest_xml_output_unittest_.cc" line="180" type_param="*" status="run" result="completed" time="*" timestamp="*" classname="Single/TypeParameterizedTestSuite/1" />
213+
<testcase name="HasTypeParamAttribute" file="gtest_xml_output_unittest_.cc" line="196" type_param="*" status="run" result="completed" time="*" timestamp="*" classname="Single/TypeParameterizedTestSuite/1" />
193214
</testsuite>
194215
</testsuites>""" % {
195216
'stack': STACK_TRACE_TEMPLATE,
@@ -205,6 +226,24 @@
205226
</testsuite>
206227
</testsuites>"""
207228

229+
ACTUAL_OUTPUT = """<?xml version="1.0" encoding="UTF-8"?>
230+
<testsuites tests="3" failures="0" disabled="0" errors="0" time="*" timestamp="*" ad_hoc_property="42" name="AllTests">
231+
<testsuite name="SuccessfulTest" tests="1" failures="0" disabled="0" skipped="0" errors="0" time="*" timestamp="*">
232+
<testcase name="Succeeds" file="gtest_xml_output_unittest_.cc" line="53" status="run" result="completed" time="*" timestamp="*" classname="SuccessfulTest"/>
233+
</testsuite>
234+
<testsuite name="PropertyRecordingTest" tests="1" failures="0" disabled="0" skipped="0" errors="0" time="*" timestamp="*" SetUpTestSuite="yes" TearDownTestSuite="aye">
235+
<testcase name="IntValuedProperty" file="gtest_xml_output_unittest_.cc" line="125" status="run" result="completed" time="*" timestamp="*" classname="PropertyRecordingTest">
236+
<properties>
237+
<property name="key_int" value="1"/>
238+
</properties>
239+
</testcase>
240+
</testsuite>
241+
<testsuite name="Single/TypeParameterizedTestSuite/0" tests="1" failures="0" disabled="0" skipped="0" errors="0" time="*" timestamp="*">
242+
<testcase name="HasTypeParamAttribute" type_param="int" file="gtest_xml_output_unittest_.cc" line="196" status="run" result="completed" time="*" timestamp="*" classname="Single/TypeParameterizedTestSuite/0"/>
243+
</testsuite>
244+
</testsuites>
245+
"""
246+
208247
EXPECTED_SHARDED_TEST_XML = """<?xml version="1.0" encoding="UTF-8"?>
209248
<testsuites tests="3" failures="0" disabled="0" errors="0" time="*" timestamp="*" name="AllTests" ad_hoc_property="42">
210249
<testsuite name="SuccessfulTest" tests="1" failures="0" disabled="0" skipped="0" errors="0" time="*" timestamp="*">
@@ -217,8 +256,8 @@
217256
</properties>
218257
</testcase>
219258
</testsuite>
220-
<testsuite name="Single/ValueParamTest" tests="1" failures="0" disabled="0" skipped="0" errors="0" time="*" timestamp="*">
221-
<testcase name="HasValueParamAttribute/0" file="gtest_xml_output_unittest_.cc" line="164" value_param="33" status="run" result="completed" time="*" timestamp="*" classname="Single/ValueParamTest" />
259+
<testsuite name="Single/TypeParameterizedTestSuite/0" tests="1" failures="0" disabled="0" skipped="0" errors="0" time="*" timestamp="*">
260+
<testcase name="HasTypeParamAttribute" type_param="*" file="gtest_xml_output_unittest_.cc" line="196" status="run" result="completed" time="*" timestamp="*" classname="Single/TypeParameterizedTestSuite/0" />
222261
</testsuite>
223262
</testsuites>"""
224263

0 commit comments

Comments
 (0)