useValgrind, valgrindLeakCheck, valgrindArgs,
noExecute, debug, isWindows,
params, config_prefix = None,
- maxIndividualTestTime = 0):
+ maxIndividualTestTime = 0,
+ maxFailures = None):
# The name of the test runner.
self.progname = progname
# The items to add to the PATH environment variable.
self.valgrindArgs.extend(self.valgrindUserArgs)
self.maxIndividualTestTime = maxIndividualTestTime
+ self.maxFailures = maxFailures
@property
def maxIndividualTestTime(self):
help="Maximum time to spend running a single test (in seconds)."
"0 means no time limit. [Default: 0]",
type=int, default=None)
+ group.add_option("", "--max-failures", dest="maxFailures",
+ help="Stop execution after the given number of failures.",
+ action="store", type=int, default=None)
parser.add_option_group(group)
group = OptionGroup(parser, "Test Selection")
else:
opts.numThreads = 1
+ if opts.maxFailures == 0:
+ parser.error("Setting --max-failures to 0 does not have any effect.")
+
inputs = args
# Create the user defined parameters.
isWindows = isWindows,
params = userParams,
config_prefix = opts.configPrefix,
- maxIndividualTestTime = maxIndividualTestTime)
+ maxIndividualTestTime = maxIndividualTestTime,
+ maxFailures = opts.maxFailures)
# Perform test discovery.
run = lit.run.Run(litConfig,
('Expected Failing Tests', lit.Test.XFAIL),
('Timed Out Tests', lit.Test.TIMEOUT)):
if (lit.Test.XFAIL == code and not opts.show_xfail) or \
- (lit.Test.UNSUPPORTED == code and not opts.show_unsupported):
+ (lit.Test.UNSUPPORTED == code and not opts.show_unsupported) or \
+ (lit.Test.UNRESOLVED == code and (opts.maxFailures is not None)):
continue
elts = byCode.get(code)
if not elts:
###
+def handleFailures(provider, consumer, maxFailures):
+ class _Display(object):
+ def __init__(self, display):
+ self.display = display
+ self.maxFailures = maxFailures or object()
+ self.failedCount = 0
+ def update(self, test):
+ self.display.update(test)
+ self.failedCount += (test.result.code == lit.Test.FAIL)
+ if self.failedCount == self.maxFailures:
+ provider.cancel()
+ consumer.display = _Display(consumer.display)
+
class Run(object):
"""
This class represents a concrete, configured testing run.
# Create the test provider.
provider = TestProvider(queue_impl, canceled_flag)
+ handleFailures(provider, consumer, self.lit_config.maxFailures)
# Queue the tests outside the main thread because we can't guarantee
# that we can put() all the tests without blocking:
--- /dev/null
+# Check the behavior of --max-failures option.
+#
+# RUN: not %{lit} -j 1 -v %{inputs}/shtest-shell > %t.out
+# RUN: not %{lit} --max-failures=1 -j 1 -v %{inputs}/shtest-shell >> %t.out
+# RUN: not %{lit} --max-failures=2 -j 1 -v %{inputs}/shtest-shell >> %t.out
+# RUN: not %{lit} --max-failures=0 -j 1 -v %{inputs}/shtest-shell 2>> %t.out
+# RUN: FileCheck < %t.out %s
+#
+# END.
+
+# CHECK: Failing Tests (3)
+# CHECK: Failing Tests (1)
+# CHECK: Failing Tests (2)
+# CHECK: error: Setting --max-failures to 0 does not have any effect.