2 # Copyright (C) 2010 Google Inc. All rights reserved.
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
8 # * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 # * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
14 # * Neither the name of Google Inc. nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32 This is a port of the existing webkit test script run-webkit-tests.
34 The TestRunner class runs a series of tests (TestType interface) against a set
35 of test files. If a test file fails a TestType, it returns a list TestFailure
36 objects to the TestRunner. The TestRunner then aggregates the TestFailures to
37 create a final report.
39 This script reads several files, if they exist in the test_lists subdirectory
40 next to this script itself. Each should contain a list of paths to individual
41 tests or entire subdirectories of tests, relative to the outermost test
42 directory. Entire lines starting with '//' (comments) will be ignored.
44 For details of the files' contents and purposes, see test_lists/README.
47 from __future__ import with_statement
66 from layout_package import dump_render_tree_thread
67 from layout_package import json_layout_results_generator
68 from layout_package import printing
69 from layout_package import test_expectations
70 from layout_package import test_failures
71 from layout_package import test_files
72 from layout_package import test_results_uploader
73 from test_types import fuzzy_image_diff
74 from test_types import image_diff
75 from test_types import text_diff
76 from test_types import test_type_base
78 from webkitpy.common.system.executive import Executive
79 from webkitpy.thirdparty import simplejson
83 _log = logging.getLogger("webkitpy.layout_tests.run_webkit_tests")
85 # Builder base URL where we have the archived test results.
86 BUILDER_BASE_URL = "http://build.chromium.org/buildbot/layout_test_results/"
88 TestExpectationsFile = test_expectations.TestExpectationsFile
92 """Groups information about a test for easy passing of data."""
94 def __init__(self, port, filename, timeout):
95 """Generates the URI and stores the filename and timeout for this test.
97 filename: Full path to the test.
98 timeout: Timeout for running the test in TestShell.
100 self.filename = filename
101 self.uri = port.filename_to_uri(filename)
102 self.timeout = timeout
103 # FIXME: Confusing that the file is .checksum and we call it "hash"
104 self._expected_hash_path = port.expected_filename(filename, '.checksum')
105 self._have_read_expected_hash = False
106 self._image_hash = None
108 def _read_image_hash(self):
110 with codecs.open(self._expected_hash_path, "r", "ascii") as hash_file:
111 return hash_file.read()
113 if errno.ENOENT != e.errno:
116 def image_hash(self):
117 # Read the image_hash lazily to reduce startup time.
118 # This class is accessed across threads, but only one thread should
119 # ever be dealing with any given TestInfo so no locking is needed.
120 if not self._have_read_expected_hash:
121 self._have_read_expected_hash = True
122 self._image_hash = self._read_image_hash()
123 return self._image_hash
126 class ResultSummary(object):
127 """A class for partitioning the test results we get into buckets.
129 This class is basically a glorified struct and it's private to this file
130 so we don't bother with any information hiding."""
132 def __init__(self, expectations, test_files):
133 self.total = len(test_files)
134 self.remaining = self.total
135 self.expectations = expectations
138 self.tests_by_expectation = {}
139 self.tests_by_timeline = {}
141 self.unexpected_results = {}
143 self.tests_by_expectation[test_expectations.SKIP] = set()
144 for expectation in TestExpectationsFile.EXPECTATIONS.values():
145 self.tests_by_expectation[expectation] = set()
146 for timeline in TestExpectationsFile.TIMELINES.values():
147 self.tests_by_timeline[timeline] = (
148 expectations.get_tests_with_timeline(timeline))
150 def add(self, result, expected):
151 """Add a TestResult into the appropriate bin.
154 result: TestResult from dump_render_tree_thread.
155 expected: whether the result was what we expected it to be.
158 self.tests_by_expectation[result.type].add(result.filename)
159 self.results[result.filename] = result
161 if len(result.failures):
162 self.failures[result.filename] = result.failures
166 self.unexpected_results[result.filename] = result.type
170 def summarize_unexpected_results(port_obj, expectations, result_summary,
172 """Summarize any unexpected results as a dict.
174 FIXME: split this data structure into a separate class?
177 port_obj: interface to port-specific hooks
178 expectations: test_expectations.TestExpectations object
179 result_summary: summary object from initial test runs
180 retry_summary: summary object from final test run of retried tests
182 A dictionary containing a summary of the unexpected results from the
183 run, with the following fields:
184 'version': a version indicator (1 in this version)
185 'fixable': # of fixable tests (NOW - PASS)
186 'skipped': # of skipped tests (NOW & SKIPPED)
187 'num_regressions': # of non-flaky failures
188 'num_flaky': # of flaky failures
189 'num_passes': # of unexpected passes
190 'tests': a dict of tests -> {'expected': '...', 'actual': '...'}
193 results['version'] = 1
195 tbe = result_summary.tests_by_expectation
196 tbt = result_summary.tests_by_timeline
197 results['fixable'] = len(tbt[test_expectations.NOW] -
198 tbe[test_expectations.PASS])
199 results['skipped'] = len(tbt[test_expectations.NOW] &
200 tbe[test_expectations.SKIP])
206 for k, v in TestExpectationsFile.EXPECTATIONS.iteritems():
207 keywords[v] = k.upper()
210 for filename, result in result_summary.unexpected_results.iteritems():
211 # Note that if a test crashed in the original run, we ignore
212 # whether or not it crashed when we retried it (if we retried it),
213 # and always consider the result not flaky.
214 test = port_obj.relative_test_filename(filename)
215 expected = expectations.get_expectations_string(filename)
216 actual = [keywords[result]]
218 if result == test_expectations.PASS:
220 elif result == test_expectations.CRASH:
223 if filename not in retry_summary.unexpected_results:
224 actual.extend(expectations.get_expectations_string(
225 filename).split(" "))
228 retry_result = retry_summary.unexpected_results[filename]
229 if result != retry_result:
230 actual.append(keywords[retry_result])
236 tests[test]['expected'] = expected
237 tests[test]['actual'] = " ".join(actual)
239 results['tests'] = tests
240 results['num_passes'] = num_passes
241 results['num_flaky'] = num_flaky
242 results['num_regressions'] = num_regressions
248 """A class for managing running a series of tests on a series of layout
251 HTTP_SUBDIR = os.sep.join(['', 'http', ''])
252 WEBSOCKET_SUBDIR = os.sep.join(['', 'websocket', ''])
254 # The per-test timeout in milliseconds, if no --time-out-ms option was
255 # given to run_webkit_tests. This should correspond to the default timeout
257 DEFAULT_TEST_TIMEOUT_MS = 6 * 1000
259 def __init__(self, port, options, printer):
260 """Initialize test runner data structures.
263 port: an object implementing port-specific
264 options: a dictionary of command line options
265 printer: a Printer object to record updates to.
268 self._options = options
269 self._printer = printer
271 # disable wss server. need to install pyOpenSSL on buildbots.
272 # self._websocket_secure_server = websocket_server.PyWebSocket(
273 # options.results_directory, use_tls=True, port=9323)
275 # a list of TestType objects
276 self._test_types = []
278 # a set of test files, and the same tests as a list
279 self._test_files = set()
280 self._test_files_list = None
281 self._result_queue = Queue.Queue()
283 self._retrying = False
285 # Hack for dumping threads on the bots
286 self._last_thread_dump = None
289 _log.debug("flushing stdout")
291 _log.debug("flushing stderr")
293 _log.debug("stopping http server")
294 self._port.stop_http_server()
295 _log.debug("stopping websocket server")
296 self._port.stop_websocket_server()
298 def gather_file_paths(self, paths):
299 """Find all the files to test.
302 paths: a list of globs to use instead of the defaults."""
303 self._test_files = test_files.gather_test_files(self._port, paths)
305 def parse_expectations(self, test_platform_name, is_debug_mode):
306 """Parse the expectations from the test_list files and return a data
307 structure holding them. Throws an error if the test_list files have
309 if self._options.lint_test_files:
312 test_files = self._test_files
315 expectations_str = self._port.test_expectations()
316 overrides_str = self._port.test_expectations_overrides()
317 self._expectations = test_expectations.TestExpectations(
318 self._port, test_files, expectations_str, test_platform_name,
319 is_debug_mode, self._options.lint_test_files,
320 tests_are_present=True, overrides=overrides_str)
321 return self._expectations
322 except SyntaxError, err:
323 if self._options.lint_test_files:
328 def prepare_lists_and_print_output(self):
329 """Create appropriate subsets of test lists and returns a
330 ResultSummary object. Also prints expected test counts.
333 # Remove skipped - both fixable and ignored - files from the
334 # top-level list of files to test.
335 num_all_test_files = len(self._test_files)
336 self._printer.print_expected("Found: %d tests" %
337 (len(self._test_files)))
338 if not num_all_test_files:
339 _log.critical("No tests to run.")
343 if num_all_test_files > 1 and not self._options.force:
344 skipped = self._expectations.get_tests_with_result_type(
345 test_expectations.SKIP)
346 self._test_files -= skipped
348 # Create a sorted list of test files so the subset chunk,
349 # if used, contains alphabetically consecutive tests.
350 self._test_files_list = list(self._test_files)
351 if self._options.randomize_order:
352 random.shuffle(self._test_files_list)
354 self._test_files_list.sort()
356 # If the user specifies they just want to run a subset of the tests,
357 # just grab a subset of the non-skipped tests.
358 if self._options.run_chunk or self._options.run_part:
359 chunk_value = self._options.run_chunk or self._options.run_part
360 test_files = self._test_files_list
362 (chunk_num, chunk_len) = chunk_value.split(":")
363 chunk_num = int(chunk_num)
364 assert(chunk_num >= 0)
365 test_size = int(chunk_len)
366 assert(test_size > 0)
368 _log.critical("invalid chunk '%s'" % chunk_value)
371 # Get the number of tests
372 num_tests = len(test_files)
374 # Get the start offset of the slice.
375 if self._options.run_chunk:
376 chunk_len = test_size
377 # In this case chunk_num can be really large. We need
378 # to make the slave fit in the current number of tests.
379 slice_start = (chunk_num * chunk_len) % num_tests
382 assert(test_size <= num_tests)
383 assert(chunk_num <= test_size)
385 # To count the chunk_len, and make sure we don't skip
386 # some tests, we round to the next value that fits exactly
388 rounded_tests = num_tests
389 if rounded_tests % test_size != 0:
390 rounded_tests = (num_tests + test_size -
391 (num_tests % test_size))
393 chunk_len = rounded_tests / test_size
394 slice_start = chunk_len * (chunk_num - 1)
395 # It does not mind if we go over test_size.
397 # Get the end offset of the slice.
398 slice_end = min(num_tests, slice_start + chunk_len)
400 files = test_files[slice_start:slice_end]
402 tests_run_msg = 'Running: %d tests (chunk slice [%d:%d] of %d)' % (
403 (slice_end - slice_start), slice_start, slice_end, num_tests)
404 self._printer.print_expected(tests_run_msg)
406 # If we reached the end and we don't have enough tests, we run some
407 # from the beginning.
408 if (self._options.run_chunk and
409 (slice_end - slice_start < chunk_len)):
410 extra = 1 + chunk_len - (slice_end - slice_start)
411 extra_msg = (' last chunk is partial, appending [0:%d]' %
413 self._printer.print_expected(extra_msg)
414 tests_run_msg += "\n" + extra_msg
415 files.extend(test_files[0:extra])
416 tests_run_filename = os.path.join(self._options.results_directory,
418 with codecs.open(tests_run_filename, "w", "utf-8") as file:
419 file.write(tests_run_msg + "\n")
421 len_skip_chunk = int(len(files) * len(skipped) /
422 float(len(self._test_files)))
423 skip_chunk_list = list(skipped)[0:len_skip_chunk]
424 skip_chunk = set(skip_chunk_list)
426 # Update expectations so that the stats are calculated correctly.
427 # We need to pass a list that includes the right # of skipped files
428 # to ParseExpectations so that ResultSummary() will get the correct
429 # stats. So, we add in the subset of skipped files, and then
430 # subtract them back out.
431 self._test_files_list = files + skip_chunk_list
432 self._test_files = set(self._test_files_list)
434 self._expectations = self.parse_expectations(
435 self._port.test_platform_name(),
436 self._options.configuration == 'Debug')
438 self._test_files = set(files)
439 self._test_files_list = files
443 result_summary = ResultSummary(self._expectations,
444 self._test_files | skip_chunk)
445 self._print_expected_results_of_type(result_summary,
446 test_expectations.PASS, "passes")
447 self._print_expected_results_of_type(result_summary,
448 test_expectations.FAIL, "failures")
449 self._print_expected_results_of_type(result_summary,
450 test_expectations.FLAKY, "flaky")
451 self._print_expected_results_of_type(result_summary,
452 test_expectations.SKIP, "skipped")
454 if self._options.force:
455 self._printer.print_expected('Running all tests, including '
458 # Note that we don't actually run the skipped tests (they were
459 # subtracted out of self._test_files, above), but we stub out the
460 # results here so the statistics can remain accurate.
461 for test in skip_chunk:
462 result = dump_render_tree_thread.TestResult(test,
463 failures=[], test_run_time=0, total_time_for_all_diffs=0,
465 result.type = test_expectations.SKIP
466 result_summary.add(result, expected=True)
467 self._printer.print_expected('')
469 return result_summary
471 def add_test_type(self, test_type):
472 """Add a TestType to the TestRunner."""
473 self._test_types.append(test_type)
475 def _get_dir_for_test_file(self, test_file):
476 """Returns the highest-level directory by which to shard the given
478 index = test_file.rfind(os.sep + 'LayoutTests' + os.sep)
480 test_file = test_file[index + len('LayoutTests/'):]
481 test_file_parts = test_file.split(os.sep, 1)
482 directory = test_file_parts[0]
483 test_file = test_file_parts[1]
485 # The http tests are very stable on mac/linux.
486 # TODO(ojan): Make the http server on Windows be apache so we can
487 # turn shard the http tests there as well. Switching to apache is
488 # what made them stable on linux/mac.
489 return_value = directory
490 while ((directory != 'http' or sys.platform in ('darwin', 'linux2'))
491 and test_file.find(os.sep) >= 0):
492 test_file_parts = test_file.split(os.sep, 1)
493 directory = test_file_parts[0]
494 return_value = os.path.join(return_value, directory)
495 test_file = test_file_parts[1]
499 def _get_test_info_for_file(self, test_file):
500 """Returns the appropriate TestInfo object for the file. Mostly this
501 is used for looking up the timeout value (in ms) to use for the given
503 if self._expectations.has_modifier(test_file, test_expectations.SLOW):
504 return TestInfo(self._port, test_file,
505 self._options.slow_time_out_ms)
506 return TestInfo(self._port, test_file, self._options.time_out_ms)
508 def _get_test_file_queue(self, test_files):
509 """Create the thread safe queue of lists of (test filenames, test URIs)
510 tuples. Each TestShellThread pulls a list from this queue and runs
511 those tests in order before grabbing the next available list.
513 Shard the lists by directory. This helps ensure that tests that depend
514 on each other (aka bad tests!) continue to run together as most
515 cross-tests dependencies tend to occur within the same directory.
518 The Queue of lists of TestInfo objects.
521 if (self._options.experimental_fully_parallel or
522 self._is_single_threaded()):
523 filename_queue = Queue.Queue()
524 for test_file in test_files:
526 ('.', [self._get_test_info_for_file(test_file)]))
527 return filename_queue
530 for test_file in test_files:
531 directory = self._get_dir_for_test_file(test_file)
532 tests_by_dir.setdefault(directory, [])
533 tests_by_dir[directory].append(
534 self._get_test_info_for_file(test_file))
536 # Sort by the number of tests in the dir so that the ones with the
537 # most tests get run first in order to maximize parallelization.
538 # Number of tests is a good enough, but not perfect, approximation
539 # of how long that set of tests will take to run. We can't just use
540 # a PriorityQueue until we move # to Python 2.6.
543 for directory in tests_by_dir:
544 test_list = tests_by_dir[directory]
545 # Keep the tests in alphabetical order.
546 # TODO: Remove once tests are fixed so they can be run in any
549 test_list_tuple = (directory, test_list)
550 if directory == 'LayoutTests' + os.sep + 'http':
551 http_tests = test_list_tuple
553 test_lists.append(test_list_tuple)
554 test_lists.sort(lambda a, b: cmp(len(b[1]), len(a[1])))
556 # Put the http tests first. There are only a couple hundred of them,
557 # but each http test takes a very long time to run, so sorting by the
558 # number of tests doesn't accurately capture how long they take to run.
560 test_lists.insert(0, http_tests)
562 filename_queue = Queue.Queue()
563 for item in test_lists:
564 filename_queue.put(item)
565 return filename_queue
567 def _get_dump_render_tree_args(self, index):
568 """Returns the tuple of arguments for tests and for DumpRenderTree."""
570 test_args = test_type_base.TestArguments()
572 if self._options.pixel_tests:
573 png_path = os.path.join(self._options.results_directory,
574 "png_result%s.png" % index)
575 shell_args.append("--pixel-tests=" + png_path)
576 test_args.png_path = png_path
578 test_args.new_baseline = self._options.new_baseline
579 test_args.reset_results = self._options.reset_results
581 test_args.show_sources = self._options.sources
583 if self._options.startup_dialog:
584 shell_args.append('--testshell-startup-dialog')
586 if self._options.gp_fault_error_box:
587 shell_args.append('--gp-fault-error-box')
589 return test_args, png_path, shell_args
591 def _contains_tests(self, subdir):
592 for test_file in self._test_files:
593 if test_file.find(subdir) >= 0:
597 def _instantiate_dump_render_tree_threads(self, test_files,
599 """Instantitates and starts the TestShellThread(s).
604 filename_queue = self._get_test_file_queue(test_files)
606 # Instantiate TestShellThreads and start them.
608 for i in xrange(int(self._options.child_processes)):
609 # Create separate TestTypes instances for each thread.
611 for test_type in self._test_types:
612 test_types.append(test_type(self._port,
613 self._options.results_directory))
615 test_args, png_path, shell_args = \
616 self._get_dump_render_tree_args(i)
617 thread = dump_render_tree_thread.TestShellThread(self._port,
618 filename_queue, self._result_queue, test_types, test_args,
619 png_path, shell_args, self._options)
620 if self._is_single_threaded():
621 thread.run_in_main_thread(self, result_summary)
624 threads.append(thread)
628 def _is_single_threaded(self):
629 """Returns whether we should run all the tests in the main thread."""
630 return int(self._options.child_processes) == 1
632 def _dump_thread_states(self):
633 for thread_id, stack in sys._current_frames().items():
634 # FIXME: Python 2.6 has thread.ident which we could
635 # use to map from thread_id back to thread.name
636 print "\n# Thread: %d" % thread_id
637 for filename, lineno, name, line in traceback.extract_stack(stack):
638 print 'File: "%s", line %d, in %s' % (filename, lineno, name)
640 print " %s" % (line.strip())
642 def _dump_thread_states_if_necessary(self):
643 # HACK: Dump thread states every minute to figure out what's
644 # hanging on the bots.
645 if not self._options.verbose:
647 dump_threads_every = 60 # Dump every minute
648 if not self._last_thread_dump:
649 self._last_thread_dump = time.time()
650 time_since_last_dump = time.time() - self._last_thread_dump
651 if time_since_last_dump > dump_threads_every:
652 self._dump_thread_states()
653 self._last_thread_dump = time.time()
655 def _run_tests(self, file_list, result_summary):
656 """Runs the tests in the file_list.
658 Return: A tuple (failures, thread_timings, test_timings,
659 individual_test_timings)
660 failures is a map from test to list of failure types
661 thread_timings is a list of dicts with the total runtime
662 of each thread with 'name', 'num_tests', 'total_time' properties
663 test_timings is a list of timings for each sharded subdirectory
664 of the form [time, directory_name, num_tests]
665 individual_test_timings is a list of run times for each test
666 in the form {filename:filename, test_run_time:test_run_time}
667 result_summary: summary object to populate with the results
669 # FIXME: We should use webkitpy.tool.grammar.pluralize here.
671 if self._options.child_processes > 1:
673 self._printer.print_update('Starting %s%s ...' %
674 (self._port.driver_name(), plural))
675 threads = self._instantiate_dump_render_tree_threads(file_list,
677 self._printer.print_update("Starting testing ...")
679 # Wait for the threads to finish and collect test failures.
682 individual_test_timings = []
684 keyboard_interrupted = False
686 # Loop through all the threads waiting for them to finish.
687 for thread in threads:
688 # FIXME: We'll end up waiting on the first thread the whole
689 # time. That means we won't notice exceptions on other
690 # threads until the first one exits.
691 # We should instead while True: in the outer loop
692 # and then loop through threads joining and checking
693 # isAlive and get_exception_info. Exiting on any exception.
694 while thread.isAlive():
695 # Wake the main thread every 0.1 seconds so we
696 # can call update_summary in a timely fashion.
698 # HACK: Used for debugging threads on the bots.
699 self._dump_thread_states_if_necessary()
700 self.update_summary(result_summary)
702 except KeyboardInterrupt:
703 keyboard_interrupted = True
704 for thread in threads:
707 if not keyboard_interrupted:
708 for thread in threads:
709 # Check whether a thread died before normal completion.
710 exception_info = thread.get_exception_info()
711 if exception_info is not None:
712 # Re-raise the thread's exception here to make it clear
713 # something went wrong. Otherwise, the tests that did not
714 # run would be assumed to have passed.
715 raise (exception_info[0], exception_info[1],
718 for thread in threads:
719 thread_timings.append({'name': thread.getName(),
720 'num_tests': thread.get_num_tests(),
721 'total_time': thread.get_total_time()})
722 test_timings.update(thread.get_directory_timing_stats())
723 individual_test_timings.extend(thread.get_test_results())
724 return (keyboard_interrupted, thread_timings, test_timings,
725 individual_test_timings)
727 def needs_http(self):
728 """Returns whether the test runner needs an HTTP server."""
729 return self._contains_tests(self.HTTP_SUBDIR)
731 def run(self, result_summary):
732 """Run all our tests on all our test files.
734 For each test file, we run each test type. If there are any failures,
735 we collect them for reporting.
738 result_summary: a summary object tracking the test results.
741 The number of unexpected results (0 == success)
743 if not self._test_files:
745 start_time = time.time()
747 if self.needs_http():
748 self._printer.print_update('Starting HTTP server ...')
750 self._port.start_http_server()
752 if self._contains_tests(self.WEBSOCKET_SUBDIR):
753 self._printer.print_update('Starting WebSocket server ...')
754 self._port.start_websocket_server()
755 # self._websocket_secure_server.Start()
757 keyboard_interrupted, thread_timings, test_timings, \
758 individual_test_timings = (
759 self._run_tests(self._test_files_list, result_summary))
761 # We exclude the crashes from the list of results to retry, because
762 # we want to treat even a potentially flaky crash as an error.
763 failures = self._get_failures(result_summary, include_crashes=False)
764 retry_summary = result_summary
765 while (len(failures) and self._options.retry_failures and
766 not self._retrying and not keyboard_interrupted):
768 _log.info("Retrying %d unexpected failure(s) ..." % len(failures))
770 self._retrying = True
771 retry_summary = ResultSummary(self._expectations, failures.keys())
772 # Note that we intentionally ignore the return value here.
773 self._run_tests(failures.keys(), retry_summary)
774 failures = self._get_failures(retry_summary, include_crashes=True)
776 end_time = time.time()
778 self._print_timing_statistics(end_time - start_time,
779 thread_timings, test_timings,
780 individual_test_timings,
783 self._print_result_summary(result_summary)
788 self._printer.print_one_line_summary(result_summary.total,
789 result_summary.expected,
790 result_summary.unexpected)
792 unexpected_results = summarize_unexpected_results(self._port,
793 self._expectations, result_summary, retry_summary)
794 self._printer.print_unexpected_results(unexpected_results)
796 # Write the same data to log files.
797 self._write_json_files(unexpected_results, result_summary,
798 individual_test_timings)
800 # Upload generated JSON files to appengine server.
801 self._upload_json_files()
803 # Write the summary to disk (results.html) and display it if requested.
804 wrote_results = self._write_results_html_file(result_summary)
805 if self._options.show_results and wrote_results:
806 self._show_results_html_file()
808 # Now that we've completed all the processing we can, we re-raise
809 # a KeyboardInterrupt if necessary so the caller can handle it.
810 if keyboard_interrupted:
811 raise KeyboardInterrupt
813 # Ignore flaky failures and unexpected passes so we don't turn the
815 return unexpected_results['num_regressions']
817 def update_summary(self, result_summary):
818 """Update the summary and print results with any completed tests."""
821 result = self._result_queue.get_nowait()
825 expected = self._expectations.matches_an_expected_result(
826 result.filename, result.type, self._options.pixel_tests)
827 result_summary.add(result, expected)
828 exp_str = self._expectations.get_expectations_string(
830 got_str = self._expectations.expectation_to_string(result.type)
831 self._printer.print_test_result(result, expected, exp_str, got_str)
832 self._printer.print_progress(result_summary, self._retrying,
833 self._test_files_list)
835 def _get_failures(self, result_summary, include_crashes):
836 """Filters a dict of results and returns only the failures.
839 result_summary: the results of the test run
840 include_crashes: whether crashes are included in the output.
841 We use False when finding the list of failures to retry
842 to see if the results were flaky. Although the crashes may also be
843 flaky, we treat them as if they aren't so that they're not ignored.
845 a dict of files -> results
848 for test, result in result_summary.unexpected_results.iteritems():
849 if (result == test_expectations.PASS or
850 result == test_expectations.CRASH and not include_crashes):
852 failed_results[test] = result
854 return failed_results
856 def _write_json_files(self, unexpected_results, result_summary,
857 individual_test_timings):
858 """Writes the results of the test run as JSON files into the results
861 There are three different files written into the results dir:
862 unexpected_results.json: A short list of any unexpected results.
863 This is used by the buildbots to display results.
864 expectations.json: This is used by the flakiness dashboard.
865 results.json: A full list of the results - used by the flakiness
866 dashboard and the aggregate results dashboard.
869 unexpected_results: dict of unexpected results
870 result_summary: full summary object
871 individual_test_timings: list of test times (used by the flakiness
874 results_directory = self._options.results_directory
875 _log.debug("Writing JSON files in %s." % results_directory)
876 unexpected_json_path = os.path.join(results_directory, "unexpected_results.json")
877 with codecs.open(unexpected_json_path, "w", "utf-8") as file:
878 simplejson.dump(unexpected_results, file, sort_keys=True, indent=2)
880 # Write a json file of the test_expectations.txt file for the layout
882 expectations_path = os.path.join(results_directory, "expectations.json")
883 expectations_json = \
884 self._expectations.get_expectations_json_for_all_platforms()
885 with codecs.open(expectations_path, "w", "utf-8") as file:
886 file.write(u"ADD_EXPECTATIONS(%s);" % expectations_json)
888 json_layout_results_generator.JSONLayoutResultsGenerator(
889 self._port, self._options.builder_name, self._options.build_name,
890 self._options.build_number, self._options.results_directory,
891 BUILDER_BASE_URL, individual_test_timings,
892 self._expectations, result_summary, self._test_files_list)
894 _log.debug("Finished writing JSON files.")
896 def _upload_json_files(self):
897 if not self._options.test_results_server:
900 _log.info("Uploading JSON files for builder: %s",
901 self._options.builder_name)
903 attrs = [('builder', self._options.builder_name)]
904 json_files = ["expectations.json", "results.json"]
906 files = [(file, os.path.join(self._options.results_directory, file))
907 for file in json_files]
909 uploader = test_results_uploader.TestResultsUploader(
910 self._options.test_results_server)
912 # Set uploading timeout in case appengine server is having problem.
913 # 120 seconds are more than enough to upload test results.
914 uploader.upload(attrs, files, 120)
915 except Exception, err:
916 _log.error("Upload failed: %s" % err)
919 _log.info("JSON files uploaded.")
921 def _print_expected_results_of_type(self, result_summary,
922 result_type, result_type_str):
923 """Print the number of the tests in a given result class.
926 result_summary - the object containing all the results to report on
927 result_type - the particular result type to report in the summary.
928 result_type_str - a string description of the result_type.
930 tests = self._expectations.get_tests_with_result_type(result_type)
931 now = result_summary.tests_by_timeline[test_expectations.NOW]
932 wontfix = result_summary.tests_by_timeline[test_expectations.WONTFIX]
933 defer = result_summary.tests_by_timeline[test_expectations.DEFER]
935 # We use a fancy format string in order to print the data out in a
936 # nicely-aligned table.
937 fmtstr = ("Expect: %%5d %%-8s (%%%dd now, %%%dd defer, %%%dd wontfix)"
938 % (self._num_digits(now), self._num_digits(defer),
939 self._num_digits(wontfix)))
940 self._printer.print_expected(fmtstr %
941 (len(tests), result_type_str, len(tests & now),
942 len(tests & defer), len(tests & wontfix)))
944 def _num_digits(self, num):
945 """Returns the number of digits needed to represent the length of a
949 ndigits = int(math.log10(len(num))) + 1
952 def _print_timing_statistics(self, total_time, thread_timings,
953 directory_test_timings, individual_test_timings,
955 """Record timing-specific information for the test run.
958 total_time: total elapsed time (in seconds) for the test run
959 thread_timings: wall clock time each thread ran for
960 directory_test_timings: timing by directory
961 individual_test_timings: timing by file
962 result_summary: summary object for the test run
964 self._printer.print_timing("Test timing:")
965 self._printer.print_timing(" %6.2f total testing time" % total_time)
966 self._printer.print_timing("")
967 self._printer.print_timing("Thread timing:")
969 for t in thread_timings:
970 self._printer.print_timing(" %10s: %5d tests, %6.2f secs" %
971 (t['name'], t['num_tests'], t['total_time']))
972 cuml_time += t['total_time']
973 self._printer.print_timing(" %6.2f cumulative, %6.2f optimal" %
974 (cuml_time, cuml_time / int(self._options.child_processes)))
975 self._printer.print_timing("")
977 self._print_aggregate_test_statistics(individual_test_timings)
978 self._print_individual_test_times(individual_test_timings,
980 self._print_directory_timings(directory_test_timings)
982 def _print_aggregate_test_statistics(self, individual_test_timings):
983 """Prints aggregate statistics (e.g. median, mean, etc.) for all tests.
985 individual_test_timings: List of dump_render_tree_thread.TestStats
988 test_types = [] # Unit tests don't actually produce any timings.
989 if individual_test_timings:
990 test_types = individual_test_timings[0].time_for_diffs.keys()
991 times_for_dump_render_tree = []
992 times_for_diff_processing = []
993 times_per_test_type = {}
994 for test_type in test_types:
995 times_per_test_type[test_type] = []
997 for test_stats in individual_test_timings:
998 times_for_dump_render_tree.append(test_stats.test_run_time)
999 times_for_diff_processing.append(
1000 test_stats.total_time_for_all_diffs)
1001 time_for_diffs = test_stats.time_for_diffs
1002 for test_type in test_types:
1003 times_per_test_type[test_type].append(
1004 time_for_diffs[test_type])
1006 self._print_statistics_for_test_timings(
1007 "PER TEST TIME IN TESTSHELL (seconds):",
1008 times_for_dump_render_tree)
1009 self._print_statistics_for_test_timings(
1010 "PER TEST DIFF PROCESSING TIMES (seconds):",
1011 times_for_diff_processing)
1012 for test_type in test_types:
1013 self._print_statistics_for_test_timings(
1014 "PER TEST TIMES BY TEST TYPE: %s" % test_type,
1015 times_per_test_type[test_type])
1017 def _print_individual_test_times(self, individual_test_timings,
1019 """Prints the run times for slow, timeout and crash tests.
1021 individual_test_timings: List of dump_render_tree_thread.TestStats
1023 result_summary: summary object for test run
1025 # Reverse-sort by the time spent in DumpRenderTree.
1026 individual_test_timings.sort(lambda a, b:
1027 cmp(b.test_run_time, a.test_run_time))
1031 timeout_or_crash_tests = []
1032 unexpected_slow_tests = []
1033 for test_tuple in individual_test_timings:
1034 filename = test_tuple.filename
1035 is_timeout_crash_or_slow = False
1036 if self._expectations.has_modifier(filename,
1037 test_expectations.SLOW):
1038 is_timeout_crash_or_slow = True
1039 slow_tests.append(test_tuple)
1041 if filename in result_summary.failures:
1042 result = result_summary.results[filename].type
1043 if (result == test_expectations.TIMEOUT or
1044 result == test_expectations.CRASH):
1045 is_timeout_crash_or_slow = True
1046 timeout_or_crash_tests.append(test_tuple)
1048 if (not is_timeout_crash_or_slow and
1049 num_printed < printing.NUM_SLOW_TESTS_TO_LOG):
1050 num_printed = num_printed + 1
1051 unexpected_slow_tests.append(test_tuple)
1053 self._printer.print_timing("")
1054 self._print_test_list_timing("%s slowest tests that are not "
1055 "marked as SLOW and did not timeout/crash:" %
1056 printing.NUM_SLOW_TESTS_TO_LOG, unexpected_slow_tests)
1057 self._printer.print_timing("")
1058 self._print_test_list_timing("Tests marked as SLOW:", slow_tests)
1059 self._printer.print_timing("")
1060 self._print_test_list_timing("Tests that timed out or crashed:",
1061 timeout_or_crash_tests)
1062 self._printer.print_timing("")
1064 def _print_test_list_timing(self, title, test_list):
1065 """Print timing info for each test.
1068 title: section heading
1069 test_list: tests that fall in this section
1071 if self._printer.disabled('slowest'):
1074 self._printer.print_timing(title)
1075 for test_tuple in test_list:
1076 filename = test_tuple.filename[len(
1077 self._port.layout_tests_dir()) + 1:]
1078 filename = filename.replace('\\', '/')
1079 test_run_time = round(test_tuple.test_run_time, 1)
1080 self._printer.print_timing(" %s took %s seconds" %
1081 (filename, test_run_time))
1083 def _print_directory_timings(self, directory_test_timings):
1084 """Print timing info by directory for any directories that
1085 take > 10 seconds to run.
1088 directory_test_timing: time info for each directory
1091 for directory in directory_test_timings:
1092 num_tests, time_for_directory = directory_test_timings[directory]
1093 timings.append((round(time_for_directory, 1), directory,
1097 self._printer.print_timing("Time to process slowest subdirectories:")
1098 min_seconds_to_print = 10
1099 for timing in timings:
1100 if timing[0] > min_seconds_to_print:
1101 self._printer.print_timing(
1102 " %s took %s seconds to run %s tests." % (timing[1],
1103 timing[0], timing[2]))
1104 self._printer.print_timing("")
1106 def _print_statistics_for_test_timings(self, title, timings):
1107 """Prints the median, mean and standard deviation of the values in
1111 title: Title for these timings.
1112 timings: A list of floats representing times.
1114 self._printer.print_timing(title)
1117 num_tests = len(timings)
1120 percentile90 = timings[int(.9 * num_tests)]
1121 percentile99 = timings[int(.99 * num_tests)]
1123 if num_tests % 2 == 1:
1124 median = timings[((num_tests - 1) / 2) - 1]
1126 lower = timings[num_tests / 2 - 1]
1127 upper = timings[num_tests / 2]
1128 median = (float(lower + upper)) / 2
1130 mean = sum(timings) / num_tests
1132 for time in timings:
1133 sum_of_deviations = math.pow(time - mean, 2)
1135 std_deviation = math.sqrt(sum_of_deviations / num_tests)
1136 self._printer.print_timing(" Median: %6.3f" % median)
1137 self._printer.print_timing(" Mean: %6.3f" % mean)
1138 self._printer.print_timing(" 90th percentile: %6.3f" % percentile90)
1139 self._printer.print_timing(" 99th percentile: %6.3f" % percentile99)
1140 self._printer.print_timing(" Standard dev: %6.3f" % std_deviation)
1141 self._printer.print_timing("")
1143 def _print_result_summary(self, result_summary):
1144 """Print a short summary about how many tests passed.
1147 result_summary: information to log
1149 failed = len(result_summary.failures)
1151 result_summary.tests_by_expectation[test_expectations.SKIP])
1152 total = result_summary.total
1153 passed = total - failed - skipped
1156 pct_passed = float(passed) * 100 / total
1158 self._printer.print_actual("")
1159 self._printer.print_actual("=> Results: %d/%d tests passed (%.1f%%)" %
1160 (passed, total, pct_passed))
1161 self._printer.print_actual("")
1162 self._print_result_summary_entry(result_summary,
1163 test_expectations.NOW, "Tests to be fixed for the current release")
1165 self._printer.print_actual("")
1166 self._print_result_summary_entry(result_summary,
1167 test_expectations.DEFER,
1168 "Tests we'll fix in the future if they fail (DEFER)")
1170 self._printer.print_actual("")
1171 self._print_result_summary_entry(result_summary,
1172 test_expectations.WONTFIX,
1173 "Tests that will only be fixed if they crash (WONTFIX)")
1174 self._printer.print_actual("")
1176 def _print_result_summary_entry(self, result_summary, timeline,
1178 """Print a summary block of results for a particular timeline of test.
1181 result_summary: summary to print results for
1182 timeline: the timeline to print results for (NOT, WONTFIX, etc.)
1183 heading: a textual description of the timeline
1185 total = len(result_summary.tests_by_timeline[timeline])
1186 not_passing = (total -
1187 len(result_summary.tests_by_expectation[test_expectations.PASS] &
1188 result_summary.tests_by_timeline[timeline]))
1189 self._printer.print_actual("=> %s (%d):" % (heading, not_passing))
1191 for result in TestExpectationsFile.EXPECTATION_ORDER:
1192 if result == test_expectations.PASS:
1194 results = (result_summary.tests_by_expectation[result] &
1195 result_summary.tests_by_timeline[timeline])
1196 desc = TestExpectationsFile.EXPECTATION_DESCRIPTIONS[result]
1197 if not_passing and len(results):
1198 pct = len(results) * 100.0 / not_passing
1199 self._printer.print_actual(" %5d %-24s (%4.1f%%)" %
1200 (len(results), desc[len(results) != 1], pct))
1202 def _results_html(self, test_files, failures, title="Test Failures", override_time=None):
1204 test_files = a list of file paths
1205 failures = dictionary mapping test paths to failure objects
1206 title = title printed at top of test
1207 override_time = current time (used by unit tests)
1211 <title>Layout Test Results (%(time)s)</title>
1214 <h2>%(title)s (%(time)s)</h2>
1215 """ % {'title': title, 'time': override_time or time.asctime()}
1217 for test_file in sorted(test_files):
1218 test_name = self._port.relative_test_filename(test_file)
1219 test_url = self._port.filename_to_uri(test_file)
1220 page += u"<p><a href='%s'>%s</a><br />\n" % (test_url, test_name)
1221 test_failures = failures.get(test_file, [])
1222 for failure in test_failures:
1223 page += u" %s<br/>" % failure.result_html_output(test_name)
1225 page += "</body></html>\n"
1228 def _write_results_html_file(self, result_summary):
1229 """Write results.html which is a summary of tests that failed.
1232 result_summary: a summary of the results :)
1235 True if any results were written (since expected failures may be
1239 if self._options.full_results_html:
1240 results_title = "Test Failures"
1241 test_files = result_summary.failures.keys()
1243 results_title = "Unexpected Test Failures"
1244 unexpected_failures = self._get_failures(result_summary,
1245 include_crashes=True)
1246 test_files = unexpected_failures.keys()
1247 if not len(test_files):
1250 out_filename = os.path.join(self._options.results_directory,
1252 with codecs.open(out_filename, "w", "utf-8") as results_file:
1253 html = self._results_html(test_files, result_summary.failures, results_title)
1254 results_file.write(html)
1258 def _show_results_html_file(self):
1259 """Shows the results.html page."""
1260 results_filename = os.path.join(self._options.results_directory,
1262 self._port.show_results_html_file(results_filename)
1265 def read_test_files(files):
1268 # FIXME: This could be cleaner using a list comprehension.
1269 for line in codecs.open(file, "r", "utf-8"):
1270 line = test_expectations.strip_comments(line)
1276 def run(port_obj, options, args, regular_output=sys.stderr,
1277 buildbot_output=sys.stdout):
1281 port_obj: Port object for port-specific behavior
1282 options: a dictionary of command line options
1283 args: a list of sub directories or files to test
1284 regular_output: a stream-like object that we can send logging/debug
1286 buildbot_output: a stream-like object that we can write all output that
1287 is intended to be parsed by the buildbot to
1289 the number of unexpected results that occurred, or -1 if there is an
1293 # Configure the printing subsystem for printing output, logging debug
1294 # info, and tracing tests.
1296 if not options.child_processes:
1297 # FIXME: Investigate perf/flakiness impact of using cpu_count + 1.
1298 options.child_processes = port_obj.default_child_processes()
1300 printer = printing.Printer(port_obj, options, regular_output=regular_output,
1301 buildbot_output=buildbot_output,
1302 child_processes=int(options.child_processes),
1303 is_fully_parallel=options.experimental_fully_parallel)
1304 if options.help_printing:
1305 printer.help_printing()
1308 executive = Executive()
1310 if not options.configuration:
1311 options.configuration = port_obj.default_configuration()
1313 if options.pixel_tests is None:
1314 options.pixel_tests = True
1316 if not options.use_apache:
1317 options.use_apache = sys.platform in ('darwin', 'linux2')
1319 if options.results_directory.startswith("/"):
1320 # Assume it's an absolute path and normalize.
1321 options.results_directory = port_obj.get_absolute_path(
1322 options.results_directory)
1324 # If it's a relative path, make the output directory relative to
1326 options.results_directory = port_obj.results_directory()
1328 last_unexpected_results = []
1329 if options.print_last_failures or options.retest_last_failures:
1330 unexpected_results_filename = os.path.join(
1331 options.results_directory, "unexpected_results.json")
1332 with codecs.open(unexpected_results_filename, "r", "utf-8") as file:
1333 results = simplejson.load(file)
1334 last_unexpected_results = results['tests'].keys()
1335 if options.print_last_failures:
1336 printer.write("\n".join(last_unexpected_results) + "\n")
1339 if options.clobber_old_results:
1340 # Just clobber the actual test results directories since the other
1341 # files in the results directory are explicitly used for cross-run
1343 printer.print_update("Clobbering old results in %s" %
1344 options.results_directory)
1345 layout_tests_dir = port_obj.layout_tests_dir()
1346 possible_dirs = os.listdir(layout_tests_dir)
1347 for dirname in possible_dirs:
1348 if os.path.isdir(os.path.join(layout_tests_dir, dirname)):
1349 shutil.rmtree(os.path.join(options.results_directory, dirname),
1352 if not options.time_out_ms:
1353 if options.configuration == "Debug":
1354 options.time_out_ms = str(2 * TestRunner.DEFAULT_TEST_TIMEOUT_MS)
1356 options.time_out_ms = str(TestRunner.DEFAULT_TEST_TIMEOUT_MS)
1358 options.slow_time_out_ms = str(5 * int(options.time_out_ms))
1359 printer.print_config("Regular timeout: %s, slow test timeout: %s" %
1360 (options.time_out_ms, options.slow_time_out_ms))
1362 if int(options.child_processes) == 1:
1363 printer.print_config("Running one %s" % port_obj.driver_name())
1365 printer.print_config("Running %s %ss in parallel" % (
1366 options.child_processes, port_obj.driver_name()))
1368 # Include all tests if none are specified.
1371 if arg and arg != '':
1372 new_args.append(arg)
1377 paths += last_unexpected_results
1378 if options.test_list:
1379 paths += read_test_files(options.test_list)
1381 # Create the output directory if it doesn't already exist.
1382 port_obj.maybe_make_directory(options.results_directory)
1383 printer.print_update("Collecting tests ...")
1385 test_runner = TestRunner(port_obj, options, printer)
1386 test_runner.gather_file_paths(paths)
1388 if options.lint_test_files:
1389 # Creating the expecations for each platform/configuration pair does
1390 # all the test list parsing and ensures it's correct syntax (e.g. no
1392 for platform_name in port_obj.test_platform_names():
1393 test_runner.parse_expectations(platform_name, is_debug_mode=True)
1394 test_runner.parse_expectations(platform_name, is_debug_mode=False)
1396 _log.info("If there are no fail messages, errors or exceptions, "
1397 "then the lint succeeded.")
1400 printer.print_config("Using port '%s'" % port_obj.name())
1401 printer.print_config("Placing test results in %s" %
1402 options.results_directory)
1403 if options.new_baseline:
1404 printer.print_config("Placing new baselines in %s" %
1405 port_obj.baseline_path())
1406 printer.print_config("Using %s build" % options.configuration)
1407 if options.pixel_tests:
1408 printer.print_config("Pixel tests enabled")
1410 printer.print_config("Pixel tests disabled")
1411 printer.print_config("")
1413 printer.print_update("Parsing expectations ...")
1414 test_runner.parse_expectations(port_obj.test_platform_name(),
1415 options.configuration == 'Debug')
1417 printer.print_update("Checking build ...")
1418 if not port_obj.check_build(test_runner.needs_http()):
1421 printer.print_update("Starting helper ...")
1422 port_obj.start_helper()
1424 # Check that the system dependencies (themes, fonts, ...) are correct.
1425 if not options.nocheck_sys_deps:
1426 printer.print_update("Checking system dependencies ...")
1427 if not port_obj.check_sys_deps(test_runner.needs_http()):
1430 printer.print_update("Preparing tests ...")
1431 result_summary = test_runner.prepare_lists_and_print_output()
1433 port_obj.setup_test_run()
1435 test_runner.add_test_type(text_diff.TestTextDiff)
1436 if options.pixel_tests:
1437 test_runner.add_test_type(image_diff.ImageDiff)
1438 if options.fuzzy_pixel_tests:
1439 test_runner.add_test_type(fuzzy_image_diff.FuzzyImageDiff)
1441 num_unexpected_results = test_runner.run(result_summary)
1443 port_obj.stop_helper()
1445 _log.debug("Exit status: %d" % num_unexpected_results)
1446 return num_unexpected_results
1449 def _compat_shim_callback(option, opt_str, value, parser):
1450 print "Ignoring unsupported option: %s" % opt_str
1453 def _compat_shim_option(option_name, **kwargs):
1454 return optparse.make_option(option_name, action="callback",
1455 callback=_compat_shim_callback,
1456 help="Ignored, for old-run-webkit-tests compat only.", **kwargs)
1459 def parse_args(args=None):
1460 """Provides a default set of command line args.
1462 Returns a tuple of options, args from optparse"""
1464 # FIXME: All of these options should be stored closer to the code which
1465 # FIXME: actually uses them. configuration_options should move
1466 # FIXME: to WebKitPort and be shared across all scripts.
1467 configuration_options = [
1468 optparse.make_option("-t", "--target", dest="configuration",
1469 help="(DEPRECATED)"),
1470 # FIXME: --help should display which configuration is default.
1471 optparse.make_option('--debug', action='store_const', const='Debug',
1472 dest="configuration",
1473 help='Set the configuration to Debug'),
1474 optparse.make_option('--release', action='store_const',
1475 const='Release', dest="configuration",
1476 help='Set the configuration to Release'),
1477 # old-run-webkit-tests also accepts -c, --configuration CONFIGURATION.
1480 print_options = printing.print_options()
1482 # FIXME: These options should move onto the ChromiumPort.
1483 chromium_options = [
1484 optparse.make_option("--chromium", action="store_true", default=False,
1485 help="use the Chromium port"),
1486 optparse.make_option("--startup-dialog", action="store_true",
1487 default=False, help="create a dialog on DumpRenderTree startup"),
1488 optparse.make_option("--gp-fault-error-box", action="store_true",
1489 default=False, help="enable Windows GP fault error box"),
1490 optparse.make_option("--nocheck-sys-deps", action="store_true",
1492 help="Don't check the system dependencies (themes)"),
1493 optparse.make_option("--use-drt", action="store_true",
1495 help="Use DumpRenderTree instead of test_shell"),
1498 # Missing Mac-specific old-run-webkit-tests options:
1499 # FIXME: Need: -g, --guard for guard malloc support on Mac.
1500 # FIXME: Need: -l --leaks Enable leaks checking.
1501 # FIXME: Need: --sample-on-timeout Run sample on timeout
1503 old_run_webkit_tests_compat = [
1504 # NRWT doesn't generate results by default anyway.
1505 _compat_shim_option("--no-new-test-results"),
1506 # NRWT doesn't sample on timeout yet anyway.
1507 _compat_shim_option("--no-sample-on-timeout"),
1508 # FIXME: NRWT needs to support remote links eventually.
1509 _compat_shim_option("--use-remote-links-to-tests"),
1510 # FIXME: NRWT doesn't need this option as much since failures are
1511 # designed to be cheap. We eventually plan to add this support.
1512 _compat_shim_option("--exit-after-n-failures", nargs=1, type="int"),
1516 # NEED for bots: --use-remote-links-to-tests Link to test files
1517 # within the SVN repository in the results.
1518 optparse.make_option("-p", "--pixel-tests", action="store_true",
1519 dest="pixel_tests", help="Enable pixel-to-pixel PNG comparisons"),
1520 optparse.make_option("--no-pixel-tests", action="store_false",
1521 dest="pixel_tests", help="Disable pixel-to-pixel PNG comparisons"),
1522 optparse.make_option("--fuzzy-pixel-tests", action="store_true",
1524 help="Also use fuzzy matching to compare pixel test outputs."),
1525 # old-run-webkit-tests allows a specific tolerance: --tolerance t
1526 # Ignore image differences less than this percentage (default: 0.1)
1527 optparse.make_option("--results-directory",
1528 default="layout-test-results",
1529 help="Output results directory source dir, relative to Debug or "
1531 optparse.make_option("--new-baseline", action="store_true",
1532 default=False, help="Save all generated results as new baselines "
1533 "into the platform directory, overwriting whatever's "
1535 optparse.make_option("--reset-results", action="store_true",
1536 default=False, help="Reset any existing baselines to the "
1537 "generated results"),
1538 optparse.make_option("--no-show-results", action="store_false",
1539 default=True, dest="show_results",
1540 help="Don't launch a browser with results after the tests "
1542 # FIXME: We should have a helper function to do this sort of
1543 # deprectated mapping and automatically log, etc.
1544 optparse.make_option("--noshow-results", action="store_false",
1545 dest="show_results",
1546 help="Deprecated, same as --no-show-results."),
1547 optparse.make_option("--no-launch-safari", action="store_false",
1548 dest="show_results",
1549 help="old-run-webkit-tests compat, same as --noshow-results."),
1550 # old-run-webkit-tests:
1551 # --[no-]launch-safari Launch (or do not launch) Safari to display
1552 # test results (default: launch)
1553 optparse.make_option("--full-results-html", action="store_true",
1555 help="Show all failures in results.html, rather than only "
1557 optparse.make_option("--clobber-old-results", action="store_true",
1558 default=False, help="Clobbers test results from previous runs."),
1559 optparse.make_option("--platform",
1560 help="Override the platform for expected results"),
1561 # old-run-webkit-tests also has HTTP toggle options:
1562 # --[no-]http Run (or do not run) http tests
1564 # --[no-]wait-for-httpd Wait for httpd if some other test
1565 # session is using it already (same
1566 # as WEBKIT_WAIT_FOR_HTTPD=1).
1571 optparse.make_option("--build", dest="build",
1572 action="store_true", default=True,
1573 help="Check to ensure the DumpRenderTree build is up-to-date "
1575 optparse.make_option("--no-build", dest="build",
1576 action="store_false", help="Don't check to see if the "
1577 "DumpRenderTree build is up-to-date."),
1578 # old-run-webkit-tests has --valgrind instead of wrapper.
1579 optparse.make_option("--wrapper",
1580 help="wrapper command to insert before invocations of "
1581 "DumpRenderTree; option is split on whitespace before "
1582 "running. (Example: --wrapper='valgrind --smc-check=all')"),
1583 # old-run-webkit-tests:
1584 # -i|--ignore-tests Comma-separated list of directories
1585 # or tests to ignore
1586 optparse.make_option("--test-list", action="append",
1587 help="read list of tests to run from file", metavar="FILE"),
1588 # old-run-webkit-tests uses --skipped==[default|ignore|only]
1589 # instead of --force:
1590 optparse.make_option("--force", action="store_true", default=False,
1591 help="Run all tests, even those marked SKIP in the test list"),
1592 optparse.make_option("--use-apache", action="store_true",
1593 default=False, help="Whether to use apache instead of lighttpd."),
1594 optparse.make_option("--time-out-ms",
1595 help="Set the timeout for each test"),
1596 # old-run-webkit-tests calls --randomize-order --random:
1597 optparse.make_option("--randomize-order", action="store_true",
1598 default=False, help=("Run tests in random order (useful "
1599 "for tracking down corruption)")),
1600 optparse.make_option("--run-chunk",
1601 help=("Run a specified chunk (n:l), the nth of len l, "
1602 "of the layout tests")),
1603 optparse.make_option("--run-part", help=("Run a specified part (n:m), "
1604 "the nth of m parts, of the layout tests")),
1605 # old-run-webkit-tests calls --batch-size: --nthly n
1606 # Restart DumpRenderTree every n tests (default: 1000)
1607 optparse.make_option("--batch-size",
1608 help=("Run a the tests in batches (n), after every n tests, "
1609 "DumpRenderTree is relaunched.")),
1610 # old-run-webkit-tests calls --run-singly: -1|--singly
1611 # Isolate each test case run (implies --nthly 1 --verbose)
1612 optparse.make_option("--run-singly", action="store_true",
1613 default=False, help="run a separate DumpRenderTree for each test"),
1614 optparse.make_option("--child-processes",
1615 help="Number of DumpRenderTrees to run in parallel."),
1616 # FIXME: Display default number of child processes that will run.
1617 optparse.make_option("--experimental-fully-parallel",
1618 action="store_true", default=False,
1619 help="run all tests in parallel"),
1620 # FIXME: Need --exit-after-n-failures N
1621 # Exit after the first N failures instead of running all tests
1622 # FIXME: Need --exit-after-n-crashes N
1623 # Exit after the first N crashes instead of running all tests
1624 # FIXME: consider: --iterations n
1625 # Number of times to run the set of tests (e.g. ABCABCABC)
1626 optparse.make_option("--print-last-failures", action="store_true",
1627 default=False, help="Print the tests in the last run that "
1628 "had unexpected failures (or passes)."),
1629 optparse.make_option("--retest-last-failures", action="store_true",
1630 default=False, help="re-test the tests in the last run that "
1631 "had unexpected failures (or passes)."),
1632 optparse.make_option("--retry-failures", action="store_true",
1634 help="Re-try any tests that produce unexpected results (default)"),
1635 optparse.make_option("--no-retry-failures", action="store_false",
1636 dest="retry_failures",
1637 help="Don't re-try any tests that produce unexpected results."),
1641 optparse.make_option("--lint-test-files", action="store_true",
1642 default=False, help=("Makes sure the test files parse for all "
1643 "configurations. Does not run any tests.")),
1646 # FIXME: Move these into json_results_generator.py
1647 results_json_options = [
1648 optparse.make_option("--builder-name", default="DUMMY_BUILDER_NAME",
1649 help=("The name of the builder shown on the waterfall running "
1650 "this script e.g. WebKit.")),
1651 optparse.make_option("--build-name", default="DUMMY_BUILD_NAME",
1652 help=("The name of the builder used in its path, e.g. "
1654 optparse.make_option("--build-number", default="DUMMY_BUILD_NUMBER",
1655 help=("The build number of the builder running this script.")),
1656 optparse.make_option("--test-results-server", default="",
1657 help=("If specified, upload results json files to this appengine "
1661 option_list = (configuration_options + print_options +
1662 chromium_options + results_options + test_options +
1663 misc_options + results_json_options +
1664 old_run_webkit_tests_compat)
1665 option_parser = optparse.OptionParser(option_list=option_list)
1667 options, args = option_parser.parse_args(args)
1669 options.verbose = True
1671 return options, args
1675 options, args = parse_args()
1676 port_obj = port.get(options.platform, options)
1677 return run(port_obj, options, args)
1679 if '__main__' == __name__:
1682 except KeyboardInterrupt:
1683 # this mirrors what the shell normally does
1684 sys.exit(signal.SIGINT + 128)