WebKitTools/Scripts/webkitpy/layout_tests/run_webkit_tests.py

   1 #!/usr/bin/env python
   2 # Copyright (C) 2010 Google Inc. All rights reserved.
   3 #
   4 # Redistribution and use in source and binary forms, with or without
   5 # modification, are permitted provided that the following conditions are
   6 # met:
   7 #
   8 #     * Redistributions of source code must retain the above copyright
   9 # notice, this list of conditions and the following disclaimer.
  10 #     * Redistributions in binary form must reproduce the above
  11 # copyright notice, this list of conditions and the following disclaimer
  12 # in the documentation and/or other materials provided with the
  13 # distribution.
  14 #     * Neither the name of Google Inc. nor the names of its
  15 # contributors may be used to endorse or promote products derived from
  16 # this software without specific prior written permission.
  17 #
  18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  29
  30 """Run layout tests.
  31
  32 This is a port of the existing webkit test script run-webkit-tests.
  33
  34 The TestRunner class runs a series of tests (TestType interface) against a set
  35 of test files.  If a test file fails a TestType, it returns a list TestFailure
  36 objects to the TestRunner.  The TestRunner then aggregates the TestFailures to
  37 create a final report.
  38
  39 This script reads several files, if they exist in the test_lists subdirectory
  40 next to this script itself.  Each should contain a list of paths to individual
  41 tests or entire subdirectories of tests, relative to the outermost test
  42 directory.  Entire lines starting with '//' (comments) will be ignored.
  43
  44 For details of the files' contents and purposes, see test_lists/README.
  45 """
  46
  47 from __future__ import with_statement
  48
  49 import codecs
  50 import errno
  51 import glob
  52 import logging
  53 import math
  54 import optparse
  55 import os
  56 import platform
  57 import Queue
  58 import random
  59 import re
  60 import shutil
  61 import signal
  62 import sys
  63 import time
  64 import traceback
  65
  66 from layout_package import dump_render_tree_thread
  67 from layout_package import json_layout_results_generator
  68 from layout_package import printing
  69 from layout_package import test_expectations
  70 from layout_package import test_failures
  71 from layout_package import test_files
  72 from layout_package import test_results_uploader
  73 from test_types import fuzzy_image_diff
  74 from test_types import image_diff
  75 from test_types import text_diff
  76 from test_types import test_type_base
  77
  78 from webkitpy.common.system.executive import Executive
  79 from webkitpy.thirdparty import simplejson
  80
  81 import port
  82
  83 _log = logging.getLogger("webkitpy.layout_tests.run_webkit_tests")
  84
  85 # Builder base URL where we have the archived test results.
  86 BUILDER_BASE_URL = "http://build.chromium.org/buildbot/layout_test_results/"
  87
  88 TestExpectationsFile = test_expectations.TestExpectationsFile
  89
  90
  91 class TestInfo:
  92     """Groups information about a test for easy passing of data."""
  93
  94     def __init__(self, port, filename, timeout):
  95         """Generates the URI and stores the filename and timeout for this test.
  96         Args:
  97           filename: Full path to the test.
  98           timeout: Timeout for running the test in TestShell.
  99           """
 100         self.filename = filename
 101         self.uri = port.filename_to_uri(filename)
 102         self.timeout = timeout
 103         # FIXME: Confusing that the file is .checksum and we call it "hash"
 104         self._expected_hash_path = port.expected_filename(filename, '.checksum')
 105         self._have_read_expected_hash = False
 106         self._image_hash = None
 107
 108     def _read_image_hash(self):
 109         try:
 110             with codecs.open(self._expected_hash_path, "r", "ascii") as hash_file:
 111                 return hash_file.read()
 112         except IOError, e:
 113             if errno.ENOENT != e.errno:
 114                 raise
 115
 116     def image_hash(self):
 117         # Read the image_hash lazily to reduce startup time.
 118         # This class is accessed across threads, but only one thread should
 119         # ever be dealing with any given TestInfo so no locking is needed.
 120         if not self._have_read_expected_hash:
 121             self._have_read_expected_hash = True
 122             self._image_hash = self._read_image_hash()
 123         return self._image_hash
 124
 125
 126 class ResultSummary(object):
 127     """A class for partitioning the test results we get into buckets.
 128
 129     This class is basically a glorified struct and it's private to this file
 130     so we don't bother with any information hiding."""
 131
 132     def __init__(self, expectations, test_files):
 133         self.total = len(test_files)
 134         self.remaining = self.total
 135         self.expectations = expectations
 136         self.expected = 0
 137         self.unexpected = 0
 138         self.tests_by_expectation = {}
 139         self.tests_by_timeline = {}
 140         self.results = {}
 141         self.unexpected_results = {}
 142         self.failures = {}
 143         self.tests_by_expectation[test_expectations.SKIP] = set()
 144         for expectation in TestExpectationsFile.EXPECTATIONS.values():
 145             self.tests_by_expectation[expectation] = set()
 146         for timeline in TestExpectationsFile.TIMELINES.values():
 147             self.tests_by_timeline[timeline] = (
 148                 expectations.get_tests_with_timeline(timeline))
 149
 150     def add(self, result, expected):
 151         """Add a TestResult into the appropriate bin.
 152
 153         Args:
 154           result: TestResult from dump_render_tree_thread.
 155           expected: whether the result was what we expected it to be.
 156         """
 157
 158         self.tests_by_expectation[result.type].add(result.filename)
 159         self.results[result.filename] = result
 160         self.remaining -= 1
 161         if len(result.failures):
 162             self.failures[result.filename] = result.failures
 163         if expected:
 164             self.expected += 1
 165         else:
 166             self.unexpected_results[result.filename] = result.type
 167             self.unexpected += 1
 168
 169
 170 def summarize_unexpected_results(port_obj, expectations, result_summary,
 171                                  retry_summary):
 172     """Summarize any unexpected results as a dict.
 173
 174     FIXME: split this data structure into a separate class?
 175
 176     Args:
 177         port_obj: interface to port-specific hooks
 178         expectations: test_expectations.TestExpectations object
 179         result_summary: summary object from initial test runs
 180         retry_summary: summary object from final test run of retried tests
 181     Returns:
 182         A dictionary containing a summary of the unexpected results from the
 183         run, with the following fields:
 184         'version': a version indicator (1 in this version)
 185         'fixable': # of fixable tests (NOW - PASS)
 186         'skipped': # of skipped tests (NOW & SKIPPED)
 187         'num_regressions': # of non-flaky failures
 188         'num_flaky': # of flaky failures
 189         'num_passes': # of unexpected passes
 190         'tests': a dict of tests -> {'expected': '...', 'actual': '...'}
 191     """
 192     results = {}
 193     results['version'] = 1
 194
 195     tbe = result_summary.tests_by_expectation
 196     tbt = result_summary.tests_by_timeline
 197     results['fixable'] = len(tbt[test_expectations.NOW] -
 198                                 tbe[test_expectations.PASS])
 199     results['skipped'] = len(tbt[test_expectations.NOW] &
 200                                 tbe[test_expectations.SKIP])
 201
 202     num_passes = 0
 203     num_flaky = 0
 204     num_regressions = 0
 205     keywords = {}
 206     for k, v in TestExpectationsFile.EXPECTATIONS.iteritems():
 207         keywords[v] = k.upper()
 208
 209     tests = {}
 210     for filename, result in result_summary.unexpected_results.iteritems():
 211         # Note that if a test crashed in the original run, we ignore
 212         # whether or not it crashed when we retried it (if we retried it),
 213         # and always consider the result not flaky.
 214         test = port_obj.relative_test_filename(filename)
 215         expected = expectations.get_expectations_string(filename)
 216         actual = [keywords[result]]
 217
 218         if result == test_expectations.PASS:
 219             num_passes += 1
 220         elif result == test_expectations.CRASH:
 221             num_regressions += 1
 222         else:
 223             if filename not in retry_summary.unexpected_results:
 224                 actual.extend(expectations.get_expectations_string(
 225                     filename).split(" "))
 226                 num_flaky += 1
 227             else:
 228                 retry_result = retry_summary.unexpected_results[filename]
 229                 if result != retry_result:
 230                     actual.append(keywords[retry_result])
 231                     num_flaky += 1
 232                 else:
 233                     num_regressions += 1
 234
 235         tests[test] = {}
 236         tests[test]['expected'] = expected
 237         tests[test]['actual'] = " ".join(actual)
 238
 239     results['tests'] = tests
 240     results['num_passes'] = num_passes
 241     results['num_flaky'] = num_flaky
 242     results['num_regressions'] = num_regressions
 243
 244     return results
 245
 246
 247 class TestRunner:
 248     """A class for managing running a series of tests on a series of layout
 249     test files."""
 250
 251     HTTP_SUBDIR = os.sep.join(['', 'http', ''])
 252     WEBSOCKET_SUBDIR = os.sep.join(['', 'websocket', ''])
 253
 254     # The per-test timeout in milliseconds, if no --time-out-ms option was
 255     # given to run_webkit_tests. This should correspond to the default timeout
 256     # in DumpRenderTree.
 257     DEFAULT_TEST_TIMEOUT_MS = 6 * 1000
 258
 259     def __init__(self, port, options, printer):
 260         """Initialize test runner data structures.
 261
 262         Args:
 263           port: an object implementing port-specific
 264           options: a dictionary of command line options
 265           printer: a Printer object to record updates to.
 266         """
 267         self._port = port
 268         self._options = options
 269         self._printer = printer
 270
 271         # disable wss server. need to install pyOpenSSL on buildbots.
 272         # self._websocket_secure_server = websocket_server.PyWebSocket(
 273         #        options.results_directory, use_tls=True, port=9323)
 274
 275         # a list of TestType objects
 276         self._test_types = []
 277
 278         # a set of test files, and the same tests as a list
 279         self._test_files = set()
 280         self._test_files_list = None
 281         self._result_queue = Queue.Queue()
 282
 283         self._retrying = False
 284
 285         # Hack for dumping threads on the bots
 286         self._last_thread_dump = None
 287
 288     def __del__(self):
 289         _log.debug("flushing stdout")
 290         sys.stdout.flush()
 291         _log.debug("flushing stderr")
 292         sys.stderr.flush()
 293         _log.debug("stopping http server")
 294         self._port.stop_http_server()
 295         _log.debug("stopping websocket server")
 296         self._port.stop_websocket_server()
 297
 298     def gather_file_paths(self, paths):
 299         """Find all the files to test.
 300
 301         Args:
 302           paths: a list of globs to use instead of the defaults."""
 303         self._test_files = test_files.gather_test_files(self._port, paths)
 304
 305     def parse_expectations(self, test_platform_name, is_debug_mode):
 306         """Parse the expectations from the test_list files and return a data
 307         structure holding them. Throws an error if the test_list files have
 308         invalid syntax."""
 309         if self._options.lint_test_files:
 310             test_files = None
 311         else:
 312             test_files = self._test_files
 313
 314         try:
 315             expectations_str = self._port.test_expectations()
 316             overrides_str = self._port.test_expectations_overrides()
 317             self._expectations = test_expectations.TestExpectations(
 318                 self._port, test_files, expectations_str, test_platform_name,
 319                 is_debug_mode, self._options.lint_test_files,
 320                 tests_are_present=True, overrides=overrides_str)
 321             return self._expectations
 322         except SyntaxError, err:
 323             if self._options.lint_test_files:
 324                 print str(err)
 325             else:
 326                 raise err
 327
 328     def prepare_lists_and_print_output(self):
 329         """Create appropriate subsets of test lists and returns a
 330         ResultSummary object. Also prints expected test counts.
 331         """
 332
 333         # Remove skipped - both fixable and ignored - files from the
 334         # top-level list of files to test.
 335         num_all_test_files = len(self._test_files)
 336         self._printer.print_expected("Found:  %d tests" %
 337                                      (len(self._test_files)))
 338         if not num_all_test_files:
 339             _log.critical("No tests to run.")
 340             sys.exit(1)
 341
 342         skipped = set()
 343         if num_all_test_files > 1 and not self._options.force:
 344             skipped = self._expectations.get_tests_with_result_type(
 345                            test_expectations.SKIP)
 346             self._test_files -= skipped
 347
 348         # Create a sorted list of test files so the subset chunk,
 349         # if used, contains alphabetically consecutive tests.
 350         self._test_files_list = list(self._test_files)
 351         if self._options.randomize_order:
 352             random.shuffle(self._test_files_list)
 353         else:
 354             self._test_files_list.sort()
 355
 356         # If the user specifies they just want to run a subset of the tests,
 357         # just grab a subset of the non-skipped tests.
 358         if self._options.run_chunk or self._options.run_part:
 359             chunk_value = self._options.run_chunk or self._options.run_part
 360             test_files = self._test_files_list
 361             try:
 362                 (chunk_num, chunk_len) = chunk_value.split(":")
 363                 chunk_num = int(chunk_num)
 364                 assert(chunk_num >= 0)
 365                 test_size = int(chunk_len)
 366                 assert(test_size > 0)
 367             except:
 368                 _log.critical("invalid chunk '%s'" % chunk_value)
 369                 sys.exit(1)
 370
 371             # Get the number of tests
 372             num_tests = len(test_files)
 373
 374             # Get the start offset of the slice.
 375             if self._options.run_chunk:
 376                 chunk_len = test_size
 377                 # In this case chunk_num can be really large. We need
 378                 # to make the slave fit in the current number of tests.
 379                 slice_start = (chunk_num * chunk_len) % num_tests
 380             else:
 381                 # Validate the data.
 382                 assert(test_size <= num_tests)
 383                 assert(chunk_num <= test_size)
 384
 385                 # To count the chunk_len, and make sure we don't skip
 386                 # some tests, we round to the next value that fits exactly
 387                 # all the parts.
 388                 rounded_tests = num_tests
 389                 if rounded_tests % test_size != 0:
 390                     rounded_tests = (num_tests + test_size -
 391                                      (num_tests % test_size))
 392
 393                 chunk_len = rounded_tests / test_size
 394                 slice_start = chunk_len * (chunk_num - 1)
 395                 # It does not mind if we go over test_size.
 396
 397             # Get the end offset of the slice.
 398             slice_end = min(num_tests, slice_start + chunk_len)
 399
 400             files = test_files[slice_start:slice_end]
 401
 402             tests_run_msg = 'Running: %d tests (chunk slice [%d:%d] of %d)' % (
 403                 (slice_end - slice_start), slice_start, slice_end, num_tests)
 404             self._printer.print_expected(tests_run_msg)
 405
 406             # If we reached the end and we don't have enough tests, we run some
 407             # from the beginning.
 408             if (self._options.run_chunk and
 409                 (slice_end - slice_start < chunk_len)):
 410                 extra = 1 + chunk_len - (slice_end - slice_start)
 411                 extra_msg = ('   last chunk is partial, appending [0:%d]' %
 412                             extra)
 413                 self._printer.print_expected(extra_msg)
 414                 tests_run_msg += "\n" + extra_msg
 415                 files.extend(test_files[0:extra])
 416             tests_run_filename = os.path.join(self._options.results_directory,
 417                                               "tests_run.txt")
 418             with codecs.open(tests_run_filename, "w", "utf-8") as file:
 419                 file.write(tests_run_msg + "\n")
 420
 421             len_skip_chunk = int(len(files) * len(skipped) /
 422                                  float(len(self._test_files)))
 423             skip_chunk_list = list(skipped)[0:len_skip_chunk]
 424             skip_chunk = set(skip_chunk_list)
 425
 426             # Update expectations so that the stats are calculated correctly.
 427             # We need to pass a list that includes the right # of skipped files
 428             # to ParseExpectations so that ResultSummary() will get the correct
 429             # stats. So, we add in the subset of skipped files, and then
 430             # subtract them back out.
 431             self._test_files_list = files + skip_chunk_list
 432             self._test_files = set(self._test_files_list)
 433
 434             self._expectations = self.parse_expectations(
 435                 self._port.test_platform_name(),
 436                 self._options.configuration == 'Debug')
 437
 438             self._test_files = set(files)
 439             self._test_files_list = files
 440         else:
 441             skip_chunk = skipped
 442
 443         result_summary = ResultSummary(self._expectations,
 444             self._test_files | skip_chunk)
 445         self._print_expected_results_of_type(result_summary,
 446             test_expectations.PASS, "passes")
 447         self._print_expected_results_of_type(result_summary,
 448             test_expectations.FAIL, "failures")
 449         self._print_expected_results_of_type(result_summary,
 450             test_expectations.FLAKY, "flaky")
 451         self._print_expected_results_of_type(result_summary,
 452             test_expectations.SKIP, "skipped")
 453
 454         if self._options.force:
 455             self._printer.print_expected('Running all tests, including '
 456                                          'skips (--force)')
 457         else:
 458             # Note that we don't actually run the skipped tests (they were
 459             # subtracted out of self._test_files, above), but we stub out the
 460             # results here so the statistics can remain accurate.
 461             for test in skip_chunk:
 462                 result = dump_render_tree_thread.TestResult(test,
 463                     failures=[], test_run_time=0, total_time_for_all_diffs=0,
 464                     time_for_diffs=0)
 465                 result.type = test_expectations.SKIP
 466                 result_summary.add(result, expected=True)
 467         self._printer.print_expected('')
 468
 469         return result_summary
 470
 471     def add_test_type(self, test_type):
 472         """Add a TestType to the TestRunner."""
 473         self._test_types.append(test_type)
 474
 475     def _get_dir_for_test_file(self, test_file):
 476         """Returns the highest-level directory by which to shard the given
 477         test file."""
 478         index = test_file.rfind(os.sep + 'LayoutTests' + os.sep)
 479
 480         test_file = test_file[index + len('LayoutTests/'):]
 481         test_file_parts = test_file.split(os.sep, 1)
 482         directory = test_file_parts[0]
 483         test_file = test_file_parts[1]
 484
 485         # The http tests are very stable on mac/linux.
 486         # TODO(ojan): Make the http server on Windows be apache so we can
 487         # turn shard the http tests there as well. Switching to apache is
 488         # what made them stable on linux/mac.
 489         return_value = directory
 490         while ((directory != 'http' or sys.platform in ('darwin', 'linux2'))
 491                 and test_file.find(os.sep) >= 0):
 492             test_file_parts = test_file.split(os.sep, 1)
 493             directory = test_file_parts[0]
 494             return_value = os.path.join(return_value, directory)
 495             test_file = test_file_parts[1]
 496
 497         return return_value
 498
 499     def _get_test_info_for_file(self, test_file):
 500         """Returns the appropriate TestInfo object for the file. Mostly this
 501         is used for looking up the timeout value (in ms) to use for the given
 502         test."""
 503         if self._expectations.has_modifier(test_file, test_expectations.SLOW):
 504             return TestInfo(self._port, test_file,
 505                             self._options.slow_time_out_ms)
 506         return TestInfo(self._port, test_file, self._options.time_out_ms)
 507
 508     def _get_test_file_queue(self, test_files):
 509         """Create the thread safe queue of lists of (test filenames, test URIs)
 510         tuples. Each TestShellThread pulls a list from this queue and runs
 511         those tests in order before grabbing the next available list.
 512
 513         Shard the lists by directory. This helps ensure that tests that depend
 514         on each other (aka bad tests!) continue to run together as most
 515         cross-tests dependencies tend to occur within the same directory.
 516
 517         Return:
 518           The Queue of lists of TestInfo objects.
 519         """
 520
 521         if (self._options.experimental_fully_parallel or
 522             self._is_single_threaded()):
 523             filename_queue = Queue.Queue()
 524             for test_file in test_files:
 525                 filename_queue.put(
 526                     ('.', [self._get_test_info_for_file(test_file)]))
 527             return filename_queue
 528
 529         tests_by_dir = {}
 530         for test_file in test_files:
 531             directory = self._get_dir_for_test_file(test_file)
 532             tests_by_dir.setdefault(directory, [])
 533             tests_by_dir[directory].append(
 534                 self._get_test_info_for_file(test_file))
 535
 536         # Sort by the number of tests in the dir so that the ones with the
 537         # most tests get run first in order to maximize parallelization.
 538         # Number of tests is a good enough, but not perfect, approximation
 539         # of how long that set of tests will take to run. We can't just use
 540         # a PriorityQueue until we move # to Python 2.6.
 541         test_lists = []
 542         http_tests = None
 543         for directory in tests_by_dir:
 544             test_list = tests_by_dir[directory]
 545             # Keep the tests in alphabetical order.
 546             # TODO: Remove once tests are fixed so they can be run in any
 547             # order.
 548             test_list.reverse()
 549             test_list_tuple = (directory, test_list)
 550             if directory == 'LayoutTests' + os.sep + 'http':
 551                 http_tests = test_list_tuple
 552             else:
 553                 test_lists.append(test_list_tuple)
 554         test_lists.sort(lambda a, b: cmp(len(b[1]), len(a[1])))
 555
 556         # Put the http tests first. There are only a couple hundred of them,
 557         # but each http test takes a very long time to run, so sorting by the
 558         # number of tests doesn't accurately capture how long they take to run.
 559         if http_tests:
 560             test_lists.insert(0, http_tests)
 561
 562         filename_queue = Queue.Queue()
 563         for item in test_lists:
 564             filename_queue.put(item)
 565         return filename_queue
 566
 567     def _get_dump_render_tree_args(self, index):
 568         """Returns the tuple of arguments for tests and for DumpRenderTree."""
 569         shell_args = []
 570         test_args = test_type_base.TestArguments()
 571         png_path = None
 572         if self._options.pixel_tests:
 573             png_path = os.path.join(self._options.results_directory,
 574                                     "png_result%s.png" % index)
 575             shell_args.append("--pixel-tests=" + png_path)
 576             test_args.png_path = png_path
 577
 578         test_args.new_baseline = self._options.new_baseline
 579         test_args.reset_results = self._options.reset_results
 580
 581         test_args.show_sources = self._options.sources
 582
 583         if self._options.startup_dialog:
 584             shell_args.append('--testshell-startup-dialog')
 585
 586         if self._options.gp_fault_error_box:
 587             shell_args.append('--gp-fault-error-box')
 588
 589         return test_args, png_path, shell_args
 590
 591     def _contains_tests(self, subdir):
 592         for test_file in self._test_files:
 593             if test_file.find(subdir) >= 0:
 594                 return True
 595         return False
 596
 597     def _instantiate_dump_render_tree_threads(self, test_files,
 598                                               result_summary):
 599         """Instantitates and starts the TestShellThread(s).
 600
 601         Return:
 602           The list of threads.
 603         """
 604         filename_queue = self._get_test_file_queue(test_files)
 605
 606         # Instantiate TestShellThreads and start them.
 607         threads = []
 608         for i in xrange(int(self._options.child_processes)):
 609             # Create separate TestTypes instances for each thread.
 610             test_types = []
 611             for test_type in self._test_types:
 612                 test_types.append(test_type(self._port,
 613                                     self._options.results_directory))
 614
 615             test_args, png_path, shell_args = \
 616                 self._get_dump_render_tree_args(i)
 617             thread = dump_render_tree_thread.TestShellThread(self._port,
 618                 filename_queue, self._result_queue, test_types, test_args,
 619                 png_path, shell_args, self._options)
 620             if self._is_single_threaded():
 621                 thread.run_in_main_thread(self, result_summary)
 622             else:
 623                 thread.start()
 624             threads.append(thread)
 625
 626         return threads
 627
 628     def _is_single_threaded(self):
 629         """Returns whether we should run all the tests in the main thread."""
 630         return int(self._options.child_processes) == 1
 631
 632     def _dump_thread_states(self):
 633         for thread_id, stack in sys._current_frames().items():
 634             # FIXME: Python 2.6 has thread.ident which we could
 635             # use to map from thread_id back to thread.name
 636             print "\n# Thread: %d" % thread_id
 637             for filename, lineno, name, line in traceback.extract_stack(stack):
 638                 print 'File: "%s", line %d, in %s' % (filename, lineno, name)
 639                 if line:
 640                     print "  %s" % (line.strip())
 641
 642     def _dump_thread_states_if_necessary(self):
 643         # HACK: Dump thread states every minute to figure out what's
 644         # hanging on the bots.
 645         if not self._options.verbose:
 646             return
 647         dump_threads_every = 60  # Dump every minute
 648         if not self._last_thread_dump:
 649             self._last_thread_dump = time.time()
 650         time_since_last_dump = time.time() - self._last_thread_dump
 651         if  time_since_last_dump > dump_threads_every:
 652             self._dump_thread_states()
 653             self._last_thread_dump = time.time()
 654
 655     def _run_tests(self, file_list, result_summary):
 656         """Runs the tests in the file_list.
 657
 658         Return: A tuple (failures, thread_timings, test_timings,
 659             individual_test_timings)
 660             failures is a map from test to list of failure types
 661             thread_timings is a list of dicts with the total runtime
 662               of each thread with 'name', 'num_tests', 'total_time' properties
 663             test_timings is a list of timings for each sharded subdirectory
 664               of the form [time, directory_name, num_tests]
 665             individual_test_timings is a list of run times for each test
 666               in the form {filename:filename, test_run_time:test_run_time}
 667             result_summary: summary object to populate with the results
 668         """
 669         # FIXME: We should use webkitpy.tool.grammar.pluralize here.
 670         plural = ""
 671         if self._options.child_processes > 1:
 672             plural = "s"
 673         self._printer.print_update('Starting %s%s ...' %
 674                                    (self._port.driver_name(), plural))
 675         threads = self._instantiate_dump_render_tree_threads(file_list,
 676                                                              result_summary)
 677         self._printer.print_update("Starting testing ...")
 678
 679         # Wait for the threads to finish and collect test failures.
 680         failures = {}
 681         test_timings = {}
 682         individual_test_timings = []
 683         thread_timings = []
 684         keyboard_interrupted = False
 685         try:
 686             # Loop through all the threads waiting for them to finish.
 687             for thread in threads:
 688                 # FIXME: We'll end up waiting on the first thread the whole
 689                 # time.  That means we won't notice exceptions on other
 690                 # threads until the first one exits.
 691                 # We should instead while True: in the outer loop
 692                 # and then loop through threads joining and checking
 693                 # isAlive and get_exception_info.  Exiting on any exception.
 694                 while thread.isAlive():
 695                     # Wake the main thread every 0.1 seconds so we
 696                     # can call update_summary in a timely fashion.
 697                     thread.join(0.1)
 698                     # HACK: Used for debugging threads on the bots.
 699                     self._dump_thread_states_if_necessary()
 700                     self.update_summary(result_summary)
 701
 702         except KeyboardInterrupt:
 703             keyboard_interrupted = True
 704             for thread in threads:
 705                 thread.cancel()
 706
 707         if not keyboard_interrupted:
 708             for thread in threads:
 709                 # Check whether a thread died before normal completion.
 710                 exception_info = thread.get_exception_info()
 711                 if exception_info is not None:
 712                     # Re-raise the thread's exception here to make it clear
 713                     # something went wrong. Otherwise, the tests that did not
 714                     # run would be assumed to have passed.
 715                     raise (exception_info[0], exception_info[1],
 716                            exception_info[2])
 717
 718         for thread in threads:
 719             thread_timings.append({'name': thread.getName(),
 720                                    'num_tests': thread.get_num_tests(),
 721                                    'total_time': thread.get_total_time()})
 722             test_timings.update(thread.get_directory_timing_stats())
 723             individual_test_timings.extend(thread.get_test_results())
 724         return (keyboard_interrupted, thread_timings, test_timings,
 725                 individual_test_timings)
 726
 727     def needs_http(self):
 728         """Returns whether the test runner needs an HTTP server."""
 729         return self._contains_tests(self.HTTP_SUBDIR)
 730
 731     def run(self, result_summary):
 732         """Run all our tests on all our test files.
 733
 734         For each test file, we run each test type. If there are any failures,
 735         we collect them for reporting.
 736
 737         Args:
 738           result_summary: a summary object tracking the test results.
 739
 740         Return:
 741           The number of unexpected results (0 == success)
 742         """
 743         if not self._test_files:
 744             return 0
 745         start_time = time.time()
 746
 747         if self.needs_http():
 748             self._printer.print_update('Starting HTTP server ...')
 749
 750             self._port.start_http_server()
 751
 752         if self._contains_tests(self.WEBSOCKET_SUBDIR):
 753             self._printer.print_update('Starting WebSocket server ...')
 754             self._port.start_websocket_server()
 755             # self._websocket_secure_server.Start()
 756
 757         keyboard_interrupted, thread_timings, test_timings, \
 758             individual_test_timings = (
 759             self._run_tests(self._test_files_list, result_summary))
 760
 761         # We exclude the crashes from the list of results to retry, because
 762         # we want to treat even a potentially flaky crash as an error.
 763         failures = self._get_failures(result_summary, include_crashes=False)
 764         retry_summary = result_summary
 765         while (len(failures) and self._options.retry_failures and
 766             not self._retrying and not keyboard_interrupted):
 767             _log.info('')
 768             _log.info("Retrying %d unexpected failure(s) ..." % len(failures))
 769             _log.info('')
 770             self._retrying = True
 771             retry_summary = ResultSummary(self._expectations, failures.keys())
 772             # Note that we intentionally ignore the return value here.
 773             self._run_tests(failures.keys(), retry_summary)
 774             failures = self._get_failures(retry_summary, include_crashes=True)
 775
 776         end_time = time.time()
 777
 778         self._print_timing_statistics(end_time - start_time,
 779                                       thread_timings, test_timings,
 780                                       individual_test_timings,
 781                                       result_summary)
 782
 783         self._print_result_summary(result_summary)
 784
 785         sys.stdout.flush()
 786         sys.stderr.flush()
 787
 788         self._printer.print_one_line_summary(result_summary.total,
 789                                              result_summary.expected,
 790                                              result_summary.unexpected)
 791
 792         unexpected_results = summarize_unexpected_results(self._port,
 793             self._expectations, result_summary, retry_summary)
 794         self._printer.print_unexpected_results(unexpected_results)
 795
 796         # Write the same data to log files.
 797         self._write_json_files(unexpected_results, result_summary,
 798                              individual_test_timings)
 799
 800         # Upload generated JSON files to appengine server.
 801         self._upload_json_files()
 802
 803         # Write the summary to disk (results.html) and display it if requested.
 804         wrote_results = self._write_results_html_file(result_summary)
 805         if self._options.show_results and wrote_results:
 806             self._show_results_html_file()
 807
 808         # Now that we've completed all the processing we can, we re-raise
 809         # a KeyboardInterrupt if necessary so the caller can handle it.
 810         if keyboard_interrupted:
 811             raise KeyboardInterrupt
 812
 813         # Ignore flaky failures and unexpected passes so we don't turn the
 814         # bot red for those.
 815         return unexpected_results['num_regressions']
 816
 817     def update_summary(self, result_summary):
 818         """Update the summary and print results with any completed tests."""
 819         while True:
 820             try:
 821                 result = self._result_queue.get_nowait()
 822             except Queue.Empty:
 823                 return
 824
 825             expected = self._expectations.matches_an_expected_result(
 826                 result.filename, result.type, self._options.pixel_tests)
 827             result_summary.add(result, expected)
 828             exp_str = self._expectations.get_expectations_string(
 829                 result.filename)
 830             got_str = self._expectations.expectation_to_string(result.type)
 831             self._printer.print_test_result(result, expected, exp_str, got_str)
 832             self._printer.print_progress(result_summary, self._retrying,
 833                                          self._test_files_list)
 834
 835     def _get_failures(self, result_summary, include_crashes):
 836         """Filters a dict of results and returns only the failures.
 837
 838         Args:
 839           result_summary: the results of the test run
 840           include_crashes: whether crashes are included in the output.
 841             We use False when finding the list of failures to retry
 842             to see if the results were flaky. Although the crashes may also be
 843             flaky, we treat them as if they aren't so that they're not ignored.
 844         Returns:
 845           a dict of files -> results
 846         """
 847         failed_results = {}
 848         for test, result in result_summary.unexpected_results.iteritems():
 849             if (result == test_expectations.PASS or
 850                 result == test_expectations.CRASH and not include_crashes):
 851                 continue
 852             failed_results[test] = result
 853
 854         return failed_results
 855
 856     def _write_json_files(self, unexpected_results, result_summary,
 857                         individual_test_timings):
 858         """Writes the results of the test run as JSON files into the results
 859         dir.
 860
 861         There are three different files written into the results dir:
 862           unexpected_results.json: A short list of any unexpected results.
 863             This is used by the buildbots to display results.
 864           expectations.json: This is used by the flakiness dashboard.
 865           results.json: A full list of the results - used by the flakiness
 866             dashboard and the aggregate results dashboard.
 867
 868         Args:
 869           unexpected_results: dict of unexpected results
 870           result_summary: full summary object
 871           individual_test_timings: list of test times (used by the flakiness
 872             dashboard).
 873         """
 874         results_directory = self._options.results_directory
 875         _log.debug("Writing JSON files in %s." % results_directory)
 876         unexpected_json_path = os.path.join(results_directory, "unexpected_results.json")
 877         with codecs.open(unexpected_json_path, "w", "utf-8") as file:
 878             simplejson.dump(unexpected_results, file, sort_keys=True, indent=2)
 879
 880         # Write a json file of the test_expectations.txt file for the layout
 881         # tests dashboard.
 882         expectations_path = os.path.join(results_directory, "expectations.json")
 883         expectations_json = \
 884             self._expectations.get_expectations_json_for_all_platforms()
 885         with codecs.open(expectations_path, "w", "utf-8") as file:
 886             file.write(u"ADD_EXPECTATIONS(%s);" % expectations_json)
 887
 888         json_layout_results_generator.JSONLayoutResultsGenerator(
 889             self._port, self._options.builder_name, self._options.build_name,
 890             self._options.build_number, self._options.results_directory,
 891             BUILDER_BASE_URL, individual_test_timings,
 892             self._expectations, result_summary, self._test_files_list)
 893
 894         _log.debug("Finished writing JSON files.")
 895
 896     def _upload_json_files(self):
 897         if not self._options.test_results_server:
 898             return
 899
 900         _log.info("Uploading JSON files for builder: %s",
 901                    self._options.builder_name)
 902
 903         attrs = [('builder', self._options.builder_name)]
 904         json_files = ["expectations.json", "results.json"]
 905
 906         files = [(file, os.path.join(self._options.results_directory, file))
 907             for file in json_files]
 908
 909         uploader = test_results_uploader.TestResultsUploader(
 910             self._options.test_results_server)
 911         try:
 912             # Set uploading timeout in case appengine server is having problem.
 913             # 120 seconds are more than enough to upload test results.
 914             uploader.upload(attrs, files, 120)
 915         except Exception, err:
 916             _log.error("Upload failed: %s" % err)
 917             return
 918
 919         _log.info("JSON files uploaded.")
 920
 921     def _print_expected_results_of_type(self, result_summary,
 922                                         result_type, result_type_str):
 923         """Print the number of the tests in a given result class.
 924
 925         Args:
 926           result_summary - the object containing all the results to report on
 927           result_type - the particular result type to report in the summary.
 928           result_type_str - a string description of the result_type.
 929         """
 930         tests = self._expectations.get_tests_with_result_type(result_type)
 931         now = result_summary.tests_by_timeline[test_expectations.NOW]
 932         wontfix = result_summary.tests_by_timeline[test_expectations.WONTFIX]
 933         defer = result_summary.tests_by_timeline[test_expectations.DEFER]
 934
 935         # We use a fancy format string in order to print the data out in a
 936         # nicely-aligned table.
 937         fmtstr = ("Expect: %%5d %%-8s (%%%dd now, %%%dd defer, %%%dd wontfix)"
 938                   % (self._num_digits(now), self._num_digits(defer),
 939                   self._num_digits(wontfix)))
 940         self._printer.print_expected(fmtstr %
 941             (len(tests), result_type_str, len(tests & now),
 942              len(tests & defer), len(tests & wontfix)))
 943
 944     def _num_digits(self, num):
 945         """Returns the number of digits needed to represent the length of a
 946         sequence."""
 947         ndigits = 1
 948         if len(num):
 949             ndigits = int(math.log10(len(num))) + 1
 950         return ndigits
 951
 952     def _print_timing_statistics(self, total_time, thread_timings,
 953                                directory_test_timings, individual_test_timings,
 954                                result_summary):
 955         """Record timing-specific information for the test run.
 956
 957         Args:
 958           total_time: total elapsed time (in seconds) for the test run
 959           thread_timings: wall clock time each thread ran for
 960           directory_test_timings: timing by directory
 961           individual_test_timings: timing by file
 962           result_summary: summary object for the test run
 963         """
 964         self._printer.print_timing("Test timing:")
 965         self._printer.print_timing("  %6.2f total testing time" % total_time)
 966         self._printer.print_timing("")
 967         self._printer.print_timing("Thread timing:")
 968         cuml_time = 0
 969         for t in thread_timings:
 970             self._printer.print_timing("    %10s: %5d tests, %6.2f secs" %
 971                   (t['name'], t['num_tests'], t['total_time']))
 972             cuml_time += t['total_time']
 973         self._printer.print_timing("   %6.2f cumulative, %6.2f optimal" %
 974               (cuml_time, cuml_time / int(self._options.child_processes)))
 975         self._printer.print_timing("")
 976
 977         self._print_aggregate_test_statistics(individual_test_timings)
 978         self._print_individual_test_times(individual_test_timings,
 979                                           result_summary)
 980         self._print_directory_timings(directory_test_timings)
 981
 982     def _print_aggregate_test_statistics(self, individual_test_timings):
 983         """Prints aggregate statistics (e.g. median, mean, etc.) for all tests.
 984         Args:
 985           individual_test_timings: List of dump_render_tree_thread.TestStats
 986               for all tests.
 987         """
 988         test_types = []  # Unit tests don't actually produce any timings.
 989         if individual_test_timings:
 990             test_types = individual_test_timings[0].time_for_diffs.keys()
 991         times_for_dump_render_tree = []
 992         times_for_diff_processing = []
 993         times_per_test_type = {}
 994         for test_type in test_types:
 995             times_per_test_type[test_type] = []
 996
 997         for test_stats in individual_test_timings:
 998             times_for_dump_render_tree.append(test_stats.test_run_time)
 999             times_for_diff_processing.append(
1000                 test_stats.total_time_for_all_diffs)
1001             time_for_diffs = test_stats.time_for_diffs
1002             for test_type in test_types:
1003                 times_per_test_type[test_type].append(
1004                     time_for_diffs[test_type])
1005
1006         self._print_statistics_for_test_timings(
1007             "PER TEST TIME IN TESTSHELL (seconds):",
1008             times_for_dump_render_tree)
1009         self._print_statistics_for_test_timings(
1010             "PER TEST DIFF PROCESSING TIMES (seconds):",
1011             times_for_diff_processing)
1012         for test_type in test_types:
1013             self._print_statistics_for_test_timings(
1014                 "PER TEST TIMES BY TEST TYPE: %s" % test_type,
1015                 times_per_test_type[test_type])
1016
1017     def _print_individual_test_times(self, individual_test_timings,
1018                                   result_summary):
1019         """Prints the run times for slow, timeout and crash tests.
1020         Args:
1021           individual_test_timings: List of dump_render_tree_thread.TestStats
1022               for all tests.
1023           result_summary: summary object for test run
1024         """
1025         # Reverse-sort by the time spent in DumpRenderTree.
1026         individual_test_timings.sort(lambda a, b:
1027             cmp(b.test_run_time, a.test_run_time))
1028
1029         num_printed = 0
1030         slow_tests = []
1031         timeout_or_crash_tests = []
1032         unexpected_slow_tests = []
1033         for test_tuple in individual_test_timings:
1034             filename = test_tuple.filename
1035             is_timeout_crash_or_slow = False
1036             if self._expectations.has_modifier(filename,
1037                                                test_expectations.SLOW):
1038                 is_timeout_crash_or_slow = True
1039                 slow_tests.append(test_tuple)
1040
1041             if filename in result_summary.failures:
1042                 result = result_summary.results[filename].type
1043                 if (result == test_expectations.TIMEOUT or
1044                     result == test_expectations.CRASH):
1045                     is_timeout_crash_or_slow = True
1046                     timeout_or_crash_tests.append(test_tuple)
1047
1048             if (not is_timeout_crash_or_slow and
1049                 num_printed < printing.NUM_SLOW_TESTS_TO_LOG):
1050                 num_printed = num_printed + 1
1051                 unexpected_slow_tests.append(test_tuple)
1052
1053         self._printer.print_timing("")
1054         self._print_test_list_timing("%s slowest tests that are not "
1055             "marked as SLOW and did not timeout/crash:" %
1056             printing.NUM_SLOW_TESTS_TO_LOG, unexpected_slow_tests)
1057         self._printer.print_timing("")
1058         self._print_test_list_timing("Tests marked as SLOW:", slow_tests)
1059         self._printer.print_timing("")
1060         self._print_test_list_timing("Tests that timed out or crashed:",
1061                                      timeout_or_crash_tests)
1062         self._printer.print_timing("")
1063
1064     def _print_test_list_timing(self, title, test_list):
1065         """Print timing info for each test.
1066
1067         Args:
1068           title: section heading
1069           test_list: tests that fall in this section
1070         """
1071         if self._printer.disabled('slowest'):
1072             return
1073
1074         self._printer.print_timing(title)
1075         for test_tuple in test_list:
1076             filename = test_tuple.filename[len(
1077                 self._port.layout_tests_dir()) + 1:]
1078             filename = filename.replace('\\', '/')
1079             test_run_time = round(test_tuple.test_run_time, 1)
1080             self._printer.print_timing("  %s took %s seconds" %
1081                                        (filename, test_run_time))
1082
1083     def _print_directory_timings(self, directory_test_timings):
1084         """Print timing info by directory for any directories that
1085         take > 10 seconds to run.
1086
1087         Args:
1088           directory_test_timing: time info for each directory
1089         """
1090         timings = []
1091         for directory in directory_test_timings:
1092             num_tests, time_for_directory = directory_test_timings[directory]
1093             timings.append((round(time_for_directory, 1), directory,
1094                             num_tests))
1095         timings.sort()
1096
1097         self._printer.print_timing("Time to process slowest subdirectories:")
1098         min_seconds_to_print = 10
1099         for timing in timings:
1100             if timing[0] > min_seconds_to_print:
1101                 self._printer.print_timing(
1102                     "  %s took %s seconds to run %s tests." % (timing[1],
1103                     timing[0], timing[2]))
1104         self._printer.print_timing("")
1105
1106     def _print_statistics_for_test_timings(self, title, timings):
1107         """Prints the median, mean and standard deviation of the values in
1108         timings.
1109
1110         Args:
1111           title: Title for these timings.
1112           timings: A list of floats representing times.
1113         """
1114         self._printer.print_timing(title)
1115         timings.sort()
1116
1117         num_tests = len(timings)
1118         if not num_tests:
1119             return
1120         percentile90 = timings[int(.9 * num_tests)]
1121         percentile99 = timings[int(.99 * num_tests)]
1122
1123         if num_tests % 2 == 1:
1124             median = timings[((num_tests - 1) / 2) - 1]
1125         else:
1126             lower = timings[num_tests / 2 - 1]
1127             upper = timings[num_tests / 2]
1128             median = (float(lower + upper)) / 2
1129
1130         mean = sum(timings) / num_tests
1131
1132         for time in timings:
1133             sum_of_deviations = math.pow(time - mean, 2)
1134
1135         std_deviation = math.sqrt(sum_of_deviations / num_tests)
1136         self._printer.print_timing("  Median:          %6.3f" % median)
1137         self._printer.print_timing("  Mean:            %6.3f" % mean)
1138         self._printer.print_timing("  90th percentile: %6.3f" % percentile90)
1139         self._printer.print_timing("  99th percentile: %6.3f" % percentile99)
1140         self._printer.print_timing("  Standard dev:    %6.3f" % std_deviation)
1141         self._printer.print_timing("")
1142
1143     def _print_result_summary(self, result_summary):
1144         """Print a short summary about how many tests passed.
1145
1146         Args:
1147           result_summary: information to log
1148         """
1149         failed = len(result_summary.failures)
1150         skipped = len(
1151             result_summary.tests_by_expectation[test_expectations.SKIP])
1152         total = result_summary.total
1153         passed = total - failed - skipped
1154         pct_passed = 0.0
1155         if total > 0:
1156             pct_passed = float(passed) * 100 / total
1157
1158         self._printer.print_actual("")
1159         self._printer.print_actual("=> Results: %d/%d tests passed (%.1f%%)" %
1160                      (passed, total, pct_passed))
1161         self._printer.print_actual("")
1162         self._print_result_summary_entry(result_summary,
1163             test_expectations.NOW, "Tests to be fixed for the current release")
1164
1165         self._printer.print_actual("")
1166         self._print_result_summary_entry(result_summary,
1167             test_expectations.DEFER,
1168             "Tests we'll fix in the future if they fail (DEFER)")
1169
1170         self._printer.print_actual("")
1171         self._print_result_summary_entry(result_summary,
1172             test_expectations.WONTFIX,
1173             "Tests that will only be fixed if they crash (WONTFIX)")
1174         self._printer.print_actual("")
1175
1176     def _print_result_summary_entry(self, result_summary, timeline,
1177                                     heading):
1178         """Print a summary block of results for a particular timeline of test.
1179
1180         Args:
1181           result_summary: summary to print results for
1182           timeline: the timeline to print results for (NOT, WONTFIX, etc.)
1183           heading: a textual description of the timeline
1184         """
1185         total = len(result_summary.tests_by_timeline[timeline])
1186         not_passing = (total -
1187            len(result_summary.tests_by_expectation[test_expectations.PASS] &
1188                result_summary.tests_by_timeline[timeline]))
1189         self._printer.print_actual("=> %s (%d):" % (heading, not_passing))
1190
1191         for result in TestExpectationsFile.EXPECTATION_ORDER:
1192             if result == test_expectations.PASS:
1193                 continue
1194             results = (result_summary.tests_by_expectation[result] &
1195                        result_summary.tests_by_timeline[timeline])
1196             desc = TestExpectationsFile.EXPECTATION_DESCRIPTIONS[result]
1197             if not_passing and len(results):
1198                 pct = len(results) * 100.0 / not_passing
1199                 self._printer.print_actual("  %5d %-24s (%4.1f%%)" %
1200                     (len(results), desc[len(results) != 1], pct))
1201
1202     def _results_html(self, test_files, failures, title="Test Failures", override_time=None):
1203         """
1204         test_files = a list of file paths
1205         failures = dictionary mapping test paths to failure objects
1206         title = title printed at top of test
1207         override_time = current time (used by unit tests)
1208         """
1209         page = """<html>
1210   <head>
1211     <title>Layout Test Results (%(time)s)</title>
1212   </head>
1213   <body>
1214     <h2>%(title)s (%(time)s)</h2>
1215         """ % {'title': title, 'time': override_time or time.asctime()}
1216
1217         for test_file in sorted(test_files):
1218             test_name = self._port.relative_test_filename(test_file)
1219             test_url = self._port.filename_to_uri(test_file)
1220             page += u"<p><a href='%s'>%s</a><br />\n" % (test_url, test_name)
1221             test_failures = failures.get(test_file, [])
1222             for failure in test_failures:
1223                 page += u"&nbsp;&nbsp;%s<br/>" % failure.result_html_output(test_name)
1224             page += "</p>\n"
1225         page += "</body></html>\n"
1226         return page
1227
1228     def _write_results_html_file(self, result_summary):
1229         """Write results.html which is a summary of tests that failed.
1230
1231         Args:
1232           result_summary: a summary of the results :)
1233
1234         Returns:
1235           True if any results were written (since expected failures may be
1236           omitted)
1237         """
1238         # test failures
1239         if self._options.full_results_html:
1240             results_title = "Test Failures"
1241             test_files = result_summary.failures.keys()
1242         else:
1243             results_title = "Unexpected Test Failures"
1244             unexpected_failures = self._get_failures(result_summary,
1245                 include_crashes=True)
1246             test_files = unexpected_failures.keys()
1247         if not len(test_files):
1248             return False
1249
1250         out_filename = os.path.join(self._options.results_directory,
1251                                     "results.html")
1252         with codecs.open(out_filename, "w", "utf-8") as results_file:
1253             html = self._results_html(test_files, result_summary.failures, results_title)
1254             results_file.write(html)
1255
1256         return True
1257
1258     def _show_results_html_file(self):
1259         """Shows the results.html page."""
1260         results_filename = os.path.join(self._options.results_directory,
1261                                         "results.html")
1262         self._port.show_results_html_file(results_filename)
1263
1264
1265 def read_test_files(files):
1266     tests = []
1267     for file in files:
1268         # FIXME: This could be cleaner using a list comprehension.
1269         for line in codecs.open(file, "r", "utf-8"):
1270             line = test_expectations.strip_comments(line)
1271             if line:
1272                 tests.append(line)
1273     return tests
1274
1275
1276 def run(port_obj, options, args, regular_output=sys.stderr,
1277         buildbot_output=sys.stdout):
1278     """Run the tests.
1279
1280     Args:
1281       port_obj: Port object for port-specific behavior
1282       options: a dictionary of command line options
1283       args: a list of sub directories or files to test
1284       regular_output: a stream-like object that we can send logging/debug
1285           output to
1286       buildbot_output: a stream-like object that we can write all output that
1287           is intended to be parsed by the buildbot to
1288     Returns:
1289       the number of unexpected results that occurred, or -1 if there is an
1290           error.
1291     """
1292
1293     # Configure the printing subsystem for printing output, logging debug
1294     # info, and tracing tests.
1295
1296     if not options.child_processes:
1297         # FIXME: Investigate perf/flakiness impact of using cpu_count + 1.
1298         options.child_processes = port_obj.default_child_processes()
1299
1300     printer = printing.Printer(port_obj, options, regular_output=regular_output,
1301         buildbot_output=buildbot_output,
1302         child_processes=int(options.child_processes),
1303         is_fully_parallel=options.experimental_fully_parallel)
1304     if options.help_printing:
1305         printer.help_printing()
1306         return 0
1307
1308     executive = Executive()
1309
1310     if not options.configuration:
1311         options.configuration = port_obj.default_configuration()
1312
1313     if options.pixel_tests is None:
1314         options.pixel_tests = True
1315
1316     if not options.use_apache:
1317         options.use_apache = sys.platform in ('darwin', 'linux2')
1318
1319     if options.results_directory.startswith("/"):
1320         # Assume it's an absolute path and normalize.
1321         options.results_directory = port_obj.get_absolute_path(
1322             options.results_directory)
1323     else:
1324         # If it's a relative path, make the output directory relative to
1325         # Debug or Release.
1326         options.results_directory = port_obj.results_directory()
1327
1328     last_unexpected_results = []
1329     if options.print_last_failures or options.retest_last_failures:
1330         unexpected_results_filename = os.path.join(
1331            options.results_directory, "unexpected_results.json")
1332         with codecs.open(unexpected_results_filename, "r", "utf-8") as file:
1333             results = simplejson.load(file)
1334         last_unexpected_results = results['tests'].keys()
1335         if options.print_last_failures:
1336             printer.write("\n".join(last_unexpected_results) + "\n")
1337             return 0
1338
1339     if options.clobber_old_results:
1340         # Just clobber the actual test results directories since the other
1341         # files in the results directory are explicitly used for cross-run
1342         # tracking.
1343         printer.print_update("Clobbering old results in %s" %
1344                              options.results_directory)
1345         layout_tests_dir = port_obj.layout_tests_dir()
1346         possible_dirs = os.listdir(layout_tests_dir)
1347         for dirname in possible_dirs:
1348             if os.path.isdir(os.path.join(layout_tests_dir, dirname)):
1349                 shutil.rmtree(os.path.join(options.results_directory, dirname),
1350                               ignore_errors=True)
1351
1352     if not options.time_out_ms:
1353         if options.configuration == "Debug":
1354             options.time_out_ms = str(2 * TestRunner.DEFAULT_TEST_TIMEOUT_MS)
1355         else:
1356             options.time_out_ms = str(TestRunner.DEFAULT_TEST_TIMEOUT_MS)
1357
1358     options.slow_time_out_ms = str(5 * int(options.time_out_ms))
1359     printer.print_config("Regular timeout: %s, slow test timeout: %s" %
1360                    (options.time_out_ms, options.slow_time_out_ms))
1361
1362     if int(options.child_processes) == 1:
1363         printer.print_config("Running one %s" % port_obj.driver_name())
1364     else:
1365         printer.print_config("Running %s %ss in parallel" % (
1366                        options.child_processes, port_obj.driver_name()))
1367
1368     # Include all tests if none are specified.
1369     new_args = []
1370     for arg in args:
1371         if arg and arg != '':
1372             new_args.append(arg)
1373
1374     paths = new_args
1375     if not paths:
1376         paths = []
1377     paths += last_unexpected_results
1378     if options.test_list:
1379         paths += read_test_files(options.test_list)
1380
1381     # Create the output directory if it doesn't already exist.
1382     port_obj.maybe_make_directory(options.results_directory)
1383     printer.print_update("Collecting tests ...")
1384
1385     test_runner = TestRunner(port_obj, options, printer)
1386     test_runner.gather_file_paths(paths)
1387
1388     if options.lint_test_files:
1389         # Creating the expecations for each platform/configuration pair does
1390         # all the test list parsing and ensures it's correct syntax (e.g. no
1391         # dupes).
1392         for platform_name in port_obj.test_platform_names():
1393             test_runner.parse_expectations(platform_name, is_debug_mode=True)
1394             test_runner.parse_expectations(platform_name, is_debug_mode=False)
1395         printer.write("")
1396         _log.info("If there are no fail messages, errors or exceptions, "
1397                   "then the lint succeeded.")
1398         return 0
1399
1400     printer.print_config("Using port '%s'" % port_obj.name())
1401     printer.print_config("Placing test results in %s" %
1402                          options.results_directory)
1403     if options.new_baseline:
1404         printer.print_config("Placing new baselines in %s" %
1405                              port_obj.baseline_path())
1406     printer.print_config("Using %s build" % options.configuration)
1407     if options.pixel_tests:
1408         printer.print_config("Pixel tests enabled")
1409     else:
1410         printer.print_config("Pixel tests disabled")
1411     printer.print_config("")
1412
1413     printer.print_update("Parsing expectations ...")
1414     test_runner.parse_expectations(port_obj.test_platform_name(),
1415                                    options.configuration == 'Debug')
1416
1417     printer.print_update("Checking build ...")
1418     if not port_obj.check_build(test_runner.needs_http()):
1419         return -1
1420
1421     printer.print_update("Starting helper ...")
1422     port_obj.start_helper()
1423
1424     # Check that the system dependencies (themes, fonts, ...) are correct.
1425     if not options.nocheck_sys_deps:
1426         printer.print_update("Checking system dependencies ...")
1427         if not port_obj.check_sys_deps(test_runner.needs_http()):
1428             return -1
1429
1430     printer.print_update("Preparing tests ...")
1431     result_summary = test_runner.prepare_lists_and_print_output()
1432
1433     port_obj.setup_test_run()
1434
1435     test_runner.add_test_type(text_diff.TestTextDiff)
1436     if options.pixel_tests:
1437         test_runner.add_test_type(image_diff.ImageDiff)
1438         if options.fuzzy_pixel_tests:
1439             test_runner.add_test_type(fuzzy_image_diff.FuzzyImageDiff)
1440
1441     num_unexpected_results = test_runner.run(result_summary)
1442
1443     port_obj.stop_helper()
1444
1445     _log.debug("Exit status: %d" % num_unexpected_results)
1446     return num_unexpected_results
1447
1448
1449 def _compat_shim_callback(option, opt_str, value, parser):
1450     print "Ignoring unsupported option: %s" % opt_str
1451
1452
1453 def _compat_shim_option(option_name, **kwargs):
1454     return optparse.make_option(option_name, action="callback",
1455         callback=_compat_shim_callback,
1456         help="Ignored, for old-run-webkit-tests compat only.", **kwargs)
1457
1458
1459 def parse_args(args=None):
1460     """Provides a default set of command line args.
1461
1462     Returns a tuple of options, args from optparse"""
1463
1464     # FIXME: All of these options should be stored closer to the code which
1465     # FIXME: actually uses them. configuration_options should move
1466     # FIXME: to WebKitPort and be shared across all scripts.
1467     configuration_options = [
1468         optparse.make_option("-t", "--target", dest="configuration",
1469                              help="(DEPRECATED)"),
1470         # FIXME: --help should display which configuration is default.
1471         optparse.make_option('--debug', action='store_const', const='Debug',
1472                              dest="configuration",
1473                              help='Set the configuration to Debug'),
1474         optparse.make_option('--release', action='store_const',
1475                              const='Release', dest="configuration",
1476                              help='Set the configuration to Release'),
1477         # old-run-webkit-tests also accepts -c, --configuration CONFIGURATION.
1478     ]
1479
1480     print_options = printing.print_options()
1481
1482     # FIXME: These options should move onto the ChromiumPort.
1483     chromium_options = [
1484         optparse.make_option("--chromium", action="store_true", default=False,
1485             help="use the Chromium port"),
1486         optparse.make_option("--startup-dialog", action="store_true",
1487             default=False, help="create a dialog on DumpRenderTree startup"),
1488         optparse.make_option("--gp-fault-error-box", action="store_true",
1489             default=False, help="enable Windows GP fault error box"),
1490         optparse.make_option("--nocheck-sys-deps", action="store_true",
1491             default=False,
1492             help="Don't check the system dependencies (themes)"),
1493         optparse.make_option("--use-drt", action="store_true",
1494             default=False,
1495             help="Use DumpRenderTree instead of test_shell"),
1496     ]
1497
1498     # Missing Mac-specific old-run-webkit-tests options:
1499     # FIXME: Need: -g, --guard for guard malloc support on Mac.
1500     # FIXME: Need: -l --leaks    Enable leaks checking.
1501     # FIXME: Need: --sample-on-timeout Run sample on timeout
1502
1503     old_run_webkit_tests_compat = [
1504         # NRWT doesn't generate results by default anyway.
1505         _compat_shim_option("--no-new-test-results"),
1506         # NRWT doesn't sample on timeout yet anyway.
1507         _compat_shim_option("--no-sample-on-timeout"),
1508         # FIXME: NRWT needs to support remote links eventually.
1509         _compat_shim_option("--use-remote-links-to-tests"),
1510         # FIXME: NRWT doesn't need this option as much since failures are
1511         # designed to be cheap.  We eventually plan to add this support.
1512         _compat_shim_option("--exit-after-n-failures", nargs=1, type="int"),
1513     ]
1514
1515     results_options = [
1516         # NEED for bots: --use-remote-links-to-tests Link to test files
1517         # within the SVN repository in the results.
1518         optparse.make_option("-p", "--pixel-tests", action="store_true",
1519             dest="pixel_tests", help="Enable pixel-to-pixel PNG comparisons"),
1520         optparse.make_option("--no-pixel-tests", action="store_false",
1521             dest="pixel_tests", help="Disable pixel-to-pixel PNG comparisons"),
1522         optparse.make_option("--fuzzy-pixel-tests", action="store_true",
1523             default=False,
1524             help="Also use fuzzy matching to compare pixel test outputs."),
1525         # old-run-webkit-tests allows a specific tolerance: --tolerance t
1526         # Ignore image differences less than this percentage (default: 0.1)
1527         optparse.make_option("--results-directory",
1528             default="layout-test-results",
1529             help="Output results directory source dir, relative to Debug or "
1530                  "Release"),
1531         optparse.make_option("--new-baseline", action="store_true",
1532             default=False, help="Save all generated results as new baselines "
1533                  "into the platform directory, overwriting whatever's "
1534                  "already there."),
1535         optparse.make_option("--reset-results", action="store_true",
1536             default=False, help="Reset any existing baselines to the "
1537                  "generated results"),
1538         optparse.make_option("--no-show-results", action="store_false",
1539             default=True, dest="show_results",
1540             help="Don't launch a browser with results after the tests "
1541                  "are done"),
1542         # FIXME: We should have a helper function to do this sort of
1543         # deprectated mapping and automatically log, etc.
1544         optparse.make_option("--noshow-results", action="store_false",
1545             dest="show_results",
1546             help="Deprecated, same as --no-show-results."),
1547         optparse.make_option("--no-launch-safari", action="store_false",
1548             dest="show_results",
1549             help="old-run-webkit-tests compat, same as --noshow-results."),
1550         # old-run-webkit-tests:
1551         # --[no-]launch-safari    Launch (or do not launch) Safari to display
1552         #                         test results (default: launch)
1553         optparse.make_option("--full-results-html", action="store_true",
1554             default=False,
1555             help="Show all failures in results.html, rather than only "
1556                  "regressions"),
1557         optparse.make_option("--clobber-old-results", action="store_true",
1558             default=False, help="Clobbers test results from previous runs."),
1559         optparse.make_option("--platform",
1560             help="Override the platform for expected results"),
1561         # old-run-webkit-tests also has HTTP toggle options:
1562         # --[no-]http                     Run (or do not run) http tests
1563         #                                 (default: run)
1564         # --[no-]wait-for-httpd           Wait for httpd if some other test
1565         #                                 session is using it already (same
1566         #                                 as WEBKIT_WAIT_FOR_HTTPD=1).
1567         #                                 (default: 0)
1568     ]
1569
1570     test_options = [
1571         optparse.make_option("--build", dest="build",
1572             action="store_true", default=True,
1573             help="Check to ensure the DumpRenderTree build is up-to-date "
1574                  "(default)."),
1575         optparse.make_option("--no-build", dest="build",
1576             action="store_false", help="Don't check to see if the "
1577                                        "DumpRenderTree build is up-to-date."),
1578         # old-run-webkit-tests has --valgrind instead of wrapper.
1579         optparse.make_option("--wrapper",
1580             help="wrapper command to insert before invocations of "
1581                  "DumpRenderTree; option is split on whitespace before "
1582                  "running. (Example: --wrapper='valgrind --smc-check=all')"),
1583         # old-run-webkit-tests:
1584         # -i|--ignore-tests               Comma-separated list of directories
1585         #                                 or tests to ignore
1586         optparse.make_option("--test-list", action="append",
1587             help="read list of tests to run from file", metavar="FILE"),
1588         # old-run-webkit-tests uses --skipped==[default|ignore|only]
1589         # instead of --force:
1590         optparse.make_option("--force", action="store_true", default=False,
1591             help="Run all tests, even those marked SKIP in the test list"),
1592         optparse.make_option("--use-apache", action="store_true",
1593             default=False, help="Whether to use apache instead of lighttpd."),
1594         optparse.make_option("--time-out-ms",
1595             help="Set the timeout for each test"),
1596         # old-run-webkit-tests calls --randomize-order --random:
1597         optparse.make_option("--randomize-order", action="store_true",
1598             default=False, help=("Run tests in random order (useful "
1599                                 "for tracking down corruption)")),
1600         optparse.make_option("--run-chunk",
1601             help=("Run a specified chunk (n:l), the nth of len l, "
1602                  "of the layout tests")),
1603         optparse.make_option("--run-part", help=("Run a specified part (n:m), "
1604                   "the nth of m parts, of the layout tests")),
1605         # old-run-webkit-tests calls --batch-size: --nthly n
1606         #   Restart DumpRenderTree every n tests (default: 1000)
1607         optparse.make_option("--batch-size",
1608             help=("Run a the tests in batches (n), after every n tests, "
1609                   "DumpRenderTree is relaunched.")),
1610         # old-run-webkit-tests calls --run-singly: -1|--singly
1611         # Isolate each test case run (implies --nthly 1 --verbose)
1612         optparse.make_option("--run-singly", action="store_true",
1613             default=False, help="run a separate DumpRenderTree for each test"),
1614         optparse.make_option("--child-processes",
1615             help="Number of DumpRenderTrees to run in parallel."),
1616         # FIXME: Display default number of child processes that will run.
1617         optparse.make_option("--experimental-fully-parallel",
1618             action="store_true", default=False,
1619             help="run all tests in parallel"),
1620         # FIXME: Need --exit-after-n-failures N
1621         #      Exit after the first N failures instead of running all tests
1622         # FIXME: Need --exit-after-n-crashes N
1623         #      Exit after the first N crashes instead of running all tests
1624         # FIXME: consider: --iterations n
1625         #      Number of times to run the set of tests (e.g. ABCABCABC)
1626         optparse.make_option("--print-last-failures", action="store_true",
1627             default=False, help="Print the tests in the last run that "
1628             "had unexpected failures (or passes)."),
1629         optparse.make_option("--retest-last-failures", action="store_true",
1630             default=False, help="re-test the tests in the last run that "
1631             "had unexpected failures (or passes)."),
1632         optparse.make_option("--retry-failures", action="store_true",
1633             default=True,
1634             help="Re-try any tests that produce unexpected results (default)"),
1635         optparse.make_option("--no-retry-failures", action="store_false",
1636             dest="retry_failures",
1637             help="Don't re-try any tests that produce unexpected results."),
1638     ]
1639
1640     misc_options = [
1641         optparse.make_option("--lint-test-files", action="store_true",
1642         default=False, help=("Makes sure the test files parse for all "
1643                             "configurations. Does not run any tests.")),
1644     ]
1645
1646     # FIXME: Move these into json_results_generator.py
1647     results_json_options = [
1648         optparse.make_option("--builder-name", default="DUMMY_BUILDER_NAME",
1649             help=("The name of the builder shown on the waterfall running "
1650                   "this script e.g. WebKit.")),
1651         optparse.make_option("--build-name", default="DUMMY_BUILD_NAME",
1652             help=("The name of the builder used in its path, e.g. "
1653                   "webkit-rel.")),
1654         optparse.make_option("--build-number", default="DUMMY_BUILD_NUMBER",
1655             help=("The build number of the builder running this script.")),
1656         optparse.make_option("--test-results-server", default="",
1657             help=("If specified, upload results json files to this appengine "
1658                   "server.")),
1659     ]
1660
1661     option_list = (configuration_options + print_options +
1662                    chromium_options + results_options + test_options +
1663                    misc_options + results_json_options +
1664                    old_run_webkit_tests_compat)
1665     option_parser = optparse.OptionParser(option_list=option_list)
1666
1667     options, args = option_parser.parse_args(args)
1668     if options.sources:
1669         options.verbose = True
1670
1671     return options, args
1672
1673
1674 def main():
1675     options, args = parse_args()
1676     port_obj = port.get(options.platform, options)
1677     return run(port_obj, options, args)
1678
1679 if '__main__' == __name__:
1680     try:
1681         sys.exit(main())
1682     except KeyboardInterrupt:
1683         # this mirrors what the shell normally does
1684         sys.exit(signal.SIGINT + 128)