Tools/Scripts/webkitpy/layout_tests/layout_package/test_expectations_unittest.py

   1 #!/usr/bin/python
   2 # Copyright (C) 2010 Google Inc. All rights reserved.
   3 #
   4 # Redistribution and use in source and binary forms, with or without
   5 # modification, are permitted provided that the following conditions are
   6 # met:
   7 #
   8 #     * Redistributions of source code must retain the above copyright
   9 # notice, this list of conditions and the following disclaimer.
  10 #     * Redistributions in binary form must reproduce the above
  11 # copyright notice, this list of conditions and the following disclaimer
  12 # in the documentation and/or other materials provided with the
  13 # distribution.
  14 #     * Neither the name of Google Inc. nor the names of its
  15 # contributors may be used to endorse or promote products derived from
  16 # this software without specific prior written permission.
  17 #
  18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  29
  30 """Unit tests for test_expectations.py."""
  31
  32 import unittest
  33
  34 from webkitpy.layout_tests import port
  35 from webkitpy.layout_tests.port import base
  36 from webkitpy.layout_tests.layout_package.test_expectations import *
  37
  38 class FunctionsTest(unittest.TestCase):
  39     def test_result_was_expected(self):
  40         # test basics
  41         self.assertEquals(result_was_expected(PASS, set([PASS]),
  42                                               False, False), True)
  43         self.assertEquals(result_was_expected(TEXT, set([PASS]),
  44                                               False, False), False)
  45
  46         # test handling of FAIL expectations
  47         self.assertEquals(result_was_expected(IMAGE_PLUS_TEXT, set([FAIL]),
  48                                               False, False), True)
  49         self.assertEquals(result_was_expected(IMAGE, set([FAIL]),
  50                                               False, False), True)
  51         self.assertEquals(result_was_expected(TEXT, set([FAIL]),
  52                                               False, False), True)
  53         self.assertEquals(result_was_expected(CRASH, set([FAIL]),
  54                                               False, False), False)
  55
  56         # test handling of SKIPped tests and results
  57         self.assertEquals(result_was_expected(SKIP, set([CRASH]),
  58                                               False, True), True)
  59         self.assertEquals(result_was_expected(SKIP, set([CRASH]),
  60                                               False, False), False)
  61
  62         # test handling of MISSING results and the REBASELINE modifier
  63         self.assertEquals(result_was_expected(MISSING, set([PASS]),
  64                                               True, False), True)
  65         self.assertEquals(result_was_expected(MISSING, set([PASS]),
  66                                               False, False), False)
  67
  68     def test_remove_pixel_failures(self):
  69         self.assertEquals(remove_pixel_failures(set([TEXT])),
  70                           set([TEXT]))
  71         self.assertEquals(remove_pixel_failures(set([PASS])),
  72                           set([PASS]))
  73         self.assertEquals(remove_pixel_failures(set([IMAGE])),
  74                           set([PASS]))
  75         self.assertEquals(remove_pixel_failures(set([IMAGE_PLUS_TEXT])),
  76                           set([TEXT]))
  77         self.assertEquals(remove_pixel_failures(set([PASS, IMAGE, CRASH])),
  78                           set([PASS, CRASH]))
  79
  80
  81 class Base(unittest.TestCase):
  82     # Note that all of these tests are written assuming the configuration
  83     # being tested is Windows XP, Release build.
  84
  85     def __init__(self, testFunc, setUp=None, tearDown=None, description=None):
  86         self._port = port.get('test-win-xp', None)
  87         self._fs = self._port._filesystem
  88         self._exp = None
  89         unittest.TestCase.__init__(self, testFunc)
  90
  91     def get_test(self, test_name):
  92         return self._fs.join(self._port.layout_tests_dir(), test_name)
  93
  94     def get_basic_tests(self):
  95         return [self.get_test('failures/expected/text.html'),
  96                 self.get_test('failures/expected/image_checksum.html'),
  97                 self.get_test('failures/expected/crash.html'),
  98                 self.get_test('failures/expected/missing_text.html'),
  99                 self.get_test('failures/expected/image.html'),
 100                 self.get_test('passes/text.html')]
 101
 102     def get_basic_expectations(self):
 103         return """
 104 BUG_TEST : failures/expected/text.html = TEXT
 105 BUG_TEST WONTFIX SKIP : failures/expected/crash.html = CRASH
 106 BUG_TEST REBASELINE : failures/expected/missing_image.html = MISSING
 107 BUG_TEST WONTFIX : failures/expected/image_checksum.html = IMAGE
 108 BUG_TEST WONTFIX MAC : failures/expected/image.html = IMAGE
 109 """
 110
 111     def parse_exp(self, expectations, overrides=None, is_lint_mode=False):
 112         test_config = self._port.test_configuration()
 113         self._exp = TestExpectations(self._port,
 114              tests=self.get_basic_tests(),
 115              expectations=expectations,
 116              test_config=test_config,
 117              is_lint_mode=is_lint_mode,
 118              overrides=overrides)
 119
 120     def assert_exp(self, test, result):
 121         self.assertEquals(self._exp.get_expectations(self.get_test(test)),
 122                           set([result]))
 123
 124
 125 class BasicTests(Base):
 126     def test_basic(self):
 127         self.parse_exp(self.get_basic_expectations())
 128         self.assert_exp('failures/expected/text.html', TEXT)
 129         self.assert_exp('failures/expected/image_checksum.html', IMAGE)
 130         self.assert_exp('passes/text.html', PASS)
 131         self.assert_exp('failures/expected/image.html', PASS)
 132
 133
 134 class MiscTests(Base):
 135     def test_multiple_results(self):
 136         self.parse_exp('BUGX : failures/expected/text.html = TEXT CRASH')
 137         self.assertEqual(self._exp.get_expectations(
 138             self.get_test('failures/expected/text.html')),
 139             set([TEXT, CRASH]))
 140
 141     def test_category_expectations(self):
 142         # This test checks unknown tests are not present in the
 143         # expectations and that known test part of a test category is
 144         # present in the expectations.
 145         exp_str = """
 146 BUGX WONTFIX : failures/expected = IMAGE
 147 """
 148         self.parse_exp(exp_str)
 149         test_name = 'failures/expected/unknown-test.html'
 150         unknown_test = self.get_test(test_name)
 151         self.assertRaises(KeyError, self._exp.get_expectations,
 152                           unknown_test)
 153         self.assert_exp('failures/expected/crash.html', IMAGE)
 154
 155     def test_get_options(self):
 156         self.parse_exp(self.get_basic_expectations())
 157         self.assertEqual(self._exp.get_options(
 158                          self.get_test('passes/text.html')), [])
 159
 160     def test_expectations_json_for_all_platforms(self):
 161         self.parse_exp(self.get_basic_expectations())
 162         json_str = self._exp.get_expectations_json_for_all_platforms()
 163         # FIXME: test actual content?
 164         self.assertTrue(json_str)
 165
 166     def test_get_expectations_string(self):
 167         self.parse_exp(self.get_basic_expectations())
 168         self.assertEquals(self._exp.get_expectations_string(
 169                           self.get_test('failures/expected/text.html')),
 170                           'TEXT')
 171
 172     def test_expectation_to_string(self):
 173         # Normal cases are handled by other tests.
 174         self.parse_exp(self.get_basic_expectations())
 175         self.assertRaises(ValueError, self._exp.expectation_to_string,
 176                           -1)
 177
 178     def test_get_test_set(self):
 179         # Handle some corner cases for this routine not covered by other tests.
 180         self.parse_exp(self.get_basic_expectations())
 181         s = self._exp._expected_failures.get_test_set(WONTFIX)
 182         self.assertEqual(s,
 183             set([self.get_test('failures/expected/crash.html'),
 184                  self.get_test('failures/expected/image_checksum.html')]))
 185         s = self._exp._expected_failures.get_test_set(WONTFIX, CRASH)
 186         self.assertEqual(s,
 187             set([self.get_test('failures/expected/crash.html')]))
 188         s = self._exp._expected_failures.get_test_set(WONTFIX, CRASH,
 189                                                       include_skips=False)
 190         self.assertEqual(s, set([]))
 191
 192     def test_parse_error_fatal(self):
 193         try:
 194             self.parse_exp("""FOO : failures/expected/text.html = TEXT
 195 SKIP : failures/expected/image.html""")
 196             self.assertFalse(True, "ParseError wasn't raised")
 197         except ParseError, e:
 198             self.assertTrue(e.fatal)
 199             exp_errors = [u"Line:1 Unrecognized option 'foo' failures/expected/text.html",
 200                           u"Line:2 Missing expectations. [' failures/expected/image.html']"]
 201             self.assertEqual(str(e), '\n'.join(map(str, exp_errors)))
 202             self.assertEqual(e.errors, exp_errors)
 203
 204     def test_parse_error_nonfatal(self):
 205         try:
 206             self.parse_exp('SKIP : failures/expected/text.html = TEXT',
 207                            is_lint_mode=True)
 208             self.assertFalse(True, "ParseError wasn't raised")
 209         except ParseError, e:
 210             self.assertFalse(e.fatal)
 211             exp_errors = [u'Line:1 Test lacks BUG modifier. failures/expected/text.html']
 212             self.assertEqual(str(e), '\n'.join(map(str, exp_errors)))
 213             self.assertEqual(e.errors, exp_errors)
 214
 215     def test_overrides(self):
 216         self.parse_exp("BUG_EXP: failures/expected/text.html = TEXT",
 217                        "BUG_OVERRIDE : failures/expected/text.html = IMAGE")
 218         self.assert_exp('failures/expected/text.html', IMAGE)
 219
 220     def test_overrides__duplicate(self):
 221         self.assertRaises(ParseError, self.parse_exp,
 222              "BUG_EXP: failures/expected/text.html = TEXT",
 223              """
 224 BUG_OVERRIDE : failures/expected/text.html = IMAGE
 225 BUG_OVERRIDE : failures/expected/text.html = CRASH
 226 """)
 227
 228     def test_pixel_tests_flag(self):
 229         def match(test, result, pixel_tests_enabled):
 230             return self._exp.matches_an_expected_result(
 231                 self.get_test(test), result, pixel_tests_enabled)
 232
 233         self.parse_exp(self.get_basic_expectations())
 234         self.assertTrue(match('failures/expected/text.html', TEXT, True))
 235         self.assertTrue(match('failures/expected/text.html', TEXT, False))
 236         self.assertFalse(match('failures/expected/text.html', CRASH, True))
 237         self.assertFalse(match('failures/expected/text.html', CRASH, False))
 238         self.assertTrue(match('failures/expected/image_checksum.html', IMAGE,
 239                               True))
 240         self.assertTrue(match('failures/expected/image_checksum.html', PASS,
 241                               False))
 242         self.assertTrue(match('failures/expected/crash.html', SKIP, False))
 243         self.assertTrue(match('passes/text.html', PASS, False))
 244
 245     def test_more_specific_override_resets_skip(self):
 246         self.parse_exp("BUGX SKIP : failures/expected = TEXT\n"
 247                        "BUGX : failures/expected/text.html = IMAGE\n")
 248         self.assert_exp('failures/expected/text.html', IMAGE)
 249         self.assertFalse(self._port._filesystem.join(self._port.layout_tests_dir(),
 250                                                      'failures/expected/text.html') in
 251                          self._exp.get_tests_with_result_type(SKIP))
 252
 253 class ExpectationSyntaxTests(Base):
 254     def test_missing_expectation(self):
 255         # This is missing the expectation.
 256         self.assertRaises(ParseError, self.parse_exp,
 257                           'BUG_TEST: failures/expected/text.html')
 258
 259     def test_missing_colon(self):
 260         # This is missing the modifiers and the ':'
 261         self.assertRaises(ParseError, self.parse_exp,
 262                           'failures/expected/text.html = TEXT')
 263
 264     def disabled_test_too_many_colons(self):
 265         # FIXME: Enable this test and fix the underlying bug.
 266         self.assertRaises(ParseError, self.parse_exp,
 267                           'BUG_TEST: failures/expected/text.html = PASS :')
 268
 269     def test_too_many_equals_signs(self):
 270         self.assertRaises(ParseError, self.parse_exp,
 271                           'BUG_TEST: failures/expected/text.html = TEXT = IMAGE')
 272
 273     def test_unrecognized_expectation(self):
 274         self.assertRaises(ParseError, self.parse_exp,
 275                           'BUG_TEST: failures/expected/text.html = UNKNOWN')
 276
 277     def test_macro(self):
 278         exp_str = """
 279 BUG_TEST WIN-XP : failures/expected/text.html = TEXT
 280 """
 281         self.parse_exp(exp_str)
 282         self.assert_exp('failures/expected/text.html', TEXT)
 283
 284
 285 class SemanticTests(Base):
 286     def test_bug_format(self):
 287         self.assertRaises(ParseError, self.parse_exp, 'BUG1234 : failures/expected/text.html = TEXT')
 288
 289     def test_missing_bugid(self):
 290         # This should log a non-fatal error.
 291         self.parse_exp('SLOW : failures/expected/text.html = TEXT')
 292         self.assertEqual(
 293             len(self._exp._expected_failures.get_non_fatal_errors()), 1)
 294
 295     def test_slow_and_timeout(self):
 296         # A test cannot be SLOW and expected to TIMEOUT.
 297         self.assertRaises(ParseError, self.parse_exp,
 298             'BUG_TEST SLOW : failures/expected/timeout.html = TIMEOUT')
 299
 300     def test_rebaseline(self):
 301         # Can't lint a file w/ 'REBASELINE' in it.
 302         self.assertRaises(ParseError, self.parse_exp,
 303             'BUG_TEST REBASELINE : failures/expected/text.html = TEXT',
 304             is_lint_mode=True)
 305
 306     def test_duplicates(self):
 307         self.assertRaises(ParseError, self.parse_exp, """
 308 BUG_EXP : failures/expected/text.html = TEXT
 309 BUG_EXP : failures/expected/text.html = IMAGE""")
 310
 311         self.assertRaises(ParseError, self.parse_exp,
 312             self.get_basic_expectations(), overrides="""
 313 BUG_OVERRIDE : failures/expected/text.html = TEXT
 314 BUG_OVERRIDE : failures/expected/text.html = IMAGE""", )
 315
 316     def test_missing_file(self):
 317         # This should log a non-fatal error.
 318         self.parse_exp('BUG_TEST : missing_file.html = TEXT')
 319         self.assertEqual(
 320             len(self._exp._expected_failures.get_non_fatal_errors()), 1)
 321
 322
 323 class PrecedenceTests(Base):
 324     def test_file_over_directory(self):
 325         # This tests handling precedence of specific lines over directories
 326         # and tests expectations covering entire directories.
 327         exp_str = """
 328 BUGX : failures/expected/text.html = TEXT
 329 BUGX WONTFIX : failures/expected = IMAGE
 330 """
 331         self.parse_exp(exp_str)
 332         self.assert_exp('failures/expected/text.html', TEXT)
 333         self.assert_exp('failures/expected/crash.html', IMAGE)
 334
 335         exp_str = """
 336 BUGX WONTFIX : failures/expected = IMAGE
 337 BUGX : failures/expected/text.html = TEXT
 338 """
 339         self.parse_exp(exp_str)
 340         self.assert_exp('failures/expected/text.html', TEXT)
 341         self.assert_exp('failures/expected/crash.html', IMAGE)
 342
 343     def test_ambiguous(self):
 344         self.assertRaises(ParseError, self.parse_exp, """
 345 BUG_TEST RELEASE : passes/text.html = PASS
 346 BUG_TEST WIN : passes/text.html = FAIL
 347 """)
 348
 349     def test_more_modifiers(self):
 350         exp_str = """
 351 BUG_TEST RELEASE : passes/text.html = PASS
 352 BUG_TEST WIN RELEASE : passes/text.html = TEXT
 353 """
 354         self.assertRaises(ParseError, self.parse_exp, exp_str)
 355
 356     def test_order_in_file(self):
 357         exp_str = """
 358 BUG_TEST WIN RELEASE : passes/text.html = TEXT
 359 BUG_TEST RELEASE : passes/text.html = PASS
 360 """
 361         self.assertRaises(ParseError, self.parse_exp, exp_str)
 362
 363     def test_version_overrides(self):
 364         exp_str = """
 365 BUG_TEST WIN : passes/text.html = PASS
 366 BUG_TEST WIN XP : passes/text.html = TEXT
 367 """
 368         self.assertRaises(ParseError, self.parse_exp, exp_str)
 369
 370     def test_macro_overrides(self):
 371         exp_str = """
 372 BUG_TEST WIN : passes/text.html = PASS
 373 BUG_TEST WIN-XP : passes/text.html = TEXT
 374 """
 375         self.assertRaises(ParseError, self.parse_exp, exp_str)
 376
 377
 378 class RebaseliningTest(Base):
 379     """Test rebaselining-specific functionality."""
 380     def assertRemove(self, input_expectations, tests, expected_expectations):
 381         self.parse_exp(input_expectations)
 382         actual_expectations = self._exp.remove_rebaselined_tests(tests)
 383         self.assertEqual(expected_expectations, actual_expectations)
 384
 385     def test_remove(self):
 386         self.assertRemove('BUGX REBASELINE : failures/expected/text.html = TEXT\n'
 387                           'BUGY : failures/expected/image.html = IMAGE\n'
 388                           'BUGZ REBASELINE : failures/expected/crash.html = CRASH\n',
 389                           ['failures/expected/text.html'],
 390                           'BUGY : failures/expected/image.html = IMAGE\n'
 391                           'BUGZ REBASELINE : failures/expected/crash.html = CRASH\n')
 392
 393     def test_no_get_rebaselining_failures(self):
 394         self.parse_exp(self.get_basic_expectations())
 395         self.assertEqual(len(self._exp.get_rebaselining_failures()), 0)
 396
 397
 398 class ModifierTests(unittest.TestCase):
 399     def setUp(self):
 400         port_obj = port.get('test-win-xp', None)
 401         self.config = port_obj.test_configuration()
 402         self.matcher = ModifierMatcher(self.config)
 403
 404     def match(self, modifiers, expected_num_matches=-1, values=None, num_errors=0):
 405         matcher = self.matcher
 406         if values:
 407             matcher = ModifierMatcher(self.FakeTestConfiguration(values))
 408         match_result = matcher.match(modifiers)
 409         self.assertEqual(len(match_result.warnings), 0)
 410         self.assertEqual(len(match_result.errors), num_errors)
 411         self.assertEqual(match_result.num_matches, expected_num_matches,
 412              'match(%s, %s) returned -> %d, expected %d' %
 413              (modifiers, str(self.config.values()),
 414               match_result.num_matches, expected_num_matches))
 415
 416     def test_bad_match_modifier(self):
 417         self.match(['foo'], num_errors=1)
 418
 419     def test_none(self):
 420         self.match([], 0)
 421
 422     def test_one(self):
 423         self.match(['xp'], 1)
 424         self.match(['win'], 1)
 425         self.match(['release'], 1)
 426         self.match(['cpu'], 1)
 427         self.match(['x86'], 1)
 428         self.match(['leopard'], -1)
 429         self.match(['gpu'], -1)
 430         self.match(['debug'], -1)
 431
 432     def test_two(self):
 433         self.match(['xp', 'release'], 2)
 434         self.match(['win7', 'release'], -1)
 435         self.match(['win7', 'xp'], 1)
 436
 437     def test_three(self):
 438         self.match(['win7', 'xp', 'release'], 2)
 439         self.match(['xp', 'debug', 'x86'], -1)
 440         self.match(['xp', 'release', 'x86'], 3)
 441         self.match(['xp', 'cpu', 'release'], 3)
 442
 443     def test_four(self):
 444         self.match(['xp', 'release', 'cpu', 'x86'], 4)
 445         self.match(['win7', 'xp', 'release', 'cpu'], 3)
 446         self.match(['win7', 'xp', 'debug', 'cpu'], -1)
 447
 448     def test_case_insensitivity(self):
 449         self.match(['Win'], num_errors=1)
 450         self.match(['WIN'], num_errors=1)
 451         self.match(['win'], 1)
 452
 453     def test_duplicates(self):
 454         self.match(['release', 'release'], num_errors=1)
 455         self.match(['win-xp', 'xp'], num_errors=1)
 456         self.match(['win-xp', 'win-xp'], num_errors=1)
 457         self.match(['xp', 'release', 'xp', 'release'], num_errors=2)
 458         self.match(['rebaseline', 'rebaseline'], num_errors=1)
 459
 460     def test_unknown_option(self):
 461         self.match(['vms'], num_errors=1)
 462
 463     def test_duplicate_bugs(self):
 464         # BUG* regexes can appear multiple times.
 465         self.match(['bugfoo', 'bugbar'], 0)
 466
 467     def test_invalid_combinations(self):
 468         # FIXME: This should probably raise an error instead of NO_MATCH.
 469         self.match(['mac', 'xp'], num_errors=0)
 470
 471     def test_regexes_are_ignored(self):
 472         self.match(['bug123xy', 'rebaseline', 'wontfix', 'slow', 'skip'], 0)
 473
 474     def test_none_is_invalid(self):
 475         self.match(['none'], num_errors=1)
 476
 477
 478 if __name__ == '__main__':
 479     unittest.main()