Tools/Scripts/webkitpy/layout_tests/rebaseline_chromium_webkit_tests_unittest.py

   1 #!/usr/bin/python
   2 # Copyright (C) 2010 Google Inc. All rights reserved.
   3 #
   4 # Redistribution and use in source and binary forms, with or without
   5 # modification, are permitted provided that the following conditions are
   6 # met:
   7 #
   8 #     * Redistributions of source code must retain the above copyright
   9 # notice, this list of conditions and the following disclaimer.
  10 #     * Redistributions in binary form must reproduce the above
  11 # copyright notice, this list of conditions and the following disclaimer
  12 # in the documentation and/or other materials provided with the
  13 # distribution.
  14 #     * Neither the name of Google Inc. nor the names of its
  15 # contributors may be used to endorse or promote products derived from
  16 # this software without specific prior written permission.
  17 #
  18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  29
  30 """Unit tests for rebaseline_chromium_webkit_tests.py."""
  31
  32 import unittest
  33
  34 from webkitpy.tool import mocktool
  35 from webkitpy.common.system import urlfetcher_mock
  36 from webkitpy.common.system import filesystem_mock
  37 from webkitpy.common.system import zipfileset_mock
  38 from webkitpy.common.system import outputcapture
  39 from webkitpy.common.system.executive import Executive, ScriptError
  40
  41 from webkitpy.layout_tests import port
  42 from webkitpy.layout_tests import rebaseline_chromium_webkit_tests
  43
  44
  45 class MockPort(object):
  46     def __init__(self, image_diff_exists):
  47         self.image_diff_exists = image_diff_exists
  48
  49     def check_image_diff(self, override_step, logging):
  50         return self.image_diff_exists
  51
  52
  53 def get_mock_get(config_expectations):
  54     def mock_get(port_name, options):
  55         return MockPort(config_expectations[options.configuration])
  56     return mock_get
  57
  58
  59 ARCHIVE_URL = 'http://localhost/layout_test_results'
  60
  61
  62 def test_options():
  63     return mocktool.MockOptions(configuration=None,
  64                                 backup=False,
  65                                 html_directory='/tmp',
  66                                 archive_url=ARCHIVE_URL,
  67                                 force_archive_url=None,
  68                                 verbose=False,
  69                                 quiet=False,
  70                                 platforms=None)
  71
  72
  73 def test_host_port_and_filesystem(options, expectations):
  74     filesystem = port.unit_test_filesystem()
  75     host_port_obj = port.get('test', options, filesystem=filesystem,
  76                              user=mocktool.MockUser())
  77
  78     expectations_path = host_port_obj.path_to_test_expectations_file()
  79     filesystem.write_text_file(expectations_path, expectations)
  80     return (host_port_obj, filesystem)
  81
  82
  83 def test_url_fetcher(filesystem):
  84     urls = {
  85         ARCHIVE_URL + '/Webkit_Mac10_6/': '<a href="4/">',
  86         ARCHIVE_URL + '/Webkit_Mac10_5/': '<a href="1/"><a href="2/">',
  87         ARCHIVE_URL + '/Webkit_Win7/': '<a href="1/">',
  88         ARCHIVE_URL + '/Webkit_Vista/': '<a href="1/">',
  89         ARCHIVE_URL + '/Webkit_Win/': '<a href="1/">',
  90         ARCHIVE_URL + '/Webkit_Linux/': '<a href="1/">',
  91     }
  92     return urlfetcher_mock.make_fetcher_cls(urls)(filesystem)
  93
  94
  95 def test_zip_factory():
  96     ziphashes = {
  97         ARCHIVE_URL + '/Webkit_Mac10_5/2/layout-test-results.zip': {
  98             'layout-test-results/failures/expected/image-actual.txt': 'new-image-txt',
  99             'layout-test-results/failures/expected/image-actual.checksum': 'new-image-checksum',
 100             'layout-test-results/failures/expected/image-actual.png': 'new-image-png',
 101             'layout-test-results/failures/expected/image_checksum-actual.txt': 'png-comment-txt',
 102             'layout-test-results/failures/expected/image_checksum-actual.checksum': '0123456789',
 103             'layout-test-results/failures/expected/image_checksum-actual.png': 'tEXtchecksum\x000123456789',
 104         },
 105         ARCHIVE_URL + '/Webkit_Mac10_6/4/layout-test-results.zip': {
 106             'layout-test-results/failures/expected/image-actual.txt': 'new-image-txt',
 107             'layout-test-results/failures/expected/image-actual.checksum': 'new-image-checksum',
 108             'layout-test-results/failures/expected/image-actual.png': 'new-image-png',
 109         },
 110          ARCHIVE_URL + '/Webkit_Vista/1/layout-test-results.zip': {
 111             'layout-test-results/failures/expected/image-actual.txt': 'win-image-txt',
 112             'layout-test-results/failures/expected/image-actual.checksum': 'win-image-checksum',
 113             'layout-test-results/failures/expected/image-actual.png': 'win-image-png',
 114         },
 115           ARCHIVE_URL + '/Webkit_Win7/1/layout-test-results.zip': {
 116             'layout-test-results/failures/expected/image-actual.txt': 'win-image-txt',
 117             'layout-test-results/failures/expected/image-actual.checksum': 'win-image-checksum',
 118             'layout-test-results/failures/expected/image-actual.png': 'win-image-png',
 119         },
 120           ARCHIVE_URL + '/Webkit_Win/1/layout-test-results.zip': {
 121             'layout-test-results/failures/expected/image-actual.txt': 'win-image-txt',
 122             'layout-test-results/failures/expected/image-actual.checksum': 'win-image-checksum',
 123             'layout-test-results/failures/expected/image-actual.png': 'win-image-png',
 124         },
 125           ARCHIVE_URL + '/Webkit_Linux/1/layout-test-results.zip': {
 126             'layout-test-results/failures/expected/image-actual.txt': 'win-image-txt',
 127             'layout-test-results/failures/expected/image-actual.checksum': 'win-image-checksum',
 128             'layout-test-results/failures/expected/image-actual.png': 'win-image-png',
 129         },
 130     }
 131     return zipfileset_mock.make_factory(ziphashes)
 132
 133
 134 def test_archive(orig_archive_dict):
 135     new_archive_dict = {}
 136     for platform, dirname in orig_archive_dict.iteritems():
 137         platform = platform.replace('chromium', 'test')
 138         new_archive_dict[platform] = dirname
 139     return new_archive_dict
 140
 141
 142 class TestGetHostPortObject(unittest.TestCase):
 143     def assert_result(self, release_present, debug_present, valid_port_obj):
 144         # Tests whether we get a valid port object returned when we claim
 145         # that Image diff is (or isn't) present in the two configs.
 146         port.get = get_mock_get({'Release': release_present,
 147                                  'Debug': debug_present})
 148         options = mocktool.MockOptions(configuration=None,
 149                                        html_directory='/tmp')
 150         port_obj = rebaseline_chromium_webkit_tests.get_host_port_object(options)
 151         if valid_port_obj:
 152             self.assertNotEqual(port_obj, None)
 153         else:
 154             self.assertEqual(port_obj, None)
 155
 156     def test_get_host_port_object(self):
 157         # Save the normal port.get() function for future testing.
 158         old_get = port.get
 159
 160         # Test whether we get a valid port object back for the four
 161         # possible cases of having ImageDiffs built. It should work when
 162         # there is at least one binary present.
 163         self.assert_result(False, False, False)
 164         self.assert_result(True, False, True)
 165         self.assert_result(False, True, True)
 166         self.assert_result(True, True, True)
 167
 168         # Restore the normal port.get() function.
 169         port.get = old_get
 170
 171
 172 class TestOptions(unittest.TestCase):
 173     def test_parse_options(self):
 174         (options, target_options) = rebaseline_chromium_webkit_tests.parse_options([])
 175         self.assertTrue(target_options.chromium)
 176         self.assertEqual(options.tolerance, 0)
 177
 178         (options, target_options) = rebaseline_chromium_webkit_tests.parse_options(['--target-platform', 'qt'])
 179         self.assertFalse(hasattr(target_options, 'chromium'))
 180         self.assertEqual(options.tolerance, 0)
 181
 182
 183 class TestRebaseliner(unittest.TestCase):
 184     def setUp(self):
 185         if not hasattr(self, '_orig_archive'):
 186             self._orig_archive = rebaseline_chromium_webkit_tests.ARCHIVE_DIR_NAME_DICT
 187             rebaseline_chromium_webkit_tests.ARCHIVE_DIR_NAME_DICT = test_archive(self._orig_archive)
 188
 189     def tearDown(self):
 190         rebaseline_chromium_webkit_tests.ARCHIVE_DIR_NAME_DICT = self._orig_archive
 191
 192     def make_rebaseliner(self, expectations):
 193         options = test_options()
 194         host_port_obj, filesystem = test_host_port_and_filesystem(options, expectations)
 195
 196         target_options = options
 197         target_port_obj = port.get('test', target_options,
 198                                    filesystem=filesystem)
 199         target_port_obj._expectations = expectations
 200         platform = target_port_obj.name()
 201
 202         url_fetcher = test_url_fetcher(filesystem)
 203         zip_factory = test_zip_factory()
 204         mock_scm = mocktool.MockSCM(filesystem)
 205         rebaseliner = rebaseline_chromium_webkit_tests.Rebaseliner(host_port_obj,
 206             target_port_obj, platform, options, url_fetcher, zip_factory, mock_scm)
 207         return rebaseliner, filesystem
 208
 209     def test_noop(self):
 210         # this method tests that was can at least instantiate an object, even
 211         # if there is nothing to do.
 212         rebaseliner, filesystem = self.make_rebaseliner("")
 213         rebaseliner.run()
 214         self.assertEqual(len(filesystem.written_files), 1)
 215
 216     def test_rebaselining_tests(self):
 217         rebaseliner, filesystem = self.make_rebaseliner(
 218             "BUGX REBASELINE MAC : failures/expected/image.html = IMAGE")
 219         compile_success = rebaseliner._compile_rebaselining_tests()
 220         self.assertTrue(compile_success)
 221         self.assertEqual(set(['failures/expected/image.html']), rebaseliner._rebaselining_tests)
 222
 223     def test_rebaselining_tests_should_ignore_reftests(self):
 224         rebaseliner, filesystem = self.make_rebaseliner(
 225             "BUGX REBASELINE : failures/expected/reftest.html = IMAGE")
 226         compile_success = rebaseliner._compile_rebaselining_tests()
 227         self.assertFalse(compile_success)
 228         self.assertFalse(rebaseliner._rebaselining_tests)
 229
 230     def test_one_platform(self):
 231         rebaseliner, filesystem = self.make_rebaseliner(
 232             "BUGX REBASELINE MAC : failures/expected/image.html = IMAGE")
 233         rebaseliner.run()
 234         # We expect to have written 12 files over the course of this rebaseline:
 235         # *) 3 files in /__im_tmp for the extracted archive members
 236         # *) 3 new baselines under '/test.checkout/LayoutTests'
 237         # *) 4 files in /tmp for the new and old baselines in the result file
 238         #    (-{old,new}.{txt,png}
 239         # *) 1 text diff in /tmp for the result file (-diff.txt). We don't
 240         #    create image diffs (FIXME?) and don't display the checksums.
 241         # *) 1 updated test_expectations file
 242         self.assertEqual(len(filesystem.written_files), 12)
 243         self.assertEqual(filesystem.files['/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image-expected.checksum'], 'new-image-checksum')
 244         self.assertEqual(filesystem.files['/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image-expected.png'], 'new-image-png')
 245         self.assertEqual(filesystem.files['/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image-expected.txt'], 'new-image-txt')
 246
 247     def test_all_platforms(self):
 248         rebaseliner, filesystem = self.make_rebaseliner(
 249             "BUGX REBASELINE : failures/expected/image.html = IMAGE")
 250         rebaseliner.run()
 251         # See comment in test_one_platform for an explanation of the 12 written tests.
 252         # Note that even though the rebaseline is marked for all platforms, each
 253         # rebaseliner only ever does one.
 254         self.assertEqual(len(filesystem.written_files), 12)
 255         self.assertEqual(filesystem.files['/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image-expected.checksum'], 'new-image-checksum')
 256         self.assertEqual(filesystem.files['/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image-expected.png'], 'new-image-png')
 257         self.assertEqual(filesystem.files['/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image-expected.txt'], 'new-image-txt')
 258
 259     def test_png_file_with_comment(self):
 260         rebaseliner, filesystem = self.make_rebaseliner(
 261             "BUGX REBASELINE MAC : failures/expected/image_checksum.html = IMAGE")
 262         compile_success = rebaseliner._compile_rebaselining_tests()
 263         self.assertTrue(compile_success)
 264         self.assertEqual(set(['failures/expected/image_checksum.html']), rebaseliner._rebaselining_tests)
 265         rebaseliner.run()
 266         # There is one less file written than |test_one_platform| because we only
 267         # write 2 expectations (the png and the txt file).
 268         self.assertEqual(len(filesystem.written_files), 11)
 269         self.assertEqual(filesystem.files['/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image_checksum-expected.png'], 'tEXtchecksum\x000123456789')
 270         self.assertEqual(filesystem.files['/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image_checksum-expected.txt'], 'png-comment-txt')
 271         self.assertFalse(filesystem.files.get('/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image_checksum-expected.checksum', None))
 272
 273     def test_png_file_with_comment_remove_old_checksum(self):
 274         rebaseliner, filesystem = self.make_rebaseliner(
 275             "BUGX REBASELINE MAC : failures/expected/image_checksum.html = IMAGE")
 276         filesystem.files['/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image_checksum-expected.png'] = 'old'
 277         filesystem.files['/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image_checksum-expected.checksum'] = 'old'
 278         filesystem.files['/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image_checksum-expected.txt'] = 'old'
 279
 280         compile_success = rebaseliner._compile_rebaselining_tests()
 281         self.assertTrue(compile_success)
 282         self.assertEqual(set(['failures/expected/image_checksum.html']), rebaseliner._rebaselining_tests)
 283         rebaseliner.run()
 284         # There is one more file written than |test_png_file_with_comment_remove_old_checksum|
 285         # because we also delete the old checksum.
 286         self.assertEqual(len(filesystem.written_files), 12)
 287         self.assertEqual(filesystem.files['/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image_checksum-expected.png'], 'tEXtchecksum\x000123456789')
 288         self.assertEqual(filesystem.files['/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image_checksum-expected.txt'], 'png-comment-txt')
 289         self.assertEqual(filesystem.files.get('/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image_checksum-expected.checksum', None), None)
 290
 291     def test_png_file_with_comment_as_duplicate(self):
 292         rebaseliner, filesystem = self.make_rebaseliner(
 293             "BUGX REBASELINE MAC : failures/expected/image_checksum.html = IMAGE")
 294         filesystem.files['/test.checkout/LayoutTests/platform/test-mac-snowleopard/failures/expected/image_checksum-expected.png'] = 'tEXtchecksum\x000123456789'
 295         filesystem.files['/test.checkout/LayoutTests/platform/test-mac-snowleopard/failures/expected/image_checksum-expected.txt'] = 'png-comment-txt'
 296
 297         compile_success = rebaseliner._compile_rebaselining_tests()
 298         self.assertTrue(compile_success)
 299         self.assertEqual(set(['failures/expected/image_checksum.html']), rebaseliner._rebaselining_tests)
 300         rebaseliner.run()
 301         self.assertEqual(filesystem.files.get('/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image_checksum-expected.png', None), None)
 302         self.assertEqual(filesystem.files.get('/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image_checksum-expected.txt', None), None)
 303         self.assertEqual(filesystem.files.get('/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image_checksum-expected.checksum', None), None)
 304
 305     def test_diff_baselines_txt(self):
 306         rebaseliner, filesystem = self.make_rebaseliner("")
 307         port = rebaseliner._port
 308         output = port.expected_text(
 309             port._filesystem.join(port.layout_tests_dir(), 'passes/text.html'))
 310         self.assertFalse(rebaseliner._diff_baselines(output, output,
 311                                                      is_image=False))
 312
 313     def test_diff_baselines_png(self):
 314         rebaseliner, filesystem = self.make_rebaseliner('')
 315         port = rebaseliner._port
 316         image = port.expected_image(
 317             port._filesystem.join(port.layout_tests_dir(), 'passes/image.html'))
 318         self.assertFalse(rebaseliner._diff_baselines(image, image,
 319                                                      is_image=True))
 320
 321
 322 class TestRealMain(unittest.TestCase):
 323     def setUp(self):
 324         if not hasattr(self, '_orig_archive'):
 325             self._orig_archive = rebaseline_chromium_webkit_tests.ARCHIVE_DIR_NAME_DICT
 326             rebaseline_chromium_webkit_tests.ARCHIVE_DIR_NAME_DICT = test_archive(self._orig_archive)
 327
 328     def tearDown(self):
 329         rebaseline_chromium_webkit_tests.ARCHIVE_DIR_NAME_DICT = self._orig_archive
 330
 331     def test_all_platforms(self):
 332         expectations = "BUGX REBASELINE : failures/expected/image.html = IMAGE"
 333
 334         options = test_options()
 335         host_port_obj, filesystem = test_host_port_and_filesystem(options, expectations)
 336         url_fetcher = test_url_fetcher(filesystem)
 337         zip_factory = test_zip_factory()
 338         mock_scm = mocktool.MockSCM()
 339         oc = outputcapture.OutputCapture()
 340         oc.capture_output()
 341         res = rebaseline_chromium_webkit_tests.real_main(options, options,
 342             host_port_obj, host_port_obj, url_fetcher, zip_factory, mock_scm)
 343         oc.restore_output()
 344
 345         # We expect to have written 36 files over the course of this rebaseline:
 346         # *) 6*3 files in /__im_tmp/ for the archived members of the 6 ports
 347         # *) 2*3 files in /test.checkout for actually differing baselines
 348         # *) 1 file in /test.checkout for the updated test_expectations file
 349         # *) 2*4 files in /tmp for the old/new baselines for the two actual ports
 350         # *) 2 files in /tmp for the text diffs for the two ports
 351         # *) 1 file in /tmp for the rebaseline results html file
 352         self.assertEqual(res, 0)
 353         self.assertEqual(len(filesystem.written_files), 36)
 354
 355
 356 class TestHtmlGenerator(unittest.TestCase):
 357     def make_generator(self, files, tests):
 358         options = mocktool.MockOptions(configuration=None, html_directory='/tmp')
 359         host_port = port.get('test', options, filesystem=port.unit_test_filesystem(files))
 360         generator = rebaseline_chromium_webkit_tests.HtmlGenerator(host_port,
 361             target_port=None, options=options, platforms=['test-mac-leopard'], rebaselining_tests=tests)
 362         return generator, host_port
 363
 364     def test_generate_baseline_links(self):
 365         files = {
 366             "/tmp/foo-expected-mac-old.txt": "",
 367             "/tmp/foo-expected-mac-new.txt": "",
 368             "/tmp/foo-expected-mac-diff.txt": "",
 369         }
 370         tests = ["foo.txt"]
 371         generator, host_port = self.make_generator(files, tests)
 372         links = generator._generate_baseline_links("foo", ".txt", "mac")
 373         expected_links = '<td align=center><a href="file:///tmp/foo-expected-mac-old.txt">foo-expected.txt</a></td><td align=center><a href="file:///tmp/foo-expected-mac-new.txt">foo-expected.txt</a></td><td align=center><a href="file:///tmp/foo-expected-mac-diff.txt">Diff</a></td>'
 374         self.assertEqual(links, expected_links)
 375
 376
 377 if __name__ == '__main__':
 378     unittest.main()