WebKitTools/Scripts/webkitpy/layout_tests/port/test.py

   1 #!/usr/bin/env python
   2 # Copyright (C) 2010 Google Inc. All rights reserved.
   3 #
   4 # Redistribution and use in source and binary forms, with or without
   5 # modification, are permitted provided that the following conditions are
   6 # met:
   7 #
   8 #     * Redistributions of source code must retain the above copyright
   9 # notice, this list of conditions and the following disclaimer.
  10 #     * Redistributions in binary form must reproduce the above
  11 # copyright notice, this list of conditions and the following disclaimer
  12 # in the documentation and/or other materials provided with the
  13 # distribution.
  14 #     * Neither the Google name nor the names of its
  15 # contributors may be used to endorse or promote products derived from
  16 # this software without specific prior written permission.
  17 #
  18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  29
  30 """Dummy Port implementation used for testing."""
  31 from __future__ import with_statement
  32
  33 import codecs
  34 import fnmatch
  35 import os
  36 import sys
  37 import time
  38
  39 import base
  40
  41
  42 # This sets basic expectations for a test. Each individual expectation
  43 # can be overridden by a keyword argument in TestList.add().
  44 class TestInstance:
  45     def __init__(self, name):
  46         self.name = name
  47         self.base = name[(name.rfind("/") + 1):name.rfind(".html")]
  48         self.crash = False
  49         self.exception = False
  50         self.hang = False
  51         self.keyboard = False
  52         self.error = ''
  53         self.timeout = False
  54         self.actual_text = self.base + '-txt\n'
  55         self.actual_checksum = self.base + '-checksum\n'
  56         self.actual_image = self.base + '-png\n'
  57         self.expected_text = self.actual_text
  58         self.expected_checksum = self.actual_checksum
  59         self.expected_image = self.actual_image
  60
  61
  62 # This is an in-memory list of tests, what we want them to produce, and
  63 # what we want to claim are the expected results.
  64 class TestList:
  65     def __init__(self, port):
  66         self.port = port
  67         self.tests = {}
  68
  69     def add(self, name, **kwargs):
  70         test = TestInstance(name)
  71         for key, value in kwargs.items():
  72             test.__dict__[key] = value
  73         self.tests[name] = test
  74
  75     def keys(self):
  76         return self.tests.keys()
  77
  78     def __contains__(self, item):
  79         return item in self.tests
  80
  81     def __getitem__(self, item):
  82         return self.tests[item]
  83
  84
  85 class TestPort(base.Port):
  86     """Test implementation of the Port interface."""
  87
  88     def __init__(self, **kwargs):
  89         base.Port.__init__(self, **kwargs)
  90         tests = TestList(self)
  91         tests.add('passes/image.html')
  92         tests.add('passes/text.html')
  93         tests.add('failures/expected/checksum.html',
  94                   actual_checksum='checksum_fail-checksum')
  95         tests.add('failures/expected/crash.html', crash=True)
  96         tests.add('failures/expected/exception.html', exception=True)
  97         tests.add('failures/expected/timeout.html', timeout=True)
  98         tests.add('failures/expected/hang.html', hang=True)
  99         tests.add('failures/expected/missing_text.html',
 100                   expected_text=None)
 101         tests.add('failures/expected/image.html',
 102                   actual_image='image_fail-png',
 103                   expected_image='image-png')
 104         tests.add('failures/expected/image_checksum.html',
 105                   actual_checksum='image_checksum_fail-checksum',
 106                   actual_image='image_checksum_fail-png')
 107         tests.add('failures/expected/keyboard.html',
 108                   keyboard=True)
 109         tests.add('failures/expected/missing_check.html',
 110                   expected_checksum=None)
 111         tests.add('failures/expected/missing_image.html',
 112                   expected_image=None)
 113         tests.add('failures/expected/missing_text.html',
 114                   expected_text=None)
 115         tests.add('failures/expected/text.html',
 116                   actual_text='text_fail-png')
 117         tests.add('failures/unexpected/text-image-checksum.html',
 118                   actual_text='text-image-checksum_fail-txt',
 119                   actual_checksum='text-image-checksum_fail-checksum')
 120         tests.add('http/tests/passes/text.html')
 121         tests.add('http/tests/ssl/text.html')
 122         tests.add('passes/error.html', error='stuff going to stderr')
 123         tests.add('passes/image.html')
 124         tests.add('passes/platform_image.html')
 125         tests.add('passes/text.html')
 126         tests.add('websocket/tests/passes/text.html')
 127         self._tests = tests
 128
 129     def baseline_path(self):
 130         return os.path.join(self.layout_tests_dir(), 'platform',
 131                             self.name() + self.version())
 132
 133     def baseline_search_path(self):
 134         return [self.baseline_path()]
 135
 136     def check_build(self, needs_http):
 137         return True
 138
 139     def diff_image(self, expected_contents, actual_contents,
 140                    diff_filename=None):
 141         diffed = actual_contents != expected_contents
 142         if diffed and diff_filename:
 143             with codecs.open(diff_filename, "w", "utf-8") as diff_fh:
 144                 diff_fh.write("< %s\n---\n> %s\n" %
 145                               (expected_contents, actual_contents))
 146         return diffed
 147
 148     def expected_checksum(self, test):
 149         test = self.relative_test_filename(test)
 150         return self._tests[test].expected_checksum
 151
 152     def expected_image(self, test):
 153         test = self.relative_test_filename(test)
 154         return self._tests[test].expected_image
 155
 156     def expected_text(self, test):
 157         test = self.relative_test_filename(test)
 158         text = self._tests[test].expected_text
 159         if not text:
 160             text = ''
 161         return text
 162
 163     def tests(self, paths):
 164         # Test the idea of port-specific overrides for test lists. Also
 165         # keep in memory to speed up the test harness.
 166         if not paths:
 167             paths = ['*']
 168
 169         matched_tests = []
 170         for p in paths:
 171             if self.path_isdir(p):
 172                 matched_tests.extend(fnmatch.filter(self._tests.keys(), p + '*'))
 173             else:
 174                 matched_tests.extend(fnmatch.filter(self._tests.keys(), p))
 175         layout_tests_dir = self.layout_tests_dir()
 176         return set([os.path.join(layout_tests_dir, p) for p in matched_tests])
 177
 178     def path_exists(self, path):
 179         # used by test_expectations.py and printing.py
 180         rpath = self.relative_test_filename(path)
 181         if rpath in self._tests:
 182             return True
 183         if self.path_isdir(rpath):
 184             return True
 185         if rpath.endswith('-expected.txt'):
 186             test = rpath.replace('-expected.txt', '.html')
 187             return (test in self._tests and
 188                     self._tests[test].expected_text)
 189         if rpath.endswith('-expected.checksum'):
 190             test = rpath.replace('-expected.checksum', '.html')
 191             return (test in self._tests and
 192                     self._tests[test].expected_checksum)
 193         if rpath.endswith('-expected.png'):
 194             test = rpath.replace('-expected.png', '.html')
 195             return (test in self._tests and
 196                     self._tests[test].expected_image)
 197         return False
 198
 199     def layout_tests_dir(self):
 200         return self.path_from_webkit_base('WebKitTools', 'Scripts',
 201                                           'webkitpy', 'layout_tests', 'data')
 202
 203     def path_isdir(self, path):
 204         # Used by test_expectations.py
 205         #
 206         # We assume that a path is a directory if we have any tests that
 207         # whose prefix matches the path plus a directory modifier.
 208         if path[-1] != '/':
 209             path += '/'
 210         return any([t.startswith(path) for t in self._tests.keys()])
 211
 212     def test_dirs(self):
 213         return ['passes', 'failures']
 214
 215     def name(self):
 216         return self._name
 217
 218     def _path_to_wdiff(self):
 219         return None
 220
 221     def results_directory(self):
 222         return '/tmp/' + self.get_option('results_directory')
 223
 224     def setup_test_run(self):
 225         pass
 226
 227     def create_driver(self, image_path, options):
 228         return TestDriver(self, image_path, options, executive=None)
 229
 230     def start_http_server(self):
 231         pass
 232
 233     def start_websocket_server(self):
 234         pass
 235
 236     def stop_http_server(self):
 237         pass
 238
 239     def stop_websocket_server(self):
 240         pass
 241
 242     def test_expectations(self):
 243         """Returns the test expectations for this port.
 244
 245         Basically this string should contain the equivalent of a
 246         test_expectations file. See test_expectations.py for more details."""
 247         return """
 248 WONTFIX : failures/expected/checksum.html = IMAGE
 249 WONTFIX : failures/expected/crash.html = CRASH
 250 // This one actually passes because the checksums will match.
 251 WONTFIX : failures/expected/image.html = PASS
 252 WONTFIX : failures/expected/image_checksum.html = IMAGE
 253 WONTFIX : failures/expected/missing_check.html = MISSING PASS
 254 WONTFIX : failures/expected/missing_image.html = MISSING PASS
 255 WONTFIX : failures/expected/missing_text.html = MISSING PASS
 256 WONTFIX : failures/expected/text.html = TEXT
 257 WONTFIX : failures/expected/timeout.html = TIMEOUT
 258 WONTFIX SKIP : failures/expected/hang.html = TIMEOUT
 259 WONTFIX SKIP : failures/expected/keyboard.html = CRASH
 260 WONTFIX SKIP : failures/expected/exception.html = CRASH
 261 """
 262
 263     def test_base_platform_names(self):
 264         return ('mac', 'win')
 265
 266     def test_platform_name(self):
 267         return 'mac'
 268
 269     def test_platform_names(self):
 270         return self.test_base_platform_names()
 271
 272     def test_platform_name_to_name(self, test_platform_name):
 273         return test_platform_name
 274
 275     def version(self):
 276         return ''
 277
 278
 279 class TestDriver(base.Driver):
 280     """Test/Dummy implementation of the DumpRenderTree interface."""
 281
 282     def __init__(self, port, image_path, options, executive):
 283         self._port = port
 284         self._image_path = image_path
 285         self._executive = executive
 286         self._image_written = False
 287
 288     def poll(self):
 289         return True
 290
 291     def run_test(self, uri, timeoutms, image_hash):
 292         test_name = self._port.uri_to_test_name(uri)
 293         test = self._port._tests[test_name]
 294         if test.keyboard:
 295             raise KeyboardInterrupt
 296         if test.exception:
 297             raise ValueError('exception from ' + test_name)
 298         if test.hang:
 299             time.sleep((float(timeoutms) * 4) / 1000.0)
 300
 301         if self._port.get_option('pixel_tests') and test.actual_image:
 302             with open(self._image_path, 'w') as file:
 303                 file.write(test.actual_image)
 304
 305         return (test.crash, test.timeout, test.actual_checksum,
 306                 test.actual_text, test.error)
 307
 308     def start(self):
 309         pass
 310
 311     def stop(self):
 312         pass