2 # Copyright (C) 2010 Google Inc. All rights reserved.
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
8 # * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 # * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
14 # * Neither the Google name nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30 """Dummy Port implementation used for testing."""
31 from __future__ import with_statement
42 # This sets basic expectations for a test. Each individual expectation
43 # can be overridden by a keyword argument in TestList.add().
45 def __init__(self, name):
47 self.base = name[(name.rfind("/") + 1):name.rfind(".html")]
49 self.exception = False
54 self.actual_text = self.base + '-txt\n'
55 self.actual_checksum = self.base + '-checksum\n'
56 self.actual_image = self.base + '-png\n'
57 self.expected_text = self.actual_text
58 self.expected_checksum = self.actual_checksum
59 self.expected_image = self.actual_image
62 # This is an in-memory list of tests, what we want them to produce, and
63 # what we want to claim are the expected results.
65 def __init__(self, port):
69 def add(self, name, **kwargs):
70 test = TestInstance(name)
71 for key, value in kwargs.items():
72 test.__dict__[key] = value
73 self.tests[name] = test
76 return self.tests.keys()
78 def __contains__(self, item):
79 return item in self.tests
81 def __getitem__(self, item):
82 return self.tests[item]
85 class TestPort(base.Port):
86 """Test implementation of the Port interface."""
88 def __init__(self, **kwargs):
89 base.Port.__init__(self, **kwargs)
90 tests = TestList(self)
91 tests.add('passes/image.html')
92 tests.add('passes/text.html')
93 tests.add('failures/expected/checksum.html',
94 actual_checksum='checksum_fail-checksum')
95 tests.add('failures/expected/crash.html', crash=True)
96 tests.add('failures/expected/exception.html', exception=True)
97 tests.add('failures/expected/timeout.html', timeout=True)
98 tests.add('failures/expected/hang.html', hang=True)
99 tests.add('failures/expected/missing_text.html',
101 tests.add('failures/expected/image.html',
102 actual_image='image_fail-png',
103 expected_image='image-png')
104 tests.add('failures/expected/image_checksum.html',
105 actual_checksum='image_checksum_fail-checksum',
106 actual_image='image_checksum_fail-png')
107 tests.add('failures/expected/keyboard.html',
109 tests.add('failures/expected/missing_check.html',
110 expected_checksum=None)
111 tests.add('failures/expected/missing_image.html',
113 tests.add('failures/expected/missing_text.html',
115 tests.add('failures/expected/text.html',
116 actual_text='text_fail-png')
117 tests.add('failures/unexpected/text-image-checksum.html',
118 actual_text='text-image-checksum_fail-txt',
119 actual_checksum='text-image-checksum_fail-checksum')
120 tests.add('http/tests/passes/text.html')
121 tests.add('http/tests/ssl/text.html')
122 tests.add('passes/error.html', error='stuff going to stderr')
123 tests.add('passes/image.html')
124 tests.add('passes/platform_image.html')
125 tests.add('passes/text.html')
126 tests.add('websocket/tests/passes/text.html')
129 def baseline_path(self):
130 return os.path.join(self.layout_tests_dir(), 'platform',
131 self.name() + self.version())
133 def baseline_search_path(self):
134 return [self.baseline_path()]
136 def check_build(self, needs_http):
139 def diff_image(self, expected_contents, actual_contents,
141 diffed = actual_contents != expected_contents
142 if diffed and diff_filename:
143 with codecs.open(diff_filename, "w", "utf-8") as diff_fh:
144 diff_fh.write("< %s\n---\n> %s\n" %
145 (expected_contents, actual_contents))
148 def expected_checksum(self, test):
149 test = self.relative_test_filename(test)
150 return self._tests[test].expected_checksum
152 def expected_image(self, test):
153 test = self.relative_test_filename(test)
154 return self._tests[test].expected_image
156 def expected_text(self, test):
157 test = self.relative_test_filename(test)
158 text = self._tests[test].expected_text
163 def tests(self, paths):
164 # Test the idea of port-specific overrides for test lists. Also
165 # keep in memory to speed up the test harness.
171 if self.path_isdir(p):
172 matched_tests.extend(fnmatch.filter(self._tests.keys(), p + '*'))
174 matched_tests.extend(fnmatch.filter(self._tests.keys(), p))
175 layout_tests_dir = self.layout_tests_dir()
176 return set([os.path.join(layout_tests_dir, p) for p in matched_tests])
178 def path_exists(self, path):
179 # used by test_expectations.py and printing.py
180 rpath = self.relative_test_filename(path)
181 if rpath in self._tests:
183 if self.path_isdir(rpath):
185 if rpath.endswith('-expected.txt'):
186 test = rpath.replace('-expected.txt', '.html')
187 return (test in self._tests and
188 self._tests[test].expected_text)
189 if rpath.endswith('-expected.checksum'):
190 test = rpath.replace('-expected.checksum', '.html')
191 return (test in self._tests and
192 self._tests[test].expected_checksum)
193 if rpath.endswith('-expected.png'):
194 test = rpath.replace('-expected.png', '.html')
195 return (test in self._tests and
196 self._tests[test].expected_image)
199 def layout_tests_dir(self):
200 return self.path_from_webkit_base('WebKitTools', 'Scripts',
201 'webkitpy', 'layout_tests', 'data')
203 def path_isdir(self, path):
204 # Used by test_expectations.py
206 # We assume that a path is a directory if we have any tests that
207 # whose prefix matches the path plus a directory modifier.
210 return any([t.startswith(path) for t in self._tests.keys()])
213 return ['passes', 'failures']
218 def _path_to_wdiff(self):
221 def results_directory(self):
222 return '/tmp/' + self.get_option('results_directory')
224 def setup_test_run(self):
227 def create_driver(self, image_path, options):
228 return TestDriver(self, image_path, options, executive=None)
230 def start_http_server(self):
233 def start_websocket_server(self):
236 def stop_http_server(self):
239 def stop_websocket_server(self):
242 def test_expectations(self):
243 """Returns the test expectations for this port.
245 Basically this string should contain the equivalent of a
246 test_expectations file. See test_expectations.py for more details."""
248 WONTFIX : failures/expected/checksum.html = IMAGE
249 WONTFIX : failures/expected/crash.html = CRASH
250 // This one actually passes because the checksums will match.
251 WONTFIX : failures/expected/image.html = PASS
252 WONTFIX : failures/expected/image_checksum.html = IMAGE
253 WONTFIX : failures/expected/missing_check.html = MISSING PASS
254 WONTFIX : failures/expected/missing_image.html = MISSING PASS
255 WONTFIX : failures/expected/missing_text.html = MISSING PASS
256 WONTFIX : failures/expected/text.html = TEXT
257 WONTFIX : failures/expected/timeout.html = TIMEOUT
258 WONTFIX SKIP : failures/expected/hang.html = TIMEOUT
259 WONTFIX SKIP : failures/expected/keyboard.html = CRASH
260 WONTFIX SKIP : failures/expected/exception.html = CRASH
263 def test_base_platform_names(self):
264 return ('mac', 'win')
266 def test_platform_name(self):
269 def test_platform_names(self):
270 return self.test_base_platform_names()
272 def test_platform_name_to_name(self, test_platform_name):
273 return test_platform_name
279 class TestDriver(base.Driver):
280 """Test/Dummy implementation of the DumpRenderTree interface."""
282 def __init__(self, port, image_path, options, executive):
284 self._image_path = image_path
285 self._executive = executive
286 self._image_written = False
291 def run_test(self, uri, timeoutms, image_hash):
292 test_name = self._port.uri_to_test_name(uri)
293 test = self._port._tests[test_name]
295 raise KeyboardInterrupt
297 raise ValueError('exception from ' + test_name)
299 time.sleep((float(timeoutms) * 4) / 1000.0)
301 if self._port.get_option('pixel_tests') and test.actual_image:
302 with open(self._image_path, 'w') as file:
303 file.write(test.actual_image)
305 return (test.crash, test.timeout, test.actual_checksum,
306 test.actual_text, test.error)