Google Git
Sign in
chromium / chromium / src / main / . / tools / autotest.py
blob: 89e56d3e2a0b016aab8bb74ae680a7be75f329e2 [file] [log] [blame]
Fumitoshi Ukaide058c1c2025-09-08 13:51:00[diff] [blame]1#!/usr/bin/env vpython3
Avi Drissmandfd880852022-09-15 20:11:09[diff] [blame]2# Copyright 2020 The Chromium Authors
Michael Thiessen09c0e1d02020-03-23 18:44:50[diff] [blame]3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5"""Builds and runs a test by filename.
6
Edman Anjosad4625e2023-06-06 21:16:49[diff] [blame]7This script finds the appropriate test suites for the specified test files or
8directories, builds it, then runs it with the (optionally) specified filter,
Dan Harrington27d104d2020-09-08 18:30:14[diff] [blame]9passing any extra args on to the test runner.
Michael Thiessen09c0e1d02020-03-23 18:44:50[diff] [blame]10
11Examples:
Dan Harrington27d104d2020-09-08 18:30:14[diff] [blame]12# Run the test target for bit_cast_unittest.cc. Use a custom test filter instead
13# of the automatically generated one.
14autotest.py -C out/Desktop bit_cast_unittest.cc --gtest_filter=BitCastTest*
15
16# Find and run UrlUtilitiesUnitTest.java's tests, pass remaining parameters to
17# the test binary.
Michael Thiessen09c0e1d02020-03-23 18:44:50[diff] [blame]18autotest.py -C out/Android UrlUtilitiesUnitTest --fast-local-dev -v
Dan Harrington27d104d2020-09-08 18:30:14[diff] [blame]19
Edman Anjosad4625e2023-06-06 21:16:49[diff] [blame]20# Run all tests under base/strings.
Dan Harrington27d104d2020-09-08 18:30:14[diff] [blame]21autotest.py -C out/foo --run-all base/strings
22
Edman Anjosad4625e2023-06-06 21:16:49[diff] [blame]23# Run tests in multiple files or directories.
24autotest.py -C out/foo base/strings base/pickle_unittest.cc
25
Dan Harrington27d104d2020-09-08 18:30:14[diff] [blame]26# Run only the test on line 11. Useful when running autotest.py from your text
27# editor.
28autotest.py -C out/foo --line 11 base/strings/strcat_unittest.cc
Michael Thiessen09c0e1d02020-03-23 18:44:50[diff] [blame]29"""
30
31import argparse
Erik Staabdbdb3e5d2024-08-26 19:23:11[diff] [blame]32import json
Michael Thiessen09c0e1d02020-03-23 18:44:50[diff] [blame]33import locale
Michael Thiessen09c0e1d02020-03-23 18:44:50[diff] [blame]34import os
35import re
Andrew Grieve911128a2023-07-10 19:06:42[diff] [blame]36import shlex
Michael Thiessen09c0e1d02020-03-23 18:44:50[diff] [blame]37import subprocess
38import sys
Terrence Reillyeab6dc22025-06-03 02:45:25[diff] [blame]39import shutil
Michael Thiessen09c0e1d02020-03-23 18:44:50[diff] [blame]40
Mario Bianucci6b545002020-12-02 01:33:39[diff] [blame]41from enum import Enum
Michael Thiessen09c0e1d02020-03-23 18:44:50[diff] [blame]42from pathlib import Path
43
Erik Staabdbdb3e5d2024-08-26 19:23:11[diff] [blame]44# Don't write pyc files to the src tree, which show up in version control
45# in some environments.
46sys.dont_write_bytecode = True
47
Michael Thiessen09c0e1d02020-03-23 18:44:50[diff] [blame]48USE_PYTHON_3 = f'This script will only run under python3.'
49
50SRC_DIR = Path(__file__).parent.parent.resolve()
Andrew Grieve911128a2023-07-10 19:06:42[diff] [blame]51sys.path.append(str(SRC_DIR / 'build'))
52import gn_helpers
53
Peter Wen1b84b4b2021-03-11 18:12:22[diff] [blame]54sys.path.append(str(SRC_DIR / 'build' / 'android'))
55from pylib import constants
56
57DEPOT_TOOLS_DIR = SRC_DIR / 'third_party' / 'depot_tools'
Michael Thiessen09c0e1d02020-03-23 18:44:50[diff] [blame]58DEBUG = False
59
Michael Thiessenf46171e2020-03-31 17:29:38[diff] [blame]60# Some test suites use suffixes that would also match non-test-suite targets.
61# Those test suites should be manually added here.
Andrew Grieve5370b0a92023-07-06 21:43:20[diff] [blame]62_TEST_TARGET_ALLOWLIST = [
Michael Thiessen09c0e1d02020-03-23 18:44:50[diff] [blame]63
Dan Harrington97d38e82025-05-21 01:05:30[diff] [blame]64 # The tests below this line were output from the ripgrep command just below:
65 '//ash:ash_pixeltests',
66 '//build/rust/tests/test_serde_json_lenient:test_serde_json_lenient',
67 '//chrome/browser/apps/app_service/app_install:app_install_fuzztests',
68 '//chrome/browser/glic/e2e_test:glic_internal_e2e_interactive_ui_tests',
69 '//chrome/browser/mac:install_sh_test',
70 '//chrome/browser/metrics/perf:profile_provider_unittest',
71 '//chrome/browser/privacy_sandbox/notice:fuzz_tests',
72 '//chrome/browser/web_applications:web_application_fuzztests',
73 '//chromecast/media/base:video_plane_controller_test',
74 '//chromecast/metrics:cast_metrics_unittest',
Dan Harrington97d38e82025-05-21 01:05:30[diff] [blame]75 '//chrome/enterprise_companion:enterprise_companion_integration_tests',
76 '//chrome/enterprise_companion:enterprise_companion_tests',
77 '//chrome/installer/gcapi:gcapi_test',
78 '//chrome/installer/test:upgrade_test',
79 '//chromeos/ash/components/kiosk/vision:kiosk_vision_unit_tests',
80 '//chrome/test/android:chrome_public_apk_baseline_profile_generator',
81 '//chrome/test:unit_tests',
82 '//clank/javatests:chrome_apk_baseline_profile_generator',
83 '//clank/javatests:chrome_smoke_test',
84 '//clank/javatests:monochrome_bundle_smoke_test',
85 '//clank/javatests:trichrome_chrome_google_bundle_smoke_test',
86 '//components/chromeos_camera:jpeg_decode_accelerator_unittest',
87 '//components/exo/wayland:wayland_client_compatibility_tests',
88 '//components/exo/wayland:wayland_client_tests',
89 '//components/facilitated_payments/core/validation:pix_code_validator_fuzzer',
90 '//components/ip_protection:components_ip_protection_fuzztests',
91 '//components/minidump_uploader:minidump_uploader_test',
92 '//components/paint_preview/browser:paint_preview_browser_unit_tests',
93 '//components/paint_preview/common:paint_preview_common_unit_tests',
94 '//components/paint_preview/renderer:paint_preview_renderer_unit_tests',
95 '//components/services/paint_preview_compositor:paint_preview_compositor_unit_tests',
96 '//components/translate/core/language_detection:language_detection_util_fuzztest',
97 '//components/webcrypto:webcrypto_testing_fuzzer',
98 '//components/zucchini:zucchini_integration_test',
99 '//content/test/fuzzer:devtools_protocol_encoding_json_fuzzer',
100 '//fuchsia_web/runners:cast_runner_integration_tests',
101 '//fuchsia_web/webengine:web_engine_integration_tests',
102 '//google_apis/gcm:gcm_unit_tests',
103 '//gpu:gl_tests',
104 '//gpu:gpu_benchmark',
105 '//gpu/vulkan/android:vk_tests',
106 '//ios/web:ios_web_inttests',
107 '//ios/web_view:ios_web_view_inttests',
108 '//media/cdm:aes_decryptor_fuzztests',
109 '//media/formats:ac3_util_fuzzer',
110 '//media/gpu/chromeos:image_processor_test',
111 '//media/gpu/v4l2:v4l2_unittest',
112 '//media/gpu/vaapi/test/fake_libva_driver:fake_libva_driver_unittest',
113 '//media/gpu/vaapi:vaapi_unittest',
114 '//native_client/tests:large_tests',
115 '//native_client/tests:medium_tests',
116 '//native_client/tests:small_tests',
117 '//sandbox/mac:sandbox_mac_fuzztests',
118 '//sandbox/win:sbox_integration_tests',
119 '//sandbox/win:sbox_validation_tests',
120 '//testing/libfuzzer/fuzzers:libyuv_scale_fuzztest',
121 '//testing/libfuzzer/fuzzers:paint_vector_icon_fuzztest',
122 '//third_party/blink/renderer/controller:blink_perf_tests',
123 '//third_party/blink/renderer/core:css_parser_fuzzer',
124 '//third_party/blink/renderer/core:inspector_ghost_rules_fuzzer',
125 '//third_party/blink/renderer/platform/loader:unencoded_digest_fuzzer',
126 '//third_party/crc32c:crc32c_benchmark',
127 '//third_party/crc32c:crc32c_tests',
128 '//third_party/dawn/src/dawn/tests/benchmarks:dawn_benchmarks',
Jay Zhou0bb653c2025-09-02 18:07:43[diff] [blame]129 '//third_party/federated_compute:federated_compute_tests',
Dan Harrington97d38e82025-05-21 01:05:30[diff] [blame]130 '//third_party/highway:highway_tests',
131 '//third_party/ipcz/src:ipcz_tests',
132 '//third_party/libaom:av1_encoder_fuzz_test',
133 '//third_party/libaom:test_libaom',
134 '//third_party/libvpx:test_libvpx',
135 '//third_party/libvpx:vp8_encoder_fuzz_test',
136 '//third_party/libvpx:vp9_encoder_fuzz_test',
137 '//third_party/libwebp:libwebp_advanced_api_fuzzer',
138 '//third_party/libwebp:libwebp_animation_api_fuzzer',
139 '//third_party/libwebp:libwebp_animencoder_fuzzer',
140 '//third_party/libwebp:libwebp_enc_dec_api_fuzzer',
141 '//third_party/libwebp:libwebp_huffman_fuzzer',
142 '//third_party/libwebp:libwebp_mux_demux_api_fuzzer',
143 '//third_party/libwebp:libwebp_simple_api_fuzzer',
144 '//third_party/opus:test_opus_api',
145 '//third_party/opus:test_opus_decode',
146 '//third_party/opus:test_opus_encode',
147 '//third_party/opus:test_opus_padding',
148 '//third_party/pdfium:pdfium_embeddertests',
149 '//third_party/pffft:pffft_unittest',
150 '//third_party/rapidhash:rapidhash_fuzztests',
151 '//ui/ozone:ozone_integration_tests',
152]
Dan Harrington254f5422025-05-21 16:05:19[diff] [blame]153r"""
Dan Harrington97d38e82025-05-21 01:05:30[diff] [blame]154 You can run this command to find test targets that do not match these regexes,
155 and use it to update _TEST_TARGET_ALLOWLIST.
156rg '^(instrumentation_test_runner|test)\("([^"]*)' -o -g'BUILD.gn' -r'$2' -N \
157 | rg -v '(_browsertests|_perftests|_wpr_tests|_unittests)$' \
158 | rg '^(.*)/BUILD.gn(.*)$' -r'\'//$1$2\',' \
159 | sort
160
161 And you can use a command like this to find source_set targets that do match
162 the test target regex (ideally this is minimal).
163rg '^source_set\("([^"]*)' -o -g'BUILD.gn' -r'$1' -N | \
164 rg '(_browsertests|_perftests|_wpr_tests|_unittests)$'
165"""
Dan Harrington31f4eb582024-01-24 16:43:47[diff] [blame]166_TEST_TARGET_REGEX = re.compile(
Dan Harrington97d38e82025-05-21 01:05:30[diff] [blame]167 r'(_browsertests|_perftests|_wpr_tests|_unittests)$')
Andrew Grievec2122d272021-02-10 16:22:29[diff] [blame]168
Edman Anjos5617af5e2024-02-01 18:08:18[diff] [blame]169_PREF_MAPPING_FILE_PATTERN = re.escape(
170 str(Path('components') / 'policy' / 'test' / 'data' / 'pref_mapping') +
171 r'/') + r'.*\.json'
172
Edman Anjos2a7daff2025-03-14 14:38:27[diff] [blame]173TEST_FILE_NAME_REGEX = re.compile(
174 r'(.*Test\.java)' +
Jonathan Leea2f47542025-08-01 16:55:44[diff] [blame]175 r'|(.*_[a-z]*test(?:_win|_mac|_linux|_chromeos|_android)?\.(cc|mm))' +
176 r'|(' + _PREF_MAPPING_FILE_PATTERN + r')')
Mario Bianucciebea79d2020-11-04 17:19:00[diff] [blame]177
178# Some tests don't directly include gtest.h and instead include it via gmock.h
179# or a test_utils.h file, so make sure these cases are captured. Also include
180# files that use <...> for #includes instead of quotes.
Tushar Agarwal9cd8e4992022-05-20 15:03:11[diff] [blame]181GTEST_INCLUDE_REGEX = re.compile(
182 r'#include.*(gtest|gmock|_test_utils|browser_test)\.h("|>)')
Dan Harrington27d104d2020-09-08 18:30:14[diff] [blame]183
184
185def ExitWithMessage(*args):
186 print(*args, file=sys.stderr)
187 sys.exit(1)
188
Mario Bianucci6b545002020-12-02 01:33:39[diff] [blame]189
190class TestValidity(Enum):
191 NOT_A_TEST = 0 # Does not match test file regex.
192 MAYBE_A_TEST = 1 # Matches test file regex, but doesn't include gtest files.
193 VALID_TEST = 2 # Matches test file regex and includes gtest files.
194
195
Terrence Reillyeab6dc22025-06-03 02:45:25[diff] [blame]196def FindRemoteCandidates(target):
197 """Find files using a remote code search utility, if installed."""
198 if not shutil.which('cs'):
199 return []
200 results = RunCommand([
201 'cs', '-l',
202 # Give the local path to the file, if the file exists.
203 '--local',
204 f'file:{target}',
205 # Restrict our search to Chromium
206 'git:chrome-internal/codesearch/chrome/src@main']).splitlines()
207 exact = set()
208 close = set()
209 for filename in results:
210 file_validity = IsTestFile(filename)
211 if file_validity is TestValidity.VALID_TEST:
212 exact.add(filename)
213 elif file_validity is TestValidity.MAYBE_A_TEST:
214 close.add(filename)
215 return list(exact), list(close)
216
217
Dan Harrington27d104d2020-09-08 18:30:14[diff] [blame]218def IsTestFile(file_path):
219 if not TEST_FILE_NAME_REGEX.match(file_path):
Mario Bianucci6b545002020-12-02 01:33:39[diff] [blame]220 return TestValidity.NOT_A_TEST
Jonathan Leea2f47542025-08-01 16:55:44[diff] [blame]221 if file_path.endswith('.cc') or file_path.endswith('.mm'):
Dan Harrington27d104d2020-09-08 18:30:14[diff] [blame]222 # Try a bit harder to remove non-test files for c++. Without this,
223 # 'autotest.py base/' finds non-test files.
224 try:
Mario Bianucciebea79d2020-11-04 17:19:00[diff] [blame]225 with open(file_path, 'r', encoding='utf-8') as f:
Dan Harrington27d104d2020-09-08 18:30:14[diff] [blame]226 if GTEST_INCLUDE_REGEX.search(f.read()) is not None:
Mario Bianucci6b545002020-12-02 01:33:39[diff] [blame]227 return TestValidity.VALID_TEST
Dan Harrington27d104d2020-09-08 18:30:14[diff] [blame]228 except IOError:
229 pass
Mario Bianucci6b545002020-12-02 01:33:39[diff] [blame]230 # It may still be a test file, even if it doesn't include a gtest file.
231 return TestValidity.MAYBE_A_TEST
232 return TestValidity.VALID_TEST
Dan Harrington27d104d2020-09-08 18:30:14[diff] [blame]233
Michael Thiessen09c0e1d02020-03-23 18:44:50[diff] [blame]234
235class CommandError(Exception):
Dan Harrington27d104d2020-09-08 18:30:14[diff] [blame]236 """Exception thrown when a subcommand fails."""
Michael Thiessen09c0e1d02020-03-23 18:44:50[diff] [blame]237
238 def __init__(self, command, return_code, output=None):
239 Exception.__init__(self)
240 self.command = command
241 self.return_code = return_code
242 self.output = output
243
244 def __str__(self):
245 message = (f'\n***\nERROR: Error while running command {self.command}'
246 f'.\nExit status: {self.return_code}\n')
247 if self.output:
248 message += f'Output:\n{self.output}\n'
249 message += '***'
250 return message
251
252
Dan Harrington27d104d2020-09-08 18:30:14[diff] [blame]253def StreamCommandOrExit(cmd, **kwargs):
Michael Thiessen09c0e1d02020-03-23 18:44:50[diff] [blame]254 try:
255 subprocess.check_call(cmd, **kwargs)
256 except subprocess.CalledProcessError as e:
Dan Harrington27d104d2020-09-08 18:30:14[diff] [blame]257 sys.exit(1)
Michael Thiessen09c0e1d02020-03-23 18:44:50[diff] [blame]258
259
260def RunCommand(cmd, **kwargs):
Michael Thiessen09c0e1d02020-03-23 18:44:50[diff] [blame]261 try:
262 # Set an encoding to convert the binary output to a string.
263 return subprocess.check_output(
264 cmd, **kwargs, encoding=locale.getpreferredencoding())
265 except subprocess.CalledProcessError as e:
266 raise CommandError(e.cmd, e.returncode, e.output) from None
267
268
Sam Maierde9930442025-06-20 15:10:15[diff] [blame]269def BuildTestTargets(out_dir, targets, dry_run, quiet):
Dan Harrington27d104d2020-09-08 18:30:14[diff] [blame]270 """Builds the specified targets with ninja"""
Andrew Grieve911128a2023-07-10 19:06:42[diff] [blame]271 cmd = gn_helpers.CreateBuildCommand(out_dir) + targets
272 print('Building: ' + shlex.join(cmd))
Michael Thiessen09c0e1d02020-03-23 18:44:50[diff] [blame]273 if (dry_run):
Michael Thiessena8a82f52020-11-30 18:05:32[diff] [blame]274 return True
Sam Maierde9930442025-06-20 15:10:15[diff] [blame]275 completed_process = subprocess.run(cmd,
276 capture_output=quiet,
277 encoding='utf-8')
278 if completed_process.returncode != 0:
279 if quiet:
280 before, _, after = completed_process.stdout.partition('stderr:')
281 if not after:
282 before, _, after = completed_process.stdout.partition('stdout:')
283 if after:
284 print(after)
285 else:
286 print(before)
Dan Harringtonaa2c7ba2020-09-16 15:34:24[diff] [blame]287 return False
288 return True
Michael Thiessen09c0e1d02020-03-23 18:44:50[diff] [blame]289
290
291def RecursiveMatchFilename(folder, filename):
292 current_dir = os.path.split(folder)[-1]
293 if current_dir.startswith('out') or current_dir.startswith('.'):
Mario Bianucci6b545002020-12-02 01:33:39[diff] [blame]294 return [[], []]
295 exact = []
296 close = []
Gary Tong21664c02025-03-06 16:17:17[diff] [blame]297 try:
298 with os.scandir(folder) as it:
299 for entry in it:
300 if (entry.is_symlink()):
301 continue
302 if (entry.is_file() and filename in entry.path and
303 not os.path.basename(entry.path).startswith('.')):
304 file_validity = IsTestFile(entry.path)
305 if file_validity is TestValidity.VALID_TEST:
306 exact.append(entry.path)
307 elif file_validity is TestValidity.MAYBE_A_TEST:
308 close.append(entry.path)
309 if entry.is_dir():
310 # On Windows, junctions are like a symlink that python interprets as a
311 # directory, leading to exceptions being thrown. We can just catch and
312 # ignore these exceptions like we would ignore symlinks.
313 try:
314 matches = RecursiveMatchFilename(entry.path, filename)
315 exact += matches[0]
316 close += matches[1]
317 except FileNotFoundError as e:
318 if DEBUG:
319 print(f'Failed to scan directory "{entry}" - junction?')
320 pass
321 except PermissionError:
322 print(f'Permission error while scanning {folder}')
323
Mario Bianucci6b545002020-12-02 01:33:39[diff] [blame]324 return [exact, close]
Michael Thiessen09c0e1d02020-03-23 18:44:50[diff] [blame]325
326
Dan Harrington27d104d2020-09-08 18:30:14[diff] [blame]327def FindTestFilesInDirectory(directory):
328 test_files = []
Mario Bianucciebea79d2020-11-04 17:19:00[diff] [blame]329 if DEBUG:
330 print('Test files:')
Peter Wen1b84b4b2021-03-11 18:12:22[diff] [blame]331 for root, _, files in os.walk(directory):
Dan Harrington27d104d2020-09-08 18:30:14[diff] [blame]332 for f in files:
333 path = os.path.join(root, f)
Mario Bianucci6b545002020-12-02 01:33:39[diff] [blame]334 file_validity = IsTestFile(path)
335 if file_validity is TestValidity.VALID_TEST:
Mario Bianucciebea79d2020-11-04 17:19:00[diff] [blame]336 if DEBUG:
337 print(path)
Dan Harrington27d104d2020-09-08 18:30:14[diff] [blame]338 test_files.append(path)
Mario Bianucci6b545002020-12-02 01:33:39[diff] [blame]339 elif DEBUG and file_validity is TestValidity.MAYBE_A_TEST:
340 print(path + ' matched but doesn\'t include gtest files, skipping.')
Dan Harrington27d104d2020-09-08 18:30:14[diff] [blame]341 return test_files
342
343
Terrence Reillyeab6dc22025-06-03 02:45:25[diff] [blame]344def FindMatchingTestFiles(target, remote_search=False):
Dan Harrington27d104d2020-09-08 18:30:14[diff] [blame]345 # Return early if there's an exact file match.
346 if os.path.isfile(target):
Jonathan Leea2f47542025-08-01 16:55:44[diff] [blame]347 if test_file := _FindTestForFile(target):
348 return [test_file]
349 ExitWithMessage(f"{target} doesn't look like a test file")
Dan Harrington27d104d2020-09-08 18:30:14[diff] [blame]350 # If this is a directory, return all the test files it contains.
351 if os.path.isdir(target):
352 files = FindTestFilesInDirectory(target)
353 if not files:
354 ExitWithMessage('No tests found in directory')
355 return files
356
Jesse McKenna83b6ac1b2020-05-07 18:25:38[diff] [blame]357 if sys.platform.startswith('win32') and os.path.altsep in target:
358 # Use backslash as the path separator on Windows to match os.scandir().
359 if DEBUG:
360 print('Replacing ' + os.path.altsep + ' with ' + os.path.sep + ' in: '
361 + target)
362 target = target.replace(os.path.altsep, os.path.sep)
Michael Thiessen09c0e1d02020-03-23 18:44:50[diff] [blame]363 if DEBUG:
364 print('Finding files with full path containing: ' + target)
Mario Bianucci6b545002020-12-02 01:33:39[diff] [blame]365
Terrence Reillyeab6dc22025-06-03 02:45:25[diff] [blame]366 if remote_search:
367 exact, close = FindRemoteCandidates(target)
368 if not exact and not close:
369 print('Failed to find remote candidates; searching recursively')
370 exact, close = RecursiveMatchFilename(SRC_DIR, target)
371 else:
372 exact, close = RecursiveMatchFilename(SRC_DIR, target)
373
Michael Thiessen09c0e1d02020-03-23 18:44:50[diff] [blame]374 if DEBUG:
Mario Bianucci6b545002020-12-02 01:33:39[diff] [blame]375 if exact:
376 print('Found exact matching file(s):')
377 print('\n'.join(exact))
378 if close:
379 print('Found possible matching file(s):')
380 print('\n'.join(close))
381
Andrew Grieve774439a2023-09-06 14:36:10[diff] [blame]382 if len(exact) >= 1:
383 # Given "Foo", don't ask to disambiguate ModFoo.java vs Foo.java.
384 more_exact = [
385 p for p in exact if os.path.basename(p) in (target, f'{target}.java')
386 ]
387 if len(more_exact) == 1:
388 test_files = more_exact
389 else:
390 test_files = exact
391 else:
392 test_files = close
393
Mario Bianucci6b545002020-12-02 01:33:39[diff] [blame]394 if len(test_files) > 1:
Andrew Grieveecc9b872023-03-27 21:09:20[diff] [blame]395 if len(test_files) < 10:
396 test_files = [HaveUserPickFile(test_files)]
397 else:
398 # Arbitrarily capping at 10 results so we don't print the name of every
399 # file in the repo if the target is poorly specified.
400 test_files = test_files[:10]
401 ExitWithMessage(f'Target "{target}" is ambiguous. Matching files: '
402 f'{test_files}')
Mario Bianucci6b545002020-12-02 01:33:39[diff] [blame]403 if not test_files:
Dan Harrington27d104d2020-09-08 18:30:14[diff] [blame]404 ExitWithMessage(f'Target "{target}" did not match any files.')
Mario Bianucci6b545002020-12-02 01:33:39[diff] [blame]405 return test_files
Michael Thiessen09c0e1d02020-03-23 18:44:50[diff] [blame]406
407
Jonathan Leea2f47542025-08-01 16:55:44[diff] [blame]408def _FindTestForFile(target: os.PathLike) -> str | None:
409 root, ext = os.path.splitext(target)
410 # If the target is a C++ implementation file, try to guess the test file.
411 # Candidates should be ordered most to least promising.
412 test_candidates = [target]
413 if ext == '.h':
414 # `*_unittest.{cc,mm}` are both possible.
415 test_candidates.append(f'{root}_unittest.cc')
416 test_candidates.append(f'{root}_unittest.mm')
417 elif ext == '.cc' or ext == '.mm':
418 test_candidates.append(f'{root}_unittest{ext}')
419 else:
420 return target
421
422 maybe_valid = []
423 for candidate in test_candidates:
424 if not os.path.isfile(candidate):
425 continue
426 validity = IsTestFile(candidate)
427 if validity is TestValidity.VALID_TEST:
428 return candidate
429 elif validity is TestValidity.MAYBE_A_TEST:
430 maybe_valid.append(candidate)
431 return maybe_valid[0] if maybe_valid else None
432
433
Andrew Grieveecc9b872023-03-27 21:09:20[diff] [blame]434def HaveUserPickFile(paths):
435 paths = sorted(paths, key=lambda p: (len(p), p))
436 path_list = '\n'.join(f'{i}. {t}' for i, t in enumerate(paths))
437
438 while True:
439 user_input = input(f'Please choose the path you mean.\n{path_list}\n')
440 try:
441 value = int(user_input)
442 return paths[value]
443 except (ValueError, IndexError):
444 print('Try again')
445
446
Dan Harrington27d104d2020-09-08 18:30:14[diff] [blame]447def HaveUserPickTarget(paths, targets):
Michael Thiessen09c0e1d02020-03-23 18:44:50[diff] [blame]448 # Cap to 10 targets for convenience [0-9].
449 targets = targets[:10]
Dan Harrington27d104d2020-09-08 18:30:14[diff] [blame]450 target_list = '\n'.join(f'{i}. {t}' for i, t in enumerate(targets))
451
452 user_input = input(f'Target "{paths}" is used by multiple test targets.\n' +
Svend Larsen1c38e2162024-12-20 15:35:08[diff] [blame]453 target_list + '\nPlease pick a target by its numeric index'
454 'listed below: ')
Michael Thiessen09c0e1d02020-03-23 18:44:50[diff] [blame]455 try:
Dan Harrington27d104d2020-09-08 18:30:14[diff] [blame]456 value = int(user_input)
Michael Thiessen09c0e1d02020-03-23 18:44:50[diff] [blame]457 return targets[value]
Dan Harrington27d104d2020-09-08 18:30:14[diff] [blame]458 except (ValueError, IndexError):
Svend Larsen1c38e2162024-12-20 15:35:08[diff] [blame]459 print('Value entered was not a numeric index listed above. Trying again.')
Dan Harrington27d104d2020-09-08 18:30:14[diff] [blame]460 return HaveUserPickTarget(paths, targets)
Michael Thiessen09c0e1d02020-03-23 18:44:50[diff] [blame]461
462
Dan Harrington27d104d2020-09-08 18:30:14[diff] [blame]463# A persistent cache to avoid running gn on repeated runs of autotest.
464class TargetCache:
465 def __init__(self, out_dir):
Dan Harringtonaa2c7ba2020-09-16 15:34:24[diff] [blame]466 self.out_dir = out_dir
Dan Harrington27d104d2020-09-08 18:30:14[diff] [blame]467 self.path = os.path.join(out_dir, 'autotest_cache')
Dan Harringtonaa2c7ba2020-09-16 15:34:24[diff] [blame]468 self.gold_mtime = self.GetBuildNinjaMtime()
Dan Harrington27d104d2020-09-08 18:30:14[diff] [blame]469 self.cache = {}
470 try:
471 mtime, cache = json.load(open(self.path, 'r'))
472 if mtime == self.gold_mtime:
473 self.cache = cache
474 except Exception:
475 pass
476
477 def Save(self):
478 with open(self.path, 'w') as f:
479 json.dump([self.gold_mtime, self.cache], f)
480
481 def Find(self, test_paths):
482 key = ' '.join(test_paths)
483 return self.cache.get(key, None)
484
485 def Store(self, test_paths, test_targets):
486 key = ' '.join(test_paths)
487 self.cache[key] = test_targets
488
Dan Harringtonaa2c7ba2020-09-16 15:34:24[diff] [blame]489 def GetBuildNinjaMtime(self):
490 return os.path.getmtime(os.path.join(self.out_dir, 'build.ninja'))
491
492 def IsStillValid(self):
493 return self.GetBuildNinjaMtime() == self.gold_mtime
494
Dan Harrington27d104d2020-09-08 18:30:14[diff] [blame]495
Andrew Grieve5370b0a92023-07-06 21:43:20[diff] [blame]496def _TestTargetsFromGnRefs(targets):
497 # First apply allowlists:
498 ret = [t for t in targets if '__' not in t]
499 ret = [
500 t for t in ret
501 if _TEST_TARGET_REGEX.search(t) or t in _TEST_TARGET_ALLOWLIST
502 ]
503 if ret:
504 return ret
505
506 _SUBTARGET_SUFFIXES = (
507 '__java_binary', # robolectric_binary()
508 '__test_runner_script', # test() targets
509 '__test_apk', # instrumentation_test_apk() targets
510 )
511 ret = []
512 for suffix in _SUBTARGET_SUFFIXES:
513 ret.extend(t[:-len(suffix)] for t in targets if t.endswith(suffix))
514
515 return ret
516
517
Andrew Grieve16b72bd2025-09-10 14:16:30[diff] [blame]518def _ParseRefsOutput(output):
519 targets = output.splitlines()
520 # Filter out any warnings messages. E.g. those about unused GN args.
521 # https://crbug.com/444024516
522 targets = [t for t in targets if t.startswith('//')]
523 return targets
524
525
Dan Harrington27d104d2020-09-08 18:30:14[diff] [blame]526def FindTestTargets(target_cache, out_dir, paths, run_all):
527 # Normalize paths, so they can be cached.
528 paths = [os.path.realpath(p) for p in paths]
529 test_targets = target_cache.Find(paths)
Dan Harringtonaa2c7ba2020-09-16 15:34:24[diff] [blame]530 used_cache = True
Dan Harrington27d104d2020-09-08 18:30:14[diff] [blame]531 if not test_targets:
Dan Harringtonaa2c7ba2020-09-16 15:34:24[diff] [blame]532 used_cache = False
Dan Harrington27d104d2020-09-08 18:30:14[diff] [blame]533
534 # Use gn refs to recursively find all targets that depend on |path|, filter
535 # internal gn targets, and match against well-known test suffixes, falling
536 # back to a list of known test targets if that fails.
Henrique Nakashimaa227aa6d2025-05-01 19:26:34[diff] [blame]537 gn_path = os.path.join(DEPOT_TOOLS_DIR, 'gn.py')
Dan Harrington27d104d2020-09-08 18:30:14[diff] [blame]538
Henrique Nakashimaa227aa6d2025-05-01 19:26:34[diff] [blame]539 cmd = [sys.executable, gn_path, 'refs', out_dir, '--all'] + paths
Andrew Grieve16b72bd2025-09-10 14:16:30[diff] [blame]540 targets = _ParseRefsOutput(RunCommand(cmd))
Andrew Grieve5370b0a92023-07-06 21:43:20[diff] [blame]541 test_targets = _TestTargetsFromGnRefs(targets)
542
Andrew Grieve16b72bd2025-09-10 14:16:30[diff] [blame]543 # If no targets were identified as tests by looking at their names, ask GN
Andrew Grieve5370b0a92023-07-06 21:43:20[diff] [blame]544 # if any are executables.
545 if not test_targets and targets:
Andrew Grieve16b72bd2025-09-10 14:16:30[diff] [blame]546 test_targets = _ParseRefsOutput(RunCommand(cmd + ['--type=executable']))
Michael Thiessen09c0e1d02020-03-23 18:44:50[diff] [blame]547
548 if not test_targets:
Dan Harrington27d104d2020-09-08 18:30:14[diff] [blame]549 ExitWithMessage(
Andrew Grieve5370b0a92023-07-06 21:43:20[diff] [blame]550 f'"{paths}" did not match any test targets. Consider adding'
551 f' one of the following targets to _TEST_TARGET_ALLOWLIST within '
552 f'{__file__}: \n' + '\n'.join(targets))
Dan Harrington27d104d2020-09-08 18:30:14[diff] [blame]553
Andrew Grievef6a1b1e2023-09-13 16:26:02[diff] [blame]554 test_targets.sort()
Dan Harrington27d104d2020-09-08 18:30:14[diff] [blame]555 target_cache.Store(paths, test_targets)
556 target_cache.Save()
557
Michael Thiessen09c0e1d02020-03-23 18:44:50[diff] [blame]558 if len(test_targets) > 1:
Dan Harrington27d104d2020-09-08 18:30:14[diff] [blame]559 if run_all:
560 print(f'Warning, found {len(test_targets)} test targets.',
561 file=sys.stderr)
562 if len(test_targets) > 10:
563 ExitWithMessage('Your query likely involves non-test sources.')
564 print('Trying to run all of them!', file=sys.stderr)
565 else:
566 test_targets = [HaveUserPickTarget(paths, test_targets)]
Michael Thiessen09c0e1d02020-03-23 18:44:50[diff] [blame]567
Andrew Grievef6a1b1e2023-09-13 16:26:02[diff] [blame]568 # Remove the // prefix to turn GN label into ninja target.
569 test_targets = [t[2:] for t in test_targets]
Dan Harrington27d104d2020-09-08 18:30:14[diff] [blame]570
Dan Harringtonaa2c7ba2020-09-16 15:34:24[diff] [blame]571 return (test_targets, used_cache)
Michael Thiessen09c0e1d02020-03-23 18:44:50[diff] [blame]572
573
Edman Anjos5617af5e2024-02-01 18:08:18[diff] [blame]574def RunTestTargets(out_dir, targets, gtest_filter, pref_mapping_filter,
575 extra_args, dry_run, no_try_android_wrappers,
576 no_fast_local_dev):
Olivier Li8ac87f412021-05-05 15:26:54[diff] [blame]577
Dan Harrington27d104d2020-09-08 18:30:14[diff] [blame]578 for target in targets:
Andrew Grievef6a1b1e2023-09-13 16:26:02[diff] [blame]579 target_binary = target.split(':')[1]
Olivier Li8ac87f412021-05-05 15:26:54[diff] [blame]580
Dan Harrington27d104d2020-09-08 18:30:14[diff] [blame]581 # Look for the Android wrapper script first.
Andrew Grievef6a1b1e2023-09-13 16:26:02[diff] [blame]582 path = os.path.join(out_dir, 'bin', f'run_{target_binary}')
Olivier Li8ac87f412021-05-05 15:26:54[diff] [blame]583 if no_try_android_wrappers or not os.path.isfile(path):
584 # If the wrapper is not found or disabled use the Desktop target
585 # which is an executable.
Andrew Grievef6a1b1e2023-09-13 16:26:02[diff] [blame]586 path = os.path.join(out_dir, target_binary)
Andrew Grievecf0727622022-02-23 16:06:06[diff] [blame]587 elif not no_fast_local_dev:
588 # Usually want this flag when developing locally.
589 extra_args = extra_args + ['--fast-local-dev']
Olivier Li8ac87f412021-05-05 15:26:54[diff] [blame]590
Edman Anjos5617af5e2024-02-01 18:08:18[diff] [blame]591 cmd = [path, f'--gtest_filter={gtest_filter}']
592 if pref_mapping_filter:
593 cmd.append(f'--test_policy_to_pref_mappings_filter={pref_mapping_filter}')
594 cmd.extend(extra_args)
595
Andrew Grieve911128a2023-07-10 19:06:42[diff] [blame]596 print('Running test: ' + shlex.join(cmd))
Dan Harrington27d104d2020-09-08 18:30:14[diff] [blame]597 if not dry_run:
598 StreamCommandOrExit(cmd)
599
600
601def BuildCppTestFilter(filenames, line):
Mario Bianucciebea79d2020-11-04 17:19:00[diff] [blame]602 make_filter_command = [
Roman Sorokin34f5e2a2022-02-02 16:31:27[diff] [blame]603 sys.executable, SRC_DIR / 'tools' / 'make_gtest_filter.py'
Mario Bianucciebea79d2020-11-04 17:19:00[diff] [blame]604 ]
Dan Harrington27d104d2020-09-08 18:30:14[diff] [blame]605 if line:
606 make_filter_command += ['--line', str(line)]
607 else:
608 make_filter_command += ['--class-only']
609 make_filter_command += filenames
610 return RunCommand(make_filter_command).strip()
611
612
613def BuildJavaTestFilter(filenames):
Michael Thiessen7bbda482020-09-19 02:07:34[diff] [blame]614 return ':'.join('*.{}*'.format(os.path.splitext(os.path.basename(f))[0])
Dan Harrington27d104d2020-09-08 18:30:14[diff] [blame]615 for f in filenames)
616
617
Edman Anjos5617af5e2024-02-01 18:08:18[diff] [blame]618_PREF_MAPPING_GTEST_FILTER = '*PolicyPrefsTest.PolicyToPrefsMapping*'
619
620_PREF_MAPPING_FILE_REGEX = re.compile(_PREF_MAPPING_FILE_PATTERN)
621
622SPECIAL_TEST_FILTERS = [(_PREF_MAPPING_FILE_REGEX, _PREF_MAPPING_GTEST_FILTER)]
623
624
Dan Harrington27d104d2020-09-08 18:30:14[diff] [blame]625def BuildTestFilter(filenames, line):
626 java_files = [f for f in filenames if f.endswith('.java')]
Jonathan Leea2f47542025-08-01 16:55:44[diff] [blame]627 # TODO(crbug.com/434009870): Support EarlGrey tests, which don't use
628 # Googletest's macros or pascal case naming convention.
629 cc_files = [
630 f for f in filenames if f.endswith('.cc') or f.endswith('_unittest.mm')
631 ]
Dan Harrington27d104d2020-09-08 18:30:14[diff] [blame]632 filters = []
633 if java_files:
634 filters.append(BuildJavaTestFilter(java_files))
635 if cc_files:
636 filters.append(BuildCppTestFilter(cc_files, line))
Edman Anjos5617af5e2024-02-01 18:08:18[diff] [blame]637 for regex, gtest_filter in SPECIAL_TEST_FILTERS:
638 if any(True for f in filenames if regex.match(f)):
639 filters.append(gtest_filter)
640 break
Dan Harrington27d104d2020-09-08 18:30:14[diff] [blame]641 return ':'.join(filters)
Michael Thiessen09c0e1d02020-03-23 18:44:50[diff] [blame]642
643
Edman Anjos5617af5e2024-02-01 18:08:18[diff] [blame]644def BuildPrefMappingTestFilter(filenames):
645 mapping_files = [f for f in filenames if _PREF_MAPPING_FILE_REGEX.match(f)]
646 if not mapping_files:
647 return None
648 names_without_extension = [Path(f).stem for f in mapping_files]
649 return ':'.join(names_without_extension)
650
651
Michael Thiessen09c0e1d02020-03-23 18:44:50[diff] [blame]652def main():
653 parser = argparse.ArgumentParser(
654 description=__doc__, formatter_class=argparse.RawTextHelpFormatter)
Andrew Grievea5193d3a2020-09-21 14:58:34[diff] [blame]655 parser.add_argument('--out-dir',
Edman Anjos7d319a63e2024-01-29 10:58:31[diff] [blame]656 '--out_dir',
Peter Wen1b84b4b2021-03-11 18:12:22[diff] [blame]657 '--output-directory',
Edman Anjos7d319a63e2024-01-29 10:58:31[diff] [blame]658 '--output_directory',
Andrew Grievea5193d3a2020-09-21 14:58:34[diff] [blame]659 '-C',
660 metavar='OUT_DIR',
661 help='output directory of the build')
Terrence Reillyeab6dc22025-06-03 02:45:25[diff] [blame]662 parser.add_argument('--remote-search',
663 '--remote_search',
664 '-r',
665 action='store_true',
666 help='Search for tests using a remote service')
Michael Thiessen09c0e1d02020-03-23 18:44:50[diff] [blame]667 parser.add_argument(
Dan Harrington27d104d2020-09-08 18:30:14[diff] [blame]668 '--run-all',
Edman Anjos7d319a63e2024-01-29 10:58:31[diff] [blame]669 '--run_all',
Dan Harrington27d104d2020-09-08 18:30:14[diff] [blame]670 action='store_true',
671 help='Run all tests for the file or directory, instead of just one')
672 parser.add_argument('--line',
673 type=int,
674 help='run only the test on this line number. c++ only.')
Edman Anjos7d319a63e2024-01-29 10:58:31[diff] [blame]675 parser.add_argument('--gtest-filter',
676 '--gtest_filter',
Michael Thiessenfe328bc2022-11-30 02:37:52[diff] [blame]677 '-f',
678 metavar='FILTER',
679 help='test filter')
Edman Anjos5617af5e2024-02-01 18:08:18[diff] [blame]680 parser.add_argument('--test-policy-to-pref-mappings-filter',
681 '--test_policy_to_pref_mappings_filter',
682 metavar='FILTER',
683 help='policy pref mappings test filter')
Michael Thiessen09c0e1d02020-03-23 18:44:50[diff] [blame]684 parser.add_argument(
Dan Harrington27d104d2020-09-08 18:30:14[diff] [blame]685 '--dry-run',
Edman Anjos7d319a63e2024-01-29 10:58:31[diff] [blame]686 '--dry_run',
Michael Thiessen09c0e1d02020-03-23 18:44:50[diff] [blame]687 '-n',
688 action='store_true',
689 help='Print ninja and test run commands without executing them.')
Olivier Li8ac87f412021-05-05 15:26:54[diff] [blame]690 parser.add_argument(
Sam Maierde9930442025-06-20 15:10:15[diff] [blame]691 '--quiet',
692 '-q',
693 action='store_true',
694 help='Do not print while building, only print if build fails.')
695 parser.add_argument(
Olivier Li8ac87f412021-05-05 15:26:54[diff] [blame]696 '--no-try-android-wrappers',
Edman Anjos7d319a63e2024-01-29 10:58:31[diff] [blame]697 '--no_try_android_wrappers',
Olivier Li8ac87f412021-05-05 15:26:54[diff] [blame]698 action='store_true',
699 help='Do not try to use Android test wrappers to run tests.')
Andrew Grievecf0727622022-02-23 16:06:06[diff] [blame]700 parser.add_argument('--no-fast-local-dev',
Edman Anjos7d319a63e2024-01-29 10:58:31[diff] [blame]701 '--no_fast_local_dev',
Andrew Grievecf0727622022-02-23 16:06:06[diff] [blame]702 action='store_true',
703 help='Do not add --fast-local-dev for Android tests.')
Edman Anjosad4625e2023-06-06 21:16:49[diff] [blame]704 parser.add_argument('files',
Dan Harringtonaa2c7ba2020-09-16 15:34:24[diff] [blame]705 metavar='FILE_NAME',
Edman Anjos7d319a63e2024-01-29 10:58:31[diff] [blame]706 nargs='+',
Dan Harringtonaa2c7ba2020-09-16 15:34:24[diff] [blame]707 help='test suite file (eg. FooTest.java)')
Michael Thiessen09c0e1d02020-03-23 18:44:50[diff] [blame]708
709 args, _extras = parser.parse_known_args()
710
Peter Wen1b84b4b2021-03-11 18:12:22[diff] [blame]711 if args.out_dir:
712 constants.SetOutputDirectory(args.out_dir)
713 constants.CheckOutputDirectory()
714 out_dir: str = constants.GetOutDirectory()
715
716 if not os.path.isdir(out_dir):
717 parser.error(f'OUT_DIR "{out_dir}" does not exist.')
718 target_cache = TargetCache(out_dir)
Edman Anjosad4625e2023-06-06 21:16:49[diff] [blame]719 filenames = []
720 for file in args.files:
Terrence Reillyeab6dc22025-06-03 02:45:25[diff] [blame]721 filenames.extend(FindMatchingTestFiles(file, args.remote_search))
Dan Harrington27d104d2020-09-08 18:30:14[diff] [blame]722
Peter Wen1b84b4b2021-03-11 18:12:22[diff] [blame]723 targets, used_cache = FindTestTargets(target_cache, out_dir, filenames,
Dan Harringtonaa2c7ba2020-09-16 15:34:24[diff] [blame]724 args.run_all)
Michael Thiessen09c0e1d02020-03-23 18:44:50[diff] [blame]725
726 gtest_filter = args.gtest_filter
727 if not gtest_filter:
Dan Harrington27d104d2020-09-08 18:30:14[diff] [blame]728 gtest_filter = BuildTestFilter(filenames, args.line)
Michael Thiessen09c0e1d02020-03-23 18:44:50[diff] [blame]729
Dan Harrington27d104d2020-09-08 18:30:14[diff] [blame]730 if not gtest_filter:
731 ExitWithMessage('Failed to derive a gtest filter')
732
Edman Anjos5617af5e2024-02-01 18:08:18[diff] [blame]733 pref_mapping_filter = args.test_policy_to_pref_mappings_filter
734 if not pref_mapping_filter:
735 pref_mapping_filter = BuildPrefMappingTestFilter(filenames)
736
Dan Harrington27d104d2020-09-08 18:30:14[diff] [blame]737 assert targets
Sam Maierde9930442025-06-20 15:10:15[diff] [blame]738 build_ok = BuildTestTargets(out_dir, targets, args.dry_run, args.quiet)
Dan Harringtonaa2c7ba2020-09-16 15:34:24[diff] [blame]739
740 # If we used the target cache, it's possible we chose the wrong target because
741 # a gn file was changed. The build step above will check for gn modifications
742 # and update build.ninja. Use this opportunity the verify the cache is still
743 # valid.
744 if used_cache and not target_cache.IsStillValid():
Peter Wen1b84b4b2021-03-11 18:12:22[diff] [blame]745 target_cache = TargetCache(out_dir)
746 new_targets, _ = FindTestTargets(target_cache, out_dir, filenames,
Dan Harringtonaa2c7ba2020-09-16 15:34:24[diff] [blame]747 args.run_all)
748 if targets != new_targets:
749 # Note that this can happen, for example, if you rename a test target.
750 print('gn config was changed, trying to build again', file=sys.stderr)
751 targets = new_targets
Sam Maierde9930442025-06-20 15:10:15[diff] [blame]752 build_ok = BuildTestTargets(out_dir, targets, args.dry_run, args.quiet)
Dan Harrington8e95b892021-05-14 21:02:10[diff] [blame]753
754 if not build_ok: sys.exit(1)
Dan Harringtonaa2c7ba2020-09-16 15:34:24[diff] [blame]755
Edman Anjos5617af5e2024-02-01 18:08:18[diff] [blame]756 RunTestTargets(out_dir, targets, gtest_filter, pref_mapping_filter, _extras,
757 args.dry_run, args.no_try_android_wrappers,
758 args.no_fast_local_dev)
Michael Thiessen09c0e1d02020-03-23 18:44:50[diff] [blame]759
760
761if __name__ == '__main__':
762 sys.exit(main())