|
16 | 16 |
|
17 | 17 | from __future__ import print_function |
18 | 18 |
|
| 19 | +import ast |
19 | 20 | import os |
20 | 21 | import subprocess |
21 | 22 |
|
22 | 23 |
|
| 24 | +PROJECT_ROOT = os.path.abspath( |
| 25 | + os.path.join(os.path.dirname(__file__), '..')) |
23 | 26 | LOCAL_REMOTE_ENV = 'GOOGLE_CLOUD_TESTING_REMOTE' |
24 | 27 | LOCAL_BRANCH_ENV = 'GOOGLE_CLOUD_TESTING_BRANCH' |
25 | 28 | IN_TRAVIS_ENV = 'TRAVIS' |
26 | 29 | TRAVIS_PR_ENV = 'TRAVIS_PULL_REQUEST' |
27 | 30 | TRAVIS_BRANCH_ENV = 'TRAVIS_BRANCH' |
| 31 | +INST_REQS_KWARG = 'install_requires' |
| 32 | +REQ_VAR = 'REQUIREMENTS' |
| 33 | +PACKAGE_PREFIX = 'google-cloud-' |
28 | 34 |
|
29 | 35 |
|
30 | 36 | def in_travis(): |
@@ -226,3 +232,114 @@ def get_affected_files(allow_limited=True): |
226 | 232 | result = subprocess.check_output(['git', 'ls-files']) |
227 | 233 |
|
228 | 234 | return result.rstrip('\n').split('\n'), diff_base |
| 235 | + |
| 236 | + |
| 237 | +def get_required_packages(file_contents): |
| 238 | + """Get required packages from a ``setup.py`` file. |
| 239 | +
|
| 240 | + Makes the following assumptions: |
| 241 | +
|
| 242 | + * ``install_requires=REQUIREMENTS`` occurs in the call to |
| 243 | + ``setup()`` in the ``file_contents``. |
| 244 | + * The text ``install_requires`` occurs nowhere else in the file. |
| 245 | + * The text ``REQUIREMENTS`` only appears when being passed to |
| 246 | + ``setup()`` (as above) and when being defined. |
| 247 | + * The ``REQUIREMENTS`` variable is a list and the text from the |
| 248 | + ``setup.py`` file containing that list can be parsed using |
| 249 | + ``ast.literal_eval()``. |
| 250 | +
|
| 251 | + :type file_contents: str |
| 252 | + :param file_contents: The contents of a ``setup.py`` file. |
| 253 | +
|
| 254 | + :rtype: list |
| 255 | + :returns: The list of required packages. |
| 256 | + :raises: :class:`~exceptions.ValueError` if the file is in an |
| 257 | + unexpected format. |
| 258 | + """ |
| 259 | + # Make sure the only ``install_requires`` happens in the |
| 260 | + # call to setup() |
| 261 | + if file_contents.count(INST_REQS_KWARG) != 1: |
| 262 | + raise ValueError('Expected only one use of keyword', |
| 263 | + INST_REQS_KWARG, file_contents) |
| 264 | + # Make sure the only usage of ``install_requires`` is to set |
| 265 | + # install_requires=REQUIREMENTS. |
| 266 | + keyword_stmt = INST_REQS_KWARG + '=' + REQ_VAR |
| 267 | + if file_contents.count(keyword_stmt) != 1: |
| 268 | + raise ValueError('Expected keyword to be set with variable', |
| 269 | + INST_REQS_KWARG, REQ_VAR, file_contents) |
| 270 | + # Split file on ``REQUIREMENTS`` variable while asserting that |
| 271 | + # it only appear twice. |
| 272 | + _, reqs_section, _ = file_contents.split(REQ_VAR) |
| 273 | + # Find ``REQUIREMENTS`` list variable defined in ``reqs_section``. |
| 274 | + reqs_begin = reqs_section.index('[') |
| 275 | + reqs_end = reqs_section.index(']') + 1 |
| 276 | + |
| 277 | + # Convert the text to an actual list, but make sure no |
| 278 | + # locals or globals can be used. |
| 279 | + reqs_list_text = reqs_section[reqs_begin:reqs_end] |
| 280 | + # We use literal_eval() because it limits to evaluating |
| 281 | + # strings that only consist of a few Python literals: strings, |
| 282 | + # numbers, tuples, lists, dicts, booleans, and None. |
| 283 | + requirements = ast.literal_eval(reqs_list_text) |
| 284 | + |
| 285 | + # Take the list of requirements and strip off the package name |
| 286 | + # from each requirement. |
| 287 | + result = [] |
| 288 | + for required in requirements: |
| 289 | + parts = required.split() |
| 290 | + result.append(parts[0]) |
| 291 | + return result |
| 292 | + |
| 293 | + |
| 294 | +def get_dependency_graph(package_list): |
| 295 | + """Get a directed graph of package dependencies. |
| 296 | +
|
| 297 | + :type package_list: list |
| 298 | + :param package_list: The list of **all** valid packages. |
| 299 | +
|
| 300 | + :rtype: dict |
| 301 | + :returns: A dictionary where keys are packages and values are |
| 302 | + the set of packages that depend on the key. |
| 303 | + """ |
| 304 | + result = {package: set() for package in package_list} |
| 305 | + for package in package_list: |
| 306 | + setup_file = os.path.join(PROJECT_ROOT, package, |
| 307 | + 'setup.py') |
| 308 | + with open(setup_file, 'r') as file_obj: |
| 309 | + file_contents = file_obj.read() |
| 310 | + |
| 311 | + requirements = get_required_packages(file_contents) |
| 312 | + for requirement in requirements: |
| 313 | + if not requirement.startswith(PACKAGE_PREFIX): |
| 314 | + continue |
| 315 | + _, req_package = requirement.split(PACKAGE_PREFIX) |
| 316 | + req_package = req_package.replace('-', '_') |
| 317 | + result[req_package].add(package) |
| 318 | + |
| 319 | + return result |
| 320 | + |
| 321 | + |
| 322 | +def follow_dependencies(subset, package_list): |
| 323 | + """Get a directed graph of package dependencies. |
| 324 | +
|
| 325 | + :type subset: list |
| 326 | + :param subset: List of a subset of package names. |
| 327 | +
|
| 328 | + :type package_list: list |
| 329 | + :param package_list: The list of **all** valid packages. |
| 330 | +
|
| 331 | + :rtype: list |
| 332 | + :returns: An expanded list of packages containing everything |
| 333 | + in ``subset`` and any packages that depend on those. |
| 334 | + """ |
| 335 | + dependency_graph = get_dependency_graph(package_list) |
| 336 | + |
| 337 | + curr_pkgs = None |
| 338 | + updated_pkgs = set(subset) |
| 339 | + while curr_pkgs != updated_pkgs: |
| 340 | + curr_pkgs = updated_pkgs |
| 341 | + updated_pkgs = set(curr_pkgs) |
| 342 | + for package in curr_pkgs: |
| 343 | + updated_pkgs.update(dependency_graph[package]) |
| 344 | + |
| 345 | + return sorted(curr_pkgs) |
0 commit comments