# Copyright 2023 The Bazel Authors. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Utility functions to discover python package types""" import os import textwrap from pathlib import Path # supported in >= 3.4 from typing import List, Optional, Set def implicit_namespace_packages( directory: str, ignored_dirnames: Optional[List[str]] = None ) -> Set[Path]: """Discovers namespace packages implemented using the 'native namespace packages' method. AKA 'implicit namespace packages', which has been supported since Python 3.3. See: https://packaging.python.org/guides/packaging-namespace-packages/#native-namespace-packages Args: directory: The root directory to recursively find packages in. ignored_dirnames: A list of directories to exclude from the search Returns: The set of directories found under root to be packages using the native namespace method. """ namespace_pkg_dirs: Set[Path] = set() standard_pkg_dirs: Set[Path] = set() directory_path = Path(directory) ignored_dirname_paths: List[Path] = [Path(p) for p in ignored_dirnames or ()] # Traverse bottom-up because a directory can be a namespace pkg because its child contains module files. for dirpath, dirnames, filenames in map( lambda t: (Path(t[0]), *t[1:]), os.walk(directory_path, topdown=False) ): if "__init__.py" in filenames: standard_pkg_dirs.add(dirpath) continue elif ignored_dirname_paths: is_ignored_dir = dirpath in ignored_dirname_paths child_of_ignored_dir = any( d in dirpath.parents for d in ignored_dirname_paths ) if is_ignored_dir or child_of_ignored_dir: continue dir_includes_py_modules = _includes_python_modules(filenames) parent_of_namespace_pkg = any( Path(dirpath, d) in namespace_pkg_dirs for d in dirnames ) parent_of_standard_pkg = any( Path(dirpath, d) in standard_pkg_dirs for d in dirnames ) parent_of_pkg = parent_of_namespace_pkg or parent_of_standard_pkg if ( (dir_includes_py_modules or parent_of_pkg) and # The root of the directory should never be an implicit namespace dirpath != directory_path ): namespace_pkg_dirs.add(dirpath) return namespace_pkg_dirs def add_pkgutil_style_namespace_pkg_init(dir_path: Path) -> None: """Adds 'pkgutil-style namespace packages' init file to the given directory See: https://packaging.python.org/guides/packaging-namespace-packages/#pkgutil-style-namespace-packages Args: dir_path: The directory to create an __init__.py for. Raises: ValueError: If the directory already contains an __init__.py file """ ns_pkg_init_filepath = os.path.join(dir_path, "__init__.py") if os.path.isfile(ns_pkg_init_filepath): raise ValueError("%s already contains an __init__.py file." % dir_path) with open(ns_pkg_init_filepath, "w") as ns_pkg_init_f: # See https://packaging.python.org/guides/packaging-namespace-packages/#pkgutil-style-namespace-packages ns_pkg_init_f.write( textwrap.dedent( """\ # __path__ manipulation added by bazelbuild/rules_python to support namespace pkgs. __path__ = __import__('pkgutil').extend_path(__path__, __name__) """ ) ) def _includes_python_modules(files: List[str]) -> bool: """ In order to only transform directories that Python actually considers namespace pkgs we need to detect if a directory includes Python modules. Which files are loadable as modules is extension based, and the particular set of extensions varies by platform. See: 1. https://github.com/python/cpython/blob/7d9d25dbedfffce61fc76bc7ccbfa9ae901bf56f/Lib/importlib/machinery.py#L19 2. PEP 420 -- Implicit Namespace Packages, Specification - https://www.python.org/dev/peps/pep-0420/#specification 3. dynload_shlib.c and dynload_win.c in python/cpython. """ module_suffixes = { ".py", # Source modules ".pyc", # Compiled bytecode modules ".so", # Unix extension modules ".pyd", # https://docs.python.org/3/faq/windows.html#is-a-pyd-file-the-same-as-a-dll } return any(Path(f).suffix in module_suffixes for f in files)