• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1from glob import glob
2from distutils.util import convert_path
3import distutils.command.build_py as orig
4import os
5import fnmatch
6import textwrap
7import io
8import distutils.errors
9import itertools
10
11from setuptools.extern import six
12from setuptools.extern.six.moves import map, filter, filterfalse
13
14try:
15    from setuptools.lib2to3_ex import Mixin2to3
16except ImportError:
17
18    class Mixin2to3:
19        def run_2to3(self, files, doctests=True):
20            "do nothing"
21
22
23class build_py(orig.build_py, Mixin2to3):
24    """Enhanced 'build_py' command that includes data files with packages
25
26    The data files are specified via a 'package_data' argument to 'setup()'.
27    See 'setuptools.dist.Distribution' for more details.
28
29    Also, this version of the 'build_py' command allows you to specify both
30    'py_modules' and 'packages' in the same setup operation.
31    """
32
33    def finalize_options(self):
34        orig.build_py.finalize_options(self)
35        self.package_data = self.distribution.package_data
36        self.exclude_package_data = (self.distribution.exclude_package_data or
37                                     {})
38        if 'data_files' in self.__dict__:
39            del self.__dict__['data_files']
40        self.__updated_files = []
41        self.__doctests_2to3 = []
42
43    def run(self):
44        """Build modules, packages, and copy data files to build directory"""
45        if not self.py_modules and not self.packages:
46            return
47
48        if self.py_modules:
49            self.build_modules()
50
51        if self.packages:
52            self.build_packages()
53            self.build_package_data()
54
55        self.run_2to3(self.__updated_files, False)
56        self.run_2to3(self.__updated_files, True)
57        self.run_2to3(self.__doctests_2to3, True)
58
59        # Only compile actual .py files, using our base class' idea of what our
60        # output files are.
61        self.byte_compile(orig.build_py.get_outputs(self, include_bytecode=0))
62
63    def __getattr__(self, attr):
64        "lazily compute data files"
65        if attr == 'data_files':
66            self.data_files = self._get_data_files()
67            return self.data_files
68        return orig.build_py.__getattr__(self, attr)
69
70    def build_module(self, module, module_file, package):
71        if six.PY2 and isinstance(package, six.string_types):
72            # avoid errors on Python 2 when unicode is passed (#190)
73            package = package.split('.')
74        outfile, copied = orig.build_py.build_module(self, module, module_file,
75                                                     package)
76        if copied:
77            self.__updated_files.append(outfile)
78        return outfile, copied
79
80    def _get_data_files(self):
81        """Generate list of '(package,src_dir,build_dir,filenames)' tuples"""
82        self.analyze_manifest()
83        return list(map(self._get_pkg_data_files, self.packages or ()))
84
85    def _get_pkg_data_files(self, package):
86        # Locate package source directory
87        src_dir = self.get_package_dir(package)
88
89        # Compute package build directory
90        build_dir = os.path.join(*([self.build_lib] + package.split('.')))
91
92        # Strip directory from globbed filenames
93        filenames = [
94            os.path.relpath(file, src_dir)
95            for file in self.find_data_files(package, src_dir)
96        ]
97        return package, src_dir, build_dir, filenames
98
99    def find_data_files(self, package, src_dir):
100        """Return filenames for package's data files in 'src_dir'"""
101        patterns = self._get_platform_patterns(
102            self.package_data,
103            package,
104            src_dir,
105        )
106        globs_expanded = map(glob, patterns)
107        # flatten the expanded globs into an iterable of matches
108        globs_matches = itertools.chain.from_iterable(globs_expanded)
109        glob_files = filter(os.path.isfile, globs_matches)
110        files = itertools.chain(
111            self.manifest_files.get(package, []),
112            glob_files,
113        )
114        return self.exclude_data_files(package, src_dir, files)
115
116    def build_package_data(self):
117        """Copy data files into build directory"""
118        for package, src_dir, build_dir, filenames in self.data_files:
119            for filename in filenames:
120                target = os.path.join(build_dir, filename)
121                self.mkpath(os.path.dirname(target))
122                srcfile = os.path.join(src_dir, filename)
123                outf, copied = self.copy_file(srcfile, target)
124                srcfile = os.path.abspath(srcfile)
125                if (copied and
126                        srcfile in self.distribution.convert_2to3_doctests):
127                    self.__doctests_2to3.append(outf)
128
129    def analyze_manifest(self):
130        self.manifest_files = mf = {}
131        if not self.distribution.include_package_data:
132            return
133        src_dirs = {}
134        for package in self.packages or ():
135            # Locate package source directory
136            src_dirs[assert_relative(self.get_package_dir(package))] = package
137
138        self.run_command('egg_info')
139        ei_cmd = self.get_finalized_command('egg_info')
140        for path in ei_cmd.filelist.files:
141            d, f = os.path.split(assert_relative(path))
142            prev = None
143            oldf = f
144            while d and d != prev and d not in src_dirs:
145                prev = d
146                d, df = os.path.split(d)
147                f = os.path.join(df, f)
148            if d in src_dirs:
149                if path.endswith('.py') and f == oldf:
150                    continue  # it's a module, not data
151                mf.setdefault(src_dirs[d], []).append(path)
152
153    def get_data_files(self):
154        pass  # Lazily compute data files in _get_data_files() function.
155
156    def check_package(self, package, package_dir):
157        """Check namespace packages' __init__ for declare_namespace"""
158        try:
159            return self.packages_checked[package]
160        except KeyError:
161            pass
162
163        init_py = orig.build_py.check_package(self, package, package_dir)
164        self.packages_checked[package] = init_py
165
166        if not init_py or not self.distribution.namespace_packages:
167            return init_py
168
169        for pkg in self.distribution.namespace_packages:
170            if pkg == package or pkg.startswith(package + '.'):
171                break
172        else:
173            return init_py
174
175        with io.open(init_py, 'rb') as f:
176            contents = f.read()
177        if b'declare_namespace' not in contents:
178            raise distutils.errors.DistutilsError(
179                "Namespace package problem: %s is a namespace package, but "
180                "its\n__init__.py does not call declare_namespace()! Please "
181                'fix it.\n(See the setuptools manual under '
182                '"Namespace Packages" for details.)\n"' % (package,)
183            )
184        return init_py
185
186    def initialize_options(self):
187        self.packages_checked = {}
188        orig.build_py.initialize_options(self)
189
190    def get_package_dir(self, package):
191        res = orig.build_py.get_package_dir(self, package)
192        if self.distribution.src_root is not None:
193            return os.path.join(self.distribution.src_root, res)
194        return res
195
196    def exclude_data_files(self, package, src_dir, files):
197        """Filter filenames for package's data files in 'src_dir'"""
198        files = list(files)
199        patterns = self._get_platform_patterns(
200            self.exclude_package_data,
201            package,
202            src_dir,
203        )
204        match_groups = (
205            fnmatch.filter(files, pattern)
206            for pattern in patterns
207        )
208        # flatten the groups of matches into an iterable of matches
209        matches = itertools.chain.from_iterable(match_groups)
210        bad = set(matches)
211        keepers = (
212            fn
213            for fn in files
214            if fn not in bad
215        )
216        # ditch dupes
217        return list(_unique_everseen(keepers))
218
219    @staticmethod
220    def _get_platform_patterns(spec, package, src_dir):
221        """
222        yield platform-specific path patterns (suitable for glob
223        or fn_match) from a glob-based spec (such as
224        self.package_data or self.exclude_package_data)
225        matching package in src_dir.
226        """
227        raw_patterns = itertools.chain(
228            spec.get('', []),
229            spec.get(package, []),
230        )
231        return (
232            # Each pattern has to be converted to a platform-specific path
233            os.path.join(src_dir, convert_path(pattern))
234            for pattern in raw_patterns
235        )
236
237
238# from Python docs
239def _unique_everseen(iterable, key=None):
240    "List unique elements, preserving order. Remember all elements ever seen."
241    # unique_everseen('AAAABBBCCDAABBB') --> A B C D
242    # unique_everseen('ABBCcAD', str.lower) --> A B C D
243    seen = set()
244    seen_add = seen.add
245    if key is None:
246        for element in filterfalse(seen.__contains__, iterable):
247            seen_add(element)
248            yield element
249    else:
250        for element in iterable:
251            k = key(element)
252            if k not in seen:
253                seen_add(k)
254                yield element
255
256
257def assert_relative(path):
258    if not os.path.isabs(path):
259        return path
260    from distutils.errors import DistutilsSetupError
261
262    msg = textwrap.dedent("""
263        Error: setup script specifies an absolute path:
264
265            %s
266
267        setup() arguments must *always* be /-separated paths relative to the
268        setup.py directory, *never* absolute paths.
269        """).lstrip() % path
270    raise DistutilsSetupError(msg)
271