• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1"""distutils.command.sdist
2
3Implements the Distutils 'sdist' command (create a source distribution)."""
4
5__revision__ = "$Id$"
6
7import os
8import string
9import sys
10from glob import glob
11from warnings import warn
12
13from distutils.core import Command
14from distutils import dir_util, dep_util, file_util, archive_util
15from distutils.text_file import TextFile
16from distutils.errors import (DistutilsPlatformError, DistutilsOptionError,
17                              DistutilsTemplateError)
18from distutils.filelist import FileList
19from distutils import log
20from distutils.util import convert_path
21
22def show_formats():
23    """Print all possible values for the 'formats' option (used by
24    the "--help-formats" command-line option).
25    """
26    from distutils.fancy_getopt import FancyGetopt
27    from distutils.archive_util import ARCHIVE_FORMATS
28    formats = []
29    for format in ARCHIVE_FORMATS.keys():
30        formats.append(("formats=" + format, None,
31                        ARCHIVE_FORMATS[format][2]))
32    formats.sort()
33    FancyGetopt(formats).print_help(
34        "List of available source distribution formats:")
35
36class sdist(Command):
37
38    description = "create a source distribution (tarball, zip file, etc.)"
39
40    def checking_metadata(self):
41        """Callable used for the check sub-command.
42
43        Placed here so user_options can view it"""
44        return self.metadata_check
45
46    user_options = [
47        ('template=', 't',
48         "name of manifest template file [default: MANIFEST.in]"),
49        ('manifest=', 'm',
50         "name of manifest file [default: MANIFEST]"),
51        ('use-defaults', None,
52         "include the default file set in the manifest "
53         "[default; disable with --no-defaults]"),
54        ('no-defaults', None,
55         "don't include the default file set"),
56        ('prune', None,
57         "specifically exclude files/directories that should not be "
58         "distributed (build tree, RCS/CVS dirs, etc.) "
59         "[default; disable with --no-prune]"),
60        ('no-prune', None,
61         "don't automatically exclude anything"),
62        ('manifest-only', 'o',
63         "just regenerate the manifest and then stop "
64         "(implies --force-manifest)"),
65        ('force-manifest', 'f',
66         "forcibly regenerate the manifest and carry on as usual. "
67         "Deprecated: now the manifest is always regenerated."),
68        ('formats=', None,
69         "formats for source distribution (comma-separated list)"),
70        ('keep-temp', 'k',
71         "keep the distribution tree around after creating " +
72         "archive file(s)"),
73        ('dist-dir=', 'd',
74         "directory to put the source distribution archive(s) in "
75         "[default: dist]"),
76        ('metadata-check', None,
77         "Ensure that all required elements of meta-data "
78         "are supplied. Warn if any missing. [default]"),
79        ('owner=', 'u',
80         "Owner name used when creating a tar file [default: current user]"),
81        ('group=', 'g',
82         "Group name used when creating a tar file [default: current group]"),
83        ]
84
85    boolean_options = ['use-defaults', 'prune',
86                       'manifest-only', 'force-manifest',
87                       'keep-temp', 'metadata-check']
88
89    help_options = [
90        ('help-formats', None,
91         "list available distribution formats", show_formats),
92        ]
93
94    negative_opt = {'no-defaults': 'use-defaults',
95                    'no-prune': 'prune' }
96
97    default_format = {'posix': 'gztar',
98                      'nt': 'zip' }
99
100    sub_commands = [('check', checking_metadata)]
101
102    def initialize_options(self):
103        # 'template' and 'manifest' are, respectively, the names of
104        # the manifest template and manifest file.
105        self.template = None
106        self.manifest = None
107
108        # 'use_defaults': if true, we will include the default file set
109        # in the manifest
110        self.use_defaults = 1
111        self.prune = 1
112
113        self.manifest_only = 0
114        self.force_manifest = 0
115
116        self.formats = None
117        self.keep_temp = 0
118        self.dist_dir = None
119
120        self.archive_files = None
121        self.metadata_check = 1
122        self.owner = None
123        self.group = None
124
125    def finalize_options(self):
126        if self.manifest is None:
127            self.manifest = "MANIFEST"
128        if self.template is None:
129            self.template = "MANIFEST.in"
130
131        self.ensure_string_list('formats')
132        if self.formats is None:
133            try:
134                self.formats = [self.default_format[os.name]]
135            except KeyError:
136                raise DistutilsPlatformError, \
137                      "don't know how to create source distributions " + \
138                      "on platform %s" % os.name
139
140        bad_format = archive_util.check_archive_formats(self.formats)
141        if bad_format:
142            raise DistutilsOptionError, \
143                  "unknown archive format '%s'" % bad_format
144
145        if self.dist_dir is None:
146            self.dist_dir = "dist"
147
148    def run(self):
149        # 'filelist' contains the list of files that will make up the
150        # manifest
151        self.filelist = FileList()
152
153        # Run sub commands
154        for cmd_name in self.get_sub_commands():
155            self.run_command(cmd_name)
156
157        # Do whatever it takes to get the list of files to process
158        # (process the manifest template, read an existing manifest,
159        # whatever).  File list is accumulated in 'self.filelist'.
160        self.get_file_list()
161
162        # If user just wanted us to regenerate the manifest, stop now.
163        if self.manifest_only:
164            return
165
166        # Otherwise, go ahead and create the source distribution tarball,
167        # or zipfile, or whatever.
168        self.make_distribution()
169
170    def check_metadata(self):
171        """Deprecated API."""
172        warn("distutils.command.sdist.check_metadata is deprecated, \
173              use the check command instead", PendingDeprecationWarning)
174        check = self.distribution.get_command_obj('check')
175        check.ensure_finalized()
176        check.run()
177
178    def get_file_list(self):
179        """Figure out the list of files to include in the source
180        distribution, and put it in 'self.filelist'.  This might involve
181        reading the manifest template (and writing the manifest), or just
182        reading the manifest, or just using the default file set -- it all
183        depends on the user's options.
184        """
185        # new behavior when using a template:
186        # the file list is recalculated every time because
187        # even if MANIFEST.in or setup.py are not changed
188        # the user might have added some files in the tree that
189        # need to be included.
190        #
191        #  This makes --force the default and only behavior with templates.
192        template_exists = os.path.isfile(self.template)
193        if not template_exists and self._manifest_is_not_generated():
194            self.read_manifest()
195            self.filelist.sort()
196            self.filelist.remove_duplicates()
197            return
198
199        if not template_exists:
200            self.warn(("manifest template '%s' does not exist " +
201                        "(using default file list)") %
202                        self.template)
203        self.filelist.findall()
204
205        if self.use_defaults:
206            self.add_defaults()
207
208        if template_exists:
209            self.read_template()
210
211        if self.prune:
212            self.prune_file_list()
213
214        self.filelist.sort()
215        self.filelist.remove_duplicates()
216        self.write_manifest()
217
218    def add_defaults(self):
219        """Add all the default files to self.filelist:
220          - README or README.txt
221          - setup.py
222          - test/test*.py
223          - all pure Python modules mentioned in setup script
224          - all files pointed by package_data (build_py)
225          - all files defined in data_files.
226          - all files defined as scripts.
227          - all C sources listed as part of extensions or C libraries
228            in the setup script (doesn't catch C headers!)
229        Warns if (README or README.txt) or setup.py are missing; everything
230        else is optional.
231        """
232
233        standards = [('README', 'README.txt'), self.distribution.script_name]
234        for fn in standards:
235            if isinstance(fn, tuple):
236                alts = fn
237                got_it = 0
238                for fn in alts:
239                    if os.path.exists(fn):
240                        got_it = 1
241                        self.filelist.append(fn)
242                        break
243
244                if not got_it:
245                    self.warn("standard file not found: should have one of " +
246                              string.join(alts, ', '))
247            else:
248                if os.path.exists(fn):
249                    self.filelist.append(fn)
250                else:
251                    self.warn("standard file '%s' not found" % fn)
252
253        optional = ['test/test*.py', 'setup.cfg']
254        for pattern in optional:
255            files = filter(os.path.isfile, glob(pattern))
256            if files:
257                self.filelist.extend(files)
258
259        # build_py is used to get:
260        #  - python modules
261        #  - files defined in package_data
262        build_py = self.get_finalized_command('build_py')
263
264        # getting python files
265        if self.distribution.has_pure_modules():
266            self.filelist.extend(build_py.get_source_files())
267
268        # getting package_data files
269        # (computed in build_py.data_files by build_py.finalize_options)
270        for pkg, src_dir, build_dir, filenames in build_py.data_files:
271            for filename in filenames:
272                self.filelist.append(os.path.join(src_dir, filename))
273
274        # getting distribution.data_files
275        if self.distribution.has_data_files():
276            for item in self.distribution.data_files:
277                if isinstance(item, str): # plain file
278                    item = convert_path(item)
279                    if os.path.isfile(item):
280                        self.filelist.append(item)
281                else:    # a (dirname, filenames) tuple
282                    dirname, filenames = item
283                    for f in filenames:
284                        f = convert_path(f)
285                        if os.path.isfile(f):
286                            self.filelist.append(f)
287
288        if self.distribution.has_ext_modules():
289            build_ext = self.get_finalized_command('build_ext')
290            self.filelist.extend(build_ext.get_source_files())
291
292        if self.distribution.has_c_libraries():
293            build_clib = self.get_finalized_command('build_clib')
294            self.filelist.extend(build_clib.get_source_files())
295
296        if self.distribution.has_scripts():
297            build_scripts = self.get_finalized_command('build_scripts')
298            self.filelist.extend(build_scripts.get_source_files())
299
300    def read_template(self):
301        """Read and parse manifest template file named by self.template.
302
303        (usually "MANIFEST.in") The parsing and processing is done by
304        'self.filelist', which updates itself accordingly.
305        """
306        log.info("reading manifest template '%s'", self.template)
307        template = TextFile(self.template,
308                            strip_comments=1,
309                            skip_blanks=1,
310                            join_lines=1,
311                            lstrip_ws=1,
312                            rstrip_ws=1,
313                            collapse_join=1)
314
315        try:
316            while 1:
317                line = template.readline()
318                if line is None:            # end of file
319                    break
320
321                try:
322                    self.filelist.process_template_line(line)
323                # the call above can raise a DistutilsTemplateError for
324                # malformed lines, or a ValueError from the lower-level
325                # convert_path function
326                except (DistutilsTemplateError, ValueError) as msg:
327                    self.warn("%s, line %d: %s" % (template.filename,
328                                                   template.current_line,
329                                                   msg))
330        finally:
331            template.close()
332
333    def prune_file_list(self):
334        """Prune off branches that might slip into the file list as created
335        by 'read_template()', but really don't belong there:
336          * the build tree (typically "build")
337          * the release tree itself (only an issue if we ran "sdist"
338            previously with --keep-temp, or it aborted)
339          * any RCS, CVS, .svn, .hg, .git, .bzr, _darcs directories
340        """
341        build = self.get_finalized_command('build')
342        base_dir = self.distribution.get_fullname()
343
344        self.filelist.exclude_pattern(None, prefix=build.build_base)
345        self.filelist.exclude_pattern(None, prefix=base_dir)
346
347        # pruning out vcs directories
348        # both separators are used under win32
349        if sys.platform == 'win32':
350            seps = r'/|\\'
351        else:
352            seps = '/'
353
354        vcs_dirs = ['RCS', 'CVS', r'\.svn', r'\.hg', r'\.git', r'\.bzr',
355                    '_darcs']
356        vcs_ptrn = r'(^|%s)(%s)(%s).*' % (seps, '|'.join(vcs_dirs), seps)
357        self.filelist.exclude_pattern(vcs_ptrn, is_regex=1)
358
359    def write_manifest(self):
360        """Write the file list in 'self.filelist' (presumably as filled in
361        by 'add_defaults()' and 'read_template()') to the manifest file
362        named by 'self.manifest'.
363        """
364        if self._manifest_is_not_generated():
365            log.info("not writing to manually maintained "
366                     "manifest file '%s'" % self.manifest)
367            return
368
369        content = self.filelist.files[:]
370        content.insert(0, '# file GENERATED by distutils, do NOT edit')
371        self.execute(file_util.write_file, (self.manifest, content),
372                     "writing manifest file '%s'" % self.manifest)
373
374    def _manifest_is_not_generated(self):
375        # check for special comment used in 2.7.1 and higher
376        if not os.path.isfile(self.manifest):
377            return False
378
379        fp = open(self.manifest, 'rU')
380        try:
381            first_line = fp.readline()
382        finally:
383            fp.close()
384        return first_line != '# file GENERATED by distutils, do NOT edit\n'
385
386    def read_manifest(self):
387        """Read the manifest file (named by 'self.manifest') and use it to
388        fill in 'self.filelist', the list of files to include in the source
389        distribution.
390        """
391        log.info("reading manifest file '%s'", self.manifest)
392        manifest = open(self.manifest)
393        for line in manifest:
394            # ignore comments and blank lines
395            line = line.strip()
396            if line.startswith('#') or not line:
397                continue
398            self.filelist.append(line)
399        manifest.close()
400
401    def make_release_tree(self, base_dir, files):
402        """Create the directory tree that will become the source
403        distribution archive.  All directories implied by the filenames in
404        'files' are created under 'base_dir', and then we hard link or copy
405        (if hard linking is unavailable) those files into place.
406        Essentially, this duplicates the developer's source tree, but in a
407        directory named after the distribution, containing only the files
408        to be distributed.
409        """
410        # Create all the directories under 'base_dir' necessary to
411        # put 'files' there; the 'mkpath()' is just so we don't die
412        # if the manifest happens to be empty.
413        self.mkpath(base_dir)
414        dir_util.create_tree(base_dir, files, dry_run=self.dry_run)
415
416        # And walk over the list of files, either making a hard link (if
417        # os.link exists) to each one that doesn't already exist in its
418        # corresponding location under 'base_dir', or copying each file
419        # that's out-of-date in 'base_dir'.  (Usually, all files will be
420        # out-of-date, because by default we blow away 'base_dir' when
421        # we're done making the distribution archives.)
422
423        if hasattr(os, 'link'):        # can make hard links on this system
424            link = 'hard'
425            msg = "making hard links in %s..." % base_dir
426        else:                           # nope, have to copy
427            link = None
428            msg = "copying files to %s..." % base_dir
429
430        if not files:
431            log.warn("no files to distribute -- empty manifest?")
432        else:
433            log.info(msg)
434        for file in files:
435            if not os.path.isfile(file):
436                log.warn("'%s' not a regular file -- skipping" % file)
437            else:
438                dest = os.path.join(base_dir, file)
439                self.copy_file(file, dest, link=link)
440
441        self.distribution.metadata.write_pkg_info(base_dir)
442
443    def make_distribution(self):
444        """Create the source distribution(s).  First, we create the release
445        tree with 'make_release_tree()'; then, we create all required
446        archive files (according to 'self.formats') from the release tree.
447        Finally, we clean up by blowing away the release tree (unless
448        'self.keep_temp' is true).  The list of archive files created is
449        stored so it can be retrieved later by 'get_archive_files()'.
450        """
451        # Don't warn about missing meta-data here -- should be (and is!)
452        # done elsewhere.
453        base_dir = self.distribution.get_fullname()
454        base_name = os.path.join(self.dist_dir, base_dir)
455
456        self.make_release_tree(base_dir, self.filelist.files)
457        archive_files = []              # remember names of files we create
458        # tar archive must be created last to avoid overwrite and remove
459        if 'tar' in self.formats:
460            self.formats.append(self.formats.pop(self.formats.index('tar')))
461
462        for fmt in self.formats:
463            file = self.make_archive(base_name, fmt, base_dir=base_dir,
464                                     owner=self.owner, group=self.group)
465            archive_files.append(file)
466            self.distribution.dist_files.append(('sdist', '', file))
467
468        self.archive_files = archive_files
469
470        if not self.keep_temp:
471            dir_util.remove_tree(base_dir, dry_run=self.dry_run)
472
473    def get_archive_files(self):
474        """Return the list of archive files created when the command
475        was run, or None if the command hasn't run yet.
476        """
477        return self.archive_files
478