• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#! /usr/bin/env python
2
3"""Mirror a remote ftp subtree into a local directory tree.
4
5usage: ftpmirror [-v] [-q] [-i] [-m] [-n] [-r] [-s pat]
6                 [-l username [-p passwd [-a account]]]
7                 hostname[:port] [remotedir [localdir]]
8-v: verbose
9-q: quiet
10-i: interactive mode
11-m: macintosh server (NCSA telnet 2.4) (implies -n -s '*.o')
12-n: don't log in
13-r: remove local files/directories no longer pertinent
14-l username [-p passwd [-a account]]: login info (default .netrc or anonymous)
15-s pat: skip files matching pattern
16hostname: remote host w/ optional port separated by ':'
17remotedir: remote directory (default initial)
18localdir: local directory (default current)
19"""
20
21import os
22import sys
23import time
24import getopt
25import ftplib
26import netrc
27from fnmatch import fnmatch
28
29# Print usage message and exit
30def usage(*args):
31    sys.stdout = sys.stderr
32    for msg in args: print msg
33    print __doc__
34    sys.exit(2)
35
36verbose = 1 # 0 for -q, 2 for -v
37interactive = 0
38mac = 0
39rmok = 0
40nologin = 0
41skippats = ['.', '..', '.mirrorinfo']
42
43# Main program: parse command line and start processing
44def main():
45    global verbose, interactive, mac, rmok, nologin
46    try:
47        opts, args = getopt.getopt(sys.argv[1:], 'a:bil:mnp:qrs:v')
48    except getopt.error, msg:
49        usage(msg)
50    login = ''
51    passwd = ''
52    account = ''
53    if not args: usage('hostname missing')
54    host = args[0]
55    port = 0
56    if ':' in host:
57        host, port = host.split(':', 1)
58        port = int(port)
59    try:
60        auth = netrc.netrc().authenticators(host)
61        if auth is not None:
62            login, account, passwd = auth
63    except (netrc.NetrcParseError, IOError):
64        pass
65    for o, a in opts:
66        if o == '-l': login = a
67        if o == '-p': passwd = a
68        if o == '-a': account = a
69        if o == '-v': verbose = verbose + 1
70        if o == '-q': verbose = 0
71        if o == '-i': interactive = 1
72        if o == '-m': mac = 1; nologin = 1; skippats.append('*.o')
73        if o == '-n': nologin = 1
74        if o == '-r': rmok = 1
75        if o == '-s': skippats.append(a)
76    remotedir = ''
77    localdir = ''
78    if args[1:]:
79        remotedir = args[1]
80        if args[2:]:
81            localdir = args[2]
82            if args[3:]: usage('too many arguments')
83    #
84    f = ftplib.FTP()
85    if verbose: print "Connecting to '%s%s'..." % (host,
86                                                   (port and ":%d"%port or ""))
87    f.connect(host,port)
88    if not nologin:
89        if verbose:
90            print 'Logging in as %r...' % (login or 'anonymous')
91        f.login(login, passwd, account)
92    if verbose: print 'OK.'
93    pwd = f.pwd()
94    if verbose > 1: print 'PWD =', repr(pwd)
95    if remotedir:
96        if verbose > 1: print 'cwd(%s)' % repr(remotedir)
97        f.cwd(remotedir)
98        if verbose > 1: print 'OK.'
99        pwd = f.pwd()
100        if verbose > 1: print 'PWD =', repr(pwd)
101    #
102    mirrorsubdir(f, localdir)
103
104# Core logic: mirror one subdirectory (recursively)
105def mirrorsubdir(f, localdir):
106    pwd = f.pwd()
107    if localdir and not os.path.isdir(localdir):
108        if verbose: print 'Creating local directory', repr(localdir)
109        try:
110            makedir(localdir)
111        except os.error, msg:
112            print "Failed to establish local directory", repr(localdir)
113            return
114    infofilename = os.path.join(localdir, '.mirrorinfo')
115    try:
116        text = open(infofilename, 'r').read()
117    except IOError, msg:
118        text = '{}'
119    try:
120        info = eval(text)
121    except (SyntaxError, NameError):
122        print 'Bad mirror info in', repr(infofilename)
123        info = {}
124    subdirs = []
125    listing = []
126    if verbose: print 'Listing remote directory %r...' % (pwd,)
127    f.retrlines('LIST', listing.append)
128    filesfound = []
129    for line in listing:
130        if verbose > 1: print '-->', repr(line)
131        if mac:
132            # Mac listing has just filenames;
133            # trailing / means subdirectory
134            filename = line.strip()
135            mode = '-'
136            if filename[-1:] == '/':
137                filename = filename[:-1]
138                mode = 'd'
139            infostuff = ''
140        else:
141            # Parse, assuming a UNIX listing
142            words = line.split(None, 8)
143            if len(words) < 6:
144                if verbose > 1: print 'Skipping short line'
145                continue
146            filename = words[-1].lstrip()
147            i = filename.find(" -> ")
148            if i >= 0:
149                # words[0] had better start with 'l'...
150                if verbose > 1:
151                    print 'Found symbolic link %r' % (filename,)
152                linkto = filename[i+4:]
153                filename = filename[:i]
154            infostuff = words[-5:-1]
155            mode = words[0]
156        skip = 0
157        for pat in skippats:
158            if fnmatch(filename, pat):
159                if verbose > 1:
160                    print 'Skip pattern', repr(pat),
161                    print 'matches', repr(filename)
162                skip = 1
163                break
164        if skip:
165            continue
166        if mode[0] == 'd':
167            if verbose > 1:
168                print 'Remembering subdirectory', repr(filename)
169            subdirs.append(filename)
170            continue
171        filesfound.append(filename)
172        if info.has_key(filename) and info[filename] == infostuff:
173            if verbose > 1:
174                print 'Already have this version of',repr(filename)
175            continue
176        fullname = os.path.join(localdir, filename)
177        tempname = os.path.join(localdir, '@'+filename)
178        if interactive:
179            doit = askabout('file', filename, pwd)
180            if not doit:
181                if not info.has_key(filename):
182                    info[filename] = 'Not retrieved'
183                continue
184        try:
185            os.unlink(tempname)
186        except os.error:
187            pass
188        if mode[0] == 'l':
189            if verbose:
190                print "Creating symlink %r -> %r" % (filename, linkto)
191            try:
192                os.symlink(linkto, tempname)
193            except IOError, msg:
194                print "Can't create %r: %s" % (tempname, msg)
195                continue
196        else:
197            try:
198                fp = open(tempname, 'wb')
199            except IOError, msg:
200                print "Can't create %r: %s" % (tempname, msg)
201                continue
202            if verbose:
203                print 'Retrieving %r from %r as %r...' % (filename, pwd, fullname)
204            if verbose:
205                fp1 = LoggingFile(fp, 1024, sys.stdout)
206            else:
207                fp1 = fp
208            t0 = time.time()
209            try:
210                f.retrbinary('RETR ' + filename,
211                             fp1.write, 8*1024)
212            except ftplib.error_perm, msg:
213                print msg
214            t1 = time.time()
215            bytes = fp.tell()
216            fp.close()
217            if fp1 != fp:
218                fp1.close()
219        try:
220            os.unlink(fullname)
221        except os.error:
222            pass            # Ignore the error
223        try:
224            os.rename(tempname, fullname)
225        except os.error, msg:
226            print "Can't rename %r to %r: %s" % (tempname, fullname, msg)
227            continue
228        info[filename] = infostuff
229        writedict(info, infofilename)
230        if verbose and mode[0] != 'l':
231            dt = t1 - t0
232            kbytes = bytes / 1024.0
233            print int(round(kbytes)),
234            print 'Kbytes in',
235            print int(round(dt)),
236            print 'seconds',
237            if t1 > t0:
238                print '(~%d Kbytes/sec)' % \
239                          int(round(kbytes/dt),)
240            print
241    #
242    # Remove files from info that are no longer remote
243    deletions = 0
244    for filename in info.keys():
245        if filename not in filesfound:
246            if verbose:
247                print "Removing obsolete info entry for",
248                print repr(filename), "in", repr(localdir or ".")
249            del info[filename]
250            deletions = deletions + 1
251    if deletions:
252        writedict(info, infofilename)
253    #
254    # Remove local files that are no longer in the remote directory
255    try:
256        if not localdir: names = os.listdir(os.curdir)
257        else: names = os.listdir(localdir)
258    except os.error:
259        names = []
260    for name in names:
261        if name[0] == '.' or info.has_key(name) or name in subdirs:
262            continue
263        skip = 0
264        for pat in skippats:
265            if fnmatch(name, pat):
266                if verbose > 1:
267                    print 'Skip pattern', repr(pat),
268                    print 'matches', repr(name)
269                skip = 1
270                break
271        if skip:
272            continue
273        fullname = os.path.join(localdir, name)
274        if not rmok:
275            if verbose:
276                print 'Local file', repr(fullname),
277                print 'is no longer pertinent'
278            continue
279        if verbose: print 'Removing local file/dir', repr(fullname)
280        remove(fullname)
281    #
282    # Recursively mirror subdirectories
283    for subdir in subdirs:
284        if interactive:
285            doit = askabout('subdirectory', subdir, pwd)
286            if not doit: continue
287        if verbose: print 'Processing subdirectory', repr(subdir)
288        localsubdir = os.path.join(localdir, subdir)
289        pwd = f.pwd()
290        if verbose > 1:
291            print 'Remote directory now:', repr(pwd)
292            print 'Remote cwd', repr(subdir)
293        try:
294            f.cwd(subdir)
295        except ftplib.error_perm, msg:
296            print "Can't chdir to", repr(subdir), ":", repr(msg)
297        else:
298            if verbose: print 'Mirroring as', repr(localsubdir)
299            mirrorsubdir(f, localsubdir)
300            if verbose > 1: print 'Remote cwd ..'
301            f.cwd('..')
302        newpwd = f.pwd()
303        if newpwd != pwd:
304            print 'Ended up in wrong directory after cd + cd ..'
305            print 'Giving up now.'
306            break
307        else:
308            if verbose > 1: print 'OK.'
309
310# Helper to remove a file or directory tree
311def remove(fullname):
312    if os.path.isdir(fullname) and not os.path.islink(fullname):
313        try:
314            names = os.listdir(fullname)
315        except os.error:
316            names = []
317        ok = 1
318        for name in names:
319            if not remove(os.path.join(fullname, name)):
320                ok = 0
321        if not ok:
322            return 0
323        try:
324            os.rmdir(fullname)
325        except os.error, msg:
326            print "Can't remove local directory %r: %s" % (fullname, msg)
327            return 0
328    else:
329        try:
330            os.unlink(fullname)
331        except os.error, msg:
332            print "Can't remove local file %r: %s" % (fullname, msg)
333            return 0
334    return 1
335
336# Wrapper around a file for writing to write a hash sign every block.
337class LoggingFile:
338    def __init__(self, fp, blocksize, outfp):
339        self.fp = fp
340        self.bytes = 0
341        self.hashes = 0
342        self.blocksize = blocksize
343        self.outfp = outfp
344    def write(self, data):
345        self.bytes = self.bytes + len(data)
346        hashes = int(self.bytes) / self.blocksize
347        while hashes > self.hashes:
348            self.outfp.write('#')
349            self.outfp.flush()
350            self.hashes = self.hashes + 1
351        self.fp.write(data)
352    def close(self):
353        self.outfp.write('\n')
354
355# Ask permission to download a file.
356def askabout(filetype, filename, pwd):
357    prompt = 'Retrieve %s %s from %s ? [ny] ' % (filetype, filename, pwd)
358    while 1:
359        reply = raw_input(prompt).strip().lower()
360        if reply in ['y', 'ye', 'yes']:
361            return 1
362        if reply in ['', 'n', 'no', 'nop', 'nope']:
363            return 0
364        print 'Please answer yes or no.'
365
366# Create a directory if it doesn't exist.  Recursively create the
367# parent directory as well if needed.
368def makedir(pathname):
369    if os.path.isdir(pathname):
370        return
371    dirname = os.path.dirname(pathname)
372    if dirname: makedir(dirname)
373    os.mkdir(pathname, 0777)
374
375# Write a dictionary to a file in a way that can be read back using
376# rval() but is still somewhat readable (i.e. not a single long line).
377# Also creates a backup file.
378def writedict(dict, filename):
379    dir, fname = os.path.split(filename)
380    tempname = os.path.join(dir, '@' + fname)
381    backup = os.path.join(dir, fname + '~')
382    try:
383        os.unlink(backup)
384    except os.error:
385        pass
386    fp = open(tempname, 'w')
387    fp.write('{\n')
388    for key, value in dict.items():
389        fp.write('%r: %r,\n' % (key, value))
390    fp.write('}\n')
391    fp.close()
392    try:
393        os.rename(filename, backup)
394    except os.error:
395        pass
396    os.rename(tempname, filename)
397
398
399if __name__ == '__main__':
400    main()
401