3 """Mirror a remote ftp subtree into a local directory tree.
5 usage: ftpmirror [-v] [-q] [-i] [-m] [-n] [-r] [-s pat]
6 [-l username [-p passwd [-a account]]]
7 hostname[:port] [remotedir [localdir]]
11 -m: macintosh server (NCSA telnet 2.4) (implies -n -s '*.o')
13 -r: remove local files/directories no longer pertinent
14 -l username [-p passwd [-a account]]: login info (default .netrc or anonymous)
15 -s pat: skip files matching pattern
16 hostname: remote host w/ optional port separated by ':'
17 remotedir: remote directory (default initial)
18 localdir: local directory (default current)
27 from fnmatch import fnmatch
29 # Print usage message and exit
31 sys.stdout = sys.stderr
32 for msg in args: print msg
36 verbose = 1 # 0 for -q, 2 for -v
41 skippats = ['.', '..', '.mirrorinfo']
43 # Main program: parse command line and start processing
45 global verbose, interactive, mac, rmok, nologin
47 opts, args = getopt.getopt(sys.argv[1:], 'a:bil:mnp:qrs:v')
48 except getopt.error, msg:
53 if not args: usage('hostname missing')
57 host, port = host.split(':', 1)
60 auth = netrc.netrc().authenticators(host)
62 login, account, passwd = auth
63 except (netrc.NetrcParseError, IOError):
66 if o == '-l': login = a
67 if o == '-p': passwd = a
68 if o == '-a': account = a
69 if o == '-v': verbose = verbose + 1
70 if o == '-q': verbose = 0
71 if o == '-i': interactive = 1
72 if o == '-m': mac = 1; nologin = 1; skippats.append('*.o')
73 if o == '-n': nologin = 1
74 if o == '-r': rmok = 1
75 if o == '-s': skippats.append(a)
82 if args[3:]: usage('too many arguments')
85 if verbose: print "Connecting to '%s%s'..." % (host,
86 (port and ":%d"%port or ""))
90 print 'Logging in as %r...' % (login or 'anonymous')
91 f.login(login, passwd, account)
92 if verbose: print 'OK.'
94 if verbose > 1: print 'PWD =', repr(pwd)
96 if verbose > 1: print 'cwd(%s)' % repr(remotedir)
98 if verbose > 1: print 'OK.'
100 if verbose > 1: print 'PWD =', repr(pwd)
102 mirrorsubdir(f, localdir)
104 # Core logic: mirror one subdirectory (recursively)
105 def mirrorsubdir(f, localdir):
107 if localdir and not os.path.isdir(localdir):
108 if verbose: print 'Creating local directory', repr(localdir)
111 except os.error, msg:
112 print "Failed to establish local directory", repr(localdir)
114 infofilename = os.path.join(localdir, '.mirrorinfo')
116 text = open(infofilename, 'r').read()
121 except (SyntaxError, NameError):
122 print 'Bad mirror info in', repr(infofilename)
126 if verbose: print 'Listing remote directory %r...' % (pwd,)
127 f.retrlines('LIST', listing.append)
130 if verbose > 1: print '-->', repr(line)
132 # Mac listing has just filenames;
133 # trailing / means subdirectory
134 filename = line.strip()
136 if filename[-1:] == '/':
137 filename = filename[:-1]
141 # Parse, assuming a UNIX listing
142 words = line.split(None, 8)
144 if verbose > 1: print 'Skipping short line'
146 filename = words[-1].lstrip()
147 i = filename.find(" -> ")
149 # words[0] had better start with 'l'...
151 print 'Found symbolic link %r' % (filename,)
152 linkto = filename[i+4:]
153 filename = filename[:i]
154 infostuff = words[-5:-1]
158 if fnmatch(filename, pat):
160 print 'Skip pattern', repr(pat),
161 print 'matches', repr(filename)
168 print 'Remembering subdirectory', repr(filename)
169 subdirs.append(filename)
171 filesfound.append(filename)
172 if info.has_key(filename) and info[filename] == infostuff:
174 print 'Already have this version of',repr(filename)
176 fullname = os.path.join(localdir, filename)
177 tempname = os.path.join(localdir, '@'+filename)
179 doit = askabout('file', filename, pwd)
181 if not info.has_key(filename):
182 info[filename] = 'Not retrieved'
190 print "Creating symlink %r -> %r" % (filename, linkto)
192 os.symlink(linkto, tempname)
194 print "Can't create %r: %s" % (tempname, msg)
198 fp = open(tempname, 'wb')
200 print "Can't create %r: %s" % (tempname, msg)
203 print 'Retrieving %r from %r as %r...' % (filename, pwd, fullname)
205 fp1 = LoggingFile(fp, 1024, sys.stdout)
210 f.retrbinary('RETR ' + filename,
212 except ftplib.error_perm, msg:
222 pass # Ignore the error
224 os.rename(tempname, fullname)
225 except os.error, msg:
226 print "Can't rename %r to %r: %s" % (tempname, fullname, msg)
228 info[filename] = infostuff
229 writedict(info, infofilename)
230 if verbose and mode[0] != 'l':
232 kbytes = bytes / 1024.0
233 print int(round(kbytes)),
235 print int(round(dt)),
238 print '(~%d Kbytes/sec)' % \
239 int(round(kbytes/dt),)
242 # Remove files from info that are no longer remote
244 for filename in info.keys():
245 if filename not in filesfound:
247 print "Removing obsolete info entry for",
248 print repr(filename), "in", repr(localdir or ".")
250 deletions = deletions + 1
252 writedict(info, infofilename)
254 # Remove local files that are no longer in the remote directory
256 if not localdir: names = os.listdir(os.curdir)
257 else: names = os.listdir(localdir)
261 if name[0] == '.' or info.has_key(name) or name in subdirs:
265 if fnmatch(name, pat):
267 print 'Skip pattern', repr(pat),
268 print 'matches', repr(name)
273 fullname = os.path.join(localdir, name)
276 print 'Local file', repr(fullname),
277 print 'is no longer pertinent'
279 if verbose: print 'Removing local file/dir', repr(fullname)
282 # Recursively mirror subdirectories
283 for subdir in subdirs:
285 doit = askabout('subdirectory', subdir, pwd)
286 if not doit: continue
287 if verbose: print 'Processing subdirectory', repr(subdir)
288 localsubdir = os.path.join(localdir, subdir)
291 print 'Remote directory now:', repr(pwd)
292 print 'Remote cwd', repr(subdir)
295 except ftplib.error_perm, msg:
296 print "Can't chdir to", repr(subdir), ":", repr(msg)
298 if verbose: print 'Mirroring as', repr(localsubdir)
299 mirrorsubdir(f, localsubdir)
300 if verbose > 1: print 'Remote cwd ..'
304 print 'Ended up in wrong directory after cd + cd ..'
305 print 'Giving up now.'
308 if verbose > 1: print 'OK.'
310 # Helper to remove a file or directory tree
311 def remove(fullname):
312 if os.path.isdir(fullname) and not os.path.islink(fullname):
314 names = os.listdir(fullname)
319 if not remove(os.path.join(fullname, name)):
325 except os.error, msg:
326 print "Can't remove local directory %r: %s" % (fullname, msg)
331 except os.error, msg:
332 print "Can't remove local file %r: %s" % (fullname, msg)
336 # Wrapper around a file for writing to write a hash sign every block.
338 def __init__(self, fp, blocksize, outfp):
342 self.blocksize = blocksize
344 def write(self, data):
345 self.bytes = self.bytes + len(data)
346 hashes = int(self.bytes) / self.blocksize
347 while hashes > self.hashes:
348 self.outfp.write('#')
350 self.hashes = self.hashes + 1
353 self.outfp.write('\n')
355 # Ask permission to download a file.
356 def askabout(filetype, filename, pwd):
357 prompt = 'Retrieve %s %s from %s ? [ny] ' % (filetype, filename, pwd)
359 reply = raw_input(prompt).strip().lower()
360 if reply in ['y', 'ye', 'yes']:
362 if reply in ['', 'n', 'no', 'nop', 'nope']:
364 print 'Please answer yes or no.'
366 # Create a directory if it doesn't exist. Recursively create the
367 # parent directory as well if needed.
368 def makedir(pathname):
369 if os.path.isdir(pathname):
371 dirname = os.path.dirname(pathname)
372 if dirname: makedir(dirname)
373 os.mkdir(pathname, 0777)
375 # Write a dictionary to a file in a way that can be read back using
376 # rval() but is still somewhat readable (i.e. not a single long line).
377 # Also creates a backup file.
378 def writedict(dict, filename):
379 dir, fname = os.path.split(filename)
380 tempname = os.path.join(dir, '@' + fname)
381 backup = os.path.join(dir, fname + '~')
386 fp = open(tempname, 'w')
388 for key, value in dict.items():
389 fp.write('%r: %r,\n' % (key, value))
393 os.rename(filename, backup)
396 os.rename(tempname, filename)
399 if __name__ == '__main__':