18def download_files(input_file, output_dir, prefix, keep_common_prefix):
19 with open(input_file, 'r') as f:
20 lines = f.readlines()
21
22 if keep_common_prefix:
23 common_prefix = os.path.commonprefix(lines)
24
25 for url in lines:
26 file_url = url.strip()
27
28 if keep_common_prefix:
29 rel_file = file_url.replace(common_prefix, '')
30 dest_dir = os.path.join(output_dir, os.path.dirname(rel_file))
31 else:
32 dest_dir = output_dir
33
34 dest_file = os.path.join(dest_dir, prefix + os.path.basename(file_url))
35 if not os.path.exists(dest_dir):
36 os.makedirs(dest_dir)
37
38 print(
'Downloading %s to %s' % (file_url, dest_file))
39 urllib.urlretrieve(file_url, dest_file)
40
41
def print(*args, **kwargs)
def download_files(input_file, output_dir, prefix, keep_common_prefix)