-
Notifications
You must be signed in to change notification settings - Fork 1
/
Copy pathcloudcrate-download.py
131 lines (110 loc) · 5.89 KB
/
cloudcrate-download.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
#!/usr/bin/python
# https://docs.python.org/2/library/os.html#os.listdir
# Getting Started with AWS - https://aws.amazon.com/articles/3998
# Introduction to boto - http://boto.cloudhackers.com/en/latest/
# How to Install boto - http://stackoverflow.com/questions/2481287/how-do-i-install-boto
# https://ariejan.net/2010/12/24/public-readable-amazon-s3-bucket-policy/
# http://aws.amazon.com/code/Amazon-S3/1713
print """
____ _ _ ____ _
/ ___| | ___ _ _ __| | / ___|_ __ __ _| |_ ___
| | | |/ _ \| | | |/ _` | | | | '__/ _` | __/ _
| |___| | (_) | |_| | (_| | | |___| | | (_| | || __/
\____|_|\___/ \__,_|\__,_| \____|_| \__,_|\__\___|
Usage : python cloudcrate.py
============================
Available tasks:
setup ................ Run 'cloudcrate setup' first to setup AWS keys.
Example: python cloudcrate.py setup
sync ................ Run Sync from the desired folder to sync to the cloud
Example: python cloudcrate.py sync
download ................ Run Download and all files are automatically downloaded to folder s3_downloads on the Desktop
Example: python cloudcrate.py download
"""
import sys
import os
from datetime import datetime
from collections import defaultdict
try:
task = str(sys.argv[1])
#args = str(sys.argv[2])
except IndexError:
print "Enter one of the above command line arguments "
sys.exit(1)
if task == 'setup' :
try:
import boto
print "=================================================================================="
print "All required libraries have already been installed , proceed to download"
print "Please note ,all downloaded files will be saved in s3_downloads folder on Desktop "
print "Example downlad command : python cloudcrate-download.py download"
print "For list of all commands,type : python cloudcrate-download.py "
print "=================================================================================="
except ImportError,e:
print "============================================================="
raw_input("Missing Libraries - Press Hit Enter to Install them")
print "============================================================="
print "Installing boto - python interface to Amazon S3"
print "============================================================="
os.system("tar -zxvf boto.0.tar.gz")
os.chdir("boto-2.34.0")
os.system("sudo python setup.py install ")
print "=================================================================================="
print "All required libraries have already been installed , proceed to sync"
print "Please note ,all downloaded files will be saved in s3_downloads folder on Desktop "
print "Example download command : python cloudcrate-download.py download"
print "For list of all commands,type : python cloudcrate-download.py "
print "=================================================================================="
if task == 'download' :
import json
from boto.s3.connection import S3Connection
from boto.s3.key import Key
download_last_modified_dict = {}
conn = S3Connection('AKIAJ332D5S6IQ7WITSQ', 'G2WNp8xGxQPSxEcurBOTI32okS/izRmz2KPAJO24')
bucket = conn.get_bucket('cloudcrate.hari')
#os.mkdir('~/Desktop/downloaded/')
if not os.path.exists(os.path.expanduser('~/Desktop/s3_downloads')):
print "===================================================="
print "Looks like you havent downloaded the files even once"
print "===================================================="
os.mkdir(os.path.expanduser('~/Desktop/s3_downloads/'))
print "Created a folder of name s3_downloads on your Desktop"
print "===================================================="
os.chdir(os.path.expanduser('~/Desktop/s3_downloads'))
for key in bucket.list():
download_last_modified_dict[key.name]= key.last_modified
downloaded_file = key.get_contents_to_filename(key.name)
#print key.last_modified
print "Downloaded file " , key.name
#print download_last_modified_dict
json.dump(download_last_modified_dict, open("download_last_modified.txt",'w'))
else:
print "============================================================================================="
print "the s3_downloads folder already exists , will now selectively download files into this folder"
print "============================================================================================="
os.chdir(os.path.expanduser('~/Desktop/s3_downloads'))
download_last_modified_dict = json.load(open("download_last_modified.txt"))
print "loaded json from file into memory and it looks like below"
# for key in download_last_modified_dict:
# print download_last_modified_dict[key]
# print type(bucket.list())
# for key in bucket.list():
# #print type(key)
# print key.name, key.last_modified , download_last_modified_dict[key.name]
try:
for key in bucket.list():
if key.last_modified > download_last_modified_dict[key.name]:
#print "from S3 " ,key.last_modified ,"from file", download_last_modified_dict[key.name]
print "Downloading file based on timestamp comparison",key.name
download_last_modified_dict[key.name]= key.last_modified
downloaded_file = key.get_contents_to_filename(key.name)
else:
#print "Skipping download of file",key.name , "last updated time that I just read from S3",key.last_modified
print "Skipping download of file",key.name
json.dump(download_last_modified_dict, open("download_last_modified.txt",'w'))
except KeyError:
#print "KeyError" , key.name
downloaded_file = key.get_contents_to_filename(key.name)
print "Downloaded in the KeyError code block",key.name
download_last_modified_dict[key.name]=key.last_modified
json.dump(download_last_modified_dict, open("download_last_modified.txt",'w'))