Skip to content
This repository has been archived by the owner on Apr 14, 2020. It is now read-only.

flake8 check #60

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

flake8 check #60

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
197 changes: 118 additions & 79 deletions cc/apollo.py
View file
Open in desktop

Large diffs are not rendered by default.

63 changes: 36 additions & 27 deletions cc/cc.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
from cc.common import BaseServerMixin, is_domain, configure_options
from cc.common import CommonHandler, kill_children


class BaseHandler(CommonHandler):

def initialize(self):
Expand Down Expand Up @@ -143,15 +144,15 @@ def put(self, target_id):
payload['weight'] = max(content['weight'], 0)
if 'options' in content:
for key, value in content['options'].items():
payload['options.'+key] = value
payload['options.' + key] = value
if not payload:
self.error('Nothing to update')
cursor = self.motor.data.targets
result = yield cursor.update({'_id': target_id}, {'$set': payload})
if result['updatedExisting']:
self.set_status(200)
else:
self.error('invalid '+target_id)
self.error('invalid ' + target_id)


class CoreAssignHandler(BaseHandler):
Expand Down Expand Up @@ -233,7 +234,7 @@ def post(self):
{'engines': 1})
if core_engine not in result['engines']:
self.error('Core engine not allowed for this target')

if target_id not in self.application.shards:
self.error('Target specified has no shards')
shards = self.application.shards[target_id]
Expand Down Expand Up @@ -271,7 +272,7 @@ def weighted_sample(d):
# exclusive prefix sum
for index, value in enumerate(values):
if index > 0:
values[index] += values[index-1]
values[index] += values[index - 1]
x = random.uniform(0, values[-1])
for index, value in enumerate(values):
if value > x:
Expand All @@ -285,7 +286,8 @@ def weighted_sample(d):
shards = self.application.shards[target_id]

def scv_online(scv_name):
if self.scvs[scv_name]['fail_count'] < self.application._max_ws_fails:
if self.scvs[scv_name][
'fail_count'] < self.application._max_ws_fails:
return True
else:
return False
Expand All @@ -298,7 +300,7 @@ def scv_online(scv_name):
if user:
msg['user'] = user
try:
password = self.scvs[scv]['password']
password = self.scvs[scv]['password']
headers = {'Authorization': password}
reply = yield self.fetch(scv, '/streams/activate',
method='POST', body=json.dumps(msg),
Expand All @@ -307,14 +309,16 @@ def scv_online(scv_name):
token = json.loads(reply.body.decode())["token"]
host = self.scvs[scv]['host']
body = {'token': token,
'url': 'https://'+host+'/core/start'}
'url': 'https://' + host + '/core/start'}
self.write(body)
return self.set_status(200)
elif reply.code == 400:
message = "Assignment returned 400, target_id: "+target_id+" scv: "+scv
message = ("Assignment returned 400, target_id: " +
target_id + " scv: " + scv)
logging.getLogger('tornado.application').critical(message)
except tornado.httpclient.HTTPError as e:
message = "Assignment failed, target_id: "+target_id+" scv: "+scv
except tornado.httpclient.HTTPError:
message = ("Assignment failed, target_id: " + target_id +
" scv: " + scv)
logging.getLogger('tornado.application').critical(message)
self.error('no streams available for the target')

Expand Down Expand Up @@ -421,12 +425,12 @@ def put(self, target_id):
streams_cursor = self.motor["streams"]
scv_names = yield streams_cursor.collection_names()
scv_names.remove('system.indexes')
shard_copy = {}
# shard_copy = {}

success = True

headers = {'Authorization': self.request.headers['Authorization']}

for scv_id in scv_names:
cursor = streams_cursor[scv_id]
results = cursor.find({'target_id': target_id}, {'_id': 1})
Expand All @@ -436,11 +440,14 @@ def put(self, target_id):

print("deleting stream", stream_id)

reply = yield self.fetch(scv_id, '/streams/delete/'+stream_id, method='PUT', headers=headers, body='')
reply = yield self.fetch(scv_id,
'/streams/delete/' + stream_id,
method='PUT', headers=headers,
body='')
if reply.code != 200:
success = False
print('\n\nCANT DELETE TARGET :(\n\n', str(reply.body))
self.error("Unable to delete, error:"+str(reply.body))
self.error("Unable to delete, error:" + str(reply.body))
else:
print('deleted', stream_id)

Expand All @@ -449,6 +456,7 @@ def put(self, target_id):
yield cursor.remove({'_id': target_id})
return self.set_status(200)


# TODO: Cache?
class TargetStreamsHandler(BaseHandler):

Expand Down Expand Up @@ -486,7 +494,7 @@ def get(self, target_id):
for scv_id in scv_names:
cursor = streams_cursor[scv_id]
results = cursor.find({'target_id': target_id},
{'_id': 1}) # stream_id
{'_id': 1}) # stream_id

while (yield results.fetch_next):
document = results.next_object()
Expand Down Expand Up @@ -600,9 +608,9 @@ def post(self):
if not is_manager:
self.error('Not a manager', code=401)
content = json.loads(self.request.body.decode())
#----------------#
# verify request #
#----------------#
# ---------------- #
# verify request #
# ---------------- #
engines = content['engines']

if 'stage' in content:
Expand All @@ -621,9 +629,9 @@ def post(self):
else:
weight = max(content['weight'], 0)

#------------#
# write data #
#------------#
# ------------ #
# write data #
# ------------ #
target_id = str(uuid.uuid4())
owner = yield self.get_current_user()
payload = {
Expand Down Expand Up @@ -685,7 +693,7 @@ def post(self):
content = json.loads(self.request.body.decode())
for required_key in ['engine', 'description']:
if required_key not in content:
self.error('missing: '+required_key)
self.error('missing: ' + required_key)
stored_id = str(uuid.uuid4())
content['_id'] = stored_id
content['creation_date'] = time.time()
Expand Down Expand Up @@ -829,16 +837,16 @@ def __init__(self, name, redis_options, mongo_options):
(r'/targets/update/(.*)', TargetUpdateHandler),
(r'/scvs/status', SCVStatusHandler),
(r'/targets/streams/(.*)', TargetStreamsHandler)
])
])

@tornado.gen.coroutine
def fetch(self, scv_id, path, **kwargs):
""" Make a request to a particular SCV and keep track of whether or not
it is alive.
it is alive.

"""
host = self.scvs[scv_id]['host']
uri = 'https://'+host+path
uri = 'https://' + host + path
client = tornado.httpclient.AsyncHTTPClient()
try:
reply = yield client.fetch(uri, validate_cert=is_domain(host),
Expand Down Expand Up @@ -872,7 +880,7 @@ def _cache_shards(self):
for scv_id in scv_names:
cursor = streams_cursor[scv_id]
results = cursor.find({'status': 'enabled'},
{'target_id': 1}) # stream_id
{'target_id': 1}) # stream_id

while (yield results.fetch_next):
document = results.next_object()
Expand All @@ -890,6 +898,7 @@ def _check_scvs(self):
for key, scv_name in self.scvs.items():
yield self.fetch(scv_name, '/')


def stop_parent(sig, frame):
kill_children()

Expand Down Expand Up @@ -955,7 +964,7 @@ def start():
pulse2 = tornado.ioloop.PeriodicCallback(app._cache_shards, 60000)
pulse2.start()
tornado.ioloop.IOLoop.instance().start()
except SystemExit as e:
except SystemExit:
print('! parent is shutting down ...')
# app.db.shutdown()
sys.exit(0)
12 changes: 5 additions & 7 deletions cc/common.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,6 @@
# License for the specific language governing permissions and limitations
# under the License.

import subprocess
import sys
import time
import tornado
import ipaddress
import os
Expand Down Expand Up @@ -54,6 +51,7 @@ def is_domain(url):
def preexec(): # Don't forward signals.
os.setpgrp()


class CommonHandler(tornado.web.RequestHandler):

def set_default_headers(self):
Expand Down Expand Up @@ -128,17 +126,17 @@ def base_init(self, name, redis_options, mongo_options):
""" A BaseServer is a server that is connected to both a redis server
and a mongo server """
self.name = name
self.data_folder = name+'_data'
self.data_folder = name + '_data'
if not os.path.exists(self.data_folder):
os.makedirs(self.data_folder)
access_channel = logging.FileHandler(os.path.join(self.data_folder,
'access.log'))
'access.log'))
logging.getLogger('tornado.access').addHandler(access_channel)
app_channel = logging.FileHandler(os.path.join(self.data_folder,
'application.log'))
'application.log'))
logging.getLogger('tornado.application').addHandler(app_channel)
general_channel = logging.FileHandler(os.path.join(self.data_folder,
'general.log'))
'general.log'))
logging.getLogger('tornado.general').addHandler(general_channel)
self._mongo_options = mongo_options

Expand Down
29 changes: 15 additions & 14 deletions docs/conf.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,18 +21,19 @@

from sphinx.ext import autodoc


class SimpleDocumenter(autodoc.MethodDocumenter):
objtype = "simple"

#do not indent the content
# do not indent the content
content_indent = ""

#do not add a header to the docstring
# do not add a header to the docstring
def add_directive_header(self, sig):
pass


#add_autodocumenter(SimpleDocumenter)
# add_autodocumenter(SimpleDocumenter)
def setup(app):
app.add_autodocumenter(SimpleDocumenter)

Expand Down Expand Up @@ -209,22 +210,22 @@ def setup(app):
# -- Options for LaTeX output ---------------------------------------------

latex_elements = {
# The paper size ('letterpaper' or 'a4paper').
#'papersize': 'letterpaper',
# The paper size ('letterpaper' or 'a4paper').
#'papersize': 'letterpaper',

# The font size ('10pt', '11pt' or '12pt').
#'pointsize': '10pt',
# The font size ('10pt', '11pt' or '12pt').
#'pointsize': '10pt',

# Additional stuff for the LaTeX preamble.
#'preamble': '',
# Additional stuff for the LaTeX preamble.
#'preamble': '',
}

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title,
# author, documentclass [howto, manual, or own class]).
latex_documents = [
('index', 'docs.tex', 'docs Documentation',
'Yutong Zhao', 'manual'),
('index', 'docs.tex', 'docs Documentation',
'Yutong Zhao', 'manual'),
]

# The name of an image file (relative to this directory) to place at the top of
Expand Down Expand Up @@ -267,9 +268,9 @@ def setup(app):
# (source start file, target name, title, author,
# dir menu entry, description, category)
texinfo_documents = [
('index', 'docs', 'docs Documentation',
'Yutong Zhao', 'docs', 'One line description of project.',
'Miscellaneous'),
('index', 'docs', 'docs Documentation',
'Yutong Zhao', 'docs', 'One line description of project.',
'Miscellaneous'),
]

# Documents to append as an appendix to all manuals.
Expand Down
17 changes: 10 additions & 7 deletions docs/docs_server.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,11 +25,13 @@ def secret_cookie():


class TestHandler(tornado.web.RequestHandler):

def get(self):
return self.write("I'm alive!")


class MainHandler(tornado.web.RequestHandler):

def get(self):
if self.get_cookie("cookie_monster") == secret_cookie():
self.redirect("/static/index.html")
Expand Down Expand Up @@ -86,17 +88,18 @@ def get(self):
body=urlencode(parameters))
access_token = dict(parse_qsl(reply.body.decode()))\
['access_token']
headers = {'Authorization': 'token '+access_token,
headers = {'Authorization': 'token ' + access_token,
'User-Agent': 'Tornado OAuth'}
client = tornado.httpclient.AsyncHTTPClient()
uri = "https://api.github.com/user"

reply = yield client.fetch(uri, headers=headers)
content = json.loads(reply.body.decode())
username = content['login']
uri = "https://api.github.com/repos/proteneer/backend"+\
"/collaborators/"+username
headers['Authorization'] = 'token '+self.proteneer_access_token
uri = "https://api.github.com/repos/proteneer/backend" +\
"/collaborators/" + username
headers['Authorization'] = 'token ' + \
self.proteneer_access_token
reply = yield client.fetch(uri, headers=headers)
self.set_cookie("cookie_monster", secret_cookie())
self.redirect('/static/index.html')
Expand All @@ -117,11 +120,11 @@ def get(self):
parameters = {
'client_id': self.client_id,
'state': self.x_site_token,
'redirect_uri': hostname()+"auth/github",
'redirect_uri': hostname() + "auth/github",
}

uri = "https://github.com/login/oauth/authorize?"
self.redirect(uri+urlencode(parameters))
self.redirect(uri + urlencode(parameters))


class AuthStaticFileHandler(tornado.web.StaticFileHandler):
Expand All @@ -142,7 +145,7 @@ def get_current_user(self):
(r"/", MainHandler),
(r"/test", TestHandler),
(r'/static/(.*)', AuthStaticFileHandler, {'path': "_build/html"})
])
])

application.listen(os.environ.get("PORT", 9430))
tornado.ioloop.IOLoop.instance().start()
2 changes: 1 addition & 1 deletion siegetank/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,4 +3,4 @@
from .base import load_target
from .base import load_stream
from .base import add_target
from .base import scvs
from .base import scvs
Loading