Skip to content

Commit

Permalink
init migration to python3
Browse files Browse the repository at this point in the history
  • Loading branch information
@rhanka
rhanka committed Apr 11, 2020
1 parent d3bcfff commit c7aa67a
Show file tree
Hide file tree
Showing 8 changed files with 36 additions and 39 deletions.
2 changes: 1 addition & 1 deletion Dockerfile
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
#######################
# Step 1: Base target #
#######################
FROM python:2-slim as base
FROM python:3-slim as base
ARG http_proxy
ARG https_proxy
ARG no_proxy
Expand Down
37 changes: 17 additions & 20 deletions code/api.py
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
#!/usr/bin/env python2
#!/usr/bin/env python3
# -*- coding: utf-8 -*-

# import basics
Expand Down Expand Up @@ -162,7 +162,7 @@ def get(self):
else:
return {
"me": str(current_user.name),
"others": config.conf["users"].keys()
"others": list(config.conf["users"].keys())
}

@api.route('/groups/', endpoint='groups')
Expand Down Expand Up @@ -234,10 +234,7 @@ def post(self):
class OAuthList(Resource):
def get(self):
return {
'providers': list(filter(
lambda x: config.conf['global']['api']['oauth'][x]['id'] != None,
config.conf['global']['api']['oauth'].keys()
))
'providers': list([x for x in list(config.conf['global']['api']['oauth'].keys()) if config.conf['global']['api']['oauth'][x]['id'] != None])
}

@api.route('/authorize/<provider>', endpoint='authorize/<provider>')
Expand Down Expand Up @@ -418,7 +415,7 @@ def put(self, project):
'''create a project'''
if (project == "conf"):
api.abort(403)
elif project in config.conf["global"]["projects"].keys():
elif project in list(config.conf["global"]["projects"].keys()):
api.abort(400, 'project "{}" already exists'.format(project))
else:
try:
Expand Down Expand Up @@ -452,7 +449,7 @@ def delete(self, project):
'''delete a project'''
if (project == "conf"):
api.abort(403)
elif project in config.conf["global"]["projects"].keys():
elif project in list(config.conf["global"]["projects"].keys()):
response = {project: "not deleted"}
try:
dirname = os.path.join(config.conf["global"][
Expand Down Expand Up @@ -566,7 +563,7 @@ class DatasetApi(Resource):
def get(self, dataset):
'''get json of a configured dataset'''
config.read_conf()
if (dataset in config.conf["datasets"].keys()):
if (dataset in list(config.conf["datasets"].keys())):
try:
response = dict(config.conf["datasets"][dataset])
try:
Expand Down Expand Up @@ -690,9 +687,9 @@ def put(self, dataset, action):
config.init()
config.read_conf()
if (action == "validation"):
if (not(dataset in config.conf["datasets"].keys())):
if (not(dataset in list(config.conf["datasets"].keys()))):
return api.abort(404, {"dataset": dataset, "status": "dataset not found"})
if not("validation" in config.conf["datasets"][dataset].keys()):
if not("validation" in list(config.conf["datasets"][dataset].keys())):
return api.abort(403, {"dataset": dataset, "status": "validation not allowed"})
if ((config.conf["datasets"][dataset]["validation"] == True) | (isinstance(config.conf["datasets"][dataset]["validation"], OrderedDict))):
try:
Expand All @@ -702,7 +699,7 @@ def put(self, dataset, action):
"datasets"][dataset]["validation"])
except:
cfg = config.conf["global"]["validation"]
for conf in cfg.keys():
for conf in list(cfg.keys()):
configfile = os.path.join(config.conf["global"]["paths"][
"validation"], secure_filename(conf + ".json"))
dic = {
Expand All @@ -719,9 +716,9 @@ def put(self, dataset, action):
else:
return api.abort(403, {"dataset": dataset, "status": "validation not allowed"})
elif (action == "search"):
if (not(dataset in config.conf["datasets"].keys())):
if (not(dataset in list(config.conf["datasets"].keys()))):
return api.abort(404, {"dataset": dataset, "status": "dataset not found"})
if not("search" in config.conf["datasets"][dataset].keys()):
if not("search" in list(config.conf["datasets"][dataset].keys())):
return api.abort(403, {"dataset": dataset, "status": "search not allowed"})
if ((config.conf["datasets"][dataset]["search"] == True) | (isinstance(config.conf["datasets"][dataset]["search"], OrderedDict))):
try:
Expand All @@ -731,7 +728,7 @@ def put(self, dataset, action):
"datasets"][dataset]["search"])
except:
cfg = config.conf["global"]["search"]
for config in cfg.keys():
for config in list(cfg.keys()):
configfile = os.path.join(config.conf["global"]["paths"][
"search"], secure_filename(config + ".json"))
dic = {
Expand Down Expand Up @@ -960,10 +957,10 @@ def post(self, recipe, action):
if isinstance(r.df, pd.DataFrame):
df = r.df.fillna("")
try:
return jsonify({"data": df.T.to_dict().values(), "log": str(r.log.writer.getvalue())})
return jsonify({"data": list(df.T.to_dict().values()), "log": str(r.log.writer.getvalue())})
except:
df = df.applymap(lambda x: str(x))
return jsonify({"data": df.T.to_dict().values(), "log": str(r.log.writer.getvalue())})
return jsonify({"data": list(df.T.to_dict().values()), "log": str(r.log.writer.getvalue())})
else:
return {"log": r.log.writer.getvalue()}

Expand Down Expand Up @@ -996,10 +993,10 @@ def put(self, recipe, action):
if (r.df.shape[0] == 0):
return {"data": [{"result": "empty"}], "log": r.callback["log"]}
try:
return jsonify({"data": df.T.to_dict().values(), "log": r.callback["log"]})
return jsonify({"data": list(df.T.to_dict().values()), "log": r.callback["log"]})
except:
df = df.applymap(lambda x: unicode_safe(x))
return jsonify({"data": df.T.to_dict().values(), "log": r.callback["log"]})
return jsonify({"data": list(df.T.to_dict().values()), "log": r.callback["log"]})
else:
return {"data": [{"result": "empty"}], "log": r.callback["log"]}
elif (action == "run"):
Expand Down Expand Up @@ -1063,7 +1060,7 @@ def get(self):

date = re.sub(
r"(\d{4}.?\d{2}.?\d{2})T(..:..).*log", r"\1-\2", file)
if (recipe in config.conf["recipes"].keys()):
if (recipe in list(config.conf["recipes"].keys())):
if (check_rights(current_user, config.conf["recipes"][recipe]["project"], "read")):
if running:
response["running"].append(
Expand Down
2 changes: 1 addition & 1 deletion code/automata.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ def next_state(self, states, input):
def get_inputs(self, states):
inputs = set()
for state in states:
inputs.update(self.transitions.get(state, {}).keys())
inputs.update(list(self.transitions.get(state, {}).keys()))
return inputs

def to_dfa(self):
Expand Down
4 changes: 2 additions & 2 deletions code/config.py
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
#!/usr/bin/env python2
#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import yaml as y
Expand Down Expand Up @@ -107,7 +107,7 @@ def deepupdate(original, update):

def check_conf(cfg, project, source):
for key in list(["recipes", "datasets", "connectors"]):
if (key in cfg.keys()):
if (key in list(cfg.keys())):
for obj in cfg[key]:
cfg[key][obj]["source"] = source
cfg[key][obj]["project"] = project
Expand Down
2 changes: 1 addition & 1 deletion code/log.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ def __init__(self,name=None,test=False):
self.writer=StringIO()
self.level=2
else:
if ("log" in config.conf["global"].keys()):
if ("log" in list(config.conf["global"].keys())):
try:
self.dir=config.conf["global"]["log"]["dir"]
except:
Expand Down
24 changes: 12 additions & 12 deletions code/recipes.py
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
#!/usr/bin/env python2
#!/usr/bin/env python3
# -*- coding: utf-8 -*-


Expand Down Expand Up @@ -971,7 +971,7 @@ def __init__(self, name=None, args={}):

# initiate input connection : creater a reader or use inmemory dataset
try:
if ("input" in self.args.keys()):
if ("input" in list(self.args.keys())):
self.input = Dataset(self.args.input, parent=self)
elif ((type(self.conf["input"]) == str) | (type(self.conf["input"]) == unicode)):
self.input = Dataset(self.conf["input"], parent=self)
Expand All @@ -989,7 +989,7 @@ def __init__(self, name=None, args={}):
self.input.select = None

try:
r = self.conf["input"]["filter"].keys()[0]
r = list(self.conf["input"]["filter"].keys())[0]
self.input.filter = Recipe(name=r, args=self.conf[
"input"]["filter"][r])
except:
Expand Down Expand Up @@ -1029,9 +1029,9 @@ def __init__(self, name=None, args={}):
self.threads = 1

try:
if ("before" in self.conf.keys()):
if ("before" in list(self.conf.keys())):
self.before = self.conf["before"]
elif ("run" in self.conf.keys()):
elif ("run" in list(self.conf.keys())):
self.before = self.conf["run"]
else:
self.before = []
Expand All @@ -1045,7 +1045,7 @@ def __init__(self, name=None, args={}):

# initiate output connection : create a writer or use current dataframe
try:
if ("output" in self.args.keys()):
if ("output" in list(self.args.keys())):
self.output = Dataset(self.args.output, parent=self)
elif ((type(self.conf["output"]) == str) | (type(self.conf["output"]) == unicode)):
self.output = Dataset(self.conf["output"], parent=self)
Expand Down Expand Up @@ -1088,7 +1088,7 @@ def __init__(self, name=None, args={}):
try:
self.steps = []
for s in self.conf["steps"]:
function = s.keys()[0]
function = list(s.keys())[0]
try:
self.steps.append(Recipe(name=function, args=s[function]))
except:
Expand Down Expand Up @@ -1234,7 +1234,7 @@ def run_chunk(self, i, df, desc=None, queue=None, supervisor=None):
df.shape[0], self.input.name, self.name))
if (self.type == "internal"):
df = getattr(self.__class__, "internal_" + self.name)(self, df=df, desc=desc)
elif((len(self.steps) > 0) | ("steps" in self.conf.keys())):
elif((len(self.steps) > 0) | ("steps" in list(self.conf.keys()))):
for recipe in self.steps:
try:
self.log.write("{} > {}".format(
Expand Down Expand Up @@ -1306,13 +1306,13 @@ def supervise(self, queue, supervisor):
self.log.chunk = "supervisor"
self.log.write("initiating supervisor")
supervisor["end"] = False
if ("supervisor_interval" in self.conf.keys()):
if ("supervisor_interval" in list(self.conf.keys())):
wait = self.conf["supervisor_interval"]
while (supervisor["end"] == False):
try:
writing = len([x for x in supervisor.keys()
writing = len([x for x in list(supervisor.keys())
if (supervisor[x] == "writing")])
running = len([x for x in supervisor.keys()
running = len([x for x in list(supervisor.keys())
if (supervisor[x] == "run_chunk")])
self.log.write("threads - running: {}/{} - writing: {}/{}/{} ".format(
running, self.threads, writing, queue.qsize(), self.output.thread_count))
Expand Down Expand Up @@ -1476,7 +1476,7 @@ def run(self, head=None, write_queue=None, supervisor=None):
self.df = df
except:
self.df = None
if ((len(self.steps) > 0) | ("steps" in self.conf.keys())):
if ((len(self.steps) > 0) | ("steps" in list(self.conf.keys()))):
self.log.write(msg="in main loop of {} {}".format(
self.name, str(self.input.select)), error=error)
else:
Expand Down
2 changes: 1 addition & 1 deletion code/security.py
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
#!/usr/bin/env python2
#!/usr/bin/env python3
# -*- coding: utf-8 -*-


Expand Down
2 changes: 1 addition & 1 deletion code/tools.py
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
#!/usr/bin/env python2
#!/usr/bin/env python3
# -*- coding: utf-8 -*-

# various libs
Expand Down

0 comments on commit c7aa67a

Please sign in to comment.