diff --git a/Dockerfile b/Dockerfile index 0c62630..c6fb63e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ ####################### # Step 1: Base target # ####################### -FROM python:2-slim as base +FROM python:3-slim as base ARG http_proxy ARG https_proxy ARG no_proxy diff --git a/code/api.py b/code/api.py index 6db059b..81a56b4 100755 --- a/code/api.py +++ b/code/api.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # import basics @@ -162,7 +162,7 @@ def get(self): else: return { "me": str(current_user.name), - "others": config.conf["users"].keys() + "others": list(config.conf["users"].keys()) } @api.route('/groups/', endpoint='groups') @@ -234,10 +234,7 @@ def post(self): class OAuthList(Resource): def get(self): return { - 'providers': list(filter( - lambda x: config.conf['global']['api']['oauth'][x]['id'] != None, - config.conf['global']['api']['oauth'].keys() - )) + 'providers': list([x for x in list(config.conf['global']['api']['oauth'].keys()) if config.conf['global']['api']['oauth'][x]['id'] != None]) } @api.route('/authorize/', endpoint='authorize/') @@ -418,7 +415,7 @@ def put(self, project): '''create a project''' if (project == "conf"): api.abort(403) - elif project in config.conf["global"]["projects"].keys(): + elif project in list(config.conf["global"]["projects"].keys()): api.abort(400, 'project "{}" already exists'.format(project)) else: try: @@ -452,7 +449,7 @@ def delete(self, project): '''delete a project''' if (project == "conf"): api.abort(403) - elif project in config.conf["global"]["projects"].keys(): + elif project in list(config.conf["global"]["projects"].keys()): response = {project: "not deleted"} try: dirname = os.path.join(config.conf["global"][ @@ -566,7 +563,7 @@ class DatasetApi(Resource): def get(self, dataset): '''get json of a configured dataset''' config.read_conf() - if (dataset in config.conf["datasets"].keys()): + if (dataset in list(config.conf["datasets"].keys())): try: response = dict(config.conf["datasets"][dataset]) try: @@ -690,9 +687,9 @@ def put(self, dataset, action): config.init() config.read_conf() if (action == "validation"): - if (not(dataset in config.conf["datasets"].keys())): + if (not(dataset in list(config.conf["datasets"].keys()))): return api.abort(404, {"dataset": dataset, "status": "dataset not found"}) - if not("validation" in config.conf["datasets"][dataset].keys()): + if not("validation" in list(config.conf["datasets"][dataset].keys())): return api.abort(403, {"dataset": dataset, "status": "validation not allowed"}) if ((config.conf["datasets"][dataset]["validation"] == True) | (isinstance(config.conf["datasets"][dataset]["validation"], OrderedDict))): try: @@ -702,7 +699,7 @@ def put(self, dataset, action): "datasets"][dataset]["validation"]) except: cfg = config.conf["global"]["validation"] - for conf in cfg.keys(): + for conf in list(cfg.keys()): configfile = os.path.join(config.conf["global"]["paths"][ "validation"], secure_filename(conf + ".json")) dic = { @@ -719,9 +716,9 @@ def put(self, dataset, action): else: return api.abort(403, {"dataset": dataset, "status": "validation not allowed"}) elif (action == "search"): - if (not(dataset in config.conf["datasets"].keys())): + if (not(dataset in list(config.conf["datasets"].keys()))): return api.abort(404, {"dataset": dataset, "status": "dataset not found"}) - if not("search" in config.conf["datasets"][dataset].keys()): + if not("search" in list(config.conf["datasets"][dataset].keys())): return api.abort(403, {"dataset": dataset, "status": "search not allowed"}) if ((config.conf["datasets"][dataset]["search"] == True) | (isinstance(config.conf["datasets"][dataset]["search"], OrderedDict))): try: @@ -731,7 +728,7 @@ def put(self, dataset, action): "datasets"][dataset]["search"]) except: cfg = config.conf["global"]["search"] - for config in cfg.keys(): + for config in list(cfg.keys()): configfile = os.path.join(config.conf["global"]["paths"][ "search"], secure_filename(config + ".json")) dic = { @@ -960,10 +957,10 @@ def post(self, recipe, action): if isinstance(r.df, pd.DataFrame): df = r.df.fillna("") try: - return jsonify({"data": df.T.to_dict().values(), "log": str(r.log.writer.getvalue())}) + return jsonify({"data": list(df.T.to_dict().values()), "log": str(r.log.writer.getvalue())}) except: df = df.applymap(lambda x: str(x)) - return jsonify({"data": df.T.to_dict().values(), "log": str(r.log.writer.getvalue())}) + return jsonify({"data": list(df.T.to_dict().values()), "log": str(r.log.writer.getvalue())}) else: return {"log": r.log.writer.getvalue()} @@ -996,10 +993,10 @@ def put(self, recipe, action): if (r.df.shape[0] == 0): return {"data": [{"result": "empty"}], "log": r.callback["log"]} try: - return jsonify({"data": df.T.to_dict().values(), "log": r.callback["log"]}) + return jsonify({"data": list(df.T.to_dict().values()), "log": r.callback["log"]}) except: df = df.applymap(lambda x: unicode_safe(x)) - return jsonify({"data": df.T.to_dict().values(), "log": r.callback["log"]}) + return jsonify({"data": list(df.T.to_dict().values()), "log": r.callback["log"]}) else: return {"data": [{"result": "empty"}], "log": r.callback["log"]} elif (action == "run"): @@ -1063,7 +1060,7 @@ def get(self): date = re.sub( r"(\d{4}.?\d{2}.?\d{2})T(..:..).*log", r"\1-\2", file) - if (recipe in config.conf["recipes"].keys()): + if (recipe in list(config.conf["recipes"].keys())): if (check_rights(current_user, config.conf["recipes"][recipe]["project"], "read")): if running: response["running"].append( diff --git a/code/automata.py b/code/automata.py index fec4852..2ae5190 100644 --- a/code/automata.py +++ b/code/automata.py @@ -57,7 +57,7 @@ def next_state(self, states, input): def get_inputs(self, states): inputs = set() for state in states: - inputs.update(self.transitions.get(state, {}).keys()) + inputs.update(list(self.transitions.get(state, {}).keys())) return inputs def to_dfa(self): diff --git a/code/config.py b/code/config.py index ed4fe1e..5f86140 100644 --- a/code/config.py +++ b/code/config.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # -*- coding: utf-8 -*- import yaml as y @@ -107,7 +107,7 @@ def deepupdate(original, update): def check_conf(cfg, project, source): for key in list(["recipes", "datasets", "connectors"]): - if (key in cfg.keys()): + if (key in list(cfg.keys())): for obj in cfg[key]: cfg[key][obj]["source"] = source cfg[key][obj]["project"] = project diff --git a/code/log.py b/code/log.py index f0da1f5..5a57626 100644 --- a/code/log.py +++ b/code/log.py @@ -34,7 +34,7 @@ def __init__(self,name=None,test=False): self.writer=StringIO() self.level=2 else: - if ("log" in config.conf["global"].keys()): + if ("log" in list(config.conf["global"].keys())): try: self.dir=config.conf["global"]["log"]["dir"] except: diff --git a/code/recipes.py b/code/recipes.py index a1024c5..fbbea73 100644 --- a/code/recipes.py +++ b/code/recipes.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # -*- coding: utf-8 -*- @@ -971,7 +971,7 @@ def __init__(self, name=None, args={}): # initiate input connection : creater a reader or use inmemory dataset try: - if ("input" in self.args.keys()): + if ("input" in list(self.args.keys())): self.input = Dataset(self.args.input, parent=self) elif ((type(self.conf["input"]) == str) | (type(self.conf["input"]) == unicode)): self.input = Dataset(self.conf["input"], parent=self) @@ -989,7 +989,7 @@ def __init__(self, name=None, args={}): self.input.select = None try: - r = self.conf["input"]["filter"].keys()[0] + r = list(self.conf["input"]["filter"].keys())[0] self.input.filter = Recipe(name=r, args=self.conf[ "input"]["filter"][r]) except: @@ -1029,9 +1029,9 @@ def __init__(self, name=None, args={}): self.threads = 1 try: - if ("before" in self.conf.keys()): + if ("before" in list(self.conf.keys())): self.before = self.conf["before"] - elif ("run" in self.conf.keys()): + elif ("run" in list(self.conf.keys())): self.before = self.conf["run"] else: self.before = [] @@ -1045,7 +1045,7 @@ def __init__(self, name=None, args={}): # initiate output connection : create a writer or use current dataframe try: - if ("output" in self.args.keys()): + if ("output" in list(self.args.keys())): self.output = Dataset(self.args.output, parent=self) elif ((type(self.conf["output"]) == str) | (type(self.conf["output"]) == unicode)): self.output = Dataset(self.conf["output"], parent=self) @@ -1088,7 +1088,7 @@ def __init__(self, name=None, args={}): try: self.steps = [] for s in self.conf["steps"]: - function = s.keys()[0] + function = list(s.keys())[0] try: self.steps.append(Recipe(name=function, args=s[function])) except: @@ -1234,7 +1234,7 @@ def run_chunk(self, i, df, desc=None, queue=None, supervisor=None): df.shape[0], self.input.name, self.name)) if (self.type == "internal"): df = getattr(self.__class__, "internal_" + self.name)(self, df=df, desc=desc) - elif((len(self.steps) > 0) | ("steps" in self.conf.keys())): + elif((len(self.steps) > 0) | ("steps" in list(self.conf.keys()))): for recipe in self.steps: try: self.log.write("{} > {}".format( @@ -1306,13 +1306,13 @@ def supervise(self, queue, supervisor): self.log.chunk = "supervisor" self.log.write("initiating supervisor") supervisor["end"] = False - if ("supervisor_interval" in self.conf.keys()): + if ("supervisor_interval" in list(self.conf.keys())): wait = self.conf["supervisor_interval"] while (supervisor["end"] == False): try: - writing = len([x for x in supervisor.keys() + writing = len([x for x in list(supervisor.keys()) if (supervisor[x] == "writing")]) - running = len([x for x in supervisor.keys() + running = len([x for x in list(supervisor.keys()) if (supervisor[x] == "run_chunk")]) self.log.write("threads - running: {}/{} - writing: {}/{}/{} ".format( running, self.threads, writing, queue.qsize(), self.output.thread_count)) @@ -1476,7 +1476,7 @@ def run(self, head=None, write_queue=None, supervisor=None): self.df = df except: self.df = None - if ((len(self.steps) > 0) | ("steps" in self.conf.keys())): + if ((len(self.steps) > 0) | ("steps" in list(self.conf.keys()))): self.log.write(msg="in main loop of {} {}".format( self.name, str(self.input.select)), error=error) else: diff --git a/code/security.py b/code/security.py index 18aca33..958738c 100644 --- a/code/security.py +++ b/code/security.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # -*- coding: utf-8 -*- diff --git a/code/tools.py b/code/tools.py index 2278daa..ea5bbe2 100644 --- a/code/tools.py +++ b/code/tools.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # various libs