From f088840c2ad22160373cf19fa5f0a008182b63e9 Mon Sep 17 00:00:00 2001 From: akhil-rana Date: Fri, 18 Dec 2020 11:00:54 +0530 Subject: [PATCH 01/12] google-auth-initial-implemented --- Dockerfile | 1 + SlideServer.py | 30 ++++++++++++++++ gDriveDownload.py | 88 +++++++++++++++++++++++++++++++++++++++++++++++ requirements.txt | 3 ++ 4 files changed, 122 insertions(+) create mode 100644 gDriveDownload.py diff --git a/Dockerfile b/Dockerfile index c8b43a3..465d957 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,6 +24,7 @@ RUN pip3 install -r requirements.txt EXPOSE 4000 +EXPOSE 4001 #debug/dev only # ENV FLASK_APP SlideServer.py diff --git a/SlideServer.py b/SlideServer.py index 98e4172..7733a93 100644 --- a/SlideServer.py +++ b/SlideServer.py @@ -21,6 +21,9 @@ import csv import pathlib import logging +from gDriveDownload import startDownload, afterUrlAuth, callApi +from flask import after_this_request +from threading import Thread try: from io import BytesIO @@ -232,6 +235,33 @@ def urlUploadStatus(): else: return flask.Response(json.dumps({"uploaded": "False"}), status=200) +class getFile2(Thread): + def __init__(self, auth_url, local_server, wsgi_app, flow, creds): + Thread.__init__(self) + self.auth_url, self.local_server, self.wsgi_app, self.flow, self.creds = auth_url, local_server, wsgi_app, flow, creds + + def run(self): + if(self.auth_url != None): + self.creds = afterUrlAuth(self.local_server, self.flow, self.wsgi_app) + listOfFiles = callApi(self.creds) + app.logger.info(listOfFiles) + # print(listOfFiles, file=sys.stderr) a + +# Route to return google-picker API credentials +@app.route('/googleDriveUpload/getFile', methods=['POST']) +def gDriveGetFile(): + creds = None + auth_url, local_server, wsgi_app, flow, creds = startDownload() + thread_a = getFile2(auth_url, local_server, wsgi_app, flow, creds) + thread_a.start() + return flask.Response(json.dumps({"authURL": auth_url}), status=200) + + +# def getFile2(auth_url, local_server, wsgi_app, flow, creds): +# if(auth_url != None): +# creds = afterUrlAuth(local_server, flow, wsgi_app) +# listOfFiles = callApi(creds) +# print(listOfFiles, file=sys.stderr) # Workbench Dataset Creation help-routes diff --git a/gDriveDownload.py b/gDriveDownload.py new file mode 100644 index 0000000..74bdbbf --- /dev/null +++ b/gDriveDownload.py @@ -0,0 +1,88 @@ +from __future__ import print_function +import pickle +import os.path +import wsgiref.simple_server +import wsgiref.util +from googleapiclient.discovery import build +from google_auth_oauthlib.flow import InstalledAppFlow, _WSGIRequestHandler, _RedirectWSGIApp +from google.auth.transport.requests import Request +import sys +import os + +# If modifying these scopes, delete the file token.pickle. +SCOPES = ['https://www.googleapis.com/auth/drive.readonly'] + +def run_local_server( + self=InstalledAppFlow, + host="0.0.0.0", + port=4001, + authorization_prompt_message=InstalledAppFlow._DEFAULT_AUTH_PROMPT_MESSAGE, + success_message=InstalledAppFlow._DEFAULT_WEB_SUCCESS_MESSAGE, +): + wsgi_app = _RedirectWSGIApp(success_message) + local_server = wsgiref.simple_server.make_server( + host, port, wsgi_app, handler_class=_WSGIRequestHandler + ) + + self.redirect_uri = "http://localhost:4010/googleAuth/" + auth_url, _ = self.authorization_url() + + + print(authorization_prompt_message.format(url=auth_url)) + + return auth_url, local_server, wsgi_app, None + +def afterUrlAuth(local_server, flow, wsgi_app): + local_server.handle_request() + + # Note: using https here because oauthlib is very picky that + # OAuth 2.0 should only occur over https. + authorization_response = wsgi_app.last_request_uri.replace("http", "https") + flow.fetch_token(authorization_response=authorization_response) + # Save the credentials for the next run + with open('token.pickle', 'wb') as token: + pickle.dump(flow.credentials, token) + return flow.credentials + +def startDownload(): + creds = None + # The file token.pickle stores the user's access and refresh tokens, and is + # created automatically when the authorization flow completes for the first + # time. + if os.path.exists('token.pickle'): + with open('token.pickle', 'rb') as token: + creds = pickle.load(token) + return None, None, None, None, creds + # If there are no (valid) credentials available, let the user log in. + if not creds or not creds.valid: + if creds and creds.expired and creds.refresh_token: + creds.refresh(Request()) + else: + flow = InstalledAppFlow.from_client_secrets_file( + './credentials/google-drive.json', SCOPES) + auth_url, local_server, wsgi_app, creds = run_local_server(self=flow) + return auth_url, local_server, wsgi_app, flow, creds + # creds = afterUrlAuth(local_server, flow, wsgi_app) + + + +def callApi(creds): + service = build('drive', 'v3', credentials=creds) + + # Call the Drive v3 API + results = service.files().list( + pageSize=10, fields="nextPageToken, files(id, name)").execute() + items = results.get('files', []) + + if not items: + print('No files found.') + return items + else: + print('Files:') + return items + # for item in items: + # print(u'{0} ({1})'.format(item['name'], item['id'])) + + + +# startDownload() diff --git a/requirements.txt b/requirements.txt index 84520dd..488377f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,3 +5,6 @@ flask-cors requests numpy Pillow +google-api-python-client +google-auth-httplib2 +google-auth-oauthlib \ No newline at end of file From ccc96a52474aa908ad9d7bc872e12c6640c599d7 Mon Sep 17 00:00:00 2001 From: akhil-rana Date: Sat, 19 Dec 2020 07:19:55 +0530 Subject: [PATCH 02/12] file download intial implementation --- SlideServer.py | 33 +++++++-------- gDriveDownload.py | 105 +++++++++++++++++++++++++++++++++------------- 2 files changed, 92 insertions(+), 46 deletions(-) diff --git a/SlideServer.py b/SlideServer.py index 7733a93..ecd33b4 100644 --- a/SlideServer.py +++ b/SlideServer.py @@ -21,7 +21,7 @@ import csv import pathlib import logging -from gDriveDownload import startDownload, afterUrlAuth, callApi +from gDriveDownload import start, afterUrlAuth, callApi from flask import after_this_request from threading import Thread @@ -43,6 +43,7 @@ app.config['SECRET_KEY'] = os.urandom(24) app.config['ROI_FOLDER'] = "/images/roiDownload" + ALLOWED_EXTENSIONS = set(['svs', 'tif', 'tiff', 'vms', 'vmu', 'ndpi', 'scn', 'mrxs', 'bif', 'svslide']) @@ -235,33 +236,31 @@ def urlUploadStatus(): else: return flask.Response(json.dumps({"uploaded": "False"}), status=200) -class getFile2(Thread): - def __init__(self, auth_url, local_server, wsgi_app, flow, creds): +class getFile(Thread): + def __init__(self, auth_url, local_server, wsgi_app, flow, creds, userId, fileId): Thread.__init__(self) - self.auth_url, self.local_server, self.wsgi_app, self.flow, self.creds = auth_url, local_server, wsgi_app, flow, creds + self.auth_url, self.local_server, self.wsgi_app, self.flow, self.creds, self.userId, self.fileId = auth_url, local_server, wsgi_app, flow, creds, userId, fileId def run(self): if(self.auth_url != None): - self.creds = afterUrlAuth(self.local_server, self.flow, self.wsgi_app) - listOfFiles = callApi(self.creds) + self.creds = afterUrlAuth(self.local_server, self.flow, self.wsgi_app, self.userId) + listOfFiles = callApi(self.creds, self.fileId) app.logger.info(listOfFiles) - # print(listOfFiles, file=sys.stderr) a + # print(listOfFiles, file=sys.stderr) # Route to return google-picker API credentials @app.route('/googleDriveUpload/getFile', methods=['POST']) def gDriveGetFile(): - creds = None - auth_url, local_server, wsgi_app, flow, creds = startDownload() - thread_a = getFile2(auth_url, local_server, wsgi_app, flow, creds) - thread_a.start() - return flask.Response(json.dumps({"authURL": auth_url}), status=200) + body = flask.request.get_json() + if not body: + return flask.Response(json.dumps({"error": "Missing JSON body"}), status=400) + creds = None + auth_url, local_server, wsgi_app, flow, creds = start(body['userId']) + thread_a = getFile(auth_url, local_server, wsgi_app, flow, creds, body['userId'], body['fileId']) + thread_a.start() + return flask.Response(json.dumps({"authURL": auth_url}), status=200) -# def getFile2(auth_url, local_server, wsgi_app, flow, creds): -# if(auth_url != None): -# creds = afterUrlAuth(local_server, flow, wsgi_app) -# listOfFiles = callApi(creds) -# print(listOfFiles, file=sys.stderr) # Workbench Dataset Creation help-routes diff --git a/gDriveDownload.py b/gDriveDownload.py index 74bdbbf..e9ce118 100644 --- a/gDriveDownload.py +++ b/gDriveDownload.py @@ -1,16 +1,27 @@ from __future__ import print_function import pickle -import os.path import wsgiref.simple_server import wsgiref.util +from googleapiclient.http import MediaIoBaseDownload from googleapiclient.discovery import build -from google_auth_oauthlib.flow import InstalledAppFlow, _WSGIRequestHandler, _RedirectWSGIApp +from google_auth_oauthlib.flow import ( + InstalledAppFlow, + _WSGIRequestHandler, + _RedirectWSGIApp, +) from google.auth.transport.requests import Request import sys import os +import io +import shutil +import random +import string +from werkzeug.utils import secure_filename + + +# If modifying these scopes, delete the file .pickle files. +SCOPES = ["https://www.googleapis.com/auth/drive.readonly"] -# If modifying these scopes, delete the file token.pickle. -SCOPES = ['https://www.googleapis.com/auth/drive.readonly'] def run_local_server( self=InstalledAppFlow, @@ -18,39 +29,45 @@ def run_local_server( port=4001, authorization_prompt_message=InstalledAppFlow._DEFAULT_AUTH_PROMPT_MESSAGE, success_message=InstalledAppFlow._DEFAULT_WEB_SUCCESS_MESSAGE, + userId=None, ): wsgi_app = _RedirectWSGIApp(success_message) local_server = wsgiref.simple_server.make_server( host, port, wsgi_app, handler_class=_WSGIRequestHandler ) - self.redirect_uri = "http://localhost:4010/googleAuth/" + self.redirect_uri = "http://localhost:4010/googleAuth/" + userId auth_url, _ = self.authorization_url() - print(authorization_prompt_message.format(url=auth_url)) return auth_url, local_server, wsgi_app, None -def afterUrlAuth(local_server, flow, wsgi_app): + +def afterUrlAuth(local_server, flow, wsgi_app, userId): local_server.handle_request() # Note: using https here because oauthlib is very picky that # OAuth 2.0 should only occur over https. authorization_response = wsgi_app.last_request_uri.replace("http", "https") flow.fetch_token(authorization_response=authorization_response) - # Save the credentials for the next run - with open('token.pickle', 'wb') as token: + # Save the credentials for the next run + with open( + "/cloud-upload-apis/tokens/" + userId + ".pickle", "wb" + ) as token: pickle.dump(flow.credentials, token) return flow.credentials -def startDownload(): + +def start(userId): creds = None # The file token.pickle stores the user's access and refresh tokens, and is # created automatically when the authorization flow completes for the first # time. - if os.path.exists('token.pickle'): - with open('token.pickle', 'rb') as token: + if os.path.exists("/cloud-upload-apis/tokens/" + userId + ".pickle"): + with open( + "/cloud-upload-apis/tokens/" + userId + ".pickle", "rb" + ) as token: creds = pickle.load(token) return None, None, None, None, creds # If there are no (valid) credentials available, let the user log in. @@ -59,30 +76,60 @@ def startDownload(): creds.refresh(Request()) else: flow = InstalledAppFlow.from_client_secrets_file( - './credentials/google-drive.json', SCOPES) - auth_url, local_server, wsgi_app, creds = run_local_server(self=flow) + "/cloud-upload-apis/credentials/google-drive.json", SCOPES + ) + auth_url, local_server, wsgi_app, creds = run_local_server( + self=flow, userId=userId + ) return auth_url, local_server, wsgi_app, flow, creds # creds = afterUrlAuth(local_server, flow, wsgi_app) - -def callApi(creds): - service = build('drive', 'v3', credentials=creds) +def callApi(creds, fileId): + + # fileId = "1HXJqXupb5L8YhCN6KV45_FUHm4K3Hp9r" + # fileId = "1NyErLXDZgv1s00-5hnyBqCd5t80SHV3g" + + service = build("drive", "v3", credentials=creds) # Call the Drive v3 API - results = service.files().list( - pageSize=10, fields="nextPageToken, files(id, name)").execute() - items = results.get('files', []) - - if not items: - print('No files found.') - return items - else: - print('Files:') - return items - # for item in items: - # print(u'{0} ({1})'.format(item['name'], item['id'])) + request = service.files().get_media(fileId=fileId) + fileName = service.files().get(fileId=fileId).execute()["name"] + token = "".join( + random.choice(string.ascii_uppercase + string.digits) for _ in range(10) + ) + token = secure_filename(token) + tmppath = os.path.join("/images/uploading/", token) + # regenerate if we happen to collide + while os.path.isfile(tmppath): + token = "".join( + random.choice(string.ascii_uppercase + string.digits) for _ in range(10) + ) + token = secure_filename(token) + tmppath = os.path.join("/images/uploading/", token) + + fh = io.BytesIO() + + # Initialise a downloader object to download the file + downloader = MediaIoBaseDownload(fh, request) + done = False + + try: + # Download the data in chunks + while not done: + status, done = downloader.next_chunk() + fh.seek(0) + # Write the received data to the file + with open("/images/uploading/" + token, "wb") as f: + shutil.copyfileobj(fh, f) + + print("File Downloaded") + # Return True if file Downloaded successfully + return {"status": True, "fileName": fileName, "token": token} + except: + # Return False if something went wrong + print("Something went wrong.") # startDownload() From f717402183775924b2cbe08b64a05dfeef9ac8d3 Mon Sep 17 00:00:00 2001 From: akhil-rana Date: Sun, 20 Dec 2020 10:24:16 +0530 Subject: [PATCH 03/12] download status rote --- SlideServer.py | 41 +++++++++++++++++++++++++++++++---------- gDriveDownload.py | 38 ++++++++++---------------------------- 2 files changed, 41 insertions(+), 38 deletions(-) diff --git a/SlideServer.py b/SlideServer.py index ecd33b4..7d79459 100644 --- a/SlideServer.py +++ b/SlideServer.py @@ -22,7 +22,6 @@ import pathlib import logging from gDriveDownload import start, afterUrlAuth, callApi -from flask import after_this_request from threading import Thread try: @@ -237,16 +236,15 @@ def urlUploadStatus(): return flask.Response(json.dumps({"uploaded": "False"}), status=200) class getFile(Thread): - def __init__(self, auth_url, local_server, wsgi_app, flow, creds, userId, fileId): + def __init__(self, auth_url, local_server, wsgi_app, flow, creds, userId, fileId, token): Thread.__init__(self) - self.auth_url, self.local_server, self.wsgi_app, self.flow, self.creds, self.userId, self.fileId = auth_url, local_server, wsgi_app, flow, creds, userId, fileId + self.auth_url, self.local_server, self.wsgi_app, self.flow, self.creds, self.userId, self.fileId , self.token = auth_url, local_server, wsgi_app, flow, creds, userId, fileId, token def run(self): if(self.auth_url != None): self.creds = afterUrlAuth(self.local_server, self.flow, self.wsgi_app, self.userId) - listOfFiles = callApi(self.creds, self.fileId) - app.logger.info(listOfFiles) - # print(listOfFiles, file=sys.stderr) + call = callApi(self.creds, self.fileId, self.token) + app.logger.info(call) # Route to return google-picker API credentials @app.route('/googleDriveUpload/getFile', methods=['POST']) @@ -254,12 +252,35 @@ def gDriveGetFile(): body = flask.request.get_json() if not body: return flask.Response(json.dumps({"error": "Missing JSON body"}), status=400) + + token = "".join(random.choice(string.ascii_uppercase + string.digits) for _ in range(10)) + token = secure_filename(token) + tmppath = os.path.join("/images/uploading/", token) + # regenerate if we happen to collide + while os.path.isfile(tmppath): + token = "".join(random.choice(string.ascii_uppercase + string.digits) for _ in range(10)) + token = secure_filename(token) + tmppath = os.path.join("/images/uploading/", token) + creds = None - auth_url, local_server, wsgi_app, flow, creds = start(body['userId']) - thread_a = getFile(auth_url, local_server, wsgi_app, flow, creds, body['userId'], body['fileId']) + try: + auth_url, local_server, wsgi_app, flow, creds = start(body['userId']) + except: + return flask.Response(json.dumps({'error': str(sys.exc_info()[0])}), status=400) + thread_a = getFile(auth_url, local_server, wsgi_app, flow, creds, body['userId'], body['fileId'], token) thread_a.start() - return flask.Response(json.dumps({"authURL": auth_url}), status=200) - + return flask.Response(json.dumps({"authURL": auth_url, "token": token}), status=200) + +@app.route('/googleDriveUpload/checkStatus', methods=['POST']) +def checkDownloadStatus(): + body = flask.request.get_json() + if not body: + return flask.Response(json.dumps({"error": "Missing JSON body"}), status=400) + token = body['token'] + path = app.config['TEMP_FOLDER']+'/'+token + if os.path.isfile(path): + return flask.Response(json.dumps({"downloadDone": True}), status=200) + return flask.Response(json.dumps({"downloadDone": False}), status=200) # Workbench Dataset Creation help-routes diff --git a/gDriveDownload.py b/gDriveDownload.py index e9ce118..aa111ee 100644 --- a/gDriveDownload.py +++ b/gDriveDownload.py @@ -14,15 +14,12 @@ import os import io import shutil -import random -import string from werkzeug.utils import secure_filename # If modifying these scopes, delete the file .pickle files. SCOPES = ["https://www.googleapis.com/auth/drive.readonly"] - def run_local_server( self=InstalledAppFlow, host="0.0.0.0", @@ -53,7 +50,7 @@ def afterUrlAuth(local_server, flow, wsgi_app, userId): flow.fetch_token(authorization_response=authorization_response) # Save the credentials for the next run with open( - "/cloud-upload-apis/tokens/" + userId + ".pickle", "wb" + "/cloud-upload-apis/tokens/googleDrive" + userId + ".pickle", "wb" ) as token: pickle.dump(flow.credentials, token) return flow.credentials @@ -64,9 +61,9 @@ def start(userId): # The file token.pickle stores the user's access and refresh tokens, and is # created automatically when the authorization flow completes for the first # time. - if os.path.exists("/cloud-upload-apis/tokens/" + userId + ".pickle"): + if os.path.exists("/cloud-upload-apis/tokens/googleDrive" + userId + ".pickle"): with open( - "/cloud-upload-apis/tokens/" + userId + ".pickle", "rb" + "/cloud-upload-apis/tokens/googleDrive" + userId + ".pickle", "rb" ) as token: creds = pickle.load(token) return None, None, None, None, creds @@ -85,8 +82,8 @@ def start(userId): # creds = afterUrlAuth(local_server, flow, wsgi_app) -def callApi(creds, fileId): - +def callApi(creds, fileId, token): + downloadDone = False # fileId = "1HXJqXupb5L8YhCN6KV45_FUHm4K3Hp9r" # fileId = "1NyErLXDZgv1s00-5hnyBqCd5t80SHV3g" @@ -96,40 +93,25 @@ def callApi(creds, fileId): request = service.files().get_media(fileId=fileId) fileName = service.files().get(fileId=fileId).execute()["name"] - token = "".join( - random.choice(string.ascii_uppercase + string.digits) for _ in range(10) - ) - token = secure_filename(token) - tmppath = os.path.join("/images/uploading/", token) - # regenerate if we happen to collide - while os.path.isfile(tmppath): - token = "".join( - random.choice(string.ascii_uppercase + string.digits) for _ in range(10) - ) - token = secure_filename(token) - tmppath = os.path.join("/images/uploading/", token) - fh = io.BytesIO() # Initialise a downloader object to download the file downloader = MediaIoBaseDownload(fh, request) - done = False try: # Download the data in chunks - while not done: - status, done = downloader.next_chunk() + while not downloadDone: + status, downloadDone = downloader.next_chunk() fh.seek(0) # Write the received data to the file with open("/images/uploading/" + token, "wb") as f: shutil.copyfileobj(fh, f) - print("File Downloaded") + # print("File Downloaded") # Return True if file Downloaded successfully return {"status": True, "fileName": fileName, "token": token} except: - # Return False if something went wrong - print("Something went wrong.") + return False + # print("Something went wrong.") -# startDownload() From 5d4a9ceced0e10b9e8905f83dc5903adee248637 Mon Sep 17 00:00:00 2001 From: akhil-rana Date: Sun, 20 Dec 2020 13:40:57 +0530 Subject: [PATCH 04/12] drive file download with status check route --- SlideServer.py | 96 ++++++++++++++++++++++++----------------------- gDriveDownload.py | 42 +++++++++------------ 2 files changed, 68 insertions(+), 70 deletions(-) diff --git a/SlideServer.py b/SlideServer.py index 7d79459..b076156 100644 --- a/SlideServer.py +++ b/SlideServer.py @@ -235,52 +235,6 @@ def urlUploadStatus(): else: return flask.Response(json.dumps({"uploaded": "False"}), status=200) -class getFile(Thread): - def __init__(self, auth_url, local_server, wsgi_app, flow, creds, userId, fileId, token): - Thread.__init__(self) - self.auth_url, self.local_server, self.wsgi_app, self.flow, self.creds, self.userId, self.fileId , self.token = auth_url, local_server, wsgi_app, flow, creds, userId, fileId, token - - def run(self): - if(self.auth_url != None): - self.creds = afterUrlAuth(self.local_server, self.flow, self.wsgi_app, self.userId) - call = callApi(self.creds, self.fileId, self.token) - app.logger.info(call) - -# Route to return google-picker API credentials -@app.route('/googleDriveUpload/getFile', methods=['POST']) -def gDriveGetFile(): - body = flask.request.get_json() - if not body: - return flask.Response(json.dumps({"error": "Missing JSON body"}), status=400) - - token = "".join(random.choice(string.ascii_uppercase + string.digits) for _ in range(10)) - token = secure_filename(token) - tmppath = os.path.join("/images/uploading/", token) - # regenerate if we happen to collide - while os.path.isfile(tmppath): - token = "".join(random.choice(string.ascii_uppercase + string.digits) for _ in range(10)) - token = secure_filename(token) - tmppath = os.path.join("/images/uploading/", token) - - creds = None - try: - auth_url, local_server, wsgi_app, flow, creds = start(body['userId']) - except: - return flask.Response(json.dumps({'error': str(sys.exc_info()[0])}), status=400) - thread_a = getFile(auth_url, local_server, wsgi_app, flow, creds, body['userId'], body['fileId'], token) - thread_a.start() - return flask.Response(json.dumps({"authURL": auth_url, "token": token}), status=200) - -@app.route('/googleDriveUpload/checkStatus', methods=['POST']) -def checkDownloadStatus(): - body = flask.request.get_json() - if not body: - return flask.Response(json.dumps({"error": "Missing JSON body"}), status=400) - token = body['token'] - path = app.config['TEMP_FOLDER']+'/'+token - if os.path.isfile(path): - return flask.Response(json.dumps({"downloadDone": True}), status=200) - return flask.Response(json.dumps({"downloadDone": False}), status=200) # Workbench Dataset Creation help-routes @@ -578,3 +532,53 @@ def roiextract(file_name): return flask.send_from_directory(app.config["ROI_FOLDER"],filename=file_name, as_attachment=True, cache_timeout=0 ) +# Google Drive API (OAuth and File Download) Routes + +# A new Thread to call the Gdrive API after an Auth Response is returned to the user. +class getFileFromGdrive(Thread): + def __init__(self, params, userId, fileId, token): + Thread.__init__(self) + self.params, self.userId, self.fileId , self.token = params, userId, fileId, token + + def run(self): + if(self.auth_url != None): + self.params["creds"] = afterUrlAuth(self.params["local_server"], self.params["flow"], self.params["wsgi_app"], self.userId) + call = callApi(self.params["creds"], self.fileId, self.token) + app.logger.info(call) + +# Route to start the OAuth Server(to listen if user is Authenticated) and start the file Download after Authentication +@app.route('/googleDriveUpload/getFile', methods=['POST']) +def gDriveGetFile(): + body = flask.request.get_json() + if not body: + return flask.Response(json.dumps({"error": "Missing JSON body"}), status=400) + + token = "".join(random.choice(string.ascii_uppercase + string.digits) for _ in range(10)) + token = secure_filename(token) + tmppath = os.path.join("/images/uploading/", token) + # regenerate if we happen to collide + while os.path.isfile(tmppath): + token = "".join(random.choice(string.ascii_uppercase + string.digits) for _ in range(10)) + token = secure_filename(token) + tmppath = os.path.join("/images/uploading/", token) + + try: + params = start(body['userId']) + except: + return flask.Response(json.dumps({'error': str(sys.exc_info()[0])}), status=400) + thread_a = getFileFromGdrive(params, body['userId'], body['fileId'], token) + thread_a.start() + return flask.Response(json.dumps({"authURL": params["auth_url"], "token": token}), status=200) + +# To check if a particular file is downloaded from Gdrive +@app.route('/googleDriveUpload/checkStatus', methods=['POST']) +def checkDownloadStatus(): + body = flask.request.get_json() + if not body: + return flask.Response(json.dumps({"error": "Missing JSON body"}), status=400) + token = body['token'] + path = app.config['TEMP_FOLDER']+'/'+token + if os.path.isfile(path): + return flask.Response(json.dumps({"downloadDone": True}), status=200) + return flask.Response(json.dumps({"downloadDone": False}), status=200) + diff --git a/gDriveDownload.py b/gDriveDownload.py index aa111ee..c37a1a9 100644 --- a/gDriveDownload.py +++ b/gDriveDownload.py @@ -4,22 +4,18 @@ import wsgiref.util from googleapiclient.http import MediaIoBaseDownload from googleapiclient.discovery import build -from google_auth_oauthlib.flow import ( - InstalledAppFlow, - _WSGIRequestHandler, - _RedirectWSGIApp, -) +from google_auth_oauthlib.flow import InstalledAppFlow, _WSGIRequestHandler, _RedirectWSGIApp from google.auth.transport.requests import Request import sys import os import io import shutil -from werkzeug.utils import secure_filename -# If modifying these scopes, delete the file .pickle files. +# If modifying these scopes, delete the file "googleDrive.pickle" SCOPES = ["https://www.googleapis.com/auth/drive.readonly"] +# Starting a local server on :4001 to listen for authentication response from user def run_local_server( self=InstalledAppFlow, host="0.0.0.0", @@ -29,10 +25,9 @@ def run_local_server( userId=None, ): wsgi_app = _RedirectWSGIApp(success_message) - local_server = wsgiref.simple_server.make_server( - host, port, wsgi_app, handler_class=_WSGIRequestHandler - ) + local_server = wsgiref.simple_server.make_server(host, port, wsgi_app, handler_class=_WSGIRequestHandler) + # Making a unique redirect URL for every user self.redirect_uri = "http://localhost:4010/googleAuth/" + userId auth_url, _ = self.authorization_url() @@ -41,6 +36,7 @@ def run_local_server( return auth_url, local_server, wsgi_app, None +# Will be called after return of Auth URL def afterUrlAuth(local_server, flow, wsgi_app, userId): local_server.handle_request() @@ -55,12 +51,11 @@ def afterUrlAuth(local_server, flow, wsgi_app, userId): pickle.dump(flow.credentials, token) return flow.credentials - +# Starting the Auth process and checking for pickle file (Token) [Creating a new token if not exists] def start(userId): creds = None - # The file token.pickle stores the user's access and refresh tokens, and is - # created automatically when the authorization flow completes for the first - # time. + # The file "googleDrive.pickle" stores the user's access and refresh tokens, and is + # created automatically when the authorization flow completes for the first time. if os.path.exists("/cloud-upload-apis/tokens/googleDrive" + userId + ".pickle"): with open( "/cloud-upload-apis/tokens/googleDrive" + userId + ".pickle", "rb" @@ -78,15 +73,18 @@ def start(userId): auth_url, local_server, wsgi_app, creds = run_local_server( self=flow, userId=userId ) - return auth_url, local_server, wsgi_app, flow, creds - # creds = afterUrlAuth(local_server, flow, wsgi_app) + return { + "auth_url": auth_url, + "local_server": local_server, + "wsgi_app": wsgi_app, + "flow": flow, + "creds": creds, + } +# Calling the Drive API to download a file def callApi(creds, fileId, token): downloadDone = False - # fileId = "1HXJqXupb5L8YhCN6KV45_FUHm4K3Hp9r" - # fileId = "1NyErLXDZgv1s00-5hnyBqCd5t80SHV3g" - service = build("drive", "v3", credentials=creds) # Call the Drive v3 API @@ -107,11 +105,7 @@ def callApi(creds, fileId, token): with open("/images/uploading/" + token, "wb") as f: shutil.copyfileobj(fh, f) - # print("File Downloaded") # Return True if file Downloaded successfully return {"status": True, "fileName": fileName, "token": token} except: - return False - # print("Something went wrong.") - - + return False From e89e0eef3df2a28ca067e8edfb8d99091530d87f Mon Sep 17 00:00:00 2001 From: akhil-rana Date: Sun, 20 Dec 2020 14:23:49 +0530 Subject: [PATCH 05/12] fix --- gDriveDownload.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/gDriveDownload.py b/gDriveDownload.py index c37a1a9..61c0e46 100644 --- a/gDriveDownload.py +++ b/gDriveDownload.py @@ -61,7 +61,14 @@ def start(userId): "/cloud-upload-apis/tokens/googleDrive" + userId + ".pickle", "rb" ) as token: creds = pickle.load(token) - return None, None, None, None, creds + return { + "auth_url": None, + "local_server": None, + "wsgi_app": None, + "flow": None, + "creds": creds, + } + # If there are no (valid) credentials available, let the user log in. if not creds or not creds.valid: if creds and creds.expired and creds.refresh_token: From 4a9ecfd5e4d2762c4c937f072dff29a5898c236b Mon Sep 17 00:00:00 2001 From: akhil-rana Date: Sun, 20 Dec 2020 14:31:27 +0530 Subject: [PATCH 06/12] fix --- SlideServer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SlideServer.py b/SlideServer.py index b076156..51869ac 100644 --- a/SlideServer.py +++ b/SlideServer.py @@ -541,7 +541,7 @@ def __init__(self, params, userId, fileId, token): self.params, self.userId, self.fileId , self.token = params, userId, fileId, token def run(self): - if(self.auth_url != None): + if(self.params["auth_url"] != None): self.params["creds"] = afterUrlAuth(self.params["local_server"], self.params["flow"], self.params["wsgi_app"], self.userId) call = callApi(self.params["creds"], self.fileId, self.token) app.logger.info(call) From 988b6c51f556dbc627d9c6ffa5695de9e92c88b8 Mon Sep 17 00:00:00 2001 From: Ryan Birmingham Date: Thu, 7 Jan 2021 14:39:06 -0500 Subject: [PATCH 07/12] use another user for slideloader --- Dockerfile | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Dockerfile b/Dockerfile index 465d957..71195ad 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,5 +30,11 @@ EXPOSE 4001 # ENV FLASK_APP SlideServer.py # CMD python -m flask run --host=0.0.0.0 --port=4000 +# non-root user +RUN chgrp -R 0 /var && \ + chmod -R g+rwX /var + +USER 1001 + #prod only CMD gunicorn -w 4 -b 0.0.0.0:4000 SlideServer:app --timeout 400 From a2b4be28008e3840d923a7e26900201ec3f95aab Mon Sep 17 00:00:00 2001 From: Ryan Birmingham Date: Thu, 21 Jan 2021 15:51:40 -0500 Subject: [PATCH 08/12] remove mongo reference search --- OmniLoad.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/OmniLoad.py b/OmniLoad.py index cd2c2ab..2952d72 100644 --- a/OmniLoad.py +++ b/OmniLoad.py @@ -26,7 +26,7 @@ parser.add_argument('-db', type=str, default="camic", help='For mongo, the db to use') # read in lookup type -parser.add_argument('-lt', type=str, help='Slide ID lookup type', default="mongo", choices=['mongo', 'jsonfile', 'api', 'pathdb']) +parser.add_argument('-lt', type=str, help='Slide ID lookup type', default="api", choices=['api', 'jsonfile', 'pathdb']) # read in lookup uri or equivalent parser.add_argument('-ld', type=str, default="mongodb://ca-mongo:27017/", help='Slide ID lookup source') @@ -79,8 +79,6 @@ def openslidedata(manifest): r = requests.get(args.ld) r.json() # put slide id in manifest - if (args.lt == "mongo"): - pass if (args.lt == "pathdb"): pass if (args.lt == "jsonfile"): From 29d05cfedfda69af8e2aca39b3786b95ee3b2457 Mon Sep 17 00:00:00 2001 From: Ryan Birmingham Date: Thu, 21 Jan 2021 17:16:32 -0500 Subject: [PATCH 09/12] add basic lookup --- OmniLoad.py | 80 +++++++++++++++++++++++++++++++---------------------- 1 file changed, 47 insertions(+), 33 deletions(-) diff --git a/OmniLoad.py b/OmniLoad.py index 2952d72..a5aeecf 100644 --- a/OmniLoad.py +++ b/OmniLoad.py @@ -6,29 +6,27 @@ import time # for timestamp import os # for os/fs systems import json # for json in and out -import pymongo # for mongo in and out import requests # for api and pathdb in and out parser = argparse.ArgumentParser(description='Load slides or results to caMicroscope.') # read in collection parser.add_argument('-i', type=str, default="slide", choices=['slide', 'heatmap', 'mark', 'user'], - help='Input type (collection)') + help='Input type') # read in filepath parser.add_argument('-f', type=str, default="manifest.csv", help='Input file') # read in dest type -parser.add_argument('-o', type=str, default="mongo", choices=['mongo', 'jsonfile', 'api', 'pathdb'], +parser.add_argument('-o', type=str, default="camic", choices=['jsonfile', 'camic', 'pathdb'], help='Output destination type') +# read in pathdb collection +parser.add_argument('-pc', type=str, help='Pathdb Collection Name') # read in dest uri or equivalent -parser.add_argument('-d', type=str, default="mongodb://ca-mongo:27017/", +parser.add_argument('-d', type=str, default="http://localhost:4010/data/Slide/post", help='Output destination') -# read in mongo database -parser.add_argument('-db', type=str, default="camic", - help='For mongo, the db to use') # read in lookup type -parser.add_argument('-lt', type=str, help='Slide ID lookup type', default="api", choices=['api', 'jsonfile', 'pathdb']) +parser.add_argument('-lt', type=str, help='Slide ID lookup type', default="camic", choices=['camic', 'pathdb']) # read in lookup uri or equivalent -parser.add_argument('-ld', type=str, default="mongodb://ca-mongo:27017/", +parser.add_argument('-ld', type=str, default="http://localhost:4010/data/Slide/find", help='Slide ID lookup source') args = parser.parse_args() @@ -41,14 +39,17 @@ def openslidedata(manifest): slide = openslide.OpenSlide(img['location']) slideData = slide.properties img['mpp-x'] = slideData.get(openslide.PROPERTY_NAME_MPP_X, None) - img['mpp-x'] = slideData.get(openslide.PROPERTY_NAME_MPP_Y, None) - img['mpp'] = img['mpp-x'] or img['mpp-x'] or None - img['height'] = slideData.get(openslide.PROPERTY_NAME_BOUNDS_HEIGHT, None) - img['width'] = slideData.get(openslide.PROPERTY_NAME_BOUNDS_WIDTH, None) + img['mpp-y'] = slideData.get(openslide.PROPERTY_NAME_MPP_Y, None) + img['height'] = slideData.get(openslide.PROPERTY_NAME_BOUNDS_HEIGHT, None) or slideData.get( + "openslide.level[0].height", None) + img['width'] = slideData.get(openslide.PROPERTY_NAME_BOUNDS_WIDTH, None) or slideData.get( + "openslide.level[0].width", None) img['vendor'] = slideData.get(openslide.PROPERTY_NAME_VENDOR, None) img['level_count'] = int(slideData.get('level_count', 1)) - img['objective'] = float(slideData.get("aperio.AppMag", None)) - img['timestamp'] = time.time() + img['objective'] = float(slideData.get(openslide.PROPERTY_NAME_OBJECTIVE_POWER, 0) or + slideData.get("aperio.AppMag", -1.0)) + img['md5sum'] = file_md5(filepath) + img['comment'] = slideData.get(openslide.PROPERTY_NAME_COMMENT, None) # required values which are often unused img['study'] = img.get('study', "") img['specimen'] = img.get('specimen', "") @@ -72,19 +73,33 @@ def openslidedata(manifest): if (args.i == "slide"): manifest = openslidedata(manifest) else: - raise NotImplementedError("Slide id lookup not implemented") - if (args.lt == "api"): + + if (args.lt == "camic"): for x in manifest: - # TODO get slide ref from manifest - r = requests.get(args.ld) - r.json() - # put slide id in manifest + # TODO more flexible with manifest fields + lookup_url = args.ld + "?name=" + x.slide + r = requests.get(lookup_url) + res = r.json() + if (len(res)) == 0: + print("[WARN] - no match for slide '" + x.slide + "', skipping") + del x + x.id = res[0]["_id"]["$oid"] if (args.lt == "pathdb"): - pass - if (args.lt == "jsonfile"): - with open(args.ld, 'r') as f: - slide_map = json.load(manifest, f) - # TODO use + raise NotImplementedError("pathdb lookup is broken now") + for x in manifest: + # TODO there's an error with the url construction when testing, something's up + lookup_url = args.ld + args.pc + "/" + lookup_url += x.get("studyid", "") or x.get("study") + lookup_url += x.get("clinicaltrialsubjectid", "") or x.get("subject") + lookup_url += x.get("imageid", "") or x.get("image", "") or x.get("slide", "") + lookup_url += "?_format=json" + r = requests.get(lookup_url) + res = r.json() + if (len(res)) == 0: + print("[WARN] - no match for slide '" + str(x) + "', skipping") + del x + else: + x.id = res[0]["PathDBID"] # perform validation (!!) @@ -95,19 +110,18 @@ def openslidedata(manifest): if (args.o == "jsonfile"): with open(args.d, 'w') as f: json.dump(manifest, f) -elif (args.o == "mongo"): - client = pymongo.MongoClient(args.d) - db = client[args.db] - col = db[args.i] - col.insert_many(manifest) -elif (args.o == "api"): +elif (args.o == "camic"): x = requests.post(args.d, json=manifest) # if we get a 401, ask the user for a token retry = True while (x.status_code == 401 and retry): token = input("API returned 401, try a (different) token? : ") if (token and token != "no" and token != "n"): - x = requests.post(args.d, json=manifest, auth=token) + if (args.i == "slide"): + x = requests.post(args.d, json=manifest, auth=token) + else: + pass + # process each file in manifest else: retry = False x.raise_for_status() From 168881fdb1c93664e2796e9de7f7854e66e5311d Mon Sep 17 00:00:00 2001 From: Ryan Birmingham Date: Thu, 21 Jan 2021 19:48:30 -0500 Subject: [PATCH 10/12] per-file for results --- OmniLoad.py | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/OmniLoad.py b/OmniLoad.py index a5aeecf..1129136 100644 --- a/OmniLoad.py +++ b/OmniLoad.py @@ -102,29 +102,35 @@ def openslidedata(manifest): x.id = res[0]["PathDBID"] -# perform validation (!!) +# TODO add validation (!!) print("[WARNING] -- Validation not Implemented") - -# take appropriate destination action -if (args.o == "jsonfile"): - with open(args.d, 'w') as f: - json.dump(manifest, f) -elif (args.o == "camic"): +def postWithAuth(data, url): x = requests.post(args.d, json=manifest) - # if we get a 401, ask the user for a token retry = True while (x.status_code == 401 and retry): token = input("API returned 401, try a (different) token? : ") if (token and token != "no" and token != "n"): - if (args.i == "slide"): - x = requests.post(args.d, json=manifest, auth=token) - else: - pass - # process each file in manifest + x = requests.post(args.d, json=manifest, auth=token) else: retry = False - x.raise_for_status() + return x + +# take appropriate destination action +if (args.o == "jsonfile"): + with open(args.d, 'w') as f: + json.dump(manifest, f) +elif (args.o == "camic"): + if (args.i == "slide"): + x = postWithAuth(args.d, manifest) + x.raise_for_status() + else: + with open(x.path) as f: + file = json.load(f) + for rec in file: + rec[slide] = x.id + x = postWithAuth(args.d, file) + x.raise_for_status() elif (args.o == "pathdb"): #! TODO if (args.i != "slide"): From ef765ae11ebe361632d74e2b2038b3ca4e15694b Mon Sep 17 00:00:00 2001 From: Ryan Birmingham Date: Mon, 25 Jan 2021 11:21:25 -0500 Subject: [PATCH 11/12] add uploaded images dir permission --- Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 71195ad..78fefd5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -32,7 +32,9 @@ EXPOSE 4001 # non-root user RUN chgrp -R 0 /var && \ - chmod -R g+rwX /var + chmod -R g+rwX /var && \ + chgrp -R 0 /images/uploading && \ + chmod -R g+rwX /images/uploading USER 1001 From 4c687b34733df7ffc3d87bc90d10fb47fdd1ca5a Mon Sep 17 00:00:00 2001 From: Ryan Birmingham Date: Mon, 25 Jan 2021 11:29:53 -0500 Subject: [PATCH 12/12] comment out user statement --- Dockerfile | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index 78fefd5..9b80f6b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,13 +30,14 @@ EXPOSE 4001 # ENV FLASK_APP SlideServer.py # CMD python -m flask run --host=0.0.0.0 --port=4000 -# non-root user -RUN chgrp -R 0 /var && \ - chmod -R g+rwX /var && \ - chgrp -R 0 /images/uploading && \ - chmod -R g+rwX /images/uploading - -USER 1001 +# The Below BROKE the ability for users to upload images. +# # non-root user +# RUN chgrp -R 0 /var && \ +# chmod -R g+rwX /var && \ +# chgrp -R 0 /images/uploading && \ +# chmod -R g+rwX /images/uploading +# +# USER 1001 #prod only CMD gunicorn -w 4 -b 0.0.0.0:4000 SlideServer:app --timeout 400