From ca06a05a542d73319f9eefd4094c3bea9c315e1f Mon Sep 17 00:00:00 2001 From: Ryan Birmingham Date: Fri, 29 Jan 2021 10:54:49 -0500 Subject: [PATCH 01/19] try fix pathdb get --- OmniLoad.py | 48 ++++++++++++++++++++++++++++++------------------ 1 file changed, 30 insertions(+), 18 deletions(-) diff --git a/OmniLoad.py b/OmniLoad.py index 1129136..73faca3 100644 --- a/OmniLoad.py +++ b/OmniLoad.py @@ -55,6 +55,31 @@ def openslidedata(manifest): img['specimen'] = img.get('specimen', "") return manifest +def getWithAuth(url): + x = requests.get(lookup_url) + retry = True + while (x.status_code == 401 and retry): + token = input("API returned 401, try a (different) token? : ") + if (token and token != "no" and token != "n"): + x = requests.get(lookup_url, auth=token) + else: + retry = False + return x + +def postWithAuth(data, url): + x = requests.post(args.d, json=manifest) + retry = True + while (x.status_code == 401 and retry): + token = input("API returned 401, try a (different) token? : ") + if (token and token != "no" and token != "n"): + x = requests.post(args.d, json=manifest, auth=token) + else: + retry = False + return x + + +## START script + manifest = [] # context for file @@ -78,44 +103,31 @@ def openslidedata(manifest): for x in manifest: # TODO more flexible with manifest fields lookup_url = args.ld + "?name=" + x.slide - r = requests.get(lookup_url) + x = getWithAuth(lookup_url) res = r.json() if (len(res)) == 0: print("[WARN] - no match for slide '" + x.slide + "', skipping") del x x.id = res[0]["_id"]["$oid"] if (args.lt == "pathdb"): - raise NotImplementedError("pathdb lookup is broken now") for x in manifest: - # TODO there's an error with the url construction when testing, something's up lookup_url = args.ld + args.pc + "/" lookup_url += x.get("studyid", "") or x.get("study") lookup_url += x.get("clinicaltrialsubjectid", "") or x.get("subject") lookup_url += x.get("imageid", "") or x.get("image", "") or x.get("slide", "") lookup_url += "?_format=json" - r = requests.get(lookup_url) + r = getWithAuth(lookup_url) res = r.json() - if (len(res)) == 0: + try: + x.id = res[0]["nid"][0].value + except: print("[WARN] - no match for slide '" + str(x) + "', skipping") del x - else: - x.id = res[0]["PathDBID"] # TODO add validation (!!) print("[WARNING] -- Validation not Implemented") -def postWithAuth(data, url): - x = requests.post(args.d, json=manifest) - retry = True - while (x.status_code == 401 and retry): - token = input("API returned 401, try a (different) token? : ") - if (token and token != "no" and token != "n"): - x = requests.post(args.d, json=manifest, auth=token) - else: - retry = False - return x - # take appropriate destination action if (args.o == "jsonfile"): with open(args.d, 'w') as f: From 5722ad3d56202ad6b68607e271461b868c244e58 Mon Sep 17 00:00:00 2001 From: Ryan Birmingham Date: Fri, 29 Jan 2021 12:01:09 -0500 Subject: [PATCH 02/19] url and param fixes --- OmniLoad.py | 35 +++++++++++++++++++---------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/OmniLoad.py b/OmniLoad.py index 73faca3..fdd7d59 100644 --- a/OmniLoad.py +++ b/OmniLoad.py @@ -21,12 +21,12 @@ # read in pathdb collection parser.add_argument('-pc', type=str, help='Pathdb Collection Name') # read in dest uri or equivalent -parser.add_argument('-d', type=str, default="http://localhost:4010/data/Slide/post", +parser.add_argument('-d', type=str, default="http://ca-back:4010/data/Slide/post", help='Output destination') # read in lookup type parser.add_argument('-lt', type=str, help='Slide ID lookup type', default="camic", choices=['camic', 'pathdb']) # read in lookup uri or equivalent -parser.add_argument('-ld', type=str, default="http://localhost:4010/data/Slide/find", +parser.add_argument('-ld', type=str, default="http://ca-back:4010/data/Slide/find", help='Slide ID lookup source') args = parser.parse_args() @@ -66,13 +66,13 @@ def getWithAuth(url): retry = False return x -def postWithAuth(data, url): +def postWithAuth(url, data): x = requests.post(args.d, json=manifest) retry = True while (x.status_code == 401 and retry): token = input("API returned 401, try a (different) token? : ") if (token and token != "no" and token != "n"): - x = requests.post(args.d, json=manifest, auth=token) + x = requests.post(args.d, json=data, auth=token) else: retry = False return x @@ -102,16 +102,17 @@ def postWithAuth(data, url): if (args.lt == "camic"): for x in manifest: # TODO more flexible with manifest fields - lookup_url = args.ld + "?name=" + x.slide - x = getWithAuth(lookup_url) + lookup_url = args.ld + "?name=" + x['slide'] + r = getWithAuth(lookup_url) res = r.json() - if (len(res)) == 0: - print("[WARN] - no match for slide '" + x.slide + "', skipping") + try: + x['id'] = res[0]["_id"]["$oid"] + except: + print("[WARN] - no match for slide '" + str(x) + "', skipping") del x - x.id = res[0]["_id"]["$oid"] if (args.lt == "pathdb"): for x in manifest: - lookup_url = args.ld + args.pc + "/" + lookup_url = args.ld + "/" + args.pc + "/" lookup_url += x.get("studyid", "") or x.get("study") lookup_url += x.get("clinicaltrialsubjectid", "") or x.get("subject") lookup_url += x.get("imageid", "") or x.get("image", "") or x.get("slide", "") @@ -119,7 +120,7 @@ def postWithAuth(data, url): r = getWithAuth(lookup_url) res = r.json() try: - x.id = res[0]["nid"][0].value + x['id'] = res[0]["nid"][0].value except: print("[WARN] - no match for slide '" + str(x) + "', skipping") del x @@ -135,13 +136,15 @@ def postWithAuth(data, url): elif (args.o == "camic"): if (args.i == "slide"): x = postWithAuth(args.d, manifest) + print(x.json()) x.raise_for_status() else: - with open(x.path) as f: - file = json.load(f) - for rec in file: - rec[slide] = x.id - x = postWithAuth(args.d, file) + with open(x['path']) as f: + fil = json.load(f) + for rec in fil: + rec['slide'] = x['id'] + x = postWithAuth(args.d, fil) + print(x.json()) x.raise_for_status() elif (args.o == "pathdb"): #! TODO From a8b223ce1d87708ec93a5bda94a25470e39eee3d Mon Sep 17 00:00:00 2001 From: Ryan Birmingham Date: Fri, 29 Jan 2021 12:36:55 -0500 Subject: [PATCH 03/19] use correct data --- OmniLoad.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/OmniLoad.py b/OmniLoad.py index fdd7d59..22466c5 100644 --- a/OmniLoad.py +++ b/OmniLoad.py @@ -3,8 +3,6 @@ import openslide # to get required slide metadata import csv # to read csv import argparse # to read arguments -import time # for timestamp -import os # for os/fs systems import json # for json in and out import requests # for api and pathdb in and out @@ -67,7 +65,7 @@ def getWithAuth(url): return x def postWithAuth(url, data): - x = requests.post(args.d, json=manifest) + x = requests.post(args.d, json=data) retry = True while (x.status_code == 401 and retry): token = input("API returned 401, try a (different) token? : ") @@ -135,17 +133,19 @@ def postWithAuth(url, data): json.dump(manifest, f) elif (args.o == "camic"): if (args.i == "slide"): - x = postWithAuth(args.d, manifest) - print(x.json()) - x.raise_for_status() + r = postWithAuth(args.d, manifest) + print(r.json()) + r.raise_for_status() else: - with open(x['path']) as f: - fil = json.load(f) - for rec in fil: - rec['slide'] = x['id'] - x = postWithAuth(args.d, fil) - print(x.json()) - x.raise_for_status() + for x in manifest: + with open(x['path']) as f: + fil = json.load(f) + for rec in fil: + # TODO safer version of this? + rec['provenance']['image']['slide'] = x['id'] + r = postWithAuth(args.d, fil) + print(r.json()) + r.raise_for_status() elif (args.o == "pathdb"): #! TODO if (args.i != "slide"): From 00a9c52c336480abd0c96452afa256754a5b7a99 Mon Sep 17 00:00:00 2001 From: Ryan Birmingham Date: Fri, 29 Jan 2021 12:41:02 -0500 Subject: [PATCH 04/19] add file_md5 --- OmniLoad.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/OmniLoad.py b/OmniLoad.py index 22466c5..ab408dd 100644 --- a/OmniLoad.py +++ b/OmniLoad.py @@ -5,6 +5,7 @@ import argparse # to read arguments import json # for json in and out import requests # for api and pathdb in and out +import hashlib parser = argparse.ArgumentParser(description='Load slides or results to caMicroscope.') # read in collection @@ -30,10 +31,21 @@ args = parser.parse_args() print(args) +def file_md5(fileName): + m = hashlib.md5() + blocksize = 2 ** 20 + with open(fileName, "rb") as f: + while True: + buf = f.read(blocksize) + if not buf: + break + m.update(buf) + return m.hexdigest() + # get fields openslide expects def openslidedata(manifest): for img in manifest: - img['location'] = img['location'] or img['filename'] or img['file'] + img['location'] = img['path'] or img['location'] or img['filename'] or img['file'] slide = openslide.OpenSlide(img['location']) slideData = slide.properties img['mpp-x'] = slideData.get(openslide.PROPERTY_NAME_MPP_X, None) From 9170abab1ddc4b501757c0cb6820bed8e7784701 Mon Sep 17 00:00:00 2001 From: Ryan Birmingham Date: Fri, 29 Jan 2021 12:41:49 -0500 Subject: [PATCH 05/19] re-add os --- OmniLoad.py | 1 + 1 file changed, 1 insertion(+) diff --git a/OmniLoad.py b/OmniLoad.py index ab408dd..04c33e5 100644 --- a/OmniLoad.py +++ b/OmniLoad.py @@ -2,6 +2,7 @@ import openslide # to get required slide metadata import csv # to read csv +import os # for os and filepath utils import argparse # to read arguments import json # for json in and out import requests # for api and pathdb in and out From 129eb000021b8ff16b51a879c4c4a30370fb8f8d Mon Sep 17 00:00:00 2001 From: Ryan Birmingham Date: Fri, 29 Jan 2021 12:43:13 -0500 Subject: [PATCH 06/19] better flexible path --- OmniLoad.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/OmniLoad.py b/OmniLoad.py index 04c33e5..c187719 100644 --- a/OmniLoad.py +++ b/OmniLoad.py @@ -46,7 +46,7 @@ def file_md5(fileName): # get fields openslide expects def openslidedata(manifest): for img in manifest: - img['location'] = img['path'] or img['location'] or img['filename'] or img['file'] + img['location'] = img.get("path", "") or img.get("location", "") or img.get("filename", "") or img.get("file", "") slide = openslide.OpenSlide(img['location']) slideData = slide.properties img['mpp-x'] = slideData.get(openslide.PROPERTY_NAME_MPP_X, None) From 1abed646aca84fd5bc1fa9e15d4ec71b0aa24a40 Mon Sep 17 00:00:00 2001 From: Ryan Birmingham Date: Fri, 29 Jan 2021 12:44:47 -0500 Subject: [PATCH 07/19] filepath and mpp fix --- OmniLoad.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/OmniLoad.py b/OmniLoad.py index c187719..aa9735d 100644 --- a/OmniLoad.py +++ b/OmniLoad.py @@ -51,6 +51,7 @@ def openslidedata(manifest): slideData = slide.properties img['mpp-x'] = slideData.get(openslide.PROPERTY_NAME_MPP_X, None) img['mpp-y'] = slideData.get(openslide.PROPERTY_NAME_MPP_Y, None) + img['mpp'] = img['mpp-x'] or img['mpp-y'] img['height'] = slideData.get(openslide.PROPERTY_NAME_BOUNDS_HEIGHT, None) or slideData.get( "openslide.level[0].height", None) img['width'] = slideData.get(openslide.PROPERTY_NAME_BOUNDS_WIDTH, None) or slideData.get( @@ -59,7 +60,7 @@ def openslidedata(manifest): img['level_count'] = int(slideData.get('level_count', 1)) img['objective'] = float(slideData.get(openslide.PROPERTY_NAME_OBJECTIVE_POWER, 0) or slideData.get("aperio.AppMag", -1.0)) - img['md5sum'] = file_md5(filepath) + img['md5sum'] = file_md5(img['location']) img['comment'] = slideData.get(openslide.PROPERTY_NAME_COMMENT, None) # required values which are often unused img['study'] = img.get('study', "") From dfb2d136f30b3c904cd3b33a66d64388f8383764 Mon Sep 17 00:00:00 2001 From: Ryan Birmingham Date: Fri, 29 Jan 2021 14:25:35 -0500 Subject: [PATCH 08/19] Start omniload segmentation support --- OmniLoad.py | 56 +++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 50 insertions(+), 6 deletions(-) diff --git a/OmniLoad.py b/OmniLoad.py index aa9735d..1c2f210 100644 --- a/OmniLoad.py +++ b/OmniLoad.py @@ -10,7 +10,7 @@ parser = argparse.ArgumentParser(description='Load slides or results to caMicroscope.') # read in collection -parser.add_argument('-i', type=str, default="slide", choices=['slide', 'heatmap', 'mark', 'user'], +parser.add_argument('-i', type=str, default="slide", choices=['slide', 'heatmap', 'mark', 'user', 'segmentation'], help='Input type') # read in filepath parser.add_argument('-f', type=str, default="manifest.csv", @@ -89,6 +89,43 @@ def postWithAuth(url, data): retry = False return x +def convertSegmentations(poly, name): + # interpret the objectively bad polygon representation + poly = poly.replace("[","") + poly = poly.replace("]","") + poly = poly.split(":") + new_poly = [] + x_max = -1. + x_min = 9e99 + y_max = -1. + y_min = 9e99 + for i in range(0,len(poly),2): + x_max = max(x_max, float(poly[i])) + x_min = min(x_min, float(poly[i])) + y_max = max(y_max, float(poly[i+1])) + y_min = min(y_min, float(poly[i+1])) + new_poly.append([float(poly[i]), float(poly[i+1])]) + # construct result + provenance = {} + provenance['image'] = {} + # may need better execution id + provenance['analysis'] = {"source":"segmentation", "coordinate":"image", "execution_id":name, "name":name} + properties = {} + properties['annotations'] = {"name": name} + geometries = {"type":"FeatureCollection"} + feature = {"type":"Feature"} + geometry = {"type":"Polygon"} + geometry['coordinates'] = [new_poly] + bound = {"type":"Polygon"} + # get bound + bound['coordinates'] = [[[x_min, y_min], [x_min, y_max], [x_max, y_max], [x_max, y_min], [x_min, y_min]]] + geometries['features'] = [feature] + geometries['bound'] = bound + res = {} + res['geometries'] = geometries + res['provenance'] = provenance + res['properties'] = properties + return res ## START script @@ -153,15 +190,22 @@ def postWithAuth(url, data): else: for x in manifest: with open(x['path']) as f: - fil = json.load(f) - for rec in fil: - # TODO safer version of this? - rec['provenance']['image']['slide'] = x['id'] + if (args.i == "segmentation"): + reader = csv.DictReader(f) + fil = [row for row in reader] + for rec in fil: + rec = convertSegmentations(rec['polygon'], x['segname']) + rec['provenance']['image']['slide'] = x['id'] + else: + fil = json.load(f) + for rec in fil: + # TODO safer version of this? + rec['provenance']['image']['slide'] = x['id'] r = postWithAuth(args.d, fil) print(r.json()) r.raise_for_status() elif (args.o == "pathdb"): - #! TODO + #! TODO - need the url and pattern for adding a slide to pathdb if (args.i != "slide"): raise AssertionError("Pathdb only holds slide data.") raise NotImplementedError("Output type: " + args.o + " not yet implemented") From c75428d37370d9e26504c3f9394ff771349eb4d6 Mon Sep 17 00:00:00 2001 From: Ryan Birmingham Date: Fri, 29 Jan 2021 15:15:13 -0500 Subject: [PATCH 09/19] use correct segmentation transformed data --- OmniLoad.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/OmniLoad.py b/OmniLoad.py index 1c2f210..448b605 100644 --- a/OmniLoad.py +++ b/OmniLoad.py @@ -2,12 +2,16 @@ import openslide # to get required slide metadata import csv # to read csv +import sys # for csv limit import os # for os and filepath utils import argparse # to read arguments import json # for json in and out import requests # for api and pathdb in and out import hashlib +# for large csv fields, especially segmentations +csv.field_size_limit(sys.maxsize) + parser = argparse.ArgumentParser(description='Load slides or results to caMicroscope.') # read in collection parser.add_argument('-i', type=str, default="slide", choices=['slide', 'heatmap', 'mark', 'user', 'segmentation'], @@ -106,6 +110,8 @@ def convertSegmentations(poly, name): y_min = min(y_min, float(poly[i+1])) new_poly.append([float(poly[i]), float(poly[i+1])]) # construct result + # complete loop + new_poly.append(new_poly[0]) provenance = {} provenance['image'] = {} # may need better execution id @@ -117,6 +123,7 @@ def convertSegmentations(poly, name): geometry = {"type":"Polygon"} geometry['coordinates'] = [new_poly] bound = {"type":"Polygon"} + feature['geometry'] = geometry # get bound bound['coordinates'] = [[[x_min, y_min], [x_min, y_max], [x_max, y_max], [x_max, y_min], [x_min, y_min]]] geometries['features'] = [feature] @@ -192,10 +199,12 @@ def convertSegmentations(poly, name): with open(x['path']) as f: if (args.i == "segmentation"): reader = csv.DictReader(f) - fil = [row for row in reader] - for rec in fil: - rec = convertSegmentations(rec['polygon'], x['segname']) - rec['provenance']['image']['slide'] = x['id'] + segs = [row for row in reader] + fil = [] + for rec in segs: + res = convertSegmentations(rec['polygon'], x['segname']) + res['provenance']['image']['slide'] = x['id'] + fil.append(res) else: fil = json.load(f) for rec in fil: From 0556c79a0593b5c7a52ba5fb056163aafdf06adb Mon Sep 17 00:00:00 2001 From: Ryan Birmingham Date: Fri, 29 Jan 2021 15:18:34 -0500 Subject: [PATCH 10/19] correct default fields for Polygon and source --- OmniLoad.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/OmniLoad.py b/OmniLoad.py index 448b605..7b8b04b 100644 --- a/OmniLoad.py +++ b/OmniLoad.py @@ -115,7 +115,7 @@ def convertSegmentations(poly, name): provenance = {} provenance['image'] = {} # may need better execution id - provenance['analysis'] = {"source":"segmentation", "coordinate":"image", "execution_id":name, "name":name} + provenance['analysis'] = {"source":"computer", "coordinate":"image", "execution_id":name, "name":name} properties = {} properties['annotations'] = {"name": name} geometries = {"type":"FeatureCollection"} @@ -202,7 +202,7 @@ def convertSegmentations(poly, name): segs = [row for row in reader] fil = [] for rec in segs: - res = convertSegmentations(rec['polygon'], x['segname']) + res = convertSegmentations(rec['Polygon'], x['segname']) res['provenance']['image']['slide'] = x['id'] fil.append(res) else: From 6deccffd0833ec1ac16e8dbef750be4c9c202d87 Mon Sep 17 00:00:00 2001 From: Ryan Birmingham Date: Fri, 29 Jan 2021 15:55:24 -0500 Subject: [PATCH 11/19] other fixes --- OmniLoad.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/OmniLoad.py b/OmniLoad.py index 7b8b04b..5ad6ecc 100644 --- a/OmniLoad.py +++ b/OmniLoad.py @@ -93,7 +93,7 @@ def postWithAuth(url, data): retry = False return x -def convertSegmentations(poly, name): +def convertSegmentations(poly, name, area): # interpret the objectively bad polygon representation poly = poly.replace("[","") poly = poly.replace("]","") @@ -115,23 +115,28 @@ def convertSegmentations(poly, name): provenance = {} provenance['image'] = {} # may need better execution id - provenance['analysis'] = {"source":"computer", "coordinate":"image", "execution_id":name, "name":name} + provenance['analysis'] = {"source":"computer", "coordinate":"image", "execution_id":name, "name":name, "computation":"segmentation"} properties = {} - properties['annotations'] = {"name": name} + properties['annotations'] = {"name": name, 'AreaInPixels':area, "PhysicalSize":area} geometries = {"type":"FeatureCollection"} feature = {"type":"Feature"} geometry = {"type":"Polygon"} geometry['coordinates'] = [new_poly] bound = {"type":"Polygon"} feature['geometry'] = geometry + feature['bound'] = bound # get bound bound['coordinates'] = [[[x_min, y_min], [x_min, y_max], [x_max, y_max], [x_max, y_min], [x_min, y_min]]] geometries['features'] = [feature] - geometries['bound'] = bound res = {} res['geometries'] = geometries res['provenance'] = provenance res['properties'] = properties + res['footprint'] = area + res['x'] = x_min + res['y'] = y_min + res['object_type'] = "unknown" + res['parent_id'] = "self" return res ## START script @@ -202,7 +207,7 @@ def convertSegmentations(poly, name): segs = [row for row in reader] fil = [] for rec in segs: - res = convertSegmentations(rec['Polygon'], x['segname']) + res = convertSegmentations(rec['Polygon'], x['segname'], x['AreaInPixels']) res['provenance']['image']['slide'] = x['id'] fil.append(res) else: From 64802052f0fd6b89e21633c2994e9c47e89666f0 Mon Sep 17 00:00:00 2001 From: Ryan Birmingham Date: Fri, 29 Jan 2021 15:56:30 -0500 Subject: [PATCH 12/19] rex not x --- OmniLoad.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/OmniLoad.py b/OmniLoad.py index 5ad6ecc..fc79b50 100644 --- a/OmniLoad.py +++ b/OmniLoad.py @@ -207,7 +207,7 @@ def convertSegmentations(poly, name, area): segs = [row for row in reader] fil = [] for rec in segs: - res = convertSegmentations(rec['Polygon'], x['segname'], x['AreaInPixels']) + res = convertSegmentations(rec['Polygon'], x['segname'], rec['AreaInPixels']) res['provenance']['image']['slide'] = x['id'] fil.append(res) else: From 609e0ea2e9b229b496e191f9d8d5aba2702cf74d Mon Sep 17 00:00:00 2001 From: Ryan Birmingham Date: Fri, 12 Feb 2021 11:55:00 -0500 Subject: [PATCH 13/19] 500 codes for errs on more routes --- SlideServer.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/SlideServer.py b/SlideServer.py index 51869ac..ad9074b 100644 --- a/SlideServer.py +++ b/SlideServer.py @@ -177,18 +177,30 @@ def testRoute(): @app.route("/data/one/", methods=['GET']) def singleSlide(filepath): - return json.dumps(dev_utils.getMetadata(filepath, app.config['UPLOAD_FOLDER'])) + res = dev_utils.getMetadata(filepath, app.config['UPLOAD_FOLDER']) + if (res.error): + return flask.Response(json.dumps(res), status=500) + else: + return flask.Response(json.dumps(res), status=200) @app.route("/data/thumbnail/", methods=['GET']) def singleThumb(filepath): + res = getThumbnail(filepath, size) size = flask.request.args.get('size', default=50, type=int) - return json.dumps(getThumbnail(filepath, size)) + if (res.error): + return flask.Response(json.dumps(res), status=500) + else: + return flask.Response(json.dumps(res), status=200) @app.route("/data/many/", methods=['GET']) def multiSlide(filepathlist): - return json.dumps(dev_utils.getMetadataList(json.loads(filepathlist), app.config['UPLOAD_FOLDER'])) + res = dev_utils.getMetadataList(json.loads(filepathlist), app.config['UPLOAD_FOLDER']) + if (res.error): + return flask.Response(json.dumps(res), status=500) + else: + return flask.Response(json.dumps(res), status=200) @app.route("/getSlide/") From ab5eea2997aee116ecd7225db3aa6023dcc467ef Mon Sep 17 00:00:00 2001 From: Ryan Birmingham Date: Fri, 12 Feb 2021 11:55:21 -0500 Subject: [PATCH 14/19] allow jpg and png pre convert --- SlideServer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SlideServer.py b/SlideServer.py index ad9074b..2c5f054 100644 --- a/SlideServer.py +++ b/SlideServer.py @@ -43,7 +43,7 @@ app.config['ROI_FOLDER'] = "/images/roiDownload" -ALLOWED_EXTENSIONS = set(['svs', 'tif', 'tiff', 'vms', 'vmu', 'ndpi', 'scn', 'mrxs', 'bif', 'svslide']) +ALLOWED_EXTENSIONS = set(['svs', 'tif', 'tiff', 'vms', 'vmu', 'ndpi', 'scn', 'mrxs', 'bif', 'svslide', 'png', 'jpg']) def allowed_file(filename): From 62794a9ac831e150ddfb1f1a963928e9af55cfc9 Mon Sep 17 00:00:00 2001 From: Ryan Birmingham Date: Fri, 12 Feb 2021 12:00:49 -0500 Subject: [PATCH 15/19] use hasattr instead --- SlideServer.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/SlideServer.py b/SlideServer.py index 2c5f054..0994059 100644 --- a/SlideServer.py +++ b/SlideServer.py @@ -178,7 +178,7 @@ def testRoute(): @app.route("/data/one/", methods=['GET']) def singleSlide(filepath): res = dev_utils.getMetadata(filepath, app.config['UPLOAD_FOLDER']) - if (res.error): + if (hasattr(res, 'error')): return flask.Response(json.dumps(res), status=500) else: return flask.Response(json.dumps(res), status=200) @@ -188,7 +188,7 @@ def singleSlide(filepath): def singleThumb(filepath): res = getThumbnail(filepath, size) size = flask.request.args.get('size', default=50, type=int) - if (res.error): + if (hasattr(res, 'error')): return flask.Response(json.dumps(res), status=500) else: return flask.Response(json.dumps(res), status=200) @@ -197,7 +197,7 @@ def singleThumb(filepath): @app.route("/data/many/", methods=['GET']) def multiSlide(filepathlist): res = dev_utils.getMetadataList(json.loads(filepathlist), app.config['UPLOAD_FOLDER']) - if (res.error): + if (hasattr(res, 'error')): return flask.Response(json.dumps(res), status=500) else: return flask.Response(json.dumps(res), status=200) From a58808d60780e9bd84d0dafc8497b784c05c69f4 Mon Sep 17 00:00:00 2001 From: Ryan Birmingham Date: Fri, 12 Feb 2021 12:03:45 -0500 Subject: [PATCH 16/19] fix line order for size --- SlideServer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SlideServer.py b/SlideServer.py index 0994059..d1f754f 100644 --- a/SlideServer.py +++ b/SlideServer.py @@ -186,8 +186,8 @@ def singleSlide(filepath): @app.route("/data/thumbnail/", methods=['GET']) def singleThumb(filepath): - res = getThumbnail(filepath, size) size = flask.request.args.get('size', default=50, type=int) + res = getThumbnail(filepath, size) if (hasattr(res, 'error')): return flask.Response(json.dumps(res), status=500) else: From 3b08961f6c4589cdbc1d9190d2712c4f5f9a36ed Mon Sep 17 00:00:00 2001 From: Ryan Birmingham Date: Fri, 12 Feb 2021 12:19:13 -0500 Subject: [PATCH 17/19] what's up with img return --- SlideServer.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/SlideServer.py b/SlideServer.py index d1f754f..92d644f 100644 --- a/SlideServer.py +++ b/SlideServer.py @@ -70,8 +70,9 @@ def makePyramid(filename, dest): try: filepath = os.path.join(app.config['UPLOAD_FOLDER'], filename) destpath = os.path.join(app.config['UPLOAD_FOLDER'], dest) - pyvips.Image.new_from_file(filepath, access='sequential').tiffsave(destpath, tile=True, compression="lzw", tile_width=256, tile_height=256, pyramid=True, bigtiff=True, xres=0.254, yres=0.254) - return flask.Response(json.dumps({"status": "OK"}), status=200) + savedImg = pyvips.Image.new_from_file(filepath, access='sequential').tiffsave(destpath, tile=True, compression="lzw", tile_width=256, tile_height=256, pyramid=True, bigtiff=True, xres=0.254, yres=0.254) + print(savedImg) + return flask.Response(json.dumps({"status": "OK", "srcFile":filename, "destFile":dest, "details":savedImg}), status=200) except BaseException as e: return flask.Response(json.dumps({"type": "pyvips", "error": str(e)}), status=500) From 8923f8b225dae8ef6b0192469bfe436b54931fe1 Mon Sep 17 00:00:00 2001 From: Ryan Birmingham Date: Fri, 12 Feb 2021 12:24:33 -0500 Subject: [PATCH 18/19] sync after pyramid --- SlideServer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SlideServer.py b/SlideServer.py index 92d644f..378e036 100644 --- a/SlideServer.py +++ b/SlideServer.py @@ -71,7 +71,7 @@ def makePyramid(filename, dest): filepath = os.path.join(app.config['UPLOAD_FOLDER'], filename) destpath = os.path.join(app.config['UPLOAD_FOLDER'], dest) savedImg = pyvips.Image.new_from_file(filepath, access='sequential').tiffsave(destpath, tile=True, compression="lzw", tile_width=256, tile_height=256, pyramid=True, bigtiff=True, xres=0.254, yres=0.254) - print(savedImg) + os.sync() return flask.Response(json.dumps({"status": "OK", "srcFile":filename, "destFile":dest, "details":savedImg}), status=200) except BaseException as e: return flask.Response(json.dumps({"type": "pyvips", "error": str(e)}), status=500) From dd9bbc7b10a1ee01f67beb40ce7e9692c16deb37 Mon Sep 17 00:00:00 2001 From: Ryan Birmingham Date: Fri, 12 Feb 2021 12:36:33 -0500 Subject: [PATCH 19/19] check for file --- SlideServer.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/SlideServer.py b/SlideServer.py index 378e036..0e02a3a 100644 --- a/SlideServer.py +++ b/SlideServer.py @@ -71,7 +71,9 @@ def makePyramid(filename, dest): filepath = os.path.join(app.config['UPLOAD_FOLDER'], filename) destpath = os.path.join(app.config['UPLOAD_FOLDER'], dest) savedImg = pyvips.Image.new_from_file(filepath, access='sequential').tiffsave(destpath, tile=True, compression="lzw", tile_width=256, tile_height=256, pyramid=True, bigtiff=True, xres=0.254, yres=0.254) - os.sync() + while not os.path.exists(filepath): + os.sync() + sleep(750) return flask.Response(json.dumps({"status": "OK", "srcFile":filename, "destFile":dest, "details":savedImg}), status=200) except BaseException as e: return flask.Response(json.dumps({"type": "pyvips", "error": str(e)}), status=500)