From bfef69d876afd576e70df06a2e31e6ceb3bab0f9 Mon Sep 17 00:00:00 2001 From: CGDogan <126820728+CGDogan@users.noreply.github.com> Date: Tue, 20 Jun 2023 16:22:33 +0100 Subject: [PATCH 01/38] Flask API typo --- SlideServer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/SlideServer.py b/SlideServer.py index c1dc9c4..566c273 100644 --- a/SlideServer.py +++ b/SlideServer.py @@ -216,7 +216,7 @@ def multiSlide(filepathlist): @app.route("/getSlide/") def getSlide(image_name): if(os.path.isfile("/images/"+image_name)): - return flask.send_from_directory(app.config["UPLOAD_FOLDER"], filename=image_name, as_attachment=True) + return flask.send_from_directory(app.config["UPLOAD_FOLDER"], image_name, as_attachment=True) else: return flask.Response(json.dumps({"error": "File does not exist"}), status=404) @@ -551,7 +551,7 @@ def roiExtract(): @app.route('/roiextract/') def roiextract(file_name): - return flask.send_from_directory(app.config["ROI_FOLDER"],filename=file_name, as_attachment=True, cache_timeout=0 ) + return flask.send_from_directory(app.config["ROI_FOLDER"], file_name, as_attachment=True, cache_timeout=0 ) # Google Drive API (OAuth and File Download) Routes From de26bdb1cad231f2751fd96274d57b9b489ca438 Mon Sep 17 00:00:00 2001 From: CGDogan <126820728+CGDogan@users.noreply.github.com> Date: Wed, 21 Jun 2023 20:22:19 +0100 Subject: [PATCH 02/38] Serve sanitized filename in begin and in final when possible --- SlideServer.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/SlideServer.py b/SlideServer.py index 566c273..dc7c95c 100644 --- a/SlideServer.py +++ b/SlideServer.py @@ -99,7 +99,11 @@ def start_upload(): tmppath = os.path.join(app.config['TEMP_FOLDER'], token) f = open(tmppath, 'a') f.close() - return flask.Response(json.dumps({"upload_token": token}), status=200) + res_body = {"upload_token": token} + body = flask.request.get_json() + if body and body.get('filename'): + res_body['filename'] = secure_filename(body['filename']) + return flask.Response(json.dumps(res_body), status=200) # using the token from the start upload endpoint, post data given offset. @@ -145,7 +149,7 @@ def finish_upload(token): else: return flask.Response(json.dumps({"error": "Token Not Recognised"}), status=400) else: - return flask.Response(json.dumps({"error": "File with name '" + filename + "' already exists"}), status=400) + return flask.Response(json.dumps({"error": "File with name '" + filename + "' already exists", "filepath": filepath}), status=400) else: return flask.Response(json.dumps({"error": "Invalid filename"}), status=400) From 65c573081547ae5303552fcd3b6ed6c974c32293 Mon Sep 17 00:00:00 2001 From: CGDogan <126820728+CGDogan@users.noreply.github.com> Date: Thu, 22 Jun 2023 11:56:46 +0100 Subject: [PATCH 03/38] Strict filename requirements for new files --- SlideServer.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/SlideServer.py b/SlideServer.py index dc7c95c..15d6460 100644 --- a/SlideServer.py +++ b/SlideServer.py @@ -50,6 +50,19 @@ ALLOWED_EXTENSIONS = set(['svs', 'tif', 'tiff', 'vms', 'vmu', 'ndpi', 'scn', 'mrxs', 'bif', 'svslide', 'png', 'jpg']) +# should be used instead of secure_filename to create new files whose extensions are important. +# use secure_filename to access previous files. +# secure_filename ensures security but may result in invalid filenames. +# secure_filename should be used to access, because users of caMicroscope +# might have already uploaded what caMicroscope cannot read, +# and allow reading those. +def secure_filename_strict(filename): + split_filename = secure_filename(filename).rsplit('.', 1) + split_filename[-1] = split_filename[-1].lower() # .SvS, .Svs, ... shouldn't be allowed + if len(split_filename) < 2: + # for example, #.svs -> .svs -> svs, which removes the extension + split_filename = ["noname", split_filename[-1]] + return '.'.join(split_filename) def allowed_file(filename): return '.' in filename and \ @@ -102,7 +115,7 @@ def start_upload(): res_body = {"upload_token": token} body = flask.request.get_json() if body and body.get('filename'): - res_body['filename'] = secure_filename(body['filename']) + res_body['filename'] = secure_filename_strict(body['filename']) return flask.Response(json.dumps(res_body), status=200) @@ -139,7 +152,7 @@ def finish_upload(token): token = secure_filename(token) filename = body['filename'] if filename and allowed_file(filename): - filename = secure_filename(filename) + filename = secure_filename_strict(filename) filepath = os.path.join(app.config['UPLOAD_FOLDER'], filename) tmppath = os.path.join(app.config['TEMP_FOLDER'], token) if not os.path.isfile(filepath): From a2a57c9aa8d156ff5bc4a5db165487cdd7f7ca43 Mon Sep 17 00:00:00 2001 From: CGDogan <126820728+CGDogan@users.noreply.github.com> Date: Thu, 29 Jun 2023 08:42:12 +0100 Subject: [PATCH 04/38] Build libvips ALSO without openslide --- Dockerfile | 52 +++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 51 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index d97ef8c..bb3510b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,30 @@ WORKDIR /var/www RUN apt-get update RUN apt-get -q update --fix-missing RUN apt-get -q install -y openslide-tools python3-openslide vim openssl -RUN apt-get -q install -y libvips libvips-dev + +# Tony has a future use case where we may adapt caMic to GIS visualization +# install libvips-dev for pyvips. No need for libvips. +RUN apt-get -q install -y libvips-dev + +# But, build libvips instead of using libvips-dev from apt +# Build without OpenSlide to open images with rather ImageMagick to handle +# images without pyramids. Otherwise opens e.g. DICOM with OpenSlide so conversion +# of files OpenSlide cannot open does not help at all. +# So, we'll have two copies on openslide on the system. +# By changing LD_LIBRARY_PATH before we launch python +# we can choose which openslide to run +# TODO: replace libjpeg-dev with libjpeg-turbo8-dev when current apt repo has it; for performance +RUN apt-get -q install -y meson libjpeg-dev libexif-dev libgsf-1-dev libtiff-dev libfftw3-dev liblcms2-dev libpng-dev libmagickcore-dev libmagickwand-dev liborc-0.4-dev libopenjp2-7 libgirepository1.0-dev +WORKDIR /root/src +RUN git clone https://github.com/libvips/libvips.git --depth=1 --branch=8.14 +RUN mkdir /root/src/libvips/build +WORKDIR /root/src/libvips +RUN mkdir /usr/local/vips-no-openslide/ +# normally --prefix=/usr/local/ --libdir=lib build +RUN meson setup -Dopenslide=disabled --buildtype=release --prefix=/usr/local/vips-no-openslide/ --libdir=lib build +RUN meson compile -C build +RUN meson test -C build +RUN meson install -C build RUN pip install pyvips RUN pip install flask @@ -12,6 +35,33 @@ RUN pip install gunicorn RUN pip install greenlet RUN pip install gunicorn[eventlet] +# verify pyvips can call libvips +RUN python3 -c "import pyvips" + +# verify that the apt libvips has openslide +ADD test_imgs/CMU-1-Small-Region.svs . +RUN python3 -c "import pyvips; pyvips.Image.openslideload(('CMU-1-Small-Region.svs'))" + +# back up previous ld_library_path +ENV LD_LIBRARY_PATH_ORIG="${LD_LIBRARY_PATH}" + +# now, prioritize openslideless libvips +# the path shown in output lines of "meson install" where .so.42 are installed +# normally /usr/local/lib/: +ENV LD_LIBRARY_PATH="/usr/local/vips-no-openslide/lib/:${LD_LIBRARY_PATH}" + +# verify that this libvips has no openslide +RUN ! python3 -c "import pyvips; pyvips.Image.openslideload(('CMU-1-Small-Region.svs'))" + +# ok, so to recap, +# there are two libvips are installed and which one pyvips connects to +# is chosen when "import pyvips" is run. +# at this point in this dockerfile, +# ld_library_path is set so that no-openslide version is run +# but if you do LD_LIBRARY_PATH="${LD_LIBRARY_PATH_ORIG}" python a.py +# or likewise using docker ENV command or os.environ in python before +# importing, this will remove the no-openslide libvips from path. + run openssl version -a ENV FLASK_ENV development From fe4f0c1a2391aa5fc626949f87bfae9a438789dd Mon Sep 17 00:00:00 2001 From: CGDogan <126820728+CGDogan@users.noreply.github.com> Date: Sun, 2 Jul 2023 15:29:11 +0100 Subject: [PATCH 05/38] Fix showing level count --- dev_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev_utils.py b/dev_utils.py index f344ac0..9180e64 100644 --- a/dev_utils.py +++ b/dev_utils.py @@ -36,7 +36,7 @@ def getMetadata(filename, upload_folder, extended): metadata['width'] = slideData.get(openslide.PROPERTY_NAME_BOUNDS_WIDTH, None) or slideData.get( "openslide.level[0].width", None) metadata['vendor'] = slideData.get(openslide.PROPERTY_NAME_VENDOR, None) - metadata['level_count'] = int(slideData.get('level_count', 1)) + metadata['level_count'] = int(slide.level_count) metadata['objective'] = float(slideData.get(openslide.PROPERTY_NAME_OBJECTIVE_POWER, 0) or slideData.get("aperio.AppMag", -1.0)) metadata['md5sum'] = file_md5(filepath) From b47ce7fe155de6ffdb6863451bee58a684e71938 Mon Sep 17 00:00:00 2001 From: CGDogan <126820728+CGDogan@users.noreply.github.com> Date: Wed, 26 Jul 2023 20:04:51 +0100 Subject: [PATCH 06/38] Use our openslide --- Dockerfile | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/Dockerfile b/Dockerfile index bb3510b..f916f51 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,10 @@ -FROM python:3 +FROM cgd30/openslide:v8main WORKDIR /var/www RUN apt-get update RUN apt-get -q update --fix-missing -RUN apt-get -q install -y openslide-tools python3-openslide vim openssl +RUN apt-get -q install -y python3-pip openslide-tools python3-openslide vim openssl +RUN apt-get -q install -y openssl libcurl4-openssl-dev libssl-dev # Tony has a future use case where we may adapt caMic to GIS visualization # install libvips-dev for pyvips. No need for libvips. @@ -16,8 +17,8 @@ RUN apt-get -q install -y libvips-dev # So, we'll have two copies on openslide on the system. # By changing LD_LIBRARY_PATH before we launch python # we can choose which openslide to run -# TODO: replace libjpeg-dev with libjpeg-turbo8-dev when current apt repo has it; for performance -RUN apt-get -q install -y meson libjpeg-dev libexif-dev libgsf-1-dev libtiff-dev libfftw3-dev liblcms2-dev libpng-dev libmagickcore-dev libmagickwand-dev liborc-0.4-dev libopenjp2-7 libgirepository1.0-dev +# TODO: use libjpeg-turbo8-dev instead of libjpeg-dev if current apt repo has it; for performance +RUN apt-get -q install -y meson libjpeg-turbo8-dev libexif-dev libgsf-1-dev libtiff-dev libfftw3-dev liblcms2-dev libpng-dev libmagickcore-dev libmagickwand-dev liborc-0.4-dev libopenjp2-7 libgirepository1.0-dev WORKDIR /root/src RUN git clone https://github.com/libvips/libvips.git --depth=1 --branch=8.14 RUN mkdir /root/src/libvips/build @@ -29,11 +30,11 @@ RUN meson compile -C build RUN meson test -C build RUN meson install -C build -RUN pip install pyvips -RUN pip install flask -RUN pip install gunicorn -RUN pip install greenlet -RUN pip install gunicorn[eventlet] +RUN pip install pyvips --break-system-packages +RUN pip install flask --break-system-packages +RUN pip install gunicorn --break-system-packages +RUN pip install greenlet --break-system-packages +RUN pip install gunicorn[eventlet] --break-system-packages # verify pyvips can call libvips RUN python3 -c "import pyvips" @@ -72,7 +73,7 @@ COPY ./ ./ RUN cp test_imgs/* /images/ -RUN pip3 install -r requirements.txt +RUN pip3 install -r requirements.txt --break-system-packages EXPOSE 4000 From c376ffcf03dbcf6f79eb531328b2a0ae41a96480 Mon Sep 17 00:00:00 2001 From: CGDogan <126820728+CGDogan@users.noreply.github.com> Date: Sat, 29 Jul 2023 09:41:45 +0100 Subject: [PATCH 07/38] camicroscope image-decoders --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index f916f51..28ee027 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM cgd30/openslide:v8main +FROM camicroscope/image-decoders:latest WORKDIR /var/www RUN apt-get update From 9ed85b7b74a7b4c99c9ab07ce8cf5b1d5580ea98 Mon Sep 17 00:00:00 2001 From: Ryan Birmingham Date: Tue, 1 Aug 2023 16:44:12 -0400 Subject: [PATCH 08/38] add dcm to allowed extensions --- SlideServer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SlideServer.py b/SlideServer.py index 15d6460..fbdef66 100644 --- a/SlideServer.py +++ b/SlideServer.py @@ -48,7 +48,7 @@ app.config['ROI_FOLDER'] = "/images/roiDownload" -ALLOWED_EXTENSIONS = set(['svs', 'tif', 'tiff', 'vms', 'vmu', 'ndpi', 'scn', 'mrxs', 'bif', 'svslide', 'png', 'jpg']) +ALLOWED_EXTENSIONS = set(['svs', 'tif', 'tiff', 'vms', 'vmu', 'ndpi', 'scn', 'mrxs', 'bif', 'svslide', 'png', 'jpg', 'dcm']) # should be used instead of secure_filename to create new files whose extensions are important. # use secure_filename to access previous files. From 4f5db8f9ff278f5a433ade47b734ff3c2ff37b77 Mon Sep 17 00:00:00 2001 From: CGDogan <126820728+CGDogan@users.noreply.github.com> Date: Mon, 14 Aug 2023 20:12:08 +0100 Subject: [PATCH 09/38] Cache pip result --- Dockerfile | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 28ee027..4461099 100644 --- a/Dockerfile +++ b/Dockerfile @@ -69,12 +69,11 @@ ENV FLASK_ENV development RUN mkdir -p /images/uploading -COPY ./ ./ - -RUN cp test_imgs/* /images/ - +COPY requirements.txt . RUN pip3 install -r requirements.txt --break-system-packages +COPY ./ ./ +RUN cp test_imgs/* /images/ EXPOSE 4000 EXPOSE 4001 From 9ff2f65a8b14f1cff97610125411703646c9040c Mon Sep 17 00:00:00 2001 From: CGDogan <126820728+CGDogan@users.noreply.github.com> Date: Tue, 15 Aug 2023 19:42:40 +0100 Subject: [PATCH 10/38] Define max limit for thumbnail size --- SlideServer.py | 1 + 1 file changed, 1 insertion(+) diff --git a/SlideServer.py b/SlideServer.py index fbdef66..b3f7d59 100644 --- a/SlideServer.py +++ b/SlideServer.py @@ -213,6 +213,7 @@ def singleSlide(filepath): @app.route("/data/thumbnail/", methods=['GET']) def singleThumb(filepath): size = flask.request.args.get('size', default=50, type=int) + size = min(500, size) res = getThumbnail(filepath, size) if (hasattr(res, 'error')): return flask.Response(json.dumps(res), status=500) From 4aa095463ab76959f33b8d09ed7f1fd5e1269288 Mon Sep 17 00:00:00 2001 From: CGDogan <126820728+CGDogan@users.noreply.github.com> Date: Wed, 16 Aug 2023 11:24:02 +0100 Subject: [PATCH 11/38] Reorder dockerfile --- Dockerfile | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 4461099..ea805aa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -31,10 +31,6 @@ RUN meson test -C build RUN meson install -C build RUN pip install pyvips --break-system-packages -RUN pip install flask --break-system-packages -RUN pip install gunicorn --break-system-packages -RUN pip install greenlet --break-system-packages -RUN pip install gunicorn[eventlet] --break-system-packages # verify pyvips can call libvips RUN python3 -c "import pyvips" @@ -63,6 +59,13 @@ RUN ! python3 -c "import pyvips; pyvips.Image.openslideload(('CMU-1-Small-Region # or likewise using docker ENV command or os.environ in python before # importing, this will remove the no-openslide libvips from path. +WORKDIR /root/src/ + +RUN pip install flask --break-system-packages +RUN pip install gunicorn --break-system-packages +RUN pip install greenlet --break-system-packages +RUN pip install gunicorn[eventlet] --break-system-package + run openssl version -a ENV FLASK_ENV development From a096f7645b3643e065a7c60fda619379f153dabf Mon Sep 17 00:00:00 2001 From: CGDogan <126820728+CGDogan@users.noreply.github.com> Date: Sat, 19 Aug 2023 10:28:07 +0100 Subject: [PATCH 12/38] getMetadata should take appended filepath --- SlideServer.py | 8 +++++--- dev_utils.py | 7 +++---- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/SlideServer.py b/SlideServer.py index b3f7d59..594ea2d 100644 --- a/SlideServer.py +++ b/SlideServer.py @@ -203,7 +203,7 @@ def testRoute(): @app.route("/data/one/", methods=['GET']) def singleSlide(filepath): extended = request.args.get('extended') - res = dev_utils.getMetadata(filepath, app.config['UPLOAD_FOLDER'], extended) + res = dev_utils.getMetadata(join(app.config['UPLOAD_FOLDER'], filepath), extended) if (hasattr(res, 'error')): return flask.Response(json.dumps(res), status=500) else: @@ -223,8 +223,10 @@ def singleThumb(filepath): @app.route("/data/many/", methods=['GET']) def multiSlide(filepathlist): - request.args.get('extended') - res = dev_utils.getMetadataList(json.loads(filepathlist), app.config['UPLOAD_FOLDER'], extended) + extended = request.args.get('extended') + filenames = json.loads(filepathlist) + paths = [join(app.config['UPLOAD_FOLDER'], filename) for filename in filenames] + res = dev_utils.getMetadataList(paths, extended) if (hasattr(res, 'error')): return flask.Response(json.dumps(res), status=500) else: diff --git a/dev_utils.py b/dev_utils.py index 9180e64..47ca303 100644 --- a/dev_utils.py +++ b/dev_utils.py @@ -10,10 +10,9 @@ # given a path, get metadata -def getMetadata(filename, upload_folder, extended): +def getMetadata(filepath, extended): # TODO consider restricting filepath metadata = {} - filepath = os.path.join(upload_folder, filename) if not os.path.isfile(filepath): msg = {"error": "No such file"} print(msg) @@ -60,10 +59,10 @@ def postslide(img, url, token=''): # given a list of path, get metadata for each -def getMetadataList(filenames, upload_folder, extended): +def getMetadataList(filenames, extended): allData = [] for filename in filenames: - allData.append(getMetadata(filename, upload_folder, extended)) + allData.append(getMetadata(filename, extended)) return allData From 3a3d0a6a82729ae2e0b6c886cbfa5af922e83ddd Mon Sep 17 00:00:00 2001 From: CGDogan <126820728+CGDogan@users.noreply.github.com> Date: Sat, 19 Aug 2023 10:50:21 +0100 Subject: [PATCH 13/38] typo --- NCISlideUtil.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NCISlideUtil.py b/NCISlideUtil.py index 78fc370..055ee72 100644 --- a/NCISlideUtil.py +++ b/NCISlideUtil.py @@ -72,7 +72,7 @@ def openslidedata(metadata): slide = openslide.OpenSlide(metadata['location']) slideData = slide.properties metadata['mpp-x'] = slideData.get(openslide.PROPERTY_NAME_MPP_X, None) - metadata['mpp-x'] = slideData.get(openslide.PROPERTY_NAME_MPP_Y, None) + metadata['mpp-y'] = slideData.get(openslide.PROPERTY_NAME_MPP_Y, None) metadata['mpp'] = metadata['mpp-x'] or metadata['mpp-x'] or None metadata['height'] = slideData.get( openslide.PROPERTY_NAME_BOUNDS_HEIGHT, None) From ddd0a140413d546ea63972b235faa88938fb1283 Mon Sep 17 00:00:00 2001 From: CGDogan <126820728+CGDogan@users.noreply.github.com> Date: Sat, 19 Aug 2023 11:17:04 +0100 Subject: [PATCH 14/38] reuse same metadata routes --- OmniLoad.py | 20 +++++--------------- SlideServer.py | 4 ++-- SlideUtil.py | 20 ++++++-------------- dev_utils.py | 10 +++++++--- 4 files changed, 20 insertions(+), 34 deletions(-) diff --git a/OmniLoad.py b/OmniLoad.py index 32a46ac..0ce0d5d 100644 --- a/OmniLoad.py +++ b/OmniLoad.py @@ -8,6 +8,7 @@ import json # for json in and out import requests # for api and pathdb in and out import hashlib +import dev_utils # for large csv fields, especially segmentations csv.field_size_limit(sys.maxsize) @@ -51,21 +52,10 @@ def file_md5(fileName): def openslidedata(manifest): for img in manifest: img['location'] = img.get("path", "") or img.get("location", "") or img.get("filename", "") or img.get("file", "") - slide = openslide.OpenSlide(img['location']) - slideData = slide.properties - img['mpp-x'] = slideData.get(openslide.PROPERTY_NAME_MPP_X, None) - img['mpp-y'] = slideData.get(openslide.PROPERTY_NAME_MPP_Y, None) - img['mpp'] = img['mpp-x'] or img['mpp-y'] - img['height'] = slideData.get(openslide.PROPERTY_NAME_BOUNDS_HEIGHT, None) or slideData.get( - "openslide.level[0].height", None) - img['width'] = slideData.get(openslide.PROPERTY_NAME_BOUNDS_WIDTH, None) or slideData.get( - "openslide.level[0].width", None) - img['vendor'] = slideData.get(openslide.PROPERTY_NAME_VENDOR, None) - img['level_count'] = int(slideData.get('level_count', 1)) - img['objective'] = float(slideData.get(openslide.PROPERTY_NAME_OBJECTIVE_POWER, 0) or - slideData.get("aperio.AppMag", -1.0)) - img['md5sum'] = file_md5(img['location']) - img['comment'] = slideData.get(openslide.PROPERTY_NAME_COMMENT, None) + metadata = dev_utils.getMetadata(img['location'], False, True) + for k, v in metadata: + if k not in img: + img[k] = v # required values which are often unused img['study'] = img.get('study', "") img['specimen'] = img.get('specimen', "") diff --git a/SlideServer.py b/SlideServer.py index 594ea2d..04576fd 100644 --- a/SlideServer.py +++ b/SlideServer.py @@ -203,7 +203,7 @@ def testRoute(): @app.route("/data/one/", methods=['GET']) def singleSlide(filepath): extended = request.args.get('extended') - res = dev_utils.getMetadata(join(app.config['UPLOAD_FOLDER'], filepath), extended) + res = dev_utils.getMetadata(join(app.config['UPLOAD_FOLDER'], filepath), extended, False) if (hasattr(res, 'error')): return flask.Response(json.dumps(res), status=500) else: @@ -226,7 +226,7 @@ def multiSlide(filepathlist): extended = request.args.get('extended') filenames = json.loads(filepathlist) paths = [join(app.config['UPLOAD_FOLDER'], filename) for filename in filenames] - res = dev_utils.getMetadataList(paths, extended) + res = dev_utils.getMetadataList(paths, extended, False) if (hasattr(res, 'error')): return flask.Response(json.dumps(res), status=500) else: diff --git a/SlideUtil.py b/SlideUtil.py index 2017f8b..63b79d1 100644 --- a/SlideUtil.py +++ b/SlideUtil.py @@ -5,7 +5,7 @@ import openslide -from dev_utils import file_md5 +from dev_utils import getMetadata from dev_utils import postslide from dev_utils import post_url @@ -34,22 +34,14 @@ def gen_thumbnail(filename, slide, size, imgtype="png"): def openslidedata(metadata): - slide = openslide.OpenSlide(metadata['location']) - slideData = slide.properties - metadata['mpp-x'] = slideData.get(openslide.PROPERTY_NAME_MPP_X, None) - metadata['mpp-x'] = slideData.get(openslide.PROPERTY_NAME_MPP_Y, None) - metadata['mpp'] = metadata['mpp-x'] or metadata['mpp-x'] or None - # metadata['height'] = slideData.get("openslide.level[0].height", None) - # metadata['width'] = slideData.get("openslide.level[0].width", None) - metadata['height'] = slideData.get(openslide.PROPERTY_NAME_BOUNDS_HEIGHT, None) - metadata['width'] = slideData.get(openslide.PROPERTY_NAME_BOUNDS_WIDTH, None) - metadata['vendor'] = slideData.get(openslide.PROPERTY_NAME_VENDOR, None) - metadata['level_count'] = int(slideData.get('level_count', 1)) - metadata['objective'] = float(slideData.get("aperio.AppMag", 0.0)) - metadata['md5sum'] = file_md5(metadata['location']) + metadata_retrieved = getMetadata(metadata['location'], False, True) + for k, v in metadata_retrieved: + if k not in metadata: + metadata[k] = v metadata['timestamp'] = time.time() thumbnail_size = config.get('thumbnail_size', None) if thumbnail_size: + slide = openslide.OpenSlide(metadata['location']) gen_thumbnail(metadata['location'], slide, thumbnail_size) return metadata diff --git a/dev_utils.py b/dev_utils.py index 47ca303..81d6f3b 100644 --- a/dev_utils.py +++ b/dev_utils.py @@ -10,10 +10,12 @@ # given a path, get metadata -def getMetadata(filepath, extended): +def getMetadata(filepath, extended, raise_exception): # TODO consider restricting filepath metadata = {} if not os.path.isfile(filepath): + if raise_exception: + raise ValueError("No such file") msg = {"error": "No such file"} print(msg) return msg @@ -21,6 +23,8 @@ def getMetadata(filepath, extended): try: slide = openslide.OpenSlide(filepath) except BaseException as e: + if raise_exception: + raise e msg = {"type": "Openslide", "error": str(e)} print(msg) return msg @@ -59,10 +63,10 @@ def postslide(img, url, token=''): # given a list of path, get metadata for each -def getMetadataList(filenames, extended): +def getMetadataList(filenames, extended, raise_exception): allData = [] for filename in filenames: - allData.append(getMetadata(filename, extended)) + allData.append(getMetadata(filename, extended, raise_exception)) return allData From 690283600edc84f32d47972cb765bf5659a4817a Mon Sep 17 00:00:00 2001 From: CGDogan <126820728+CGDogan@users.noreply.github.com> Date: Sat, 19 Aug 2023 11:35:50 +0100 Subject: [PATCH 15/38] fix my typo --- OmniLoad.py | 2 +- SlideUtil.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/OmniLoad.py b/OmniLoad.py index 0ce0d5d..d5e3a69 100644 --- a/OmniLoad.py +++ b/OmniLoad.py @@ -53,7 +53,7 @@ def openslidedata(manifest): for img in manifest: img['location'] = img.get("path", "") or img.get("location", "") or img.get("filename", "") or img.get("file", "") metadata = dev_utils.getMetadata(img['location'], False, True) - for k, v in metadata: + for k, v in metadata.items(): if k not in img: img[k] = v # required values which are often unused diff --git a/SlideUtil.py b/SlideUtil.py index 63b79d1..08d4e9f 100644 --- a/SlideUtil.py +++ b/SlideUtil.py @@ -35,7 +35,7 @@ def gen_thumbnail(filename, slide, size, imgtype="png"): def openslidedata(metadata): metadata_retrieved = getMetadata(metadata['location'], False, True) - for k, v in metadata_retrieved: + for k, v in metadata_retrieved.items(): if k not in metadata: metadata[k] = v metadata['timestamp'] = time.time() From d6eac56b839c2daf082d6e72855062e6e14661c7 Mon Sep 17 00:00:00 2001 From: CGDogan <126820728+CGDogan@users.noreply.github.com> Date: Tue, 22 Aug 2023 07:57:34 +0100 Subject: [PATCH 16/38] join -> os.path.join --- SlideServer.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/SlideServer.py b/SlideServer.py index 04576fd..8a2fe5c 100644 --- a/SlideServer.py +++ b/SlideServer.py @@ -6,8 +6,7 @@ import string import sys import pyvips -from os import listdir -from os.path import isfile, join +import os from spritemaker import createSpritesheet from PIL import Image import urllib @@ -203,7 +202,7 @@ def testRoute(): @app.route("/data/one/", methods=['GET']) def singleSlide(filepath): extended = request.args.get('extended') - res = dev_utils.getMetadata(join(app.config['UPLOAD_FOLDER'], filepath), extended, False) + res = dev_utils.getMetadata(os.path.join(app.config['UPLOAD_FOLDER'], filepath), extended, False) if (hasattr(res, 'error')): return flask.Response(json.dumps(res), status=500) else: @@ -225,7 +224,7 @@ def singleThumb(filepath): def multiSlide(filepathlist): extended = request.args.get('extended') filenames = json.loads(filepathlist) - paths = [join(app.config['UPLOAD_FOLDER'], filename) for filename in filenames] + paths = [os.path.join(app.config['UPLOAD_FOLDER'], filename) for filename in filenames] res = dev_utils.getMetadataList(paths, extended, False) if (hasattr(res, 'error')): return flask.Response(json.dumps(res), status=500) From c12b8ae0cdf4642de2a155219a6532ab44eecaa6 Mon Sep 17 00:00:00 2001 From: CGDogan <126820728+CGDogan@users.noreply.github.com> Date: Wed, 23 Aug 2023 16:33:58 +0100 Subject: [PATCH 17/38] mimetypes --- SlideServer.py | 104 ++++++++++++++++++++++++------------------------- 1 file changed, 52 insertions(+), 52 deletions(-) diff --git a/SlideServer.py b/SlideServer.py index b3f7d59..f641270 100644 --- a/SlideServer.py +++ b/SlideServer.py @@ -92,9 +92,9 @@ def makePyramid(filename, dest): while not os.path.exists(filepath): os.sync() sleep(750) - return flask.Response(json.dumps({"status": "OK", "srcFile":filename, "destFile":dest, "details":savedImg}), status=200) + return flask.Response(json.dumps({"status": "OK", "srcFile":filename, "destFile":dest, "details":savedImg}), status=200, mimetype='text/json') except BaseException as e: - return flask.Response(json.dumps({"type": "pyvips", "error": str(e)}), status=500) + return flask.Response(json.dumps({"type": "pyvips", "error": str(e)}), status=500, mimetype='text/json') # routes @@ -116,7 +116,7 @@ def start_upload(): body = flask.request.get_json() if body and body.get('filename'): res_body['filename'] = secure_filename_strict(body['filename']) - return flask.Response(json.dumps(res_body), status=200) + return flask.Response(json.dumps(res_body), status=200, mimetype='text/json') # using the token from the start upload endpoint, post data given offset. @@ -128,19 +128,19 @@ def continue_file(token): if os.path.isfile(tmppath): body = flask.request.get_json() if not body: - return flask.Response(json.dumps({"error": "Missing JSON body"}), status=400) + return flask.Response(json.dumps({"error": "Missing JSON body"}), status=400, mimetype='text/json') offset = body['offset'] or 0 if not 'data' in body: - return flask.Response(json.dumps({"error": "File data not found in body"}), status=400) + return flask.Response(json.dumps({"error": "File data not found in body"}), status=400, mimetype='text/json') else: data = base64.b64decode(body['data']) f = open(tmppath, "ab") f.seek(int(offset)) f.write(data) f.close() - return flask.Response(json.dumps({"status": "OK"}), status=200) + return flask.Response(json.dumps({"status": "OK"}), status=200, mimetype='text/json') else: - return flask.Response(json.dumps({"error": "Token Not Recognised"}), status=400) + return flask.Response(json.dumps({"error": "Token Not Recognised"}), status=400, mimetype='text/json') # end the upload, by removing the in progress indication; locks further modification @@ -148,7 +148,7 @@ def continue_file(token): def finish_upload(token): body = flask.request.get_json() if not body: - return flask.Response(json.dumps({"error": "Missing JSON body"}), status=400) + return flask.Response(json.dumps({"error": "Missing JSON body"}), status=400, mimetype='text/json') token = secure_filename(token) filename = body['filename'] if filename and allowed_file(filename): @@ -158,14 +158,14 @@ def finish_upload(token): if not os.path.isfile(filepath): if os.path.isfile(tmppath): shutil.move(tmppath, filepath) - return flask.Response(json.dumps({"ended": token, "filepath": filepath})) + return flask.Response(json.dumps({"ended": token, "filepath": filepath}), status=200, mimetype='text/json') else: - return flask.Response(json.dumps({"error": "Token Not Recognised"}), status=400) + return flask.Response(json.dumps({"error": "Token Not Recognised"}), status=400, mimetype='text/json') else: - return flask.Response(json.dumps({"error": "File with name '" + filename + "' already exists", "filepath": filepath}), status=400) + return flask.Response(json.dumps({"error": "File with name '" + filename + "' already exists", "filepath": filepath}), status=400, mimetype='text/json') else: - return flask.Response(json.dumps({"error": "Invalid filename"}), status=400) + return flask.Response(json.dumps({"error": "Invalid filename"}), status=400, mimetype='text/json') # check for token # get info associated with token @@ -177,7 +177,7 @@ def slide_delete(): body = flask.request.get_json() if not body: - return flask.Response(json.dumps({"error": "Missing JSON body"}), status=400) + return flask.Response(json.dumps({"error": "Missing JSON body"}), status=400, mimetype='text/json') filename = body['filename'] if filename and allowed_file(filename): @@ -185,12 +185,12 @@ def slide_delete(): filepath = os.path.join(app.config['UPLOAD_FOLDER'], filename) if os.path.isfile(filepath): os.remove(os.path.join(app.config['UPLOAD_FOLDER'], filename)) - return flask.Response(json.dumps({"deleted": filename, "success": True})) + return flask.Response(json.dumps({"deleted": filename, "success": True}), mimetype='text/json') else: - return flask.Response(json.dumps({"error": "File with name '" + filename + "' does not exist"}), status=400) + return flask.Response(json.dumps({"error": "File with name '" + filename + "' does not exist"}), status=400, mimetype='text/json') else: - return flask.Response(json.dumps({"error": "Invalid filename"}), status=400) + return flask.Response(json.dumps({"error": "Invalid filename"}), status=400, mimetype='text/json') # check for file if it exists or not # delete the file @@ -205,9 +205,9 @@ def singleSlide(filepath): extended = request.args.get('extended') res = dev_utils.getMetadata(filepath, app.config['UPLOAD_FOLDER'], extended) if (hasattr(res, 'error')): - return flask.Response(json.dumps(res), status=500) + return flask.Response(json.dumps(res), status=500, mimetype='text/json') else: - return flask.Response(json.dumps(res), status=200) + return flask.Response(json.dumps(res), status=200, mimetype='text/json') @app.route("/data/thumbnail/", methods=['GET']) @@ -216,9 +216,9 @@ def singleThumb(filepath): size = min(500, size) res = getThumbnail(filepath, size) if (hasattr(res, 'error')): - return flask.Response(json.dumps(res), status=500) + return flask.Response(json.dumps(res), status=500, mimetype='text/json') else: - return flask.Response(json.dumps(res), status=200) + return flask.Response(json.dumps(res), status=200, mimetype='text/json') @app.route("/data/many/", methods=['GET']) @@ -226,9 +226,9 @@ def multiSlide(filepathlist): request.args.get('extended') res = dev_utils.getMetadataList(json.loads(filepathlist), app.config['UPLOAD_FOLDER'], extended) if (hasattr(res, 'error')): - return flask.Response(json.dumps(res), status=500) + return flask.Response(json.dumps(res), status=500, mimetype='text/json') else: - return flask.Response(json.dumps(res), status=200) + return flask.Response(json.dumps(res), status=200, mimetype='text/json') @app.route("/getSlide/") @@ -236,7 +236,7 @@ def getSlide(image_name): if(os.path.isfile("/images/"+image_name)): return flask.send_from_directory(app.config["UPLOAD_FOLDER"], image_name, as_attachment=True) else: - return flask.Response(json.dumps({"error": "File does not exist"}), status=404) + return flask.Response(json.dumps({"error": "File does not exist"}), status=404, mimetype='text/json') # using the token from the start url upload endpoint @app.route('/urlupload/continue/', methods=['POST']) @@ -246,19 +246,19 @@ def continue_urlfile(token): if os.path.isfile(tmppath): body = flask.request.get_json() if not body: - return flask.Response(json.dumps({"error": "Missing JSON body"}), status=400) + return flask.Response(json.dumps({"error": "Missing JSON body"}), status=400, mimetype='text/json') if not 'url' in body: - return flask.Response(json.dumps({"error": "File url not present in body"}), status=400) + return flask.Response(json.dumps({"error": "File url not present in body"}), status=400, mimetype='text/json') else: url = body['url'] try: url = urllib.parse.unquote(url) urllib.request.urlretrieve(url, tmppath) - return flask.Response(json.dumps({"status": "OK Uploaded"}), status=200) + return flask.Response(json.dumps({"status": "OK Uploaded"}), status=200, mimetype='text/json') except: - return flask.Response(json.dumps({"error": "URL invalid"}), status=400) + return flask.Response(json.dumps({"error": "URL invalid"}), status=400, mimetype='text/json') else: - return flask.Response(json.dumps({"error": "Token Not Recognised"}), status=400) + return flask.Response(json.dumps({"error": "Token Not Recognised"}), status=400, mimetype='text/json') # Route to check if the URL file has completely uploaded to the server # Query Params: 'url', 'token' @@ -271,9 +271,9 @@ def urlUploadStatus(): urlFileSize = int(info.headers['Content-Length']) fileSize = os.path.getsize(app.config['TEMP_FOLDER']+'/'+token) if(fileSize >= urlFileSize): - return flask.Response(json.dumps({"uploaded": "True"}), status=200) + return flask.Response(json.dumps({"uploaded": "True"}), status=200, mimetype='text/json') else: - return flask.Response(json.dumps({"uploaded": "False"}), status=200) + return flask.Response(json.dumps({"uploaded": "False"}), status=200, mimetype='text/json') @@ -299,13 +299,13 @@ def getLabelsZips(): file.close() if(zipfile.is_zipfile(tmppath) == False): deleteDataset(userFolder) - return flask.Response(json.dumps({'error': 'Not a valid zip file/files'}), status=400) + return flask.Response(json.dumps({'error': 'Not a valid zip file/files'}), status=400, mimetype='text/json') with zipfile.ZipFile(tmppath, 'r') as zip_ref: zip_ref.extractall(tmppath[0:-4]) csvFile = pathlib.Path(tmppath[0:-4]+'/patches.csv') if not csvFile.is_file(): deleteDataset(userFolder) - return flask.Response(json.dumps({'error': 'Not a valid labels zip/zips'}), status=400) + return flask.Response(json.dumps({'error': 'Not a valid labels zip/zips'}), status=400, mimetype='text/json') csvFile = tmppath[0:-4]+'/patches.csv' with open(csvFile, 'r') as data1: i = 0 @@ -318,7 +318,7 @@ def getLabelsZips(): labelsData['counts'][labelsData['labels'].index( line[2])] += 1 i += 1 - return flask.Response(json.dumps(labelsData), status=200) + return flask.Response(json.dumps(labelsData), status=200, mimetype='text/json') # Route to receive base64 encoded zip file for custom dataset. @@ -340,7 +340,7 @@ def getCustomData(): if(final == 'true'): if(zipfile.is_zipfile(path) == False): deleteDataset(userFolder) - return flask.Response(json.dumps({'error': 'Not a valid zip file/files'}), status=400) + return flask.Response(json.dumps({'error': 'Not a valid zip file/files'}), status=400, mimetype='text/json') file = zipfile.ZipFile(path, 'r') # app.logger.info(file.namelist()) contents = file.namelist() @@ -348,10 +348,10 @@ def getCustomData(): for item in contents: if '/' not in item: deleteDataset(userFolder) - return flask.Response(json.dumps({'error': 'zip should contain only folders!'}), status=400) + return flask.Response(json.dumps({'error': 'zip should contain only folders!'}), status=400, mimetype='text/json') if item.endswith('/') == False and item.endswith('.jpg') == False and item.endswith('.jpeg') == False and item.endswith('.png') == False and item.endswith('.tif') == False and item.endswith('.tiff') == False: deleteDataset(userFolder) - return flask.Response(json.dumps({'error': 'Dataset zip should have only png/jpg/tif files!'}), status=400) + return flask.Response(json.dumps({'error': 'Dataset zip should have only png/jpg/tif files!'}), status=400, mimetype='text/json') if item.split('/')[0] not in labelsData['labels']: labelsData['labels'].append(item.split('/')[0]) for label in labelsData['labels']: @@ -360,9 +360,9 @@ def getCustomData(): if item.startswith(label+'/'): count+=1 labelsData['counts'].append(count) - return flask.Response(json.dumps(labelsData), status=200) + return flask.Response(json.dumps(labelsData), status=200, mimetype='text/json') else: - return flask.Response(json.dumps({'status' : 'pending'}), status=200) + return flask.Response(json.dumps({'status' : 'pending'}), status=200, mimetype='text/json') # Route to organise the extracted zip file data according to user sent customized labels and create a spritesheet @@ -394,7 +394,7 @@ def generateSprite(): file = pathlib.Path(path+line[8][1:]) if not file.is_file(): deleteDataset(userFolder) - return flask.Response(json.dumps({'error': 'Images are missing from one or more zip files'}), status=400) + return flask.Response(json.dumps({'error': 'Images are missing from one or more zip files'}), status=400, mimetype='text/json') file = path+line[8][1:] fileName = ''.join(random.choice( string.ascii_lowercase + string.digits) for _ in range(40)) @@ -405,7 +405,7 @@ def generateSprite(): try: createSpritesheet(app.config['DATASET_FOLDER']+userFolder, selectedLabels, width, height) except: - return flask.Response(json.dumps({'error': str(sys.exc_info()[0])}), status=400) + return flask.Response(json.dumps({'error': str(sys.exc_info()[0])}), status=400, mimetype='text/json') with open(app.config['DATASET_FOLDER']+userFolder+'/spritesheet/labelnames.csv', 'w', newline='') as labelsnamesCSVfile: writer = csv.writer(labelsnamesCSVfile) writer.writerow(selectedLabels) @@ -419,7 +419,7 @@ def generateSprite(): '/spritesheet/labelnames.csv', '/labelnames.csv') download_link = '/workbench/sprite/download/'+userFolder download_file(userFolder) - return flask.Response(json.dumps({'status': 'done', 'userFolder': userFolder, 'download': download_link}), status=200) + return flask.Response(json.dumps({'status': 'done', 'userFolder': userFolder, 'download': download_link}), status=200, mimetype='text/json') # Route to extract images and create the spritesheet according to user defined labels incase of custom data @@ -450,7 +450,7 @@ def generateCustomSprite(): try: createSpritesheet(app.config['DATASET_FOLDER']+userFolder, selectedLabels, width, height) except: - return flask.Response(json.dumps({'error': str(sys.exc_info()[0])}), status=400) + return flask.Response(json.dumps({'error': str(sys.exc_info()[0])}), status=400, mimetype='text/json') with open(app.config['DATASET_FOLDER']+userFolder+'/spritesheet/labelnames.csv', 'w', newline='') as labelsnamesCSVfile: writer = csv.writer(labelsnamesCSVfile) writer.writerow(selectedLabels) @@ -463,7 +463,7 @@ def generateCustomSprite(): '/spritesheet/labelnames.csv', '/labelnames.csv') download_link = '/workbench/sprite/download/'+userFolder download_file(userFolder) - return flask.Response(json.dumps({'status': 'done', 'userFolder': userFolder, 'download': download_link}), status=200) + return flask.Response(json.dumps({'status': 'done', 'userFolder': userFolder, 'download': download_link}), status=200, mimetype='text/json') # Dynamic download route for dataset.zip @@ -477,9 +477,9 @@ def download_file(userFolder): @app.route('/workbench/deleteDataset/', methods=['POST']) def deleteDataset(userFolder): if '/' in userFolder or '..' in userFolder or len(userFolder) != 20: - return flask.Response(json.dumps({"deleted": "false", 'message': 'Traversal detected or invalid foldername'}), status=403) + return flask.Response(json.dumps({"deleted": "false", 'message': 'Traversal detected or invalid foldername'}), status=403, mimetype='text/json') shutil.rmtree(app.config['DATASET_FOLDER']+userFolder) - return flask.Response(json.dumps({"deleted": "true"}), status=200) + return flask.Response(json.dumps({"deleted": "true"}), status=200, mimetype='text/json') # Helper function for converting slides into jpg @@ -563,7 +563,7 @@ def roiExtract(): # img = slide.open_image(img_path) # res= { "data" :"" } # res['data']= pred - return flask.Response(json.dumps({"extracted": "true"}), status=200) + return flask.Response(json.dumps({"extracted": "true"}), status=200, mimetype='text/json') # Route to send back the extracted @app.route('/roiextract/') @@ -591,7 +591,7 @@ def run(self): def gDriveGetFile(): body = flask.request.get_json() if not body: - return flask.Response(json.dumps({"error": "Missing JSON body"}), status=400) + return flask.Response(json.dumps({"error": "Missing JSON body"}), status=400, mimetype='text/json') token = "".join(random.choice(string.ascii_uppercase + string.digits) for _ in range(10)) token = secure_filename(token) @@ -605,20 +605,20 @@ def gDriveGetFile(): try: params = start(body['userId']) except: - return flask.Response(json.dumps({'error': str(sys.exc_info()[0])}), status=400) + return flask.Response(json.dumps({'error': str(sys.exc_info()[0])}), status=400, mimetype='text/json') thread_a = getFileFromGdrive(params, body['userId'], body['fileId'], token) thread_a.start() - return flask.Response(json.dumps({"authURL": params["auth_url"], "token": token}), status=200) + return flask.Response(json.dumps({"authURL": params["auth_url"], "token": token}), status=200, mimetype='text/json') # To check if a particular file is downloaded from Gdrive @app.route('/googleDriveUpload/checkStatus', methods=['POST']) def checkDownloadStatus(): body = flask.request.get_json() if not body: - return flask.Response(json.dumps({"error": "Missing JSON body"}), status=400) + return flask.Response(json.dumps({"error": "Missing JSON body"}), status=400, mimetype='text/json') token = body['token'] path = app.config['TEMP_FOLDER']+'/'+token if os.path.isfile(path): - return flask.Response(json.dumps({"downloadDone": True}), status=200) - return flask.Response(json.dumps({"downloadDone": False}), status=200) + return flask.Response(json.dumps({"downloadDone": True}), status=200, mimetype='text/json') + return flask.Response(json.dumps({"downloadDone": False}), status=200, mimetype='text/json') From 1f8ec29cd668727c7ae83a8118aa7b76c54972f7 Mon Sep 17 00:00:00 2001 From: CGDogan <126820728+CGDogan@users.noreply.github.com> Date: Wed, 23 Aug 2023 16:43:51 +0100 Subject: [PATCH 18/38] small deprecation --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index ea805aa..940e2ba 100644 --- a/Dockerfile +++ b/Dockerfile @@ -68,7 +68,7 @@ RUN pip install gunicorn[eventlet] --break-system-package run openssl version -a -ENV FLASK_ENV development +ENV FLASK_DEBUG True RUN mkdir -p /images/uploading From 0c9687bafb3368f4b4d5d3c9daffaa5bef28b4b0 Mon Sep 17 00:00:00 2001 From: CGDogan <126820728+CGDogan@users.noreply.github.com> Date: Thu, 24 Aug 2023 21:05:13 +0100 Subject: [PATCH 19/38] Use abstract classes --- OmniLoad.py | 3 +- OpenSlideReader.py | 61 +++++++++++++++++++++++++++++++ SlideServer.py | 15 +++++--- SlideUtil.py | 11 ++++-- file_extensions.py | 25 +++++++++++++ image_reader.py | 91 ++++++++++++++++++++++++++++++++++++++++++++++ make_thumbs.py | 4 +- thumbsFromUrl.py | 4 +- 8 files changed, 199 insertions(+), 15 deletions(-) create mode 100644 OpenSlideReader.py create mode 100644 file_extensions.py create mode 100644 image_reader.py diff --git a/OmniLoad.py b/OmniLoad.py index d5e3a69..cadcb4d 100644 --- a/OmniLoad.py +++ b/OmniLoad.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -import openslide # to get required slide metadata +import dev_utils # to get required slide metadata import csv # to read csv import sys # for csv limit import os # for os and filepath utils @@ -8,7 +8,6 @@ import json # for json in and out import requests # for api and pathdb in and out import hashlib -import dev_utils # for large csv fields, especially segmentations csv.field_size_limit(sys.maxsize) diff --git a/OpenSlideReader.py b/OpenSlideReader.py new file mode 100644 index 0000000..9e28ecc --- /dev/null +++ b/OpenSlideReader.py @@ -0,0 +1,61 @@ +import openslide +import image_reader +import dev_utils +from file_extensions import OPENSLIDE_EXTENSIONS + +class OpenSlideReader(image_reader.ImageReader): + @staticmethod + def reader_name(): + return "openslide" + + @staticmethod + def extensions_set(): + return OPENSLIDE_EXTENSIONS + + def __init__(self, imagepath): + self._image_path = imagepath + self._reader = openslide.OpenSlide(imagepath) + + @property + def level_count(self): + return self._reader.level_count + + @property + def dimensions(self): + return self._reader.dimensions + + @property + def level_dimensions(self): + return self._reader.level_dimensions + + @property + def associated_images(self): + return self._reader.associated_images + + def read_region(self, location, level, size): + return self._reader.read_region(location, level, size) + + def get_thumbnail(self, max_size): + return self._reader.get_thumbnail(max_size) + + def get_basic_metadata(self, extended): + slideData = self._reader.properties + if extended: + metadata = {k:v for (k,v) in slideData.items()} + else: + metadata = {} + metadata['width'] = slideData.get(openslide.PROPERTY_NAME_BOUNDS_WIDTH, None) \ + or slideData.get( "openslide.level[0].width", None) + metadata['height'] = slideData.get(openslide.PROPERTY_NAME_BOUNDS_HEIGHT, None) \ + or slideData.get("openslide.level[0].height", None) + metadata['mpp-x'] = slideData.get(openslide.PROPERTY_NAME_MPP_X, None) + metadata['mpp-y'] = slideData.get(openslide.PROPERTY_NAME_MPP_Y, None) + metadata['vendor'] = slideData.get(openslide.PROPERTY_NAME_VENDOR, None) + metadata['level_count'] = int(self._reader.level_count) + metadata['objective'] = float(slideData.get(openslide.PROPERTY_NAME_OBJECTIVE_POWER, 0) \ + or slideData.get("aperio.AppMag", -1.0)) + metadata['comment'] = slideData.get(openslide.PROPERTY_NAME_COMMENT, None) + metadata['study'] = "" + metadata['specimen'] = "" + metadata['md5'] = dev_utils.file_md5(self._image_path) + return metadata diff --git a/SlideServer.py b/SlideServer.py index 9e519ce..41908d5 100644 --- a/SlideServer.py +++ b/SlideServer.py @@ -13,7 +13,7 @@ import flask import flask_cors from flask import request -import openslide +from image_reader import construct_reader from werkzeug.utils import secure_filename import dev_utils import requests @@ -23,6 +23,7 @@ import logging from gDriveDownload import start, afterUrlAuth, callApi from threading import Thread +from file_extensions import ALLOWED_EXTENSIONS try: from io import BytesIO @@ -47,7 +48,6 @@ app.config['ROI_FOLDER'] = "/images/roiDownload" -ALLOWED_EXTENSIONS = set(['svs', 'tif', 'tiff', 'vms', 'vmu', 'ndpi', 'scn', 'mrxs', 'bif', 'svslide', 'png', 'jpg', 'dcm']) # should be used instead of secure_filename to create new files whose extensions are important. # use secure_filename to access previous files. @@ -73,14 +73,19 @@ def getThumbnail(filename, size=50): if not os.path.isfile(filepath): return {"error": "No such file"} try: - slide = openslide.OpenSlide(filepath) + slide = construct_reader(filepath) + except BaseException as e: + # here, e has attribute "error" + return e + + try: thumb = slide.get_thumbnail((size, size)) buffer = BytesIO() thumb.save(buffer, format="PNG") data = 'data:image/png;base64,' + str(base64.b64encode(buffer.getvalue()))[2:-1] return {"slide": data, "size": size} except BaseException as e: - return {"type": "Openslide", "error": str(e)} + return {"type": slide.reader_name(), "error": str(e)} @app.route('/slide//pyramid/', methods=['POST']) def makePyramid(filename, dest): @@ -507,7 +512,7 @@ def convert(fname, input_dir , output_dir): try: save_name = fname.split(".")[0] + ".jpg" - os_obj = openslide.OpenSlide(input_dir+"/"+fname) + os_obj = construct_reader(input_dir+"/"+fname) w, h = os_obj.dimensions w_rep, h_rep = int(w/UNIT_X)+1, int(h/UNIT_Y)+1 w_end, h_end = w%UNIT_X, h%UNIT_Y diff --git a/SlideUtil.py b/SlideUtil.py index 08d4e9f..9aa232f 100644 --- a/SlideUtil.py +++ b/SlideUtil.py @@ -1,9 +1,9 @@ import csv -import subprocess +import os import time from multiprocessing.pool import ThreadPool -import openslide +from image_reader import construct_reader from dev_utils import getMetadata from dev_utils import postslide @@ -34,14 +34,17 @@ def gen_thumbnail(filename, slide, size, imgtype="png"): def openslidedata(metadata): - metadata_retrieved = getMetadata(metadata['location'], False, True) + if not os.path.isfile(metadata['location']): + raise IOError("No such file") + + slide = construct_reader(metadata['location']) + metadata_retrieved = slide.get_basic_metadata(False) for k, v in metadata_retrieved.items(): if k not in metadata: metadata[k] = v metadata['timestamp'] = time.time() thumbnail_size = config.get('thumbnail_size', None) if thumbnail_size: - slide = openslide.OpenSlide(metadata['location']) gen_thumbnail(metadata['location'], slide, thumbnail_size) return metadata diff --git a/file_extensions.py b/file_extensions.py new file mode 100644 index 0000000..c060924 --- /dev/null +++ b/file_extensions.py @@ -0,0 +1,25 @@ +# Clone github.com/ome/bioformats and run: +# grep -r -e "static final String.*\[\].*SUFFIXES" -e "super.*new String\[\]" -e "super(.*," -A 2 -h | grep " .* .* " | grep -v "=$" | grep -v "{$" | sort -u +# then manually copying extensions, keeping them in order +# The command says: match 'static final String[] XYZ_SUFFIXES' or 'super("XYZFILES", ".xyz")' +# and print next two lines but keep only lines with at least two space and discard those +# ending with "=" or "{" (because in this case only the next line(s) are relevant +# then sort and keep unique lines + + +# An alternative would be using https://github.com/ome/bioformats/blob/19d7fb9cbfdc1/components/formats-api/src/loci/formats/readers.txt +# Or, https://bio-formats.readthedocs.io/en/latest/supported-formats.html +# For file extensions with multiple dots, here only last part is recorded +# Video, compression formats and duplicates excluded +# https://github.com/ome/bioformats/blob/19d7fb9cbfdc1/components/formats-api/src/loci/formats/readers.txt +# External readers listed on that webpage need to be excluded +# All the rest needs to be included +BIOFORMATS_EXTENSIONS = set(['v3draw', 'ano', 'cfg', 'csv', 'htm', 'rec', 'tim', 'zpo', 'tif', 'dic', 'dcm', 'dicom', 'jp2', 'j2ki', 'j2kr', 'raw', 'ima', 'cr2', 'crw', 'jpg', 'thm', 'wav', 'tiff', 'dv', 'r3d', 'r3d_d3d', 'log', 'mvd2', 'aisf', 'aiix', 'dat', 'atsf', 'tf2', 'tf8', 'btf', 'pbm', 'pgm', 'ppm', 'xdce', 'xml', 'xlog', 'apl', 'tnb', 'mtb', 'im', 'mea', 'res', 'aim', 'arf', 'psd', 'al3d', 'gel', 'am', 'amiramesh', 'grey', 'hx', 'labels', 'img', 'hdr', 'sif', 'afi', 'svs', 'exp', 'h5', '1sc', 'pic', 'scn', 'ims', 'ch5', 'vsi', 'ets', 'pnl', 'htd', 'c01', 'dib', 'cxd', 'v', 'eps', 'epsi', 'ps', 'flex', 'xlef', 'fits', 'fts', 'dm2', 'dm3', 'dm4', 'naf', 'his', 'ndpi', 'ndpis', 'vms', 'txt', 'i2i', 'hed', 'mod', 'inr', 'ipl', 'ipm', 'fff', 'ics', 'ids', 'seq', 'ips', 'ipw', 'frm', 'par', 'j2k', 'jpf', 'jpk', 'jpx', 'klb', 'xv', 'bip', 'sxm', 'fli', 'lim', 'msr', 'lif', 'lof', 'lei', 'l2d', 'mnc', 'stk', 'nd', 'scan', 'vff', 'mrw', 'stp', 'mng', 'nii', 'nrrd', 'nhdr', 'nd2', 'nef', 'obf', 'omp2info', 'oib', 'oif', 'pty', 'lut', 'oir', 'sld', 'spl', 'liff', 'top', 'pcoraw', 'pcx', 'pict', 'pct', 'df3', 'im3', 'qptiff', 'bin', 'env', 'spe', 'afm', 'sm2', 'sm3', 'spc', 'set', 'sdt', 'spi', 'xqd', 'xqf', 'db', 'vws', 'pst', 'inf', 'tfr', 'ffr', 'zfr', 'zfp', '2fl', 'tga', 'pr3', 'dti', 'fdf', 'hdf', 'bif', 'xys', 'html', 'acff', 'wat', 'bmp', 'wpi', 'czi', 'lms', 'lsm', 'mdb', 'zvi', 'mrc', 'st', 'ali', 'map', 'mrcs', 'jpeg', 'png', 'gif', 'ptif']) + +OPENSLIDE_EXTENSIONS = set(["svs", "tif", "tiff", "vms", "vmu", "ndpi", "scn", "mrxs", "bif", "svslide", 'dcm', 'dicom']) + +# iipsrv support jpg, jpg2000, tiff +# we can also produce pyramids +OTHER_EXTENSIONS = ["jp2", "j2k", "jpx", "tif", "tiff", "tif", "png", "jpg", "jpeg"] + +ALLOWED_EXTENSIONS = BIOFORMATS_EXTENSIONS.union(OPENSLIDE_EXTENSIONS).union(OTHER_EXTENSIONS) diff --git a/image_reader.py b/image_reader.py new file mode 100644 index 0000000..a196119 --- /dev/null +++ b/image_reader.py @@ -0,0 +1,91 @@ +# Please note: if you would like to import a specific reader, +# you should "import image_reader" first to avoid a cyclic dependency error + +from abc import ABCMeta, abstractmethod + +# Allow near drop-in replacements for OpenSlide-Python +class ImageReader(metaclass=ABCMeta): + # currently: "openslide", "bioformats" + @staticmethod + @abstractmethod + def reader_name(): + pass + + @staticmethod + @abstractmethod + def extensions_set(): + pass + + # Raises exception on error + @abstractmethod + def __init__(self): + pass + + @property + @abstractmethod + def level_count(self): + pass + + @property + @abstractmethod + def dimensions(self): + pass + + @property + @abstractmethod + def level_dimensions(self): + pass + + @abstractmethod + def associated_images(self): + pass + + @abstractmethod + def read_region(self, location, level, size): + pass + + @abstractmethod + def get_thumbnail(self, max_size): + pass + + # raises exception + @abstractmethod + def get_basic_metadata(self, extended): + pass + +from OpenSlideReader import OpenSlideReader +from BioFormatsReader import BioFormatsReader + +# Decreasing order of importance +readers = [OpenSlideReader, BioFormatsReader] + + +# Replaces the constructor of the abstract class +# Usage: +# image = ImageReader.construct_reader("/file/path") +# Returns a reader +# Otherwise raises an object with attribute "error" +def construct_reader(imagepath): + relevant_readers = [] + extension = imagepath.split(".")[-1].lower() + + for r in readers: + if extension in r.extensions_set(): + relevant_readers.append(r) + if len(relevant_readers) == 0: + raise RuntimeError({"error": "File extension unsupported, no readers are compatible"}) + + reader_names = [] + reader = None + errors = [] + for r in relevant_readers: + try: + reader = r(imagepath) + break + except Exception as e: + reader_names.append(r.reader_name()) + errors.append(r.reader_name() + ": " + str(e)) + continue + if reader is None: + raise RuntimeError({"type": ",".join(reader_names), "error": ", ".join(errors)}) + return reader diff --git a/make_thumbs.py b/make_thumbs.py index 0d1bf8a..bcaf9ce 100644 --- a/make_thumbs.py +++ b/make_thumbs.py @@ -1,5 +1,5 @@ import requests -import openslide +from image_reader import construct_reader import pycurl import os @@ -33,7 +33,7 @@ def process(record): # skip ones which already have a thumbnail, unless otherwise specified if REGNERATE or not record.get("thumbnail", False): try: - with openslide.OpenSlide(file) as slide: + with construct_reader(file) as slide: gen_thumbnail(name, slide, IM_SIZE, imgtype="png") setThumb(record['_id']["$oid"], name+".png") # return empty to denote no issue. diff --git a/thumbsFromUrl.py b/thumbsFromUrl.py index a2c16fe..581cbc5 100644 --- a/thumbsFromUrl.py +++ b/thumbsFromUrl.py @@ -1,4 +1,4 @@ -import openslide +from image_reader import construct_reader from multiprocessing.pool import ThreadPool import requests import sys @@ -18,7 +18,7 @@ def get_thumbnail(obj): slide = obj['file-location'] dest = obj['case_id'] + '.png' try: - image = openslide.open_slide(slide) + image = construct_reader(slide) image.get_thumbnail([200,200]).save(dest, "PNG") return dest except BaseException as e: From f0b83a374b969fed37daf2a9010c5edc2b035e75 Mon Sep 17 00:00:00 2001 From: CGDogan <126820728+CGDogan@users.noreply.github.com> Date: Thu, 24 Aug 2023 21:12:10 +0100 Subject: [PATCH 20/38] Update dev_utils.py --- dev_utils.py | 40 +++++++++++++--------------------------- 1 file changed, 13 insertions(+), 27 deletions(-) diff --git a/dev_utils.py b/dev_utils.py index 81d6f3b..75cfc79 100644 --- a/dev_utils.py +++ b/dev_utils.py @@ -2,8 +2,7 @@ import os import json import requests - -import openslide +import image_reader post_url = "http://ca-back:4010/data/Slide/post" @@ -12,41 +11,28 @@ # given a path, get metadata def getMetadata(filepath, extended, raise_exception): # TODO consider restricting filepath - metadata = {} if not os.path.isfile(filepath): if raise_exception: raise ValueError("No such file") msg = {"error": "No such file"} print(msg) return msg - metadata['location'] = filepath try: - slide = openslide.OpenSlide(filepath) + slide = image_reader.construct_reader(filepath) except BaseException as e: if raise_exception: raise e - msg = {"type": "Openslide", "error": str(e)} - print(msg) - return msg - slideData = slide.properties - if extended: - return {k:v for (k,v) in slideData.items()} - else: - metadata['mpp-x'] = slideData.get(openslide.PROPERTY_NAME_MPP_X, None) - metadata['mpp-y'] = slideData.get(openslide.PROPERTY_NAME_MPP_Y, None) - metadata['height'] = slideData.get(openslide.PROPERTY_NAME_BOUNDS_HEIGHT, None) or slideData.get( - "openslide.level[0].height", None) - metadata['width'] = slideData.get(openslide.PROPERTY_NAME_BOUNDS_WIDTH, None) or slideData.get( - "openslide.level[0].width", None) - metadata['vendor'] = slideData.get(openslide.PROPERTY_NAME_VENDOR, None) - metadata['level_count'] = int(slide.level_count) - metadata['objective'] = float(slideData.get(openslide.PROPERTY_NAME_OBJECTIVE_POWER, 0) or - slideData.get("aperio.AppMag", -1.0)) - metadata['md5sum'] = file_md5(filepath) - metadata['comment'] = slideData.get(openslide.PROPERTY_NAME_COMMENT, None) - metadata['study'] = "" - metadata['specimen'] = "" - return metadata + # here, e has attribute "error" + return str(e) + + try: + metadata = slide.get_basic_metadata(extended) + except BaseException as e: + if raise_exception: + raise e + return {'error': str(e)} + metadata['location'] = filepath + return metadata def postslide(img, url, token=''): From 590afb4c103bd2cba7b1ad748426dcdf3ebf09a6 Mon Sep 17 00:00:00 2001 From: CGDogan <126820728+CGDogan@users.noreply.github.com> Date: Thu, 24 Aug 2023 21:17:51 +0100 Subject: [PATCH 21/38] bioformats not ready --- SlideServer.py | 1 + image_reader.py | 3 +-- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/SlideServer.py b/SlideServer.py index 41908d5..3b822a9 100644 --- a/SlideServer.py +++ b/SlideServer.py @@ -24,6 +24,7 @@ from gDriveDownload import start, afterUrlAuth, callApi from threading import Thread from file_extensions import ALLOWED_EXTENSIONS +from time import sleep try: from io import BytesIO diff --git a/image_reader.py b/image_reader.py index a196119..7ac73d0 100644 --- a/image_reader.py +++ b/image_reader.py @@ -54,10 +54,9 @@ def get_basic_metadata(self, extended): pass from OpenSlideReader import OpenSlideReader -from BioFormatsReader import BioFormatsReader # Decreasing order of importance -readers = [OpenSlideReader, BioFormatsReader] +readers = [OpenSlideReader] # Replaces the constructor of the abstract class From 473bdaf96b3eb0be85ea4325388f76b332aa95c2 Mon Sep 17 00:00:00 2001 From: CGDogan <126820728+CGDogan@users.noreply.github.com> Date: Thu, 24 Aug 2023 22:00:13 +0100 Subject: [PATCH 22/38] Add BioFormats --- .gitignore | 3 ++ BioFormatsReader.py | 110 ++++++++++++++++++++++++++++++++++++++++++++ Dockerfile | 11 ++++- README.md | 4 ++ dev_utils.py | 4 ++ image_reader.py | 5 +- 6 files changed, 134 insertions(+), 3 deletions(-) create mode 100644 BioFormatsReader.py diff --git a/.gitignore b/.gitignore index 7bbc71c..76cb2c0 100644 --- a/.gitignore +++ b/.gitignore @@ -99,3 +99,6 @@ ENV/ # mypy .mypy_cache/ + +# BFBridge - should always stay in a separate repo but copied manually or by Docker +BFBridge/ diff --git a/BioFormatsReader.py b/BioFormatsReader.py new file mode 100644 index 0000000..d28a188 --- /dev/null +++ b/BioFormatsReader.py @@ -0,0 +1,110 @@ +import image_reader +import dev_utils +import ome_types +from file_extensions import BIOFORMATS_EXTENSIONS +import BFBridge.python as bfbridge + + +jvm = bfbridge.BFBridgeVM() + +class BioFormatsReader(image_reader.ImageReader): + @staticmethod + def reader_name(): + return "bioformats" + + @staticmethod + def extensions_set(): + return BIOFORMATS_EXTENSIONS + + def __init__(self, imagepath): + if not hasattr(dev_utils.keep_alive_for_thread, "bfthread"): + dev_utils.keep_alive_for_thread.bfthread = bfbridge.BFBridgeThread(jvm) + + # Conventionally internal attributes start with underscore. + # When using them without underscore, there's the risk that + # a property has the same name as a/the getter, which breaks + # the abstract class. Hence all internal attributes start with underscore. + self._bfreader = bfbridge.BFBridgeInstance(dev_utils.keep_alive_for_thread.bfthread) + if self._bfreader is None: + raise RuntimeError("cannot make bioformats instance") + self._image_path = imagepath + code = self._bfreader.open(imagepath) + if code < 0: + raise IOError("Could not open file " + imagepath + ": " + self._bfreader.get_error_string()) + # Note: actually stores the format, not the vendor ("Hamamatsu NDPI" instead of "Hamamatsu") + self._vendor = self._bfreader.get_format() + self._level_count = self._bfreader.get_resolution_count() + self._dimensions = (self._bfreader.get_size_x(), self._bfreader.get_size_y()) + self._level_dimensions = [self._dimensions] + for l in range(1, self._level_count): + self._bfreader.set_current_resolution(l) + self._level_dimensions.append( \ + (self._bfreader.get_size_x(), self._bfreader.get_size_y())) + + @property + def level_count(self): + return self._level_count + + @property + def dimensions(self): + return self._dimensions + + @property + def level_dimensions(self): + return self._level_dimensions + + @property + def associated_images(self): + return None + + def read_region(self, location, level, size): + self._bfreader.set_current_resolution(level) + return self._bfreader.open_bytes_pil_image(0, \ + location[0], location[1], size[0], size[1]) + + def get_thumbnail(self, max_size): + return self._bfreader.open_thumb_bytes_pil_image(0, max_size[0], max_size[1]) + + def get_basic_metadata(self, extended): + metadata = {} + + try: + ome_xml_raw = self._bfreader.dump_ome_xml_metadata() + except BaseException as e: + raise OverflowError("XML metadata too large for file considering the preallocated buffer length. " + str(e)) + try: + ome_xml = ome_types.from_xml(ome_xml_raw) + except BaseException as e: + raise RuntimeError("get_basic_metadata: OME-XML parsing of metadata failed, error: " + \ + str(e) + " when parsing: " + ome_xml_raw) + + # https://www.openmicroscopy.org/Schemas/Documentation/Generated/OME-2016-06/ome_xsd.html + # https://bio-formats.readthedocs.io/en/latest/metadata-summary.html + + if extended: + return {"ome-xml": ome_xml_raw} + + metadata['width'] = str(self._dimensions[0]) + metadata['height'] = str(self._dimensions[1]) + try: + metadata['mpp-x'] = str(ome_xml.images[0].pixels.physical_size_x) + metadata['mpp-y'] = str(ome_xml.images[0].pixels.physical_size_y) + except: + metadata['mpp-x'] = "0" + metadata['mpp-y'] = "0" + metadata['vendor'] = self._vendor + metadata['level_count'] = int(self._level_count) + try: + metadata['objective'] = ome_xml.instruments[0].objectives[0].nominal_magnification + except: + try: + metadata['objective'] = ome_xml.instruments[0].objectives[0].calibrated_magnification + except: + metadata['objective'] = -1.0 + + metadata['comment'] = "" + metadata['study'] = "" + metadata['specimen'] = "" + metadata['md5sum'] = dev_utils.file_md5(self._image_path) + + return metadata diff --git a/Dockerfile b/Dockerfile index 940e2ba..546afc0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,6 +6,8 @@ RUN apt-get -q update --fix-missing RUN apt-get -q install -y python3-pip openslide-tools python3-openslide vim openssl RUN apt-get -q install -y openssl libcurl4-openssl-dev libssl-dev +### Install libvips + # Tony has a future use case where we may adapt caMic to GIS visualization # install libvips-dev for pyvips. No need for libvips. RUN apt-get -q install -y libvips-dev @@ -59,6 +61,13 @@ RUN ! python3 -c "import pyvips; pyvips.Image.openslideload(('CMU-1-Small-Region # or likewise using docker ENV command or os.environ in python before # importing, this will remove the no-openslide libvips from path. +### Install BioFormats wrapper + +WORKDIR /root/src/BFBridge/python +RUN python3 compile_bfbridge.py + +### Set up the server + WORKDIR /root/src/ RUN pip install flask --break-system-packages @@ -83,7 +92,7 @@ EXPOSE 4001 #debug/dev only # ENV FLASK_APP SlideServer.py -# CMD python -m flask run --host=0.0.0.0 --port=4000 +# CMD python3 -m flask run --host=0.0.0.0 --port=4000 # The Below BROKE the ability for users to upload images. # # non-root user diff --git a/README.md b/README.md index a7a7c3b..c05dfe2 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,10 @@ # SlideLoader Tool for loading slides and getting slide metadata using openslide +## Setting up + +When used outside of caMicroscope/Distro Docker, the [BFBridge](https://github.com/camicroscope/BFBridge) folder needs to copied to this repository manually. (Not only its contents, but making it a subfolder of this repository) + ## Usage ### Upload diff --git a/dev_utils.py b/dev_utils.py index 75cfc79..da0ff1a 100644 --- a/dev_utils.py +++ b/dev_utils.py @@ -3,11 +3,15 @@ import json import requests import image_reader +import threading post_url = "http://ca-back:4010/data/Slide/post" +# Keep BioFormats Java thread alive; to minimize reattaching thread +keep_alive_for_thread = threading.local() + # given a path, get metadata def getMetadata(filepath, extended, raise_exception): # TODO consider restricting filepath diff --git a/image_reader.py b/image_reader.py index 7ac73d0..8b7a3bc 100644 --- a/image_reader.py +++ b/image_reader.py @@ -54,9 +54,10 @@ def get_basic_metadata(self, extended): pass from OpenSlideReader import OpenSlideReader +from BioFormatsReader import BioFormatsReader -# Decreasing order of importance -readers = [OpenSlideReader] +# Decreasing order of performance +readers = [OpenSlideReader, BioFormatsReader] # Replaces the constructor of the abstract class From 9eb6004b6734071e1f733b307faf2111d79ee0f4 Mon Sep 17 00:00:00 2001 From: CGDogan <126820728+CGDogan@users.noreply.github.com> Date: Fri, 25 Aug 2023 09:37:20 +0100 Subject: [PATCH 23/38] Update requirements.txt --- requirements.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/requirements.txt b/requirements.txt index 7e6e95e..0f31178 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,6 +5,8 @@ flask-cors requests numpy Pillow +cffi +ome-xml google-api-python-client google-auth-httplib2 google-auth-oauthlib From 234a2fe665f44b72349a32c2a7227334b4c8f374 Mon Sep 17 00:00:00 2001 From: CGDogan <126820728+CGDogan@users.noreply.github.com> Date: Fri, 25 Aug 2023 09:39:38 +0100 Subject: [PATCH 24/38] Update requirements.txt --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 0f31178..8a6525b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ requests numpy Pillow cffi -ome-xml +ome-types google-api-python-client google-auth-httplib2 google-auth-oauthlib From edef85ecfc3b14a4225154b6bf5f15296b2653fe Mon Sep 17 00:00:00 2001 From: Ryan Birmingham Date: Fri, 25 Aug 2023 11:40:02 -0400 Subject: [PATCH 25/38] explain the specimen/study field empty strings --- OpenSlideReader.py | 1 + 1 file changed, 1 insertion(+) diff --git a/OpenSlideReader.py b/OpenSlideReader.py index 9e28ecc..c4deef4 100644 --- a/OpenSlideReader.py +++ b/OpenSlideReader.py @@ -55,6 +55,7 @@ def get_basic_metadata(self, extended): metadata['objective'] = float(slideData.get(openslide.PROPERTY_NAME_OBJECTIVE_POWER, 0) \ or slideData.get("aperio.AppMag", -1.0)) metadata['comment'] = slideData.get(openslide.PROPERTY_NAME_COMMENT, None) + # caMicroscope expects some value for study and specimen for slides, add empty string as defauly. metadata['study'] = "" metadata['specimen'] = "" metadata['md5'] = dev_utils.file_md5(self._image_path) From ef61a684de19bc39cdab3145a5a3f4cfd01cd4f2 Mon Sep 17 00:00:00 2001 From: Ryan Birmingham Date: Fri, 25 Aug 2023 11:58:39 -0400 Subject: [PATCH 26/38] add develop to container build and push --- .github/workflows/container.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/container.yml b/.github/workflows/container.yml index cbc6b65..c91bcf2 100644 --- a/.github/workflows/container.yml +++ b/.github/workflows/container.yml @@ -7,7 +7,7 @@ name: Container Publish on: push: - branches: ['master', 'auto-build'] + branches: ['master', 'develop'] env: REGISTRY: ghcr.io From 0af970a9ed0a36317c69ea321a91a5d0e1cae762 Mon Sep 17 00:00:00 2001 From: CGDogan <126820728+CGDogan@users.noreply.github.com> Date: Sat, 2 Sep 2023 06:52:59 +0100 Subject: [PATCH 27/38] Add note about updating BioFormats --- _file_extensions_bioformatsv7.0.0.txt | 348 ++++++++++++++++++++++++++ file_extensions.py | 5 +- 2 files changed, 352 insertions(+), 1 deletion(-) create mode 100644 _file_extensions_bioformatsv7.0.0.txt diff --git a/_file_extensions_bioformatsv7.0.0.txt b/_file_extensions_bioformatsv7.0.0.txt new file mode 100644 index 0000000..28a2293 --- /dev/null +++ b/_file_extensions_bioformatsv7.0.0.txt @@ -0,0 +1,348 @@ + domains = new String[] {FormatTools.LM_DOMAIN}; + super("SlideBook 7 SLD (native)", new String[] {"sldy"}); + CompressionType.J2K.getCompression()}; + CompressionType.UNCOMPRESSED.getCompression()}; + super("Vaa3d", new String[]{"v3draw"}); + "ano", "cfg", "csv", "htm", "rec", "tim", "zpo", "tif"}); + "dic", "dcm", "dicom", "jp2", "j2ki", "j2kr", "raw", "ima"}); + AWTImageTools.makeColorSpace(nChannels), nChannels == 4, false, + COMPRESSION_UNCOMPRESSED, + ColorModel.TRANSLUCENT, dataType); + CompressionType.UNCOMPRESSED.getCompression(), + FormatTools.UNKNOWN_DOMAIN}; + new String[] {"cr2", "crw", "jpg", "thm", "wav", "tif", "tiff"}); + new String[] {"dv", "r3d", "r3d_d3d", "dv.log", "r3d.log"}); + new String[] {"mvd2", "aisf", "aiix", "dat", "atsf"}); + new String[] {"ome.tif", "ome.tiff", "ome.tf2", "ome.tf8", "ome.btf"}); + new String[] {"pbm", "pgm", "ppm"}); + new String[] {"xdce", "xml", "tiff", "tif", "xlog"}); + new String[] {CompressionType.UNCOMPRESSED.getCompression(), + new String[] {FormatTools.ASTRONOMY_DOMAIN, FormatTools.UNKNOWN_DOMAIN}; + null, path); + suffixNecessary = false; + suffixSufficient = false; + super("JPEG", new String[] {"jpg", "jpeg", "jpe"}); + super(name, recType, offset, recLength); + super(name, type, offset, length); + "dic", "dcm", "dicom", "j2ki", "j2kr" + /** + // call getStatistics() to ensure that single-slice stacks have the + // correct pixel type + // remove unavailable stack formats + ClassList classes = ImageReader.getDefaultReaderClasses(); + LMSFileReader.log = LOGGER; + bankData = dataArray; + bankData = new byte[1][]; + bankData = new int[1][]; + bankData[0] = dataArray; + canSeparateSeries = false; + compressionTypes = new String[] {CompressionType.J2K_LOSSY.getCompression(), + datasetDescription = "One .hed file plus one similarly-named .img file"; + datasetDescription = "One .htd file plus one or more .tif files"; + datasetDescription = "One .ndpis file and at least one .ndpi file"; + datasetDescription = "One .omp2info file and at least one .oir or .vsi file"; + datasetDescription = "One .vms file plus several .jpg files"; + datasetDescription = "One or more .c01 files"; + domains = FormatTools.NON_GRAPHICS_DOMAINS; + domains = new String[] {FormatTools.ASTRONOMY_DOMAIN}; + domains = new String[] {FormatTools.EM_DOMAIN}; + domains = new String[] {FormatTools.FLIM_DOMAIN}; + domains = new String[] {FormatTools.GEL_DOMAIN}; + domains = new String[] {FormatTools.GRAPHICS_DOMAIN}; + domains = new String[] {FormatTools.HCS_DOMAIN}; + domains = new String[] {FormatTools.HISTOLOGY_DOMAIN, FormatTools.LM_DOMAIN}; + domains = new String[] {FormatTools.HISTOLOGY_DOMAIN}; + domains = new String[] {FormatTools.LM_DOMAIN, FormatTools.FLIM_DOMAIN, + domains = new String[] {FormatTools.LM_DOMAIN, FormatTools.HCS_DOMAIN}; + domains = new String[] {FormatTools.LM_DOMAIN, FormatTools.HISTOLOGY_DOMAIN}; + domains = new String[] {FormatTools.LM_DOMAIN}; + domains = new String[] {FormatTools.MEDICAL_DOMAIN, + domains = new String[] {FormatTools.MEDICAL_DOMAIN, FormatTools.LM_DOMAIN}; + domains = new String[] {FormatTools.MEDICAL_DOMAIN}; + domains = new String[] {FormatTools.SEM_DOMAIN}; + domains = new String[] {FormatTools.SPM_DOMAIN}; + domains = new String[] {FormatTools.UNKNOWN_DOMAIN}; + domains = new String[]{FormatTools.FLIM_DOMAIN}; + handle = new ByteArrayHandle(INITIAL_CAPACITY); + hasCompanionFiles = true; + helper = new DataBufferShort(data, size); + helper = new DataBufferShort(data, size, offset); + helper = new DataBufferShort(data, size, offsets); + helper = new DataBufferShort(size); + helper = new DataBufferShort(size, numbanks); + ifds = new IFDList(); + imageFormat = checkImageFormat(); + imageFormat = xlif.getImageFormat(); + index = -1; + initChildren(); + initImagePaths(); + inputOrder = dimensionOrder; + legacyReader = new TiffJAIReader(); + legacyReader = new TileJPEGReader(); + nativeReader = new DefaultJPEGReader(); + nativeReader = new TiffReader(); + new String[] {"apl", "tnb", "mtb" }; + new String[] {"tif", "tiff", "im"}; + new String[] {"xml", "xlog"}; + new String[] {FLEX_SUFFIX, MEA_SUFFIX, RES_SUFFIX}; + new String[] {MEA_SUFFIX, RES_SUFFIX}; + noSubresolutions = true; + panel = getTextPanel(); + reader = new ImageProcessorReader(r); + suffixNecessary = false; + suffixNecessary = false; // allow extensionless IPLab files + suffixNecessary = true; + suffixSufficient = false; + suffixSufficient = true; + super( "CellVoyager", new String[] { "tif", "xml" } ); + super("AIM", "aim"); + super("ARF", "arf"); + super("Adobe Photoshop TIFF", new String[] {"tif", "tiff"}); + super("Adobe Photoshop", "psd"); + super("Alicona AL3D", "al3d"); + super("Amersham Biosciences GEL", new String[] {"gel"}); + super("Amira", new String[] {"am", "amiramesh", "grey", "hx", "labels"}); + super("Analyze 7.5", new String[] {"img", "hdr"}); + super("Andor SIF", "sif"); + super("Animated PNG", "png"); + super("Aperio AFI", "afi"); + super("Aperio SVS", new String[] {"svs"}); + super("Audio Video Interleave", "avi"); + super("BD Pathway", new String[] {"exp", "tif"}); + super("BDV", new String[] {"xml", "h5"}); + super("Bio-Rad GEL", "1sc"); + super("Bio-Rad PIC", new String[] {"pic", "xml", "raw"}); + super("Bio-Rad SCN", "scn"); + super("Bitplane Imaris 3 (TIFF)", "ims"); + super("Bitplane Imaris 5.5 (HDF)", "ims"); + super("Bitplane Imaris", "ims"); + super("Bruker", ""); + super("Burleigh", "img"); + super("Canon RAW", new String[] {"cr2", "crw", "jpg", "thm", "wav"}); + super("CellH5 (HDF)", "ch5"); + super("CellSens VSI", new String[] {"vsi", "ets"}); + super("CellWorx", new String[] {"pnl", "htd", "log"}); + super("Cellomics C01", new String[] {"c01", "dib"}); + super("Compix Simple-PCI", "cxd"); + super("DICOM", "dcm"); + super("DNG", + super("Deltavision", + super("ECAT7", "v"); + super("Encapsulated PostScript", new String[] {"eps", "epsi", "ps"}); + super("Encapsulated PostScript", new String[] {"eps", "epsi"}); + super("Evotec Flex", SUFFIXES); + super("Extended leica file", "xlef"); + super("FEI TIFF", new String[] {"tif", "tiff"}); + super("FEI/Philips", "img"); + super("File pattern", new String[] {"pattern"}); + super("Flexible Image Transport System", new String[] {"fits", "fts"}); + super("FlowSight", "cif"); + super("Fuji LAS 3000", new String[] {"img", "inf"}); + super("Gatan DM2", "dm2"); + super("Gatan Digital Micrograph", new String[] {"dm3", "dm4"}); + super("Graphics Interchange Format", "gif"); + super("Hamamatsu Aquacosmos", "naf"); + super("Hamamatsu HIS", "his"); + super("Hamamatsu NDPI", new String[] {"ndpi"}); + super("Hamamatsu NDPIS", "ndpis"); + super("Hamamatsu VMS", "vms"); + super("Hitachi", "txt"); + super("I2I", new String[] {"i2i"}); + super("IMAGIC", new String[] {"hed", "img"}); + super("IMOD", "mod"); + super("INR", "inr"); + super("IPLab", "ipl"); + super("IVision", "ipm"); + super("Imacon", new String[] {"fff"}); + super("Image Cytometry Standard", new String[] {"ics", "ids"}); + super("Image Cytometry Standard", new String[] {"ids", "ics"}); + super("Image-Pro Sequence", new String[] {"seq", "ips"}); + super("Image-Pro Workspace", "ipw"); + super("Improvision TIFF", new String[] {"tif", "tiff"}); + super("InCell 1000/2000", + super("InCell 3000", "frm"); + super("Inveon", new String[] {"hdr"}); + super("Ionpath MIBI", new String[] {"tif, tiff"}); + super("JEOL", new String[] {"dat", "img", "par"}); + super("JPEG", new String[] {"jpg", "jpeg", "jpe"}); + super("JPEG", new String[] {"jpg", "jpeg", "jpe"}, "jpeg"); + super("JPEG-2000", "jp2"); + super("JPEG-2000", new String[] {"jp2", "j2k", "jpf"}); + super("JPK Instruments", "jpk"); + super("JPX", "jpx"); + super("KLB", "klb"); + super("Khoros XV", "xv"); + super("Kodak Molecular Imaging", "bip"); + super("LEO", new String[] {"sxm", "tif", "tiff"}); + super("LI-FLIM", "fli"); + super("Laboratory Imaging", "lim"); + super("Lavision Imspector", "msr"); + super("Leica Image File Format", "lif"); + super("Leica Object Format", "lof"); + super("Leica SCN", new String[] {"scn"}); + super("Leica TCS TIFF", new String[] {"tif", "tiff", "xml"}); + super("Leica", new String[] {"lei", "tif", "tiff", "raw"}); + super("Li-Cor L2D", new String[] {"l2d", "scn", "tif"}); + super("MIAS", new String[] {"tif", "tiff", "txt"}); + super("MINC MRI", "mnc"); + super("Medical Research Council", MRC_SUFFIXES); + super("MetaXpress TIFF", new String[] {"htd", "tif"}); + super("Metamorph STK", new String[] {"stk", "nd", "scan", "tif", "tiff"}); + super("Metamorph TIFF", new String[] {"tif", "tiff"}); + super("Micro-Manager", new String[] {"tif", "tiff", "txt", "xml"}); + super("MicroCT", "vff"); + super("Mikroscan TIFF", new String[] {"tif", "tiff"}); + super("Minolta MRW", "mrw"); + super("Molecular Imaging", "stp"); + super("Multiple images", ".*"); + super("Multiple-image Network Graphics", "mng"); + super("NIfTI", new String[] {"nii", "img", "hdr", "nii.gz"}); + super("NOAA-HRD Gridded Data Format", ""); + super("NRRD", new String[] {"nrrd", "nhdr"}); + super("Nikon Elements TIFF", new String[] {"tif", "tiff"}); + super("Nikon ND2", new String[] {"nd2", "jp2"}); + super("Nikon NEF", new String[] {"nef", "tif", "tiff"}); + super("Nikon TIFF", new String[] {"tif", "tiff"}); + super("OBF", new String[] {"obf", "msr"}); + super("OME-TIFF", + super("OME-TIFF", OME_TIFF_SUFFIXES); + super("OME-XML", new String[] {"ome", "ome.xml"}); + super("Olympus .omp2info", "omp2info"); + super("Olympus APL", new String[] {"apl", "tnb", "mtb", "tif"}); + super("Olympus FV1000", new String[] {"oib", "oif", "pty", "lut"}); + super("Olympus Fluoview/ABD TIFF", new String[] {"tif", "tiff"}); + super("Olympus OIR", "oir"); + super("Olympus SIS TIFF", new String[] {"tif", "tiff"}); + super("Olympus ScanR", new String[] {"dat", "xml", "tif"}); + super("Olympus Slidebook", new String[] {"sld", "spl"}); + super("Openlab LIFF", "liff"); + super("Openlab RAW", "raw"); + super("Oxford Instruments", "top"); + super("PCO-RAW", new String[] {"pcoraw", "rec"}); + super("PCX", "pcx"); + super("PICT", new String[] {"pict", "pct"}); + super("POV-Ray", "df3"); + super("Perkin Elmer Densitometer", new String[] {"hdr", "img"}); + super("Perkin-Elmer Nuance IM3", "im3"); + super("PerkinElmer Columbus", new String[] {"xml"}); + super("PerkinElmer Operetta", new String[] {"tif", "tiff", "xml"}); + super("PerkinElmer Vectra/QPTIFF", new String[] {"tiff", "tif", "qptiff"}); + super("PicoQuant Bin", "bin"); + super("Portable Any Map", + super("Prairie TIFF", new String[] {"tif", "tiff", "cfg", "env", "xml"}); + super("Princeton Instruments SPE", "spe"); + super("Pyramid TIFF", new String[] {"tif", "tiff"}); + super("Quesant AFM", "afm"); + super("QuickTime", "mov"); + super("RHK Technologies", new String[] {"sm2", "sm3"}); + super("SBIG", ""); + super("SM Camera", ""); + super("SPC FIFO Data", new String[] {"spc", "set"}); + super("SPCImage Data", "sdt"); + super("SPIDER", "spi"); + super("Seiko", new String[] {"xqd", "xqf"}); + super("SimplePCI TIFF", new String[] {"tif", "tiff"}); + super("Simulated data", "fake"); + super("Slidebook TIFF", new String[] {"tif", "tiff"}); + super("Tagged Image File Format", TIFF_SUFFIXES); + super("Tagged Image File Format", TiffReader.TIFF_SUFFIXES); + super("Tecan Spark Cyto", new String[] {"db"}); + super("Text", new String[] {"txt", "csv"}); + super("Tile JPEG", new String[] {"jpg", "jpeg"}); + super("TillVision", new String[] {"vws", "pst", "inf"}); + super("TopoMetrix", new String[] {"tfr", "ffr", "zfr", "zfp", "2fl"}); + super("Trestle", new String[] {"tif"}); + super("Truevision Targa", "tga"); + super("UBM", "pr3"); + super("Unisoku STM", new String[] {"hdr", "dat"}); + super("VG SAM", "dti"); + super("Varian FDF", "fdf"); + super("Veeco", "hdf"); + super("Ventana .bif", new String[] {"bif"}); + super("Visitech XYS", new String[] {"xys", "html"}); + super("Volocity Library Clipping", "acff"); + super("Volocity Library", + super("WA Technology TOP", "wat"); + super("Windows Bitmap", "bmp"); + super("Yokogawa CV7000", new String[] {"wpi"}); + super("Zeiss AxioVision TIFF", TIFF_SUFFIXES); + super("Zeiss CZI", "czi"); + super("Zeiss LMS", "lms"); + super("Zeiss Laser-Scanning Microscopy", new String[] {"lsm", "mdb"}); + super("Zeiss Vision Image (ZVI)", "zvi"); + super("Zip", "zip"); + super("importer-options.txt", ImporterOptions.class); + super(DataBuffer.TYPE_BYTE, size); + super(DataBuffer.TYPE_INT, size); + super(DataBuffer.TYPE_USHORT, size); + super(DataBuffer.TYPE_USHORT, size, 1, offset); + super(DataBuffer.TYPE_USHORT, size, data.length); + super(DataBuffer.TYPE_USHORT, size, data.length, offsets); + super(DataBuffer.TYPE_USHORT, size, numbanks); + super(filepath, null); + super(filepath, parent); + super(format, exts); + super(format, suffix); + super(format, suffixes); + super(getWidth(r, path, r.getSeries()), getHeight(r, path, r.getSeries()), + super(imp, ic); + super(key, save, label, info); + super(message, cause); + super(name, extensions); + super(name, suffix); + super(name, suffixes); + super(pixelBits, makeBitArray(nChannels, pixelBits), + super(r, seriesNo); + super(s, cause); + super(store, filter); + super(title, headings, data, w, h); + super(title, stack); + super(type, components); + super(x, y, w, h); + this.datasetDescription = "Directory with 2 master files 'MeasurementResult.xml' and 'MeasurementResult.ome.xml', used to stitch together several TIF files."; + this.defaultValue = defaultValue; + this.hasCompanionFiles = true; + this.kind = kind; + this.op = op; this.options = options; this.cf = cf; + this.possibleValues = possibleValues; + this.value = defaultValue; + tileCount = xlif.getTileCount(); + {"mrc", "st", "ali", "map", "rec", "mrcs"}; + {"ome.tiff", "ome.tif", "ome.tf2", "ome.tf8", "ome.btf", "companion.ome"}; + {"tif", "tiff", "tf2", "tf8", "btf"}; + } + /** Constructs a format reader with the given name and default suffixes. */ + /** Constructs a format writer with the given name and default suffixes. */ + /** Constructs a new BaseTiffReader. */ + /** TIFF tiles must be of a height and width divisible by 16. */ + // -- Fields -- + // -- IFormatWriter API methods -- + // Private tags present in Prairie TIFF files + private static final String[] BIG_TIFF_SUFFIXES = {"tf2", "tf8", "btf"}; + private static final String[] METADATA_SUFFIXES = new String[] {"dat", "xml"}; + public AVIWriter() { super("Audio Video Interleave", "avi"); } + public BaseTiffReader(String name, String suffix) { super(name, suffix); } + public CacheException(String s, Throwable cause) { super(s, cause); } + public CacheException(Throwable cause) { super(cause); } + public CoreMetadataList(int size1, int size2) { super(size1, size2); } + public FormatException(String s, Throwable cause) { super(s, cause); } + public FormatException(Throwable cause) { super(cause); } + public FormatReader(String format, String suffix) { super(format, suffix); } + public FormatWriter(String format, String suffix) { super(format, suffix); } + public JavaWriter() { super("Java source code", "java"); } + public MissingLibraryException(String s, Throwable cause) { super(s, cause); } + public MissingLibraryException(Throwable cause) { super(cause); } + public SubResolutionFormatReader(String format, String suffix) { super(format, suffix); } + public UnknownFormatException(String s, Throwable cause) { super(s, cause); } + public UnknownFormatException(Throwable cause) { super(cause); } + public UnsupportedCompressionException(Throwable cause) { super(cause); } + public static final String SKIP_MISSING_WELLS = "scanr.skip_missing_wells"; + public static final String XML_NAME = "_meta.xml"; + public static final String[] COMPANION_SUFFIXES = {"xml", "txt"}; + public static final String[] COMPRESSION_SUFFIXES = {"bz2", "gz"}; + public static final String[] FV1000_SUFFIXES = {"oib", "oif"}; + public static final String[] PRAIRIE_SUFFIXES = {"cfg", "env", "xml"}; + public static final String[] TIFF_SUFFIXES = {"tif", "xml"}; + public static final boolean SKIP_MISSING_WELLS_DEFAULT = true; + public static final int IMAGEJ_TAG = 50839; + } diff --git a/file_extensions.py b/file_extensions.py index c060924..bf03b36 100644 --- a/file_extensions.py +++ b/file_extensions.py @@ -1,6 +1,9 @@ +# Updating BioFormats: # Clone github.com/ome/bioformats and run: # grep -r -e "static final String.*\[\].*SUFFIXES" -e "super.*new String\[\]" -e "super(.*," -A 2 -h | grep " .* .* " | grep -v "=$" | grep -v "{$" | sort -u -# then manually copying extensions, keeping them in order +# then save the output to a file and "diff" it with _file_extensions_bioformatsvX.X.X.txt +# then manually copy the new extensions, making an attempt to preserve the order +# then delete the previous _file_extensions_bioformatsvX.X.X.txt and rename the new output # The command says: match 'static final String[] XYZ_SUFFIXES' or 'super("XYZFILES", ".xyz")' # and print next two lines but keep only lines with at least two space and discard those # ending with "=" or "{" (because in this case only the next line(s) are relevant From 7046db9044573602e8d235af79b32ebc2099df6c Mon Sep 17 00:00:00 2001 From: CGDogan <126820728+CGDogan@users.noreply.github.com> Date: Tue, 5 Sep 2023 11:17:35 +0100 Subject: [PATCH 28/38] Revert #66 build libvips twice --- Dockerfile | 57 ++---------------------------------------------------- 1 file changed, 2 insertions(+), 55 deletions(-) diff --git a/Dockerfile b/Dockerfile index 546afc0..4dbfb67 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,61 +5,7 @@ RUN apt-get update RUN apt-get -q update --fix-missing RUN apt-get -q install -y python3-pip openslide-tools python3-openslide vim openssl RUN apt-get -q install -y openssl libcurl4-openssl-dev libssl-dev - -### Install libvips - -# Tony has a future use case where we may adapt caMic to GIS visualization -# install libvips-dev for pyvips. No need for libvips. -RUN apt-get -q install -y libvips-dev - -# But, build libvips instead of using libvips-dev from apt -# Build without OpenSlide to open images with rather ImageMagick to handle -# images without pyramids. Otherwise opens e.g. DICOM with OpenSlide so conversion -# of files OpenSlide cannot open does not help at all. -# So, we'll have two copies on openslide on the system. -# By changing LD_LIBRARY_PATH before we launch python -# we can choose which openslide to run -# TODO: use libjpeg-turbo8-dev instead of libjpeg-dev if current apt repo has it; for performance -RUN apt-get -q install -y meson libjpeg-turbo8-dev libexif-dev libgsf-1-dev libtiff-dev libfftw3-dev liblcms2-dev libpng-dev libmagickcore-dev libmagickwand-dev liborc-0.4-dev libopenjp2-7 libgirepository1.0-dev -WORKDIR /root/src -RUN git clone https://github.com/libvips/libvips.git --depth=1 --branch=8.14 -RUN mkdir /root/src/libvips/build -WORKDIR /root/src/libvips -RUN mkdir /usr/local/vips-no-openslide/ -# normally --prefix=/usr/local/ --libdir=lib build -RUN meson setup -Dopenslide=disabled --buildtype=release --prefix=/usr/local/vips-no-openslide/ --libdir=lib build -RUN meson compile -C build -RUN meson test -C build -RUN meson install -C build - -RUN pip install pyvips --break-system-packages - -# verify pyvips can call libvips -RUN python3 -c "import pyvips" - -# verify that the apt libvips has openslide -ADD test_imgs/CMU-1-Small-Region.svs . -RUN python3 -c "import pyvips; pyvips.Image.openslideload(('CMU-1-Small-Region.svs'))" - -# back up previous ld_library_path -ENV LD_LIBRARY_PATH_ORIG="${LD_LIBRARY_PATH}" - -# now, prioritize openslideless libvips -# the path shown in output lines of "meson install" where .so.42 are installed -# normally /usr/local/lib/: -ENV LD_LIBRARY_PATH="/usr/local/vips-no-openslide/lib/:${LD_LIBRARY_PATH}" - -# verify that this libvips has no openslide -RUN ! python3 -c "import pyvips; pyvips.Image.openslideload(('CMU-1-Small-Region.svs'))" - -# ok, so to recap, -# there are two libvips are installed and which one pyvips connects to -# is chosen when "import pyvips" is run. -# at this point in this dockerfile, -# ld_library_path is set so that no-openslide version is run -# but if you do LD_LIBRARY_PATH="${LD_LIBRARY_PATH_ORIG}" python a.py -# or likewise using docker ENV command or os.environ in python before -# importing, this will remove the no-openslide libvips from path. +RUN apt-get -q install -y libvips libvips-dev ### Install BioFormats wrapper @@ -70,6 +16,7 @@ RUN python3 compile_bfbridge.py WORKDIR /root/src/ +RUN pip install pyvips --break-system-packages RUN pip install flask --break-system-packages RUN pip install gunicorn --break-system-packages RUN pip install greenlet --break-system-packages From daaf690172516327c56ae2462e75e1f00fed394d Mon Sep 17 00:00:00 2001 From: CGDogan <126820728+CGDogan@users.noreply.github.com> Date: Tue, 5 Sep 2023 17:31:45 +0100 Subject: [PATCH 29/38] fix build --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 4dbfb67..550ce32 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,6 +10,7 @@ RUN apt-get -q install -y libvips libvips-dev ### Install BioFormats wrapper WORKDIR /root/src/BFBridge/python +RUN pip install -r requirements.txt --break-system-packages RUN python3 compile_bfbridge.py ### Set up the server From 24661f5a396d7bde54b5f12b0cf10d5ce8b60bc8 Mon Sep 17 00:00:00 2001 From: CGDogan <126820728+CGDogan@users.noreply.github.com> Date: Wed, 6 Sep 2023 11:47:08 +0100 Subject: [PATCH 30/38] Move dicoms to subdir --- SlideServer.py | 79 +++++++++++++++++++++++++++++++++++------------- dev_utils.py | 2 +- image_reader.py | 36 ++++++++++++++++++++++ requirements.txt | 1 + 4 files changed, 96 insertions(+), 22 deletions(-) diff --git a/SlideServer.py b/SlideServer.py index 3b822a9..d0743c4 100644 --- a/SlideServer.py +++ b/SlideServer.py @@ -13,7 +13,7 @@ import flask import flask_cors from flask import request -from image_reader import construct_reader +from image_reader import construct_reader, suggest_folder_name from werkzeug.utils import secure_filename import dev_utils import requests @@ -64,10 +64,27 @@ def secure_filename_strict(filename): split_filename = ["noname", split_filename[-1]] return '.'.join(split_filename) -def allowed_file(filename): +def verify_extension(filename): return '.' in filename and \ filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS +def secure_relative_path(filename): + if filename[0] == os.sep: + raise ValueError("Filepath starts from the root directory which is forbidden") + if os.sep + os.sep in filename: + raise ValueError("Filepath contains '//' which is forbidden") + if ".." in filename: + raise ValueError("Filepath contains '..' which is forbidden") + level_names = filename.split(os.sep) + filename = "" + for name in level_names: + name = secure_filename(name) + if len(name) == 0: + name = "noname" + filename += name + filename += os.sep + return filename[:-1] + def getThumbnail(filename, size=50): filepath = os.path.join(app.config['UPLOAD_FOLDER'], filename) @@ -155,20 +172,26 @@ def finish_upload(token): if not body: return flask.Response(json.dumps({"error": "Missing JSON body"}), status=400, mimetype='text/json') token = secure_filename(token) + tmppath = os.path.join(app.config['TEMP_FOLDER'], token) + if not os.path.isfile(tmppath): + return flask.Response(json.dumps({"error": "Token Not Recognised"}), status=400, mimetype='text/json') filename = body['filename'] - if filename and allowed_file(filename): + if filename and verify_extension(filename): filename = secure_filename_strict(filename) - filepath = os.path.join(app.config['UPLOAD_FOLDER'], filename) - tmppath = os.path.join(app.config['TEMP_FOLDER'], token) + foldername = suggest_folder_name(tmppath, filename.rsplit('.', 1)[1]) + if foldername != "": + folderpath = os.path.join(app.config['UPLOAD_FOLDER'], foldername) + if not os.path.isdir(folderpath): + os.mkdir(folderpath) + relpath = os.path.join(foldername, filename) + else: + relpath = filename + filepath = os.path.join(app.config['UPLOAD_FOLDER'], relpath) if not os.path.isfile(filepath): - if os.path.isfile(tmppath): - shutil.move(tmppath, filepath) - return flask.Response(json.dumps({"ended": token, "filepath": filepath}), status=200, mimetype='text/json') - else: - return flask.Response(json.dumps({"error": "Token Not Recognised"}), status=400, mimetype='text/json') + shutil.move(tmppath, filepath) + return flask.Response(json.dumps({"ended": token, "filepath": filepath, "filename": filename, "relpath": relpath}), status=200, mimetype='text/json') else: - return flask.Response(json.dumps({"error": "File with name '" + filename + "' already exists", "filepath": filepath}), status=400, mimetype='text/json') - + return flask.Response(json.dumps({"error": "File with name '" + filename + "' already exists", "filepath": filepath, "filename": filename}), status=400, mimetype='text/json') else: return flask.Response(json.dumps({"error": "Invalid filename"}), status=400, mimetype='text/json') @@ -185,8 +208,8 @@ def slide_delete(): return flask.Response(json.dumps({"error": "Missing JSON body"}), status=400, mimetype='text/json') filename = body['filename'] - if filename and allowed_file(filename): - filename = secure_filename(filename) + if filename and verify_extension(filename): + filename = secure_relative_path(filename) filepath = os.path.join(app.config['UPLOAD_FOLDER'], filename) if os.path.isfile(filepath): os.remove(os.path.join(app.config['UPLOAD_FOLDER'], filename)) @@ -205,18 +228,21 @@ def testRoute(): return '{"Status":"up"}' -@app.route("/data/one/", methods=['GET']) +@app.route("/data/one/", methods=['GET']) def singleSlide(filepath): + filepath = secure_relative_path(filepath) extended = request.args.get('extended') res = dev_utils.getMetadata(os.path.join(app.config['UPLOAD_FOLDER'], filepath), extended, False) + res["filepath"] = filepath if (hasattr(res, 'error')): return flask.Response(json.dumps(res), status=500, mimetype='text/json') else: return flask.Response(json.dumps(res), status=200, mimetype='text/json') -@app.route("/data/thumbnail/", methods=['GET']) +@app.route("/data/thumbnail/", methods=['GET']) def singleThumb(filepath): + filepath = secure_relative_path(filepath) size = flask.request.args.get('size', default=50, type=int) size = min(500, size) res = getThumbnail(filepath, size) @@ -230,18 +256,29 @@ def singleThumb(filepath): def multiSlide(filepathlist): extended = request.args.get('extended') filenames = json.loads(filepathlist) - paths = [os.path.join(app.config['UPLOAD_FOLDER'], filename) for filename in filenames] - res = dev_utils.getMetadataList(paths, extended, False) + paths = [secure_relative_path(filename) for filename in filenames] + absolute_paths = [os.path.join(app.config['UPLOAD_FOLDER'], path) for path in paths] + res = dev_utils.getMetadataList(absolute_paths, extended, False) + for i in range(len(absolute_paths)): + res[i]["filepath"] = paths[i] if (hasattr(res, 'error')): return flask.Response(json.dumps(res), status=500, mimetype='text/json') else: return flask.Response(json.dumps(res), status=200, mimetype='text/json') -@app.route("/getSlide/") +@app.route("/getSlide/") def getSlide(image_name): - if(os.path.isfile("/images/"+image_name)): - return flask.send_from_directory(app.config["UPLOAD_FOLDER"], image_name, as_attachment=True) + image_name = secure_relative_path(image_name) + if not verify_extension(image_name): + return flask.Response(json.dumps({"error": "Bad image type requested"}), status=400, mimetype='text/json') + folder = app.config['UPLOAD_FOLDER'] + if os.sep in image_name: + folder_and_file = image_name.rsplit(os.sep, 1) + folder = os.path.join(folder, folder_and_file[0]) + image_name = folder_and_file[1] + if(os.path.isfile(os.path.join(folder, image_name))): + return flask.send_from_directory(folder, image_name, as_attachment=True) else: return flask.Response(json.dumps({"error": "File does not exist"}), status=404, mimetype='text/json') diff --git a/dev_utils.py b/dev_utils.py index da0ff1a..60bfdc9 100644 --- a/dev_utils.py +++ b/dev_utils.py @@ -27,7 +27,7 @@ def getMetadata(filepath, extended, raise_exception): if raise_exception: raise e # here, e has attribute "error" - return str(e) + return e.args[0] try: metadata = slide.get_basic_metadata(extended) diff --git a/image_reader.py b/image_reader.py index 8b7a3bc..3dcfaed 100644 --- a/image_reader.py +++ b/image_reader.py @@ -89,3 +89,39 @@ def construct_reader(imagepath): if reader is None: raise RuntimeError({"type": ",".join(reader_names), "error": ", ".join(errors)}) return reader + + +from pydicom import dcmread +import base64 +dicom_extensions = set(["dcm", "dic", "dicom"]) + +# For file formats where multiple files are opened together, +# we should move them to a directory. This function infers a common name. +def suggest_folder_name(filepath, extension): + try: + if extension in dicom_extensions: + ds = dcmread(filepath) + #study_instance_uid = ds[0x0020,0x000D].repval + series_instance_uid = ds[0x0020,0x000E].repval + #uid = study_instance_uid + ".." + series_instance_uid + uid = series_instance_uid + summary = 0 + + for c in uid: + if c == '.': + c = 10 + else: + c = int(c) + summary *= 11 + summary += c + + # make it a byte array + summary = hex(summary)[2:] + if len(summary) % 2 == 1: + summary = '0' + summary + summary = bytes.fromhex(summary) + summary = base64.urlsafe_b64encode(summary).decode("ascii").replace("=", "") + return summary + return "" + except: + return "" diff --git a/requirements.txt b/requirements.txt index 8a6525b..c9064f3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,3 +11,4 @@ google-api-python-client google-auth-httplib2 google-auth-oauthlib pycurl +pydicom From 281cea143269f291af31a067cd8463ec118bb08c Mon Sep 17 00:00:00 2001 From: CGDogan <126820728+CGDogan@users.noreply.github.com> Date: Wed, 6 Sep 2023 15:48:41 +0100 Subject: [PATCH 31/38] update with comments from tony --- image_reader.py | 22 ++++------------------ 1 file changed, 4 insertions(+), 18 deletions(-) diff --git a/image_reader.py b/image_reader.py index 3dcfaed..1031039 100644 --- a/image_reader.py +++ b/image_reader.py @@ -92,7 +92,7 @@ def construct_reader(imagepath): from pydicom import dcmread -import base64 +import hashlib dicom_extensions = set(["dcm", "dic", "dicom"]) # For file formats where multiple files are opened together, @@ -105,23 +105,9 @@ def suggest_folder_name(filepath, extension): series_instance_uid = ds[0x0020,0x000E].repval #uid = study_instance_uid + ".." + series_instance_uid uid = series_instance_uid - summary = 0 - - for c in uid: - if c == '.': - c = 10 - else: - c = int(c) - summary *= 11 - summary += c - - # make it a byte array - summary = hex(summary)[2:] - if len(summary) % 2 == 1: - summary = '0' + summary - summary = bytes.fromhex(summary) - summary = base64.urlsafe_b64encode(summary).decode("ascii").replace("=", "") - return summary + s = hashlib.md5() + s.update(uid.encode('ascii')) + return s.hexdigest()[:10] return "" except: return "" From eed317b84d604f13a45e45159de01bf2406bcd9b Mon Sep 17 00:00:00 2001 From: CGDogan <126820728+CGDogan@users.noreply.github.com> Date: Sun, 10 Sep 2023 12:13:20 +0100 Subject: [PATCH 32/38] unnecessary param --- SlideServer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SlideServer.py b/SlideServer.py index d0743c4..cb4ba76 100644 --- a/SlideServer.py +++ b/SlideServer.py @@ -191,7 +191,7 @@ def finish_upload(token): shutil.move(tmppath, filepath) return flask.Response(json.dumps({"ended": token, "filepath": filepath, "filename": filename, "relpath": relpath}), status=200, mimetype='text/json') else: - return flask.Response(json.dumps({"error": "File with name '" + filename + "' already exists", "filepath": filepath, "filename": filename}), status=400, mimetype='text/json') + return flask.Response(json.dumps({"error": "File with name '" + filename + "' already exists", "filename": filename}), status=400, mimetype='text/json') else: return flask.Response(json.dumps({"error": "Invalid filename"}), status=400, mimetype='text/json') From eeed0952327f67e4e79c0e47d3d590e5441e05f0 Mon Sep 17 00:00:00 2001 From: CGDogan <126820728+CGDogan@users.noreply.github.com> Date: Sun, 10 Sep 2023 21:05:53 +0100 Subject: [PATCH 33/38] one more case --- SlideServer.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/SlideServer.py b/SlideServer.py index cb4ba76..e2a60f3 100644 --- a/SlideServer.py +++ b/SlideServer.py @@ -73,6 +73,8 @@ def secure_relative_path(filename): raise ValueError("Filepath starts from the root directory which is forbidden") if os.sep + os.sep in filename: raise ValueError("Filepath contains '//' which is forbidden") + if os.sep + '.' + os.sep in filename: + raise ValueError("Filepath contains '/./' which is forbidden") if ".." in filename: raise ValueError("Filepath contains '..' which is forbidden") level_names = filename.split(os.sep) From ce79a40acc931abfd3e4aabbb3f5b162241f2820 Mon Sep 17 00:00:00 2001 From: Naymul Islam Date: Tue, 12 Sep 2023 19:12:57 +0600 Subject: [PATCH 34/38] Google drive related function is transfered into new file named gdrive_utils.py Google drive related file is being transfer into a new file named gdrive_utils.py --- SlideServer.py | 33 ++++++--------------------------- gdrive_utils.py | 45 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 27 deletions(-) create mode 100644 gdrive_utils.py diff --git a/SlideServer.py b/SlideServer.py index c1dc9c4..4c41a6d 100644 --- a/SlideServer.py +++ b/SlideServer.py @@ -22,7 +22,7 @@ import csv import pathlib import logging -from gDriveDownload import start, afterUrlAuth, callApi +from gdrive_utils import getFileFromGdrive, gDriveGetFile, checkDownloadStatus from threading import Thread try: @@ -48,7 +48,7 @@ app.config['ROI_FOLDER'] = "/images/roiDownload" -ALLOWED_EXTENSIONS = set(['svs', 'tif', 'tiff', 'vms', 'vmu', 'ndpi', 'scn', 'mrxs', 'bif', 'svslide', 'png', 'jpg']) +ALLOWED_EXTENSIONS = set(['svs', 'tif', 'tiff', 'vms', 'vmu', 'ndpi', 'scn', 'mrxs', 'bif', 'svslide', 'png', 'jpg']) def allowed_file(filename): @@ -570,37 +570,16 @@ def run(self): # Route to start the OAuth Server(to listen if user is Authenticated) and start the file Download after Authentication @app.route('/googleDriveUpload/getFile', methods=['POST']) -def gDriveGetFile(): +def gDriveGetFileRoute(): body = flask.request.get_json() if not body: return flask.Response(json.dumps({"error": "Missing JSON body"}), status=400) - - token = "".join(random.choice(string.ascii_uppercase + string.digits) for _ in range(10)) - token = secure_filename(token) - tmppath = os.path.join("/images/uploading/", token) - # regenerate if we happen to collide - while os.path.isfile(tmppath): - token = "".join(random.choice(string.ascii_uppercase + string.digits) for _ in range(10)) - token = secure_filename(token) - tmppath = os.path.join("/images/uploading/", token) - - try: - params = start(body['userId']) - except: - return flask.Response(json.dumps({'error': str(sys.exc_info()[0])}), status=400) - thread_a = getFileFromGdrive(params, body['userId'], body['fileId'], token) - thread_a.start() - return flask.Response(json.dumps({"authURL": params["auth_url"], "token": token}), status=200) + return gDriveGetFile(body) # To check if a particular file is downloaded from Gdrive @app.route('/googleDriveUpload/checkStatus', methods=['POST']) -def checkDownloadStatus(): +def checkDownloadStatusRoute(): body = flask.request.get_json() if not body: return flask.Response(json.dumps({"error": "Missing JSON body"}), status=400) - token = body['token'] - path = app.config['TEMP_FOLDER']+'/'+token - if os.path.isfile(path): - return flask.Response(json.dumps({"downloadDone": True}), status=200) - return flask.Response(json.dumps({"downloadDone": False}), status=200) - + return checkDownloadStatus(body) \ No newline at end of file diff --git a/gdrive_utils.py b/gdrive_utils.py new file mode 100644 index 0000000..e1de7a4 --- /dev/null +++ b/gdrive_utils.py @@ -0,0 +1,45 @@ +from threading import Thread +from gDriveDownload import start, afterUrlAuth, callApi +import flask +import os +import random +import string +from werkzeug.utils import secure_filename +import sys + +# A new Thread to call the Gdrive API after an Auth Response is returned to the user. +class getFileFromGdrive(Thread): + def __init__(self, params, userId, fileId, token): + Thread.__init__(self) + self.params, self.userId, self.fileId , self.token = params, userId, fileId, token + + def run(self): + if(self.params["auth_url"] != None): + self.params["creds"] = afterUrlAuth(self.params["local_server"], self.params["flow"], self.params["wsgi_app"], self.userId) + call = callApi(self.params["creds"], self.fileId, self.token) + app.logger.info(call) + +def gDriveGetFile(body): + token = "".join(random.choice(string.ascii_uppercase + string.digits) for _ in range(10)) + token = secure_filename(token) + tmppath = os.path.join("/images/uploading/", token) + # regenerate if we happen to collide + while os.path.isfile(tmppath): + token = "".join(random.choice(string.ascii_uppercase + string.digits) for _ in range(10)) + token = secure_filename(token) + tmppath = os.path.join("/images/uploading/", token) + + try: + params = start(body['userId']) + except: + return flask.Response(json.dumps({'error': str(sys.exc_info()[0])}), status=400) + thread_a = getFileFromGdrive(params, body['userId'], body['fileId'], token) + thread_a.start() + return flask.Response(json.dumps({"authURL": params["auth_url"], "token": token}), status=200) + +def checkDownloadStatus(body): + token = body['token'] + path = app.config['TEMP_FOLDER']+'/'+token + if os.path.isfile(path): + return flask.Response(json.dumps({"downloadDone": True}), status=200) + return flask.Response(json.dumps({"downloadDone": False}), status=200) \ No newline at end of file From 6f1105dcd78a089977aeca8300a5658646d8674b Mon Sep 17 00:00:00 2001 From: CGDogan <126820728+CGDogan@users.noreply.github.com> Date: Sun, 17 Sep 2023 14:28:11 +0100 Subject: [PATCH 35/38] Dicom server --- SlideServer.py | 52 +++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 51 insertions(+), 1 deletion(-) diff --git a/SlideServer.py b/SlideServer.py index e2a60f3..843374d 100644 --- a/SlideServer.py +++ b/SlideServer.py @@ -77,6 +77,8 @@ def secure_relative_path(filename): raise ValueError("Filepath contains '/./' which is forbidden") if ".." in filename: raise ValueError("Filepath contains '..' which is forbidden") + if filename[0] == '.': + raise ValueError("Filepath starts with '.' (or is '.') which is forbidden") level_names = filename.split(os.sep) filename = "" for name in level_names: @@ -193,7 +195,9 @@ def finish_upload(token): shutil.move(tmppath, filepath) return flask.Response(json.dumps({"ended": token, "filepath": filepath, "filename": filename, "relpath": relpath}), status=200, mimetype='text/json') else: - return flask.Response(json.dumps({"error": "File with name '" + filename + "' already exists", "filename": filename}), status=400, mimetype='text/json') + return flask.Response(json.dumps({"error": "File with name '" + filename + "' already exists", "filepath": filepath, "filename": filename}), status=400, mimetype='text/json') + # The above return "filename" to show the user the sanitized filename + # and on success, return relpath for subsequent SlideLoader calls by the frontend. else: return flask.Response(json.dumps({"error": "Invalid filename"}), status=400, mimetype='text/json') @@ -268,6 +272,24 @@ def multiSlide(filepathlist): else: return flask.Response(json.dumps(res), status=200, mimetype='text/json') +# Used by Caracal; may be removed after our schema fully supports multifile formats in a subdir +@app.route("/data/folder/", methods=['GET']) +def listFolderContents(relpath): + res = {} + try: + relpath = secure_relative_path(relpath) + absolutepath = os.path.join(app.config['UPLOAD_FOLDER'], relpath) + except BaseException as e: + res['error'] = "bad folderpath: " + str(e) + return flask.Response(json.dumps(res), status=400, mimetype='text/json') + + try: + res['contents'] = os.listdir(absolutepath) + res['contents'] = [filename for filename in res['contents'] if not filename.startswith('.')] + return flask.Response(json.dumps(res), status=200, mimetype='text/json') + except: + res['contents'] = [] + return flask.Response(json.dumps(res), status=200, mimetype='text/json') @app.route("/getSlide/") def getSlide(image_name): @@ -668,3 +690,31 @@ def checkDownloadStatus(): return flask.Response(json.dumps({"downloadDone": True}), status=200, mimetype='text/json') return flask.Response(json.dumps({"downloadDone": False}), status=200, mimetype='text/json') +# DICOM Explorer UI and DICOM server hostname and port +@app.route('/dicomsrv/location', methods=['GET']) +def guiLocation(): + port = os.getenv("DICOM_PORT") + hostname = os.getenv("DICOM_HOSTNAME") + ui_port = os.getenv("DICOM_UI_PORT") + ui_hostname = os.getenv("DICOM_UI_HOSTNAME") + res = {} + if port is not None: + res["port"] = int(port) + else: + print("DICOM_PORT env variable not found") + + if ui_port is not None: + res["ui_port"] = int(ui_port) + else: + print("DICOM_UI_PORT env variable not found") + + + # If the DICOM server is on a different computer, this can be uncommented, + # the frontend will parse this, but it's better to keep this in a comment against env var poisoning + # if hostname is not None: + # res["hostname"] = hostname + # if ui_hostname is not None: + # res["ui_hostname"] = ui_hostname + + success = "port" in res and "ui_port" in res + return flask.Response(json.dumps(res), status=200 if success else 500, mimetype='text/json') From d88d42706e274a0dab4f9e1a0e86a455c21f0cf1 Mon Sep 17 00:00:00 2001 From: CGDogan <126820728+CGDogan@users.noreply.github.com> Date: Mon, 18 Sep 2023 17:46:07 +0100 Subject: [PATCH 36/38] Fix bioformats spam --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index d97ef8c..c466841 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,6 +15,7 @@ RUN pip install gunicorn[eventlet] run openssl version -a ENV FLASK_ENV development +ENV BFBRIDGE_LOGLEVEL=WARN RUN mkdir -p /images/uploading From 521bb6a4eeeb6a2ee117acea3436d0dee0bf343d Mon Sep 17 00:00:00 2001 From: Birm Date: Fri, 29 Sep 2023 16:36:35 -0400 Subject: [PATCH 37/38] don't pretend to know xres and yres --- SlideServer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SlideServer.py b/SlideServer.py index 843374d..648f23b 100644 --- a/SlideServer.py +++ b/SlideServer.py @@ -114,7 +114,7 @@ def makePyramid(filename, dest): try: filepath = os.path.join(app.config['UPLOAD_FOLDER'], filename) destpath = os.path.join(app.config['UPLOAD_FOLDER'], dest) - savedImg = pyvips.Image.new_from_file(filepath, access='sequential').tiffsave(destpath, tile=True, compression="lzw", tile_width=256, tile_height=256, pyramid=True, bigtiff=True, xres=0.254, yres=0.254) + savedImg = pyvips.Image.new_from_file(filepath, access='sequential').tiffsave(destpath, tile=True, compression="lzw", tile_width=256, tile_height=256, pyramid=True, bigtiff=True) while not os.path.exists(filepath): os.sync() sleep(750) From 57480b6fe49a2c9b0a1bce2f4261840a65d0fe30 Mon Sep 17 00:00:00 2001 From: Birm Date: Thu, 9 Nov 2023 14:20:47 -0500 Subject: [PATCH 38/38] Update LICENSE years --- LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index 4213d86..b60c65b 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ BSD 3-Clause License -Copyright (c) 2019, caMicroscope +Copyright (c) 2018-2023, caMicroscope All rights reserved. Redistribution and use in source and binary forms, with or without