Skip to content

Commit

Permalink
Merge pull request #86 from camicroscope/develop
Browse files Browse the repository at this point in the history
For 3.11
  • Loading branch information
birm authored Nov 17, 2023
2 parents 128587f + a5c7745 commit 0684457
Show file tree
Hide file tree
Showing 19 changed files with 981 additions and 185 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/container.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ name: Container Publish

on:
push:
branches: ['master', 'auto-build']
branches: ['master', 'develop']

env:
REGISTRY: ghcr.io
Expand Down
3 changes: 3 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -99,3 +99,6 @@ ENV/

# mypy
.mypy_cache/

# BFBridge - should always stay in a separate repo but copied manually or by Docker
BFBridge/
110 changes: 110 additions & 0 deletions BioFormatsReader.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,110 @@
import image_reader
import dev_utils
import ome_types
from file_extensions import BIOFORMATS_EXTENSIONS
import BFBridge.python as bfbridge


jvm = bfbridge.BFBridgeVM()

class BioFormatsReader(image_reader.ImageReader):
@staticmethod
def reader_name():
return "bioformats"

@staticmethod
def extensions_set():
return BIOFORMATS_EXTENSIONS

def __init__(self, imagepath):
if not hasattr(dev_utils.keep_alive_for_thread, "bfthread"):
dev_utils.keep_alive_for_thread.bfthread = bfbridge.BFBridgeThread(jvm)

# Conventionally internal attributes start with underscore.
# When using them without underscore, there's the risk that
# a property has the same name as a/the getter, which breaks
# the abstract class. Hence all internal attributes start with underscore.
self._bfreader = bfbridge.BFBridgeInstance(dev_utils.keep_alive_for_thread.bfthread)
if self._bfreader is None:
raise RuntimeError("cannot make bioformats instance")
self._image_path = imagepath
code = self._bfreader.open(imagepath)
if code < 0:
raise IOError("Could not open file " + imagepath + ": " + self._bfreader.get_error_string())
# Note: actually stores the format, not the vendor ("Hamamatsu NDPI" instead of "Hamamatsu")
self._vendor = self._bfreader.get_format()
self._level_count = self._bfreader.get_resolution_count()
self._dimensions = (self._bfreader.get_size_x(), self._bfreader.get_size_y())
self._level_dimensions = [self._dimensions]
for l in range(1, self._level_count):
self._bfreader.set_current_resolution(l)
self._level_dimensions.append( \
(self._bfreader.get_size_x(), self._bfreader.get_size_y()))

@property
def level_count(self):
return self._level_count

@property
def dimensions(self):
return self._dimensions

@property
def level_dimensions(self):
return self._level_dimensions

@property
def associated_images(self):
return None

def read_region(self, location, level, size):
self._bfreader.set_current_resolution(level)
return self._bfreader.open_bytes_pil_image(0, \
location[0], location[1], size[0], size[1])

def get_thumbnail(self, max_size):
return self._bfreader.open_thumb_bytes_pil_image(0, max_size[0], max_size[1])

def get_basic_metadata(self, extended):
metadata = {}

try:
ome_xml_raw = self._bfreader.dump_ome_xml_metadata()
except BaseException as e:
raise OverflowError("XML metadata too large for file considering the preallocated buffer length. " + str(e))
try:
ome_xml = ome_types.from_xml(ome_xml_raw)
except BaseException as e:
raise RuntimeError("get_basic_metadata: OME-XML parsing of metadata failed, error: " + \
str(e) + " when parsing: " + ome_xml_raw)

# https://www.openmicroscopy.org/Schemas/Documentation/Generated/OME-2016-06/ome_xsd.html
# https://bio-formats.readthedocs.io/en/latest/metadata-summary.html

if extended:
return {"ome-xml": ome_xml_raw}

metadata['width'] = str(self._dimensions[0])
metadata['height'] = str(self._dimensions[1])
try:
metadata['mpp-x'] = str(ome_xml.images[0].pixels.physical_size_x)
metadata['mpp-y'] = str(ome_xml.images[0].pixels.physical_size_y)
except:
metadata['mpp-x'] = "0"
metadata['mpp-y'] = "0"
metadata['vendor'] = self._vendor
metadata['level_count'] = int(self._level_count)
try:
metadata['objective'] = ome_xml.instruments[0].objectives[0].nominal_magnification
except:
try:
metadata['objective'] = ome_xml.instruments[0].objectives[0].calibrated_magnification
except:
metadata['objective'] = -1.0

metadata['comment'] = ""
metadata['study'] = ""
metadata['specimen'] = ""
metadata['md5sum'] = dev_utils.file_md5(self._image_path)

return metadata
37 changes: 24 additions & 13 deletions Dockerfile
Original file line number Diff line number Diff line change
@@ -1,36 +1,47 @@
FROM python:3
FROM camicroscope/image-decoders:latest

WORKDIR /var/www
RUN apt-get update
RUN apt-get -q update --fix-missing
RUN apt-get -q install -y openslide-tools python3-openslide vim openssl
RUN apt-get -q install -y python3-pip openslide-tools python3-openslide vim openssl
RUN apt-get -q install -y openssl libcurl4-openssl-dev libssl-dev
RUN apt-get -q install -y libvips libvips-dev

RUN pip install pyvips
RUN pip install flask
RUN pip install gunicorn
RUN pip install greenlet
RUN pip install gunicorn[eventlet]
### Install BioFormats wrapper

WORKDIR /root/src/BFBridge/python
RUN pip install -r requirements.txt --break-system-packages
RUN python3 compile_bfbridge.py

### Set up the server

WORKDIR /root/src/

RUN pip install pyvips --break-system-packages
RUN pip install flask --break-system-packages
RUN pip install gunicorn --break-system-packages
RUN pip install greenlet --break-system-packages
RUN pip install gunicorn[eventlet] --break-system-package

run openssl version -a

ENV FLASK_ENV development
ENV FLASK_DEBUG True
ENV BFBRIDGE_LOGLEVEL=WARN

RUN mkdir -p /images/uploading

COPY ./ ./
COPY requirements.txt .
RUN pip3 install -r requirements.txt --break-system-packages

COPY ./ ./
RUN cp test_imgs/* /images/

RUN pip3 install -r requirements.txt


EXPOSE 4000
EXPOSE 4001

#debug/dev only
# ENV FLASK_APP SlideServer.py
# CMD python -m flask run --host=0.0.0.0 --port=4000
# CMD python3 -m flask run --host=0.0.0.0 --port=4000

# The Below BROKE the ability for users to upload images.
# # non-root user
Expand Down
2 changes: 1 addition & 1 deletion LICENSE
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
BSD 3-Clause License

Copyright (c) 2019, caMicroscope
Copyright (c) 2018-2023, caMicroscope
All rights reserved.

Redistribution and use in source and binary forms, with or without
Expand Down
2 changes: 1 addition & 1 deletion NCISlideUtil.py
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ def openslidedata(metadata):
slide = openslide.OpenSlide(metadata['location'])
slideData = slide.properties
metadata['mpp-x'] = slideData.get(openslide.PROPERTY_NAME_MPP_X, None)
metadata['mpp-x'] = slideData.get(openslide.PROPERTY_NAME_MPP_Y, None)
metadata['mpp-y'] = slideData.get(openslide.PROPERTY_NAME_MPP_Y, None)
metadata['mpp'] = metadata['mpp-x'] or metadata['mpp-x'] or None
metadata['height'] = slideData.get(
openslide.PROPERTY_NAME_BOUNDS_HEIGHT, None)
Expand Down
21 changes: 5 additions & 16 deletions OmniLoad.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
#!/usr/bin/env python3

import openslide # to get required slide metadata
import dev_utils # to get required slide metadata
import csv # to read csv
import sys # for csv limit
import os # for os and filepath utils
Expand Down Expand Up @@ -51,21 +51,10 @@ def file_md5(fileName):
def openslidedata(manifest):
for img in manifest:
img['location'] = img.get("path", "") or img.get("location", "") or img.get("filename", "") or img.get("file", "")
slide = openslide.OpenSlide(img['location'])
slideData = slide.properties
img['mpp-x'] = slideData.get(openslide.PROPERTY_NAME_MPP_X, None)
img['mpp-y'] = slideData.get(openslide.PROPERTY_NAME_MPP_Y, None)
img['mpp'] = img['mpp-x'] or img['mpp-y']
img['height'] = slideData.get(openslide.PROPERTY_NAME_BOUNDS_HEIGHT, None) or slideData.get(
"openslide.level[0].height", None)
img['width'] = slideData.get(openslide.PROPERTY_NAME_BOUNDS_WIDTH, None) or slideData.get(
"openslide.level[0].width", None)
img['vendor'] = slideData.get(openslide.PROPERTY_NAME_VENDOR, None)
img['level_count'] = int(slideData.get('level_count', 1))
img['objective'] = float(slideData.get(openslide.PROPERTY_NAME_OBJECTIVE_POWER, 0) or
slideData.get("aperio.AppMag", -1.0))
img['md5sum'] = file_md5(img['location'])
img['comment'] = slideData.get(openslide.PROPERTY_NAME_COMMENT, None)
metadata = dev_utils.getMetadata(img['location'], False, True)
for k, v in metadata.items():
if k not in img:
img[k] = v
# required values which are often unused
img['study'] = img.get('study', "")
img['specimen'] = img.get('specimen', "")
Expand Down
62 changes: 62 additions & 0 deletions OpenSlideReader.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
import openslide
import image_reader
import dev_utils
from file_extensions import OPENSLIDE_EXTENSIONS

class OpenSlideReader(image_reader.ImageReader):
@staticmethod
def reader_name():
return "openslide"

@staticmethod
def extensions_set():
return OPENSLIDE_EXTENSIONS

def __init__(self, imagepath):
self._image_path = imagepath
self._reader = openslide.OpenSlide(imagepath)

@property
def level_count(self):
return self._reader.level_count

@property
def dimensions(self):
return self._reader.dimensions

@property
def level_dimensions(self):
return self._reader.level_dimensions

@property
def associated_images(self):
return self._reader.associated_images

def read_region(self, location, level, size):
return self._reader.read_region(location, level, size)

def get_thumbnail(self, max_size):
return self._reader.get_thumbnail(max_size)

def get_basic_metadata(self, extended):
slideData = self._reader.properties
if extended:
metadata = {k:v for (k,v) in slideData.items()}
else:
metadata = {}
metadata['width'] = slideData.get(openslide.PROPERTY_NAME_BOUNDS_WIDTH, None) \
or slideData.get( "openslide.level[0].width", None)
metadata['height'] = slideData.get(openslide.PROPERTY_NAME_BOUNDS_HEIGHT, None) \
or slideData.get("openslide.level[0].height", None)
metadata['mpp-x'] = slideData.get(openslide.PROPERTY_NAME_MPP_X, None)
metadata['mpp-y'] = slideData.get(openslide.PROPERTY_NAME_MPP_Y, None)
metadata['vendor'] = slideData.get(openslide.PROPERTY_NAME_VENDOR, None)
metadata['level_count'] = int(self._reader.level_count)
metadata['objective'] = float(slideData.get(openslide.PROPERTY_NAME_OBJECTIVE_POWER, 0) \
or slideData.get("aperio.AppMag", -1.0))
metadata['comment'] = slideData.get(openslide.PROPERTY_NAME_COMMENT, None)
# caMicroscope expects some value for study and specimen for slides, add empty string as defauly.
metadata['study'] = ""
metadata['specimen'] = ""
metadata['md5'] = dev_utils.file_md5(self._image_path)
return metadata
4 changes: 4 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
@@ -1,6 +1,10 @@
# SlideLoader
Tool for loading slides and getting slide metadata using openslide

## Setting up

When used outside of caMicroscope/Distro Docker, the [BFBridge](https://github.com/camicroscope/BFBridge) folder needs to copied to this repository manually. (Not only its contents, but making it a subfolder of this repository)

## Usage

### Upload
Expand Down
Loading

0 comments on commit 0684457

Please sign in to comment.