separate prefect and tiled

.github/workflows/python-app.yml

@@ -0,0 +1,38 @@
+name: build and test
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python 3.9
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.11
+        cache: 'pip'
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install flake8 pytest
+        pip install .
+        pip install .[dev]
+        pip install .[prefect_tiled]
+    - name: Lint with flake8
+      run: |
+        # stop the build if there are Python syntax errors or undefined names
+        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+    - name: Test with pytest
+      run: |
+        pytest
+    - name: Build MkDocs site
+      if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+      run: mkdocs gh-deploy -f docs/mkdocs/mkdocs.yml --force

README.md

@@ -9,9 +9,20 @@ git clone [email protected]:als-computing/SAXSWAXS_workflows.git
 cd SAXSWAXS_workflows
 python -m venv saxswaxs-workflows-env
 source saxswaxs-workflows-env/bin/activate
-pip install -r requirements.txt
+pip install .
 ```
 
+If you are using this in a prefect and tiled environment, also run (quotes are required for zsh):
+```bash
+pip install '.[prefect_tiled]'
+```
+
+If you are using this in a development environment, also run:
+```bash
+pip install '.[dev]'
+```
+
+
 The command `source saxswaxs-workflows-env/bin/activate` may need to be adapted for the specific operating system, see the [venv](https://docs.python.org/3/library/venv.html) documentation
 
 ## Set up environment file with relevant paths

pyproject.toml

@@ -1,3 +1,49 @@
+[project]
+name = "saxswaxs_workflows"
+version = "1.0"
+description = "Data analysis workflows for SAXS/WAXS. See README.md for details."
+authors = [
+  { name = "Wiebke Koepp", email = "[email protected]" }
+]
+urls = { homepage = "https://github.com/als-computing/SAXSWAXS_workflows" }
+requires-python = ">=3.10,<=3.12"
+dependencies = [
+  "silx==1.1.2",
+  "pyFAI==2023.9.0",
+  "h5py==3.9.0",
+  "fabio==2023.6.0",
+  "numpy==1.26.1",
+  "Pillow==10.0.1",
+  "scipy==1.11.3",
+  "matplotlib==3.8.0",
+  "pyyaml==6.0.1",
+  "python-dotenv==1.0.0",
+  "astropy==5.3.4",
+]
+
+[project.optional-dependencies]
+dev = [
+  "black",
+  "flake8",
+  "isort",
+  "nbstripout",
+  "pre-commit",
+  "pre-commit-hooks",
+  "pytest"
+]
+
+prefect_tiled = [
+  "prefect==2.13.4",
+  "tiled[all]==0.1.0a116",
+]
+
+gpcam = [
+  "gpcam==8.1.7"
+]
+
+[tool.setuptools.packages.find]
+where = ["."]
+
 [tool.black]
 line-length = 88
 exclude = '''
@@ -8,6 +54,7 @@ exclude = '''
   | dist
 )/
 '''
+
 [tool.isort]
 profile = "black"
-multi_line_output = 3
+multi_line_output = 3

saxswaxs_workflows/flows/reduction.py

@@ -3,14 +3,9 @@
 
 import numpy as np
 import zmq
-from conversions import (
+from saxswaxs_workflows.reduction.conversions import (
     filter_nans,
-    get_azimuthal_integrator,
     mask_image,
-    pix_to_alpha_f,
-    pix_to_theta_f,
-    q_parallel,
-    q_z,
 )
 from file_handling import (
     open_cbf,
@@ -22,6 +17,8 @@
 
 from prefect import flow, get_run_logger
 
+from ..reduction import reduce
+
 host = "127.0.0.1"
 port = "5001"
 
@@ -44,34 +41,8 @@ def pixel_roi_vertical_cut(
     y_max,
     output_unit,
 ):
-    shape = masked_image.shape
-
-    y_min = max(0, y_min)
-    y_max = min(shape[0], y_max + 1)
-
-    x_min = max(0, int(beamcenter_x - cut_half_width))
-    x_max = min(shape[1], int(beamcenter_x + cut_half_width + 1))
-
-    cut_data = masked_image[
-        y_min:y_max,
-        x_min:x_max,
-    ]
-
-    cut_average = np.average(cut_data, axis=1)
-    errors = np.sqrt(cut_average)
-
-    pix = np.arange(y_min, y_max)
-    if output_unit == "pixel":
-        return (pix, cut_average, errors)
-    else:
-        # Set pixel coordinates in reference to beam center
-        pix = pix - beamcenter_y
-        af = pix_to_alpha_f(pix, sample_detector_dist, pix_size, incident_angle)
-    if output_unit == "angle":
-        return (af, cut_average, errors)
-    elif output_unit == "q":
-        qz = q_z(wavelength, af, incident_angle)
-        return (qz, cut_average, errors)
+    params = locals().copy()
+    return reduce.pixel_roi_vertical_cut(**params)
 
 
 @flow(name="pixel-roi-vertical-cut-tiled")
@@ -114,39 +85,8 @@ def pixel_roi_horizontal_cut(
     """
     Extract a cut in horizontal direction on the detector with width=2*cut_half_width.
     """
-    shape = masked_image.shape
-
-    x_min = max(0, x_min)
-    x_max = min(shape[1], x_max + 1)
-    y_min = max(0, int(cut_pos_y - cut_half_width))
-    y_max = min(shape[0], int(cut_pos_y + cut_half_width + 1))
-
-    cut_data = masked_image[
-        y_min:y_max,
-        x_min:x_max,
-    ]
-
-    cut_average = np.average(cut_data, axis=0)
-    errors = np.sqrt(cut_average)
-
-    pix = np.arange(x_min, x_max)
-
-    if output_unit == "pixel":
-        return (pix, cut_average, errors)
-    else:
-        pix = pix - beamcenter_x
-        af = pix_to_alpha_f(
-            beamcenter_y - cut_pos_y,
-            sample_detector_dist,
-            pix_size,
-            incident_angle,
-        )
-        tf = pix_to_theta_f(pix, sample_detector_dist, pix_size)
-    if output_unit == "angle":
-        return (tf, cut_average, errors)
-    elif output_unit == "q":
-        qp = q_parallel(wavelength, tf, af, incident_angle)
-        return (qp, cut_average, errors)
+    param = locals().copy()
+    return reduce.pixel_roi_horizontal_cut(**param)
 
 
 @flow(name="pixel-roi-horizontal-cut-tiled")
@@ -191,28 +131,8 @@ def integrate1d_azimuthal(
     outer_radius,
     output_unit,  # Always q for now
 ):
-    azimuthal_integrator = get_azimuthal_integrator(
-        beamcenter_x,
-        beamcenter_y,
-        wavelength,
-        sample_detector_dist,
-        tilt,
-        rotation,
-        pix_size,
-    )
-
-    result = azimuthal_integrator.integrate1d(
-        data=np.copy(image),
-        mask=np.copy(mask),
-        npt=num_bins,
-        correctSolidAngle=True,
-        error_model="poisson",
-        # radial_range=(inner_radius, outer_radius),
-        azimuth_range=(chi_min, chi_max),
-        polarization_factor=polarization_factor,
-    )
-
-    return result
+    params = locals().copy()
+    return reduce.integrate1d_azimuthal(**params)
 
 
 @flow(name="saxs-waxs-azimuthal-integration-tiled")
@@ -286,30 +206,8 @@ def integrate1d_radial(
     outer_radius,
     output_unit,  # Always q for now
 ):
-    azimuthal_integrator = get_azimuthal_integrator(
-        beamcenter_x,
-        beamcenter_y,
-        wavelength,
-        sample_detector_dist,
-        tilt,
-        rotation,
-        pix_size,
-    )
-
-    result = azimuthal_integrator.integrate_radial(
-        # Copying here due to issue with memory ownership
-        # line 323, in pyFAI.ext.splitBBoxCSR.CsrIntegrator.integrate_ng
-        # "ValueError: buffer source array is read-only"
-        data=np.copy(image),
-        mask=mask,
-        npt=num_bins,
-        correctSolidAngle=True,
-        # radial_range=(inner_radius, outer_radius),
-        azimuth_range=(chi_min, chi_max),
-        polarization_factor=polarization_factor,
-    )
-
-    return result
+    params = locals().copy()
+    return reduce.integrate1d_radial(**params)
 
 
 @flow(name="saxs-waxs-integration-radial-tiled")
@@ -466,3 +364,22 @@ def reduction_files_wrapper(
         "output_unit": "q",
     }
     pixel_roi_horizontal_cut_tiled(**parameters_horizontal)
+
+if __name__ == "__main__":
+    # integrate1d_azimuthal_tiled(**parameters_azimuthal)
+    parameters_horizontal = {
+        "input_uri_data": "raw/ALS-S2VP42/218_A0p160_A0p160_sfloat_2m",
+        "input_uri_mask": "processed/masks/ALS_BCP_Mixing_inverted",
+        "beamcenter_x": 670.976,
+        "beamcenter_y": 1180.42,
+        "incident_angle": 0.16,
+        "sample_detector_dist": 3513.21,
+        "wavelength": 1.2398,
+        "pix_size": 172,
+        "cut_half_width": 10,
+        "cut_pos_y": 1180 - 105,
+        "x_min": 670 - 250,
+        "x_max": 670 + 250,
+        "output_unit": "q",
+    }
+    pixel_roi_horizontal_cut_tiled(**parameters_horizontal)