Merge pull request #471 from ssl-hep/3.0_develop

3.0!!!!!

.devcontainer/devcontainer.json

@@ -35,13 +35,14 @@
 	"extensions": [
 		"ms-python.python",
 		"ms-python.vscode-pylance",
+		"ms-python.flake8",
 	],
 
 	// Use 'forwardPorts' to make a list of ports inside the container available locally.
 	// "forwardPorts": [],
 
 	// Use 'postCreateCommand' to run commands after the container is created.
-	"postCreateCommand": "pip3 install --user -e .[test]",
+	"postCreateCommand": "pip3 install --user -e .[develop]",
 
 	// Comment out connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root.
 	"remoteUser": "vscode"

.github/workflows/ci.yaml

@@ -14,59 +14,51 @@ jobs:
 
     steps:
       - uses: actions/checkout@v3
+
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: "3.11"
-      - name: Install dependencies
-        env:
-          servicex_version: 1.0a1
-        run: |
-          python -m pip install --upgrade pip setuptools wheel
-          python -m pip install --no-cache-dir -e .[test]
-          python -m pip list
+          python-version: "3.x"
+
       - name: Lint with Flake8
         run: |
-          flake8 --exclude=tests/* --ignore=E501,W503
-      - name: Check for vulnerable libraries
-        run: |
-          python -m pip install safety
-          python -m pip freeze | safety check
+          pipx run flake8
 
   test:
     needs:
       - flake8
 
     strategy:
       matrix:
-        platform: [ubuntu-latest, windows-latest, macOS-latest]
-        python-version: [3.8, 3.9, "3.10", "3.11", "3.12"]
+        platform: [ubuntu-latest, windows-latest, macos-latest]
+        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
     runs-on: ${{ matrix.platform }}
 
     steps:
       - uses: actions/checkout@v3
+
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v3
+
       - name: Install dependencies
-        env:
-          servicex_version: 1.0a1
         run: |
-          python -m pip install --upgrade pip setuptools wheel
-          python -m pip install --no-cache-dir -e .[test]
-          python -m pip list
+          uv pip install --system --upgrade '.[test,docs]'
+          uv pip list --system
+
       - name: Test with pytest
         run: |
-          python -m pytest
-      - name: Code coverage with pytest
-        if: github.event_name == 'push' && matrix.platform == 'ubuntu-latest'
-        run: |
-          python -m pytest --ignore=setup.py --cov=servicex --cov-report=term-missing --cov-config=.coveragerc --cov-report xml
-      - name: Report coverage with Codecov
-        if: github.event_name == 'push' && matrix.platform == 'ubuntu-latest'
-        uses: codecov/codecov-action@v3
+          coverage run -m pytest tests --cov=./servicex/ --cov-report=xml
+
+      - name: Upload coverage to Codecov
+        uses: codecov/[email protected]
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
-          file: ./coverage.xml # optional
-          flags: unittests-${{ matrix.python-version }} # optional
+          directory: ./coverage/reports/
+          env_vars: OS,PYTHON
+          files: ./coverage.xml
+          flags: unittests

.github/workflows/ci_production.yaml

@@ -9,14 +9,9 @@ jobs:
   test:
     runs-on: ubuntu-latest
     environment: production-service
-    strategy:
-      matrix:
-        example: [Uproot_UprootRaw_Dict.py, Uproot_PythonFunction_Dict.py, Uproot_FuncADL_Dict.py]
 
     steps:
       - uses: actions/checkout@v3
-        with:
-          ref: 3.0_develop
 
       - name: Set up Python 3.12
         uses: actions/setup-python@v4
@@ -29,13 +24,15 @@ jobs:
         env:
           SERVICEX_YAML: ${{ secrets.SERVICEX_YAML }}
 
+      - name: Install uv
+        uses: astral-sh/setup-uv@v3
+
       - name: Install package
         run: |
-          python -m pip install --upgrade uv
-          uv pip install --system --upgrade pip setuptools wheel
-          uv pip install --system --upgrade '.[databinder,pandas,test]'
+          uv pip install --system --upgrade '.[test]'
           uv pip list --system
 
-      - name: Run examples
+      - name: Run example
         working-directory: examples
-        run: python3 ${{ matrix.example }}
+        run: python3 Uproot_UprootRaw_Dict.py
+

.github/workflows/docs.yml

@@ -0,0 +1,79 @@
+name: Docs
+
+on:
+  push:
+    branches:
+    - main
+    - 3.0_develop
+  pull_request:
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    name: Build docs
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.12'
+
+    - name: Install uv
+      uses: astral-sh/setup-uv@v3
+
+    - name: Install Python dependencies
+      run: |
+        uv pip install --system --upgrade ".[docs]"
+        uv pip list --system
+
+    - name: Test and build docs
+      run: |
+        pushd docs
+        make html
+
+    - name: Fix permissions if needed
+      run: |
+        chmod -c -R +rX "docs/_build/html/" | while read line; do
+          echo "::warning title=Invalid file permissions automatically fixed::$line"
+        done
+
+    - name: Upload artifact
+      uses: actions/upload-pages-artifact@v3
+      with:
+        path: 'docs/_build/html'
+
+  deploy:
+    name: Deploy docs to GitHub Pages
+    if: github.event_name == 'push'
+    needs: build
+    # Set permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages
+    permissions:
+      contents: read
+      pages: write
+      id-token: write
+
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Setup Pages
+      uses: actions/configure-pages@v5
+
+    - name: Deploy to GitHub Pages
+      id: deployment
+      uses: actions/deploy-pages@v4

.github/workflows/pypi.yaml

@@ -7,23 +7,62 @@ on:
 jobs:
   publish:
     runs-on: ubuntu-latest
+    # Mandatory for publishing with a trusted publisher
+    # c.f. https://docs.pypi.org/trusted-publishers/using-a-publisher/
+    permissions:
+      id-token: write
 
     steps:
-      - uses: actions/checkout@v3
-      - name: Set up Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: "3.10"
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip build
-      - name: Build the servicex wheel
-        env:
-          servicex_version: ${{ github.ref }}
-        run: |
-          python -m build --sdist --wheel
-      - name: Publish servicex to PyPI
-        uses: pypa/[email protected]
-        with:
-          password: ${{ secrets.PYPI_PASSWORD_SERVICEX }}
-          print_hash: true
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+
+    - name: Set up Python 3.12
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.12'
+
+    - name: Install uv
+      uses: astral-sh/setup-uv@v3
+
+    - name: Install python-build and twine
+      run: |
+        uv pip install --system build twine
+        uv pip list --system
+
+    - name: Set env
+      run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
+
+    - name: Test
+      run: |
+        echo $RELEASE_VERSION
+        echo ${{ env.RELEASE_VERSION }}
+
+    - name: Update Version
+      run: |
+        sed -i '/version =/ s/= "[^"][^"]*"/ = "${{ env.RELEASE_VERSION }}"/' pyproject.toml
+
+    - name: Build a sdist and wheel
+      run: |
+        python -m build .
+
+    - name: Verify the distribution
+      run: twine check --strict dist/*
+
+    - name: List contents of sdist
+      run: python -m tarfile --list dist/servicex-*.tar.gz
+
+    - name: List contents of wheel
+      run: python -m zipfile --list dist/servicex-*.whl
+
+    - name: Upload distribution artifact
+      uses: actions/upload-artifact@v4
+      with:
+        name: dist-artifact
+        path: dist
+
+    - name: Publish distribution 📦 to PyPI
+      if: github.repository == 'ssl-hep/ServiceX_frontend'
+      uses: pypa/[email protected]
+      with:
+        print-hash: true

.gitignore

@@ -125,3 +125,6 @@ dmypy.json
 .pyre/
 
 .idea/
+
+.servicex
+

.readthedocs.yaml

@@ -0,0 +1,33 @@
+# Read the Docs configuration file for Sphinx projects
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+# Required
+version: 2
+
+# Set the OS, Python version and other tools you might need
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.12"
+
+# Build documentation in the "docs/" directory with Sphinx
+sphinx:
+  configuration: docs/conf.py
+#   # You can configure Sphinx to use a different builder, for instance use the dirhtml builder for simpler URLs
+  builder: html
+  # Fail on all warnings to avoid broken references
+  # fail_on_warning: true
+
+# Optionally build your docs in additional formats such as PDF and ePub
+formats:
+  - pdf
+
+# Optional but recommended, declare the Python requirements required
+# to build your documentation
+# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html
+python:
+  install:
+    - method: pip
+      path: .
+      extra_requirements:
+        - docs