From 675c8951e1147f5c280cb31d87c185eac4257461 Mon Sep 17 00:00:00 2001 From: jenniferliddle Date: Tue, 14 Jun 2022 14:22:43 +0100 Subject: [PATCH 1/5] Imported a client script from GitLab. Imported a client script for interaction with the npg_porch server from an internal GitLab project. To avoid hardcoding pipeline details, created two additional script arguments, pipeline_url and pipeline_version. Replaced real token values with placeholders. --- pow | 114 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 114 insertions(+) create mode 100755 pow diff --git a/pow b/pow new file mode 100755 index 0000000..7fc41ff --- /dev/null +++ b/pow @@ -0,0 +1,114 @@ +#!/usr/bin/env python3 + +# Copyright (C) 2022, 2024 Genome Research Ltd. +# +# Author: Jennifer Liddle +# +# npg_porch_client is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation; either version 3 of the License, or (at your option) any +# later version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +# details. +# +# You should have received a copy of the GNU General Public License along with +# this program. If not, see . + +import requests +import argparse + +# Certification file for https requests +# if we don't have one, we can set certfile = False +certfile = '/usr/share/ca-certificates/sanger.ac.uk/Genome_Research_Ltd_Certificate_Authority-cert.pem' + +# tokens from Porch +admin_headers = {"Authorization":"Bearer a"} +pipeline_headers = {"Authorization":"Bearer b"} + +# Command line arguments +parser = argparse.ArgumentParser(description='Pipeline Orchestration Wrapper') +parser.add_argument('--baseurl', type=str, help="base URL") +parser.add_argument('--pipeline_url', type=str, help="Pipeline git project URL") +parser.add_argument('--pipeline_version', type=str, help="Pipeline version") +parser.add_argument('--pipeline', type=str, help="pipeline name") +parser.add_argument('--idrun', type=int, help="id_run") +parser.add_argument('--sample', type=str, help="sample name") +parser.add_argument('--study', type=str, help="study ID") +parser.add_argument('--status', type=str, help="new status to set", default="DONE", + choices = ['PENDING', 'CLAIMED', 'RUNNING', 'DONE', 'FAILED', 'CANCELLED']) +parser.add_argument('command', type=str, help="command to send to npg_porch", + choices = ['list', 'plist', 'register', 'add', 'claim', 'update']) +args = parser.parse_args() + +if (args.command == 'list'): + response = requests.get(args.baseurl+'/tasks', verify=certfile, headers=pipeline_headers) + if not response.ok: + print(f"\"{response.reason}\" received from {response.url}") + exit(1) + + x=response.json() + for p in x: + if (p['pipeline']['name'] == args.pipeline): + print(f"{p['task_input']}\t{p['status']}") + +if (args.command == 'plist'): + response = requests.get(args.baseurl+'/pipelines', verify=certfile, headers=admin_headers) + if not response.ok: + print(f"\"{response.reason}\" received from {response.url}") + exit(1) + + x=response.json() + pipelines = {} + for p in x: + pname = p['name'] + if pname not in pipelines: + print(pname) + pipelines[pname] = 1 + +if (args.command == 'register'): + data = { 'name': args.pipeline, 'uri': args.pipeline_url, "version": args.pipeline_version } + response = requests.post(args.baseurl+'/pipelines', json=data, verify=certfile, headers=admin_headers) + if not response.ok: + print(f"\"{response.reason}\" received from {response.url}") + exit(1) + + print(response.json()) + +if (args.command == 'add'): + data = { 'pipeline': + { 'name': args.pipeline, 'uri': args.pipeline_url, "version": args.pipeline_version }, + 'task_input': { 'id_run': args.idrun, 'sample': args.sample, 'id_study': args.study } + } + response = requests.post(args.baseurl+'/tasks', json=data, verify=certfile, headers=pipeline_headers) + if not response.ok: + print(f"\"{response.reason}\" received from {response.url}") + exit(1) + + print(response.json()) + +if (args.command == 'claim'): + data = { 'name': args.pipeline, 'uri': args.pipeline_url, "version": args.pipeline_version } + response = requests.post(args.baseurl+'/tasks/claim', json=data, verify=certfile, headers=pipeline_headers) + if not response.ok: + print(f"\"{response.reason}\" received from {response.url}") + exit(1) + + print(response.json()) + +if (args.command == 'update'): + data = { 'pipeline': + { 'name': args.pipeline, 'uri': args.pipeline_url, "version": args.pipeline_version }, + 'task_input': { 'id_run': args.idrun, 'sample': args.sample, 'id_study': args.study }, + 'status': args.status + } + response = requests.put(args.baseurl+'/tasks', json=data, verify=certfile, headers=pipeline_headers) + if not response.ok: + print(f"\"{response.reason}\" received from {response.url}") + exit(1) + + print(data) + print(response.json()) + From 9e6b1dadd8611245f9966edb25d3012439ce13a4 Mon Sep 17 00:00:00 2001 From: mgcam Date: Wed, 24 Apr 2024 18:32:24 +0100 Subject: [PATCH 2/5] Renamed and reformatted the imported script. Updated project dependencies. --- .flake8 | 2 + .github/workflows/test.yml | 6 +- pow | 114 ------------------ pyproject.toml | 5 + src/npg_porch_cli/npg_porch_client.py | 160 ++++++++++++++++++++++++++ tests/test_api.py | 1 + 6 files changed, 173 insertions(+), 115 deletions(-) delete mode 100755 pow create mode 100755 src/npg_porch_cli/npg_porch_client.py diff --git a/.flake8 b/.flake8 index c201160..45aa47f 100644 --- a/.flake8 +++ b/.flake8 @@ -1,4 +1,6 @@ [flake8] max-line-length = 100 +extend-select = B950 +extend-ignore = E203, E501, E701 ignore = E251, E265, E261, E302, W503 per-file-ignores = __init__.py:F401 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 40019bc..b697fb7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -33,10 +33,14 @@ jobs: run: | poetry run pytest + - name: Run import checker + run: | + poetry run isort --check . + - name: Run code style checker run: | poetry run flake8 - name: Run code formatter run: | - poetry run black --check src + poetry run black --check src tests diff --git a/pow b/pow deleted file mode 100755 index 7fc41ff..0000000 --- a/pow +++ /dev/null @@ -1,114 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright (C) 2022, 2024 Genome Research Ltd. -# -# Author: Jennifer Liddle -# -# npg_porch_client is free software: you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by the Free -# Software Foundation; either version 3 of the License, or (at your option) any -# later version. -# -# This program is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more -# details. -# -# You should have received a copy of the GNU General Public License along with -# this program. If not, see . - -import requests -import argparse - -# Certification file for https requests -# if we don't have one, we can set certfile = False -certfile = '/usr/share/ca-certificates/sanger.ac.uk/Genome_Research_Ltd_Certificate_Authority-cert.pem' - -# tokens from Porch -admin_headers = {"Authorization":"Bearer a"} -pipeline_headers = {"Authorization":"Bearer b"} - -# Command line arguments -parser = argparse.ArgumentParser(description='Pipeline Orchestration Wrapper') -parser.add_argument('--baseurl', type=str, help="base URL") -parser.add_argument('--pipeline_url', type=str, help="Pipeline git project URL") -parser.add_argument('--pipeline_version', type=str, help="Pipeline version") -parser.add_argument('--pipeline', type=str, help="pipeline name") -parser.add_argument('--idrun', type=int, help="id_run") -parser.add_argument('--sample', type=str, help="sample name") -parser.add_argument('--study', type=str, help="study ID") -parser.add_argument('--status', type=str, help="new status to set", default="DONE", - choices = ['PENDING', 'CLAIMED', 'RUNNING', 'DONE', 'FAILED', 'CANCELLED']) -parser.add_argument('command', type=str, help="command to send to npg_porch", - choices = ['list', 'plist', 'register', 'add', 'claim', 'update']) -args = parser.parse_args() - -if (args.command == 'list'): - response = requests.get(args.baseurl+'/tasks', verify=certfile, headers=pipeline_headers) - if not response.ok: - print(f"\"{response.reason}\" received from {response.url}") - exit(1) - - x=response.json() - for p in x: - if (p['pipeline']['name'] == args.pipeline): - print(f"{p['task_input']}\t{p['status']}") - -if (args.command == 'plist'): - response = requests.get(args.baseurl+'/pipelines', verify=certfile, headers=admin_headers) - if not response.ok: - print(f"\"{response.reason}\" received from {response.url}") - exit(1) - - x=response.json() - pipelines = {} - for p in x: - pname = p['name'] - if pname not in pipelines: - print(pname) - pipelines[pname] = 1 - -if (args.command == 'register'): - data = { 'name': args.pipeline, 'uri': args.pipeline_url, "version": args.pipeline_version } - response = requests.post(args.baseurl+'/pipelines', json=data, verify=certfile, headers=admin_headers) - if not response.ok: - print(f"\"{response.reason}\" received from {response.url}") - exit(1) - - print(response.json()) - -if (args.command == 'add'): - data = { 'pipeline': - { 'name': args.pipeline, 'uri': args.pipeline_url, "version": args.pipeline_version }, - 'task_input': { 'id_run': args.idrun, 'sample': args.sample, 'id_study': args.study } - } - response = requests.post(args.baseurl+'/tasks', json=data, verify=certfile, headers=pipeline_headers) - if not response.ok: - print(f"\"{response.reason}\" received from {response.url}") - exit(1) - - print(response.json()) - -if (args.command == 'claim'): - data = { 'name': args.pipeline, 'uri': args.pipeline_url, "version": args.pipeline_version } - response = requests.post(args.baseurl+'/tasks/claim', json=data, verify=certfile, headers=pipeline_headers) - if not response.ok: - print(f"\"{response.reason}\" received from {response.url}") - exit(1) - - print(response.json()) - -if (args.command == 'update'): - data = { 'pipeline': - { 'name': args.pipeline, 'uri': args.pipeline_url, "version": args.pipeline_version }, - 'task_input': { 'id_run': args.idrun, 'sample': args.sample, 'id_study': args.study }, - 'status': args.status - } - response = requests.put(args.baseurl+'/tasks', json=data, verify=certfile, headers=pipeline_headers) - if not response.ok: - print(f"\"{response.reason}\" received from {response.url}") - exit(1) - - print(data) - print(response.json()) - diff --git a/pyproject.toml b/pyproject.toml index 3ae58a1..338747b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,10 +12,12 @@ license = "GPL-3.0-or-later" [tool.poetry.dependencies] python = "^3.10" +requests = "^2.31.0" [tool.poetry.dev-dependencies] black = "^22.3.0" flake8 = "^4.0.1" +flake8-bugbear = "^18.2.0" pytest = "^7.1.1" isort = { version = "^5.10.1", extras = ["colors"] } @@ -26,6 +28,9 @@ build-backend = "poetry.core.masonry.api" [tool.isort] profile = "black" +[tool.black] +line_length = 100 + [tool.pytest.ini_options] addopts = [ "--import-mode=importlib", diff --git a/src/npg_porch_cli/npg_porch_client.py b/src/npg_porch_cli/npg_porch_client.py new file mode 100755 index 0000000..abc212c --- /dev/null +++ b/src/npg_porch_cli/npg_porch_client.py @@ -0,0 +1,160 @@ +#!/usr/bin/env python3 + +# Copyright (C) 2022, 2024 Genome Research Ltd. +# +# Author: Jennifer Liddle +# +# npg_porch_client is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation; either version 3 of the License, or (at your option) any +# later version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +# details. +# +# You should have received a copy of the GNU General Public License along with +# this program. If not, see . + +import argparse + +import requests + +# Certification file for https requests +# if we don't have one, we can set certfile = False +certfile = ( + "/usr/share/ca-certificates/sanger.ac.uk/Genome_Research_Ltd_Certificate_Authority-cert.pem" +) + +# tokens from Porch +admin_headers = {"Authorization": "Bearer a"} +pipeline_headers = {"Authorization": "Bearer b"} + +# Command line arguments +parser = argparse.ArgumentParser(description="Pipeline Orchestration Wrapper") +parser.add_argument("--baseurl", type=str, help="base URL") +parser.add_argument("--pipeline_url", type=str, help="Pipeline git project URL") +parser.add_argument("--pipeline_version", type=str, help="Pipeline version") +parser.add_argument("--pipeline", type=str, help="pipeline name") +parser.add_argument("--idrun", type=int, help="id_run") +parser.add_argument("--sample", type=str, help="sample name") +parser.add_argument("--study", type=str, help="study ID") +parser.add_argument( + "--status", + type=str, + help="new status to set", + default="DONE", + choices=["PENDING", "CLAIMED", "RUNNING", "DONE", "FAILED", "CANCELLED"], +) +parser.add_argument( + "command", + type=str, + help="command to send to npg_porch", + choices=["list", "plist", "register", "add", "claim", "update"], +) +args = parser.parse_args() + +if args.command == "list": + response = requests.get(args.baseurl + "/tasks", verify=certfile, headers=pipeline_headers) + if not response.ok: + print(f'"{response.reason}" received from {response.url}') + exit(1) + + x = response.json() + for p in x: + if p["pipeline"]["name"] == args.pipeline: + print(f"{p['task_input']}\t{p['status']}") + +if args.command == "plist": + response = requests.get(args.baseurl + "/pipelines", verify=certfile, headers=admin_headers) + if not response.ok: + print(f'"{response.reason}" received from {response.url}') + exit(1) + + x = response.json() + pipelines = {} + for p in x: + pname = p["name"] + if pname not in pipelines: + print(pname) + pipelines[pname] = 1 + +if args.command == "register": + data = { + "name": args.pipeline, + "uri": args.pipeline_url, + "version": args.pipeline_version, + } + response = requests.post( + args.baseurl + "/pipelines", json=data, verify=certfile, headers=admin_headers + ) + if not response.ok: + print(f'"{response.reason}" received from {response.url}') + exit(1) + + print(response.json()) + +if args.command == "add": + data = { + "pipeline": { + "name": args.pipeline, + "uri": args.pipeline_url, + "version": args.pipeline_version, + }, + "task_input": { + "id_run": args.idrun, + "sample": args.sample, + "id_study": args.study, + }, + } + response = requests.post( + args.baseurl + "/tasks", json=data, verify=certfile, headers=pipeline_headers + ) + if not response.ok: + print(f'"{response.reason}" received from {response.url}') + exit(1) + + print(response.json()) + +if args.command == "claim": + data = { + "name": args.pipeline, + "uri": args.pipeline_url, + "version": args.pipeline_version, + } + response = requests.post( + args.baseurl + "/tasks/claim", + json=data, + verify=certfile, + headers=pipeline_headers, + ) + if not response.ok: + print(f'"{response.reason}" received from {response.url}') + exit(1) + + print(response.json()) + +if args.command == "update": + data = { + "pipeline": { + "name": args.pipeline, + "uri": args.pipeline_url, + "version": args.pipeline_version, + }, + "task_input": { + "id_run": args.idrun, + "sample": args.sample, + "id_study": args.study, + }, + "status": args.status, + } + response = requests.put( + args.baseurl + "/tasks", json=data, verify=certfile, headers=pipeline_headers + ) + if not response.ok: + print(f'"{response.reason}" received from {response.url}') + exit(1) + + print(data) + print(response.json()) diff --git a/tests/test_api.py b/tests/test_api.py index 4e2a9c0..0ae0c2f 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1,5 +1,6 @@ import npg_porch_cli.api as porchApi + def test_addition(): assert porchApi.add_two(1, 2) == 3 From aa594ee323a880fbe29a7be12410b7cb8b54eb73 Mon Sep 17 00:00:00 2001 From: mgcam Date: Thu, 25 Apr 2024 13:02:46 +0100 Subject: [PATCH 3/5] Manage flake8 via pyproject-flake8 And configure flake8 in pyproject.toml --- .flake8 | 6 ------ .github/workflows/test.yml | 2 +- pyproject.toml | 6 ++++-- 3 files changed, 5 insertions(+), 9 deletions(-) delete mode 100644 .flake8 diff --git a/.flake8 b/.flake8 deleted file mode 100644 index 45aa47f..0000000 --- a/.flake8 +++ /dev/null @@ -1,6 +0,0 @@ -[flake8] -max-line-length = 100 -extend-select = B950 -extend-ignore = E203, E501, E701 -ignore = E251, E265, E261, E302, W503 -per-file-ignores = __init__.py:F401 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b697fb7..5a44a45 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -39,7 +39,7 @@ jobs: - name: Run code style checker run: | - poetry run flake8 + poetry run pflake8 - name: Run code formatter run: | diff --git a/pyproject.toml b/pyproject.toml index 338747b..c769f61 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,8 +16,7 @@ requests = "^2.31.0" [tool.poetry.dev-dependencies] black = "^22.3.0" -flake8 = "^4.0.1" -flake8-bugbear = "^18.2.0" +pyproject-flake8 = "^7.0.0" pytest = "^7.1.1" isort = { version = "^5.10.1", extras = ["colors"] } @@ -31,6 +30,9 @@ profile = "black" [tool.black] line_length = 100 +[tool.flake8] +max-line-length = 100 + [tool.pytest.ini_options] addopts = [ "--import-mode=importlib", From 3d0a05dd146363311f03570c3dab682a8f2aa9e1 Mon Sep 17 00:00:00 2001 From: mgcam Date: Thu, 25 Apr 2024 13:15:27 +0100 Subject: [PATCH 4/5] Reduced max line length to 88. --- pyproject.toml | 4 ++-- src/npg_porch_cli/npg_porch_client.py | 12 +++++++----- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index c769f61..5078197 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,10 +28,10 @@ build-backend = "poetry.core.masonry.api" profile = "black" [tool.black] -line_length = 100 +line_length = 88 [tool.flake8] -max-line-length = 100 +max-line-length = 88 [tool.pytest.ini_options] addopts = [ diff --git a/src/npg_porch_cli/npg_porch_client.py b/src/npg_porch_cli/npg_porch_client.py index abc212c..1d7e57b 100755 --- a/src/npg_porch_cli/npg_porch_client.py +++ b/src/npg_porch_cli/npg_porch_client.py @@ -23,9 +23,7 @@ # Certification file for https requests # if we don't have one, we can set certfile = False -certfile = ( - "/usr/share/ca-certificates/sanger.ac.uk/Genome_Research_Ltd_Certificate_Authority-cert.pem" -) +certfile = "/usr/share/ca-certificates/sanger.ac.uk/Genome_Research_Ltd_Certificate_Authority-cert.pem" # noqa: E501 # tokens from Porch admin_headers = {"Authorization": "Bearer a"} @@ -56,7 +54,9 @@ args = parser.parse_args() if args.command == "list": - response = requests.get(args.baseurl + "/tasks", verify=certfile, headers=pipeline_headers) + response = requests.get( + args.baseurl + "/tasks", verify=certfile, headers=pipeline_headers + ) if not response.ok: print(f'"{response.reason}" received from {response.url}') exit(1) @@ -67,7 +67,9 @@ print(f"{p['task_input']}\t{p['status']}") if args.command == "plist": - response = requests.get(args.baseurl + "/pipelines", verify=certfile, headers=admin_headers) + response = requests.get( + args.baseurl + "/pipelines", verify=certfile, headers=admin_headers + ) if not response.ok: print(f'"{response.reason}" received from {response.url}') exit(1) From b5e1a4d9ca45daa4225dda19e256b04c408049a7 Mon Sep 17 00:00:00 2001 From: mgcam Date: Thu, 25 Apr 2024 13:50:15 +0100 Subject: [PATCH 5/5] The bugbear returns. --- pyproject.toml | 3 +++ src/npg_porch_cli/npg_porch_client.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 5078197..ded1888 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,6 +17,7 @@ requests = "^2.31.0" [tool.poetry.dev-dependencies] black = "^22.3.0" pyproject-flake8 = "^7.0.0" +flake8-bugbear = "^24.4.0" pytest = "^7.1.1" isort = { version = "^5.10.1", extras = ["colors"] } @@ -32,6 +33,8 @@ line_length = 88 [tool.flake8] max-line-length = 88 +extend-select = ["B950"] +extend-ignore = ["E501"] [tool.pytest.ini_options] addopts = [ diff --git a/src/npg_porch_cli/npg_porch_client.py b/src/npg_porch_cli/npg_porch_client.py index 1d7e57b..773b4a4 100755 --- a/src/npg_porch_cli/npg_porch_client.py +++ b/src/npg_porch_cli/npg_porch_client.py @@ -23,7 +23,7 @@ # Certification file for https requests # if we don't have one, we can set certfile = False -certfile = "/usr/share/ca-certificates/sanger.ac.uk/Genome_Research_Ltd_Certificate_Authority-cert.pem" # noqa: E501 +certfile = "/usr/share/ca-certificates/sanger.ac.uk/Genome_Research_Ltd_Certificate_Authority-cert.pem" # noqa: B950 # tokens from Porch admin_headers = {"Authorization": "Bearer a"}