From b43ac1a7d2a03acf45543deec44ecb0dd577f8ed Mon Sep 17 00:00:00 2001 From: Marjorie Lucas Date: Tue, 30 Jul 2024 17:51:05 -0700 Subject: [PATCH 01/10] added more query params --- maap/maap.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/maap/maap.py b/maap/maap.py index 8b91695..421bdd9 100644 --- a/maap/maap.py +++ b/maap/maap.py @@ -256,13 +256,23 @@ def cancelJob(self, jobid): job.id = jobid return job.cancel_job() - def listJobs(self, username=None, page_size=None, offset=None): + def listJobs(self, username=None, **kwargs): if username==None and self.profile is not None and 'username' in self.profile.account_info().keys(): username = self.profile.account_info()['username'] url = os.path.join(self.config.dps_job, username, endpoints.DPS_JOB_LIST) - params = {k: v for k, v in (("page_size", page_size), ("offset", offset)) if v} - + + valid_keys = ['algo_id', 'end_time', 'offset', 'page_size', 'priority', 'queue', 'start_time', 'status', 'tag', 'version'] + + params = {k: v for k, v in kwargs.items() if k in valid_keys and v} + + # DPS requests use 'job_type', which is a concatenation of 'algo_id' and 'version' + if 'algo_id' in params and 'version' in params: + params['job_type'] = params['algo_id'] + ':' + params['version'] + + params.pop('algo_id', None) + params.pop('version', None) + headers = self._get_api_header() logger.debug('GET request sent to {}'.format(url)) logger.debug('headers:') From f84971970cb90d3de003ac5015588e431e3859ac Mon Sep 17 00:00:00 2001 From: Marjorie Lucas Date: Tue, 30 Jul 2024 18:32:48 -0700 Subject: [PATCH 02/10] added get_job_details flag --- maap/maap.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/maap/maap.py b/maap/maap.py index 421bdd9..ee9d40e 100644 --- a/maap/maap.py +++ b/maap/maap.py @@ -261,11 +261,10 @@ def listJobs(self, username=None, **kwargs): username = self.profile.account_info()['username'] url = os.path.join(self.config.dps_job, username, endpoints.DPS_JOB_LIST) + valid_keys = ['algo_id', 'end_time', 'get_job_details', 'offset', 'page_size', 'priority', 'queue', 'start_time', 'status', 'tag', 'version'] - valid_keys = ['algo_id', 'end_time', 'offset', 'page_size', 'priority', 'queue', 'start_time', 'status', 'tag', 'version'] - - params = {k: v for k, v in kwargs.items() if k in valid_keys and v} - + params = {k: v for k, v in kwargs.items() if k in valid_keys and v is not None} + # DPS requests use 'job_type', which is a concatenation of 'algo_id' and 'version' if 'algo_id' in params and 'version' in params: params['job_type'] = params['algo_id'] + ':' + params['version'] @@ -282,6 +281,7 @@ def listJobs(self, username=None, **kwargs): headers=headers, params=params, ) + print(response.url) return response def submitJob(self, identifier, algo_id, version, queue, retrieve_attributes=False, **kwargs): From ba2fbad01974637b69a10fa251cb2d9c0bad34c4 Mon Sep 17 00:00:00 2001 From: Marjorie Lucas Date: Tue, 30 Jul 2024 18:37:02 -0700 Subject: [PATCH 03/10] cleanup --- maap/maap.py | 1 - 1 file changed, 1 deletion(-) diff --git a/maap/maap.py b/maap/maap.py index ee9d40e..9b9749b 100644 --- a/maap/maap.py +++ b/maap/maap.py @@ -281,7 +281,6 @@ def listJobs(self, username=None, **kwargs): headers=headers, params=params, ) - print(response.url) return response def submitJob(self, identifier, algo_id, version, queue, retrieve_attributes=False, **kwargs): From ae91831ee14d5ff7387aa1dd9bcfddce24f2cd02 Mon Sep 17 00:00:00 2001 From: Marjorie Lucas <47004511+marjo-luc@users.noreply.github.com> Date: Thu, 1 Aug 2024 10:14:42 -0700 Subject: [PATCH 04/10] Update maap/maap.py Co-authored-by: Chuck Daniels --- maap/maap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maap/maap.py b/maap/maap.py index 9b9749b..c84eb63 100644 --- a/maap/maap.py +++ b/maap/maap.py @@ -257,7 +257,7 @@ def cancelJob(self, jobid): return job.cancel_job() def listJobs(self, username=None, **kwargs): - if username==None and self.profile is not None and 'username' in self.profile.account_info().keys(): + if username is None and self.profile is not None and 'username' in self.profile.account_info().keys(): username = self.profile.account_info()['username'] url = os.path.join(self.config.dps_job, username, endpoints.DPS_JOB_LIST) From 1e2426a888250109b46bab7f09b4ecd28527d074 Mon Sep 17 00:00:00 2001 From: Marjorie Lucas <47004511+marjo-luc@users.noreply.github.com> Date: Thu, 1 Aug 2024 10:16:31 -0700 Subject: [PATCH 05/10] Update maap/maap.py Co-authored-by: Chuck Daniels --- maap/maap.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/maap/maap.py b/maap/maap.py index c84eb63..da61798 100644 --- a/maap/maap.py +++ b/maap/maap.py @@ -260,7 +260,10 @@ def listJobs(self, username=None, **kwargs): if username is None and self.profile is not None and 'username' in self.profile.account_info().keys(): username = self.profile.account_info()['username'] - url = os.path.join(self.config.dps_job, username, endpoints.DPS_JOB_LIST) + url = "/".join( + segment.strip("/") + for segment in (self.config.dps_job, username, endpoints.DPS_JOB_LIST) + ) valid_keys = ['algo_id', 'end_time', 'get_job_details', 'offset', 'page_size', 'priority', 'queue', 'start_time', 'status', 'tag', 'version'] params = {k: v for k, v in kwargs.items() if k in valid_keys and v is not None} From 187838931a24cef8ee83e365358f0066d275e3ee Mon Sep 17 00:00:00 2001 From: Marjorie Lucas <47004511+marjo-luc@users.noreply.github.com> Date: Thu, 1 Aug 2024 10:18:08 -0700 Subject: [PATCH 06/10] Update maap/maap.py Co-authored-by: Chuck Daniels --- maap/maap.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/maap/maap.py b/maap/maap.py index da61798..c423e86 100644 --- a/maap/maap.py +++ b/maap/maap.py @@ -269,11 +269,15 @@ def listJobs(self, username=None, **kwargs): params = {k: v for k, v in kwargs.items() if k in valid_keys and v is not None} # DPS requests use 'job_type', which is a concatenation of 'algo_id' and 'version' - if 'algo_id' in params and 'version' in params: - params['job_type'] = params['algo_id'] + ':' + params['version'] + algo_id = params.pop('algo_id', None) + version = params.pop('version', None) + + if (not algo_id) != (not version): + # Either algo_id or version was supplied as a non-empty string, but not both. + # Either both must be non-empty strings or both must be None. + raise ValueError("Either supply non-empty strings for both algo_id and version, or supply neither.") - params.pop('algo_id', None) - params.pop('version', None) + params['job_type'] = f"{algo_id}:{version}" headers = self._get_api_header() logger.debug('GET request sent to {}'.format(url)) From d7f3f056c4bb8ba09f16e87dbee7496ca3a55cf6 Mon Sep 17 00:00:00 2001 From: Marjorie Lucas Date: Thu, 1 Aug 2024 12:17:43 -0700 Subject: [PATCH 07/10] added docstring | updated param handling --- maap/maap.py | 55 ++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 51 insertions(+), 4 deletions(-) diff --git a/maap/maap.py b/maap/maap.py index c423e86..d706ae3 100644 --- a/maap/maap.py +++ b/maap/maap.py @@ -256,7 +256,38 @@ def cancelJob(self, jobid): job.id = jobid return job.cancel_job() - def listJobs(self, username=None, **kwargs): + def listJobs(self, username=None, + algo_id=None, + end_time=None, + get_job_details=True, + offset=None, + page_size=None, + priority=None, + queue=None, + start_time=None, + status=None, + tag=None, + version=None): + """ + Returns a list of jobs for a given user that matches query params provided. + + Args: + username (str): Platform user. + algo_id (str): Algorithm type. + end_time (str, optional): Specifying this parameter will return all jobs that have completed from the provided end time to now. e.g. 2024-01-01 or 2024-01-01T00:00:00.000000Z. + get_job_details (bool, optional): Flag that determines whether to return a detailed job list or a compact list containing just the job ids and their associated job tags. Default is True. + offset (int, optional): Offset for pagination. + page_size (int, optional): Page size for pagination. + priority (int, optional): Job processing priority. Valid values are integers from 0-9. + queue (str, optional): Job processing resource. + start_time (str, optional): Specifying this parameter will return all jobs that have started from the provided start time to now. e.g. 2024-01-01 or 2024-01-01T00:00:00.000000Z. + status (str, optional): Job status e.g. job-completed, job-failed, job-started, job-queued. + tag (str, optional): User job tag. + version (str, optional): Algorithm version i.e. GitHub branch. + + Returns: + list: List of jobs for a given user that matches query params provided. + """ if username is None and self.profile is not None and 'username' in self.profile.account_info().keys(): username = self.profile.account_info()['username'] @@ -264,9 +295,25 @@ def listJobs(self, username=None, **kwargs): segment.strip("/") for segment in (self.config.dps_job, username, endpoints.DPS_JOB_LIST) ) - valid_keys = ['algo_id', 'end_time', 'get_job_details', 'offset', 'page_size', 'priority', 'queue', 'start_time', 'status', 'tag', 'version'] - - params = {k: v for k, v in kwargs.items() if k in valid_keys and v is not None} + + params = { + k: v + for k, v in ( + ("algo_id", algo_id), + ("end_time", end_time), + ("get_job_details", get_job_details), + ("offset", offset), + ("page_size", page_size), + ("priority", priority), + ("queue", queue), + ("start_time", start_time), + ("status", status), + ("tag", tag), + ("username", username), + ("version", version), + ) + if v is not None + } # DPS requests use 'job_type', which is a concatenation of 'algo_id' and 'version' algo_id = params.pop('algo_id', None) From 7e91c6d764917395817dcfa95a90ceb6745aadb0 Mon Sep 17 00:00:00 2001 From: Marjorie Lucas Date: Mon, 5 Aug 2024 13:22:45 -0700 Subject: [PATCH 08/10] review updates --- maap/maap.py | 41 +++++++++++++++++++++++------------------ maap/utils/JobUtils.py | 18 ++++++++++++++++++ 2 files changed, 41 insertions(+), 18 deletions(-) create mode 100644 maap/utils/JobUtils.py diff --git a/maap/maap.py b/maap/maap.py index d706ae3..1226f9d 100644 --- a/maap/maap.py +++ b/maap/maap.py @@ -19,6 +19,7 @@ from maap.AWS import AWS from maap.dps.DpsHelper import DpsHelper from maap.utils import endpoints +from maap.utils import JobUtils logger = logging.getLogger(__name__) @@ -256,13 +257,12 @@ def cancelJob(self, jobid): job.id = jobid return job.cancel_job() - def listJobs(self, username=None, + def listJobs(self, username=None, *, algo_id=None, end_time=None, get_job_details=True, - offset=None, - page_size=None, - priority=None, + offset=0, + page_size=10, queue=None, start_time=None, status=None, @@ -272,18 +272,17 @@ def listJobs(self, username=None, Returns a list of jobs for a given user that matches query params provided. Args: - username (str): Platform user. - algo_id (str): Algorithm type. + username (str, optional): Platform user. If no username is provided, the profile username will be used. + algo_id (str, optional): Algorithm type. end_time (str, optional): Specifying this parameter will return all jobs that have completed from the provided end time to now. e.g. 2024-01-01 or 2024-01-01T00:00:00.000000Z. get_job_details (bool, optional): Flag that determines whether to return a detailed job list or a compact list containing just the job ids and their associated job tags. Default is True. - offset (int, optional): Offset for pagination. - page_size (int, optional): Page size for pagination. - priority (int, optional): Job processing priority. Valid values are integers from 0-9. + offset (int, optional): Offset for pagination. Default is 0. + page_size (int, optional): Page size for pagination. Default is 10. queue (str, optional): Job processing resource. start_time (str, optional): Specifying this parameter will return all jobs that have started from the provided start time to now. e.g. 2024-01-01 or 2024-01-01T00:00:00.000000Z. - status (str, optional): Job status e.g. job-completed, job-failed, job-started, job-queued. - tag (str, optional): User job tag. - version (str, optional): Algorithm version i.e. GitHub branch. + status (str, optional): Job status, e.g. job-completed, job-failed, job-started, job-queued. + tag (str, optional): User job tag/identifier. + version (str, optional): Algorithm version, e.g. GitHub branch or tag. Returns: list: List of jobs for a given user that matches query params provided. @@ -291,6 +290,9 @@ def listJobs(self, username=None, if username is None and self.profile is not None and 'username' in self.profile.account_info().keys(): username = self.profile.account_info()['username'] + if username is None: + raise ValueError("Username must be supplied.") + url = "/".join( segment.strip("/") for segment in (self.config.dps_job, username, endpoints.DPS_JOB_LIST) @@ -304,7 +306,6 @@ def listJobs(self, username=None, ("get_job_details", get_job_details), ("offset", offset), ("page_size", page_size), - ("priority", priority), ("queue", queue), ("start_time", start_time), ("status", status), @@ -315,16 +316,20 @@ def listJobs(self, username=None, if v is not None } - # DPS requests use 'job_type', which is a concatenation of 'algo_id' and 'version' - algo_id = params.pop('algo_id', None) - version = params.pop('version', None) - if (not algo_id) != (not version): # Either algo_id or version was supplied as a non-empty string, but not both. # Either both must be non-empty strings or both must be None. raise ValueError("Either supply non-empty strings for both algo_id and version, or supply neither.") - params['job_type'] = f"{algo_id}:{version}" + # DPS requests use 'job_type', which is a concatenation of 'algo_id' and 'version' + if algo_id and version: + params['job_type'] = f"{algo_id}:{version}" + + algo_id = params.pop('algo_id', None) + version = params.pop('version', None) + + if status is not None: + params['status'] = JobUtils.validate_job_status(status) headers = self._get_api_header() logger.debug('GET request sent to {}'.format(url)) diff --git a/maap/utils/JobUtils.py b/maap/utils/JobUtils.py new file mode 100644 index 0000000..a5021f2 --- /dev/null +++ b/maap/utils/JobUtils.py @@ -0,0 +1,18 @@ +# Valid job statuses (loosely based on OGC job status types) +JOB_STATUSES = ['Accepted', 'Running', 'Succeeded', 'Failed', 'Dismissed', 'Deduped', 'Offline'] + +def validate_job_status(status): + ''' + Validates job status + + Args: + status (str): Job status. Accepted values are: 'Accepted', 'Running', 'Succeeded', 'Failed, 'Dismissed', 'Deduped', and 'Offline'. + + Returns: + status (str): Returns unmodified job status if job status is valid. + ''' + if status not in JOB_STATUSES: + valid_statuses = ", ".join(str(status) for status in JOB_STATUSES) + raise ValueError("Invalid job status: '{}'. Job status must be one of the following: {}".format(status, valid_statuses)) + + return status \ No newline at end of file From 343ee538d9572bfa15874b1d9cf459581d80f0bc Mon Sep 17 00:00:00 2001 From: Marjorie Lucas Date: Tue, 6 Aug 2024 10:43:47 -0700 Subject: [PATCH 09/10] review updates --- maap/maap.py | 10 +++++++--- maap/utils/{JobUtils.py => job_utils.py} | 5 ++++- 2 files changed, 11 insertions(+), 4 deletions(-) rename maap/utils/{JobUtils.py => job_utils.py} (79%) diff --git a/maap/maap.py b/maap/maap.py index 1226f9d..6ae3221 100644 --- a/maap/maap.py +++ b/maap/maap.py @@ -19,7 +19,7 @@ from maap.AWS import AWS from maap.dps.DpsHelper import DpsHelper from maap.utils import endpoints -from maap.utils import JobUtils +from maap.utils import job_utils logger = logging.getLogger(__name__) @@ -286,12 +286,16 @@ def listJobs(self, username=None, *, Returns: list: List of jobs for a given user that matches query params provided. + + Raises: + ValueError: If username is not provided and cannot be obtained from the user's profile. + ValueError: If either algo_id or version is provided, but not both. """ if username is None and self.profile is not None and 'username' in self.profile.account_info().keys(): username = self.profile.account_info()['username'] if username is None: - raise ValueError("Username must be supplied.") + raise ValueError("Unable to determine username from profile. Please provide a username.") url = "/".join( segment.strip("/") @@ -329,7 +333,7 @@ def listJobs(self, username=None, *, version = params.pop('version', None) if status is not None: - params['status'] = JobUtils.validate_job_status(status) + params['status'] = job_utils.validate_job_status(status) headers = self._get_api_header() logger.debug('GET request sent to {}'.format(url)) diff --git a/maap/utils/JobUtils.py b/maap/utils/job_utils.py similarity index 79% rename from maap/utils/JobUtils.py rename to maap/utils/job_utils.py index a5021f2..549d5e1 100644 --- a/maap/utils/JobUtils.py +++ b/maap/utils/job_utils.py @@ -1,5 +1,5 @@ # Valid job statuses (loosely based on OGC job status types) -JOB_STATUSES = ['Accepted', 'Running', 'Succeeded', 'Failed', 'Dismissed', 'Deduped', 'Offline'] +JOB_STATUSES = {'Accepted', 'Running', 'Succeeded', 'Failed', 'Dismissed', 'Deduped', 'Offline'} def validate_job_status(status): ''' @@ -10,6 +10,9 @@ def validate_job_status(status): Returns: status (str): Returns unmodified job status if job status is valid. + + Raises: + ValueError: If invalid job status is provided. ''' if status not in JOB_STATUSES: valid_statuses = ", ".join(str(status) for status in JOB_STATUSES) From 5b12d72a79dabeeeb6ca1c9dd1fe688d9d3edac1 Mon Sep 17 00:00:00 2001 From: Marjorie Lucas Date: Tue, 3 Sep 2024 16:54:51 -0700 Subject: [PATCH 10/10] renamed file --- maap/maap.py | 4 ++-- maap/utils/{job_utils.py => job.py} | 0 2 files changed, 2 insertions(+), 2 deletions(-) rename maap/utils/{job_utils.py => job.py} (100%) diff --git a/maap/maap.py b/maap/maap.py index 6ae3221..7b2858d 100644 --- a/maap/maap.py +++ b/maap/maap.py @@ -19,7 +19,7 @@ from maap.AWS import AWS from maap.dps.DpsHelper import DpsHelper from maap.utils import endpoints -from maap.utils import job_utils +from maap.utils import job logger = logging.getLogger(__name__) @@ -333,7 +333,7 @@ def listJobs(self, username=None, *, version = params.pop('version', None) if status is not None: - params['status'] = job_utils.validate_job_status(status) + params['status'] = job.validate_job_status(status) headers = self._get_api_header() logger.debug('GET request sent to {}'.format(url)) diff --git a/maap/utils/job_utils.py b/maap/utils/job.py similarity index 100% rename from maap/utils/job_utils.py rename to maap/utils/job.py