From 902557f78c8fa1288bc6112eebd8ecaea31e668f Mon Sep 17 00:00:00 2001 From: Cheick Keita Date: Tue, 31 Oct 2023 09:23:21 -0700 Subject: [PATCH] Create job after the upload is done (#3608) * Create job after the upload is done - Decoupled the job creation from the JobHelper initialization - Move the job creation after the all the files have been uploaded * build fix * isort --- src/cli/examples/domato.py | 5 ++- src/cli/examples/honggfuzz.py | 6 ++- .../source-coverage-libfuzzer.py | 6 +-- .../llvm-source-coverage/source-coverage.py | 5 ++- src/cli/onefuzz/api.py | 2 +- src/cli/onefuzz/templates/__init__.py | 41 +++++++++---------- src/cli/onefuzz/templates/afl.py | 9 ++-- src/cli/onefuzz/templates/libfuzzer.py | 36 +++++++++------- src/cli/onefuzz/templates/ossfuzz.py | 10 ++--- src/cli/onefuzz/templates/radamsa.py | 12 +++--- src/cli/onefuzz/templates/regression.py | 6 ++- src/cli/tests/test_template_helper.py | 8 ---- 12 files changed, 75 insertions(+), 71 deletions(-) diff --git a/src/cli/examples/domato.py b/src/cli/examples/domato.py index 4bdf2a297c..5e2db0a8b7 100755 --- a/src/cli/examples/domato.py +++ b/src/cli/examples/domato.py @@ -156,8 +156,9 @@ def main() -> None: ] of.logger.info("Creating generic_crash_report task") + job = helper.create_job() of.tasks.create( - helper.job.job_id, + job.job_id, TaskType.generic_crash_report, helper.setup_relative_blob_name(args.target_exe, args.setup_dir), containers, @@ -202,7 +203,7 @@ def main() -> None: ] of.tasks.create( - helper.job.job_id, + job.job_id, TaskType.generic_generator, helper.setup_relative_blob_name(args.target_exe, args.setup_dir), containers, diff --git a/src/cli/examples/honggfuzz.py b/src/cli/examples/honggfuzz.py index 9466716d98..4b104b54d5 100644 --- a/src/cli/examples/honggfuzz.py +++ b/src/cli/examples/honggfuzz.py @@ -101,8 +101,10 @@ def main() -> None: ] of.logger.info("Creating generic_crash_report task") + + job = helper.create_job() of.tasks.create( - helper.job.job_id, + job.job_id, TaskType.generic_crash_report, helper.setup_relative_blob_name(args.target_exe, args.setup_dir), containers, @@ -133,7 +135,7 @@ def main() -> None: ] of.tasks.create( - helper.job.job_id, + job.job_id, TaskType.generic_supervisor, helper.setup_relative_blob_name(args.target_exe, args.setup_dir), containers, diff --git a/src/cli/examples/llvm-source-coverage/source-coverage-libfuzzer.py b/src/cli/examples/llvm-source-coverage/source-coverage-libfuzzer.py index b8ab5f347a..2e9cd9a0d6 100755 --- a/src/cli/examples/llvm-source-coverage/source-coverage-libfuzzer.py +++ b/src/cli/examples/llvm-source-coverage/source-coverage-libfuzzer.py @@ -62,7 +62,6 @@ def main() -> None: args.duration, pool_name=args.pool_name, target_exe=args.target_exe, - job=job, ) helper.define_containers( @@ -86,8 +85,9 @@ def main() -> None: ] of.logger.info("Creating generic_analysis task") + job = helper.create_job() of.tasks.create( - helper.job.job_id, + job.job_id, TaskType.generic_analysis, helper.setup_relative_blob_name(args.target_coverage_exe, args.setup_dir), containers, @@ -97,7 +97,7 @@ def main() -> None: analyzer_options=["{target_exe}", "{output_dir}", "{input}"], ) - print(f"job:{helper.job.json(indent=4)}") + print(f"job:{job.json(indent=4)}") if __name__ == "__main__": diff --git a/src/cli/examples/llvm-source-coverage/source-coverage.py b/src/cli/examples/llvm-source-coverage/source-coverage.py index ae903cd3b5..ccdbedfd9b 100755 --- a/src/cli/examples/llvm-source-coverage/source-coverage.py +++ b/src/cli/examples/llvm-source-coverage/source-coverage.py @@ -73,8 +73,9 @@ def main() -> None: ] of.logger.info("Creating generic_analysis task") + job = helper.create_job() of.tasks.create( - helper.job.job_id, + job.job_id, TaskType.generic_analysis, helper.setup_relative_blob_name(args.target_coverage_exe, args.setup_dir), containers, @@ -84,7 +85,7 @@ def main() -> None: analyzer_options=["{target_exe}", "{output_dir}", "{input}"], ) - print(f"job:{helper.job.json(indent=4)}") + print(f"job:{job.json(indent=4)}") if __name__ == "__main__": diff --git a/src/cli/onefuzz/api.py b/src/cli/onefuzz/api.py index 4f4e152484..b1973798c8 100644 --- a/src/cli/onefuzz/api.py +++ b/src/cli/onefuzz/api.py @@ -215,7 +215,7 @@ def get(self, container: primitives.Container, filename: str) -> bytes: """get a file from a container""" self.logger.debug("getting file from container: %s:%s", container, filename) client = self._get_client(container) - downloaded = client.download_blob(filename) + downloaded: bytes = client.download_blob(filename) return downloaded def download( diff --git a/src/cli/onefuzz/templates/__init__.py b/src/cli/onefuzz/templates/__init__.py index a88db46303..d8064af9b2 100644 --- a/src/cli/onefuzz/templates/__init__.py +++ b/src/cli/onefuzz/templates/__init__.py @@ -60,7 +60,6 @@ def __init__( pool_name: Optional[str] = None, target_exe: File, platform: Optional[OS] = None, - job: Optional[Job] = None, ): self.onefuzz = onefuzz self.logger = logger @@ -86,15 +85,16 @@ def __init__( self.wait_for_stopped: bool = False self.containers: Dict[ContainerType, ContainerTemplate] = {} self.tags: Dict[str, str] = {"project": project, "name": name, "build": build} - if job is None: - self.onefuzz.versions.check() - self.logger.info("creating job (runtime: %d hours)", duration) - self.job = self.onefuzz.jobs.create( - self.project, self.name, self.build, duration=duration - ) - self.logger.info("created job: %s" % self.job.job_id) - else: - self.job = job + self.duration = duration + + def create_job(self) -> Job: + self.onefuzz.versions.check() + self.logger.info("creating job (runtime: %d hours)", self.duration) + job = self.onefuzz.jobs.create( + self.project, self.name, self.build, duration=self.duration + ) + self.logger.info("created job: %s" % job) + return job def add_existing_container( self, container_type: ContainerType, container: Container @@ -259,14 +259,13 @@ def wait_on( } self.wait_for_running = wait_for_running - def check_current_job(self) -> Job: - job = self.onefuzz.jobs.get(self.job.job_id) + def check_current_job(self, job: Job) -> Job: if job.state in JobState.shutting_down(): raise StoppedEarly("job unexpectedly stopped early") errors = [] for task in self.onefuzz.tasks.list( - job_id=self.job.job_id, state=TaskState.shutting_down() + job_id=job.job_id, state=TaskState.shutting_down() ): if task.error: errors.append("%s: %s" % (task.config.task.type, task.error)) @@ -277,8 +276,8 @@ def check_current_job(self) -> Job: raise StoppedEarly("tasks stopped unexpectedly.\n%s" % "\n".join(errors)) return job - def get_waiting(self) -> List[str]: - tasks = self.onefuzz.tasks.list(job_id=self.job.job_id) + def get_waiting(self, job: Job) -> List[str]: + tasks = self.onefuzz.tasks.list(job_id=job.job_id) waiting = [ "%s:%s" % (x.config.task.type.name, x.state.name) for x in tasks @@ -286,12 +285,12 @@ def get_waiting(self) -> List[str]: ] return waiting - def is_running(self) -> Tuple[bool, str, Any]: - waiting = self.get_waiting() + def is_running(self, job: Job) -> Tuple[bool, str, Any]: + waiting = self.get_waiting(job) return (not waiting, "waiting on: %s" % ", ".join(sorted(waiting)), None) - def has_files(self) -> Tuple[bool, str, Any]: - self.check_current_job() + def has_files(self, job: Job) -> Tuple[bool, str, Any]: + self.check_current_job(job) new = { x: len(self.onefuzz.containers.files.list(x).files) @@ -307,8 +306,8 @@ def has_files(self) -> Tuple[bool, str, Any]: None, ) - def wait(self) -> None: - JobMonitor(self.onefuzz, self.job).wait( + def wait(self, job: Job) -> None: + JobMonitor(self.onefuzz, job).wait( wait_for_running=self.wait_for_running, wait_for_files=self.to_monitor, wait_for_stopped=self.wait_for_stopped, diff --git a/src/cli/onefuzz/templates/afl.py b/src/cli/onefuzz/templates/afl.py index 339b457652..707a80bcbb 100644 --- a/src/cli/onefuzz/templates/afl.py +++ b/src/cli/onefuzz/templates/afl.py @@ -146,8 +146,9 @@ def basic( ) self.logger.info("creating afl fuzz task") + job = helper.create_job() fuzzer_task = self.onefuzz.tasks.create( - helper.job.job_id, + job.job_id, TaskType.generic_supervisor, target_exe_blob_name, containers, @@ -190,7 +191,7 @@ def basic( self.logger.info("creating generic_crash_report task") self.onefuzz.tasks.create( - helper.job.job_id, + job.job_id, TaskType.generic_crash_report, target_exe_blob_name, report_containers, @@ -206,5 +207,5 @@ def basic( ) self.logger.info("done creating tasks") - helper.wait() - return helper.job + helper.wait(job) + return job diff --git a/src/cli/onefuzz/templates/libfuzzer.py b/src/cli/onefuzz/templates/libfuzzer.py index 4b32cac0cd..b4ff2d3849 100644 --- a/src/cli/onefuzz/templates/libfuzzer.py +++ b/src/cli/onefuzz/templates/libfuzzer.py @@ -484,8 +484,9 @@ def basic( ContainerType.extra_output, extra_output_container ) + job = helper.create_job() self._create_tasks( - job=helper.job, + job=job, containers=helper.container_names(), pool_name=pool_name, target_exe=target_exe_blob_name, @@ -518,8 +519,8 @@ def basic( ) self.logger.info("done creating tasks") - helper.wait() - return helper.job + helper.wait(job) + return job def merge( self, @@ -648,8 +649,9 @@ def merge( merge_containers.append((ContainerType.inputs, existing_container)) self.logger.info("creating libfuzzer_merge task") + job = helper.create_job() self.onefuzz.tasks.create( - helper.job.job_id, + job.job_id, TaskType.libfuzzer_merge, target_exe_blob_name, merge_containers, @@ -670,8 +672,8 @@ def merge( ) self.logger.info("done creating tasks") - helper.wait() - return helper.job + helper.wait(job) + return job def dotnet( self, @@ -810,8 +812,10 @@ def dotnet( helper.wait_on(wait_for_files, wait_for_running) + job = helper.create_job() + fuzzer_task = self.onefuzz.tasks.create( - helper.job.job_id, + job.job_id, TaskType.libfuzzer_dotnet_fuzz, target_dll_blob_name, # Not used fuzzer_containers, @@ -870,7 +874,7 @@ def dotnet( self.logger.info("creating `dotnet_coverage` task") self.onefuzz.tasks.create( - helper.job.job_id, + job.job_id, TaskType.dotnet_coverage, libfuzzer_dotnet_loader_dll, coverage_containers, @@ -911,7 +915,7 @@ def dotnet( self.logger.info("creating `dotnet_crash_report` task") self.onefuzz.tasks.create( - helper.job.job_id, + job.job_id, TaskType.dotnet_crash_report, libfuzzer_dotnet_loader_dll, report_containers, @@ -932,8 +936,8 @@ def dotnet( ) self.logger.info("done creating tasks") - helper.wait() - return helper.job + helper.wait(job) + return job def qemu_user( self, @@ -1121,8 +1125,10 @@ def qemu_user( libfuzzer_fuzz_target_options += fuzzing_target_options self.logger.info("creating libfuzzer_fuzz task") + + job = helper.create_job() fuzzer_task = self.onefuzz.tasks.create( - helper.job.job_id, + job.job_id, TaskType.libfuzzer_fuzz, wrapper_name, fuzzer_containers, @@ -1164,7 +1170,7 @@ def qemu_user( self.logger.info("creating libfuzzer_crash_report task") self.onefuzz.tasks.create( - helper.job.job_id, + job.job_id, TaskType.libfuzzer_crash_report, wrapper_name, report_containers, @@ -1187,5 +1193,5 @@ def qemu_user( ) self.logger.info("done creating tasks") - helper.wait() - return helper.job + helper.wait(job) + return job diff --git a/src/cli/onefuzz/templates/ossfuzz.py b/src/cli/onefuzz/templates/ossfuzz.py index 2b76969561..c446a7c4a5 100644 --- a/src/cli/onefuzz/templates/ossfuzz.py +++ b/src/cli/onefuzz/templates/ossfuzz.py @@ -185,7 +185,8 @@ def libfuzzer( ) base_helper.platform = platform - helpers = [] + job = base_helper.create_job() + for fuzzer in [File(x) for x in fuzzers]: fuzzer_name = fuzzer.replace(".exe", "").replace("_fuzzer", "") self.logger.info("creating tasks for %s", fuzzer) @@ -197,14 +198,12 @@ def libfuzzer( fuzzer_name, build, duration, - job=base_helper.job, pool_name=pool_name, target_exe=fuzzer, ) helper.platform = platform helper.add_tags(tags) helper.platform = base_helper.platform - helper.job = base_helper.job helper.define_containers( ContainerType.setup, ContainerType.inputs, @@ -247,7 +246,7 @@ def libfuzzer( fuzzer_blob_name = helper.setup_relative_blob_name(fuzzer, None) self.onefuzz.template.libfuzzer._create_tasks( - job=base_helper.job, + job=job, containers=helper.container_names(), pool_name=pool_name, target_exe=fuzzer_blob_name, @@ -260,5 +259,4 @@ def libfuzzer( ensemble_sync_delay=ensemble_sync_delay, min_available_memory_mb=min_available_memory_mb, ) - helpers.append(helper) - base_helper.wait() + base_helper.wait(job) diff --git a/src/cli/onefuzz/templates/radamsa.py b/src/cli/onefuzz/templates/radamsa.py index 369851fb10..efe74cbc88 100644 --- a/src/cli/onefuzz/templates/radamsa.py +++ b/src/cli/onefuzz/templates/radamsa.py @@ -168,8 +168,10 @@ def basic( ) ) + job = helper.create_job() + fuzzer_task = self.onefuzz.tasks.create( - helper.job.job_id, + job.job_id, TaskType.generic_generator, target_exe_blob_name, containers, @@ -212,7 +214,7 @@ def basic( self.logger.info("creating generic_crash_report task") self.onefuzz.tasks.create( - helper.job.job_id, + job.job_id, TaskType.generic_crash_report, target_exe_blob_name, report_containers, @@ -263,7 +265,7 @@ def basic( ) self.onefuzz.tasks.create( - helper.job.job_id, + job.job_id, TaskType.generic_analysis, target_exe_blob_name, analysis_containers, @@ -284,5 +286,5 @@ def basic( ) self.logger.info("done creating tasks") - helper.wait() - return helper.job + helper.wait(job) + return job diff --git a/src/cli/onefuzz/templates/regression.py b/src/cli/onefuzz/templates/regression.py index c8f47ea801..cf871389e1 100644 --- a/src/cli/onefuzz/templates/regression.py +++ b/src/cli/onefuzz/templates/regression.py @@ -257,9 +257,11 @@ def _create_job( helper.upload_setup(setup_dir, target_exe) target_exe_blob_name = helper.setup_relative_blob_name(target_exe, setup_dir) + job = helper.create_job() + self.logger.info("creating regression task") task = self.onefuzz.tasks.create( - helper.job.job_id, + job.job_id, task_type, target_exe_blob_name, containers, @@ -280,7 +282,7 @@ def _create_job( helper.wait_for_stopped = check_regressions self.logger.info("done creating tasks") - helper.wait() + helper.wait(job) if check_regressions: task = self.onefuzz.tasks.get(task.task_id) diff --git a/src/cli/tests/test_template_helper.py b/src/cli/tests/test_template_helper.py index 69c1002257..b7575005f9 100644 --- a/src/cli/tests/test_template_helper.py +++ b/src/cli/tests/test_template_helper.py @@ -6,10 +6,7 @@ import logging import sys import unittest -from uuid import UUID -from onefuzztypes.enums import JobState -from onefuzztypes.models import Job, JobConfig from onefuzztypes.primitives import Directory, File from onefuzz.api import Onefuzz @@ -26,11 +23,6 @@ def test_path_resolution(self) -> None: "c", False, target_exe=File("README.md"), - job=Job( - job_id=UUID("0" * 32), - state=JobState.init, - config=JobConfig(project="a", name="a", build="a", duration=1), - ), ) values = { (File("filename.txt"), None): "filename.txt",