From e1839ec47255c583ce66f5815893014d8ce5c2b7 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Tue, 23 Jan 2024 19:01:23 +0100 Subject: [PATCH 01/42] OP-7687 - not applicable for Ayon Maybe AYON_BUNDLE_NAME should be used instead of Version if it would be possible to parse it out. --- .../plugins/publish/create_publish_royalrender_job.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py b/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py index d4af1c2aeef..95ebe82c149 100644 --- a/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py +++ b/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py @@ -270,6 +270,8 @@ def _sanitize_version(self, version): 3.15.7-nightly.2 >> 3.157 """ + if not version: + return VERSION_REGEX = re.compile( r"(?P0|[1-9]\d*)" r"\.(?P0|[1-9]\d*)" From aed2d0651ea36cd75646c3e56e23c8f753f5225c Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Tue, 23 Jan 2024 19:05:04 +0100 Subject: [PATCH 02/42] OP-7687 - added Royal Render env var injection script for Ayon --- .../perjob/ayon_inject_envvar.py | 212 ++++++++++++++++++ 1 file changed, 212 insertions(+) create mode 100644 openpype/modules/royalrender/rr_root/plugins/submitter_onsubmission/perjob/ayon_inject_envvar.py diff --git a/openpype/modules/royalrender/rr_root/plugins/submitter_onsubmission/perjob/ayon_inject_envvar.py b/openpype/modules/royalrender/rr_root/plugins/submitter_onsubmission/perjob/ayon_inject_envvar.py new file mode 100644 index 00000000000..3c7831f665b --- /dev/null +++ b/openpype/modules/royalrender/rr_root/plugins/submitter_onsubmission/perjob/ayon_inject_envvar.py @@ -0,0 +1,212 @@ +import os +import json +import glob +from datetime import datetime +import tempfile +import subprocess +import uuid + +logs = [] + + +class InjectEnvironment: + """Creates rrEnv file and adds it to the rendered job. + + RR evnList has limitation on 2000 characters, which might not be enough. + This script should be triggered by render jobs that were published from + Ayon, it uses .json metadata to parse context and required Ayon launch + environments to generate environment variable file for particular context. + + This file is converted into rrEnv file and added to the job to be picked up + by RR. + + Expected set environments on RR worker: + - AYON_API_KEY - API key to Ayon server, most likely from service account + - AYON_EXECUTABLE_PATH - locally accessible path for `ayon_console` + (could be removed if it would be possible to have it in renderApps config + and to be accessible from there as there it is required for publish jobs). + + Ayon submission job must be adding this line to .xml submission file: + OSperjob_ayon_inject_envvar=1~1 + + Scripts logs into folder with metadata json - could be removed if there + is a way how to log into RR output. + + """ + + def __init__(self): + self.meta_dir = None + + def inject(self): + logs.append("InjectEnvironment starting") + meta_dir = self._get_metadata_dir() + + metadata = self._get_metadata(meta_dir) + if not metadata: + logs.append("No metadata json found {}".format(meta_dir)) + return + self.meta_dir = meta_dir + + if not self._is_required_environment(): + return + + context = self._get_context(metadata) + logs.append("context {}".format(context)) + + executable = self._get_executable() + logs.append("executable {}".format(executable)) + + extracted_env = self._extract_environments(executable, context, + metadata) + + rrEnv_path = self._create_rrEnv(meta_dir, extracted_env) + + self._set_rrEnv_to_job(rrEnv_path) + logs.append("InjectEnvironment ending") + + def _get_metadata_dir(self): + job = rr.getJob() + image_dir = job.imageDir + logs.append(f"_get_metadata_dir::{image_dir}") + + # return "C:/projects_local/ayon_test/assets/characters/characterA/work/lookdev/renders/maya/at_characterA_lookdev_v019/Main" + return image_dir + + def _get_metadata(self, meta_dir): + search_frm = "{}{}*_metadata.json".format(meta_dir, os.path.sep) + metadata_files = glob.glob(search_frm) + if not metadata_files: + return {} + + logs.append(f"_get_metadata::{metadata_files[0]}") + with open(metadata_files[0]) as json_file: + return json.load(json_file) + + def _is_required_environment(self): + if (not os.environ.get("AYON_API_KEY") or + not os.path.exists(os.environ.get("AYON_EXECUTABLE_PATH", ""))): + msg = ("AYON_API_KEY and AYON_EXECUTABLE_PATHenv var must be set " + "for Ayon jobs!") + logs.append(msg) + return False + return True + + def _get_context(self, metadata_content): + envs = self._get_job_environments(metadata_content) + + return {"project": envs["AVALON_PROJECT"], + "asset": envs["AVALON_ASSET"], + "task": envs["AVALON_TASK"], + "app": envs["AVALON_APP_NAME"], + "envgroup": "farm"} + + def _get_job_environments(self, metadata_content): + """Gets environments set on job. + + It seems that it is not possible to query "rrEnvList" on job directly, + it must be parsed from .json document. + """ + job = metadata_content["job"] + env_list = job["rrEnvList"] + envs = {} + for env in env_list.split("~~~"): + key, value = env.split("=") + envs[key] = value + return envs + + def _get_executable(self): + # rr_python_utils.cache.get_rr_bin_folder() # TODO maybe useful + return os.environ["AYON_EXECUTABLE_PATH"] + + def _extract_environments(self, executable, context, metadata_content): + # tempfile.TemporaryFile cannot be used because of locking + export_url = self._get_export_url() + + args = [ + executable, + "--headless", + "extractenvironments", + export_url + ] + + if all(context.values()): + for key, value in context.items(): + args.extend(["--{}".format(key), value]) + + environments = self._get_launch_environments(metadata_content) + + logs.append("Running:: {}".format(args)) + proc = subprocess.Popen(args, env=environments, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + output, error = proc.communicate() + + if not os.path.exists(export_url): + raise RuntimeError("Extract failed with {}".format(error)) + + with open(export_url) as json_file: + return json.load(json_file) + + def _get_launch_environments(self, metadata_content): + """ Enhances environemnt with required for Ayon to be launched.""" + job_envs = self._get_job_environments(metadata_content) + ayon_environment = { + "AYON_SERVER_URL": job_envs["AYON_SERVER_URL"], + "AYON_API_KEY": os.environ["AYON_API_KEY"], + "AYON_BUNDLE_NAME": job_envs["AYON_BUNDLE_NAME"], + } + + environment = os.environ.copy() + environment.update(ayon_environment) + return environment + + def _get_export_url(self): + temp_file_name = "{}_{}.json".format( + datetime.utcnow().strftime('%Y%m%d%H%M%S%f'), + str(uuid.uuid1()) + ) + export_url = os.path.join(tempfile.gettempdir(), temp_file_name) + return export_url + + def _create_rrEnv(self, meta_dir, extracted_env): + filter_out = os.environ.get("AYON_FILTER_ENVIRONMENTS") + filter_envs = set() + if filter_out: + filter_envs = set(filter_out.split(";")) + + lines = [] + for key, value in extracted_env.items(): + if key in filter_envs: + continue + + line = f"{key} = {value}" + lines.append(line) + + rrenv_path = os.path.join(meta_dir, "rrEnv.rrEnv") + with open(rrenv_path, "w") as fp: + fp.writelines(s + '\n' for s in lines) + + return rrenv_path + + def _set_rrEnv_to_job(self, rrEnv_path): + logs.append(f"_set_rrEnv_to_job::{rrEnv_path}") + job = rr.getJob() + + job.customDataSet_Str("rrEnvFile", rrEnv_path) + rr.setJob(job) + + + +if __name__ == "__main__": + os.environ["AYON_API_KEY"] = "d84bde8c757540ec907e96edb58455bd" # TEMP + os.environ["AYON_EXECUTABLE_PATH"] = "c:/Users/petrk/Documents/ayon/ayon-launcher/build/output/ayon_console.exe" # TEMP + + try: + injector = InjectEnvironment() + injector.inject() + except Exception as exp: + logs.append(f"Error happened::{str(exp)}") + + log_path = os.path.join(injector.meta_dir, "log.txt") + with open(log_path, "w") as fp: + fp.writelines(s + '\n' for s in logs) From 0ec505ac4bfec7bd4ca8e4ffba4e64fa58df19e4 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 24 Jan 2024 12:42:31 +0100 Subject: [PATCH 03/42] OP-7687 - removed obsolete notworking scripts --- .../_prepost_scripts/OpenPypeEnvironment.cfg | 11 ----------- .../_prepost_scripts/PreOpenPypeInjectEnvironments.py | 4 ---- 2 files changed, 15 deletions(-) delete mode 100644 openpype/modules/royalrender/rr_root/render_apps/_prepost_scripts/OpenPypeEnvironment.cfg delete mode 100644 openpype/modules/royalrender/rr_root/render_apps/_prepost_scripts/PreOpenPypeInjectEnvironments.py diff --git a/openpype/modules/royalrender/rr_root/render_apps/_prepost_scripts/OpenPypeEnvironment.cfg b/openpype/modules/royalrender/rr_root/render_apps/_prepost_scripts/OpenPypeEnvironment.cfg deleted file mode 100644 index 70f0bc2e244..00000000000 --- a/openpype/modules/royalrender/rr_root/render_apps/_prepost_scripts/OpenPypeEnvironment.cfg +++ /dev/null @@ -1,11 +0,0 @@ -PrePostType= pre -CommandLine= - -CommandLine= rrPythonconsole" > "render_apps/_prepost_scripts/PreOpenPypeInjectEnvironments.py" - -CommandLine= - - -CommandLine= "" -CommandLine= - diff --git a/openpype/modules/royalrender/rr_root/render_apps/_prepost_scripts/PreOpenPypeInjectEnvironments.py b/openpype/modules/royalrender/rr_root/render_apps/_prepost_scripts/PreOpenPypeInjectEnvironments.py deleted file mode 100644 index 891de9594c5..00000000000 --- a/openpype/modules/royalrender/rr_root/render_apps/_prepost_scripts/PreOpenPypeInjectEnvironments.py +++ /dev/null @@ -1,4 +0,0 @@ -# -*- coding: utf-8 -*- -import os - -os.environ["OPENYPYPE_TESTVAR"] = "OpenPype was here" From e0cf62f7520f01e6d2836fba5ef2f2282c9327e8 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 24 Jan 2024 12:47:03 +0100 Subject: [PATCH 04/42] OP-7687 - removed obsolete Openpype --- .../perjob/m50__openpype_publish_render.py | 197 ------------------ .../_config/E01__OpenPype__PublishJob.cfg | 71 ------- .../_config/E01__OpenPype___global.inc | 2 - .../render_apps/_install_paths/OpenPype.cfg | 12 -- 4 files changed, 282 deletions(-) delete mode 100644 openpype/modules/royalrender/rr_root/plugins/control_job/perjob/m50__openpype_publish_render.py delete mode 100644 openpype/modules/royalrender/rr_root/render_apps/_config/E01__OpenPype__PublishJob.cfg delete mode 100644 openpype/modules/royalrender/rr_root/render_apps/_config/E01__OpenPype___global.inc delete mode 100644 openpype/modules/royalrender/rr_root/render_apps/_install_paths/OpenPype.cfg diff --git a/openpype/modules/royalrender/rr_root/plugins/control_job/perjob/m50__openpype_publish_render.py b/openpype/modules/royalrender/rr_root/plugins/control_job/perjob/m50__openpype_publish_render.py deleted file mode 100644 index cdc37588cd0..00000000000 --- a/openpype/modules/royalrender/rr_root/plugins/control_job/perjob/m50__openpype_publish_render.py +++ /dev/null @@ -1,197 +0,0 @@ -# -*- coding: utf-8 -*- -"""This is RR control plugin that runs on the job by user interaction. - -It asks user for context to publish, getting it from OpenPype. In order to -run it needs `OPENPYPE_ROOT` to be set to know where to execute OpenPype. - -""" -import rr # noqa -import rrGlobal # noqa -import subprocess -import os -import glob -import platform -import tempfile -import json - - -class OpenPypeContextSelector: - """Class to handle publishing context determination in RR.""" - - def __init__(self): - self.job = rr.getJob() - self.context = {} - - self.openpype_executable = "openpype_gui" - if platform.system().lower() == "windows": - self.openpype_executable = "{}.exe".format( - self.openpype_executable) - - op_path = os.environ.get("OPENPYPE_ROOT") - print("initializing ... {}".format(op_path)) - if not op_path: - print("Warning: OpenPype root is not found.") - - if platform.system().lower() == "windows": - print(" * trying to find OpenPype on local computer.") - op_path = os.path.join( - os.environ.get("PROGRAMFILES"), - "OpenPype", "openpype_console.exe" - ) - if not os.path.exists(op_path): - # try to find in user local context - op_path = os.path.join( - os.environ.get("LOCALAPPDATA"), - "Programs", - "OpenPype", "openpype_console.exe" - ) - if not os.path.exists(op_path): - raise Exception("Error: OpenPype was not found.") - - op_path = os.path.dirname(op_path) - print(" - found OpenPype installation {}".format(op_path)) - - self.openpype_root = op_path - - def _process_metadata_file(self): - """Find and process metadata file. - - Try to find metadata json file in job folder to get context from. - - Returns: - dict: Context from metadata json file. - - """ - image_dir = self.job.imageDir - metadata_files = glob.glob( - "{}{}*_metadata.json".format(image_dir, os.path.sep)) - if not metadata_files: - return {} - - raise NotImplementedError( - "Processing existing metadata not implemented yet.") - - def process_job(self): - """Process selected job. - - This should process selected job. If context can be determined - automatically, no UI will be show and publishing will directly - proceed. - """ - if not self.context and not self.show(): - return - - self.context["user"] = self.job.userName - self.run_publish() - - def show(self): - """Show UI for context selection. - - Because of RR UI limitations, this must be done using OpenPype - itself. - - """ - tf = tempfile.TemporaryFile(delete=False) - context_file = tf.name - op_args = [os.path.join(self.openpype_root, self.openpype_executable), - "contextselection", tf.name] - - tf.close() - print(">>> running {}".format(" ".join(op_args))) - - subprocess.call(op_args) - - with open(context_file, "r") as cf: - self.context = json.load(cf) - - os.unlink(context_file) - print(">>> context: {}".format(self.context)) - - if not self.context or \ - not self.context.get("project") or \ - not self.context.get("asset") or \ - not self.context.get("task"): - self._show_rr_warning("Context selection failed.") - return False - - # self.context["app_name"] = self.job.renderer.name - # there should be mapping between OpenPype and Royal Render - # app names and versions, but since app_name is not used - # currently down the line (but it is required by OP publish command - # right now). - # self.context["app_name"] = "maya/2022" - return True - - @staticmethod - def _show_rr_warning(text): - warning_dialog = rrGlobal.getGenericUI() - warning_dialog.addItem(rrGlobal.genUIType.label, "infoLabel", "") - warning_dialog.setText("infoLabel", text) - warning_dialog.addItem( - rrGlobal.genUIType.layoutH, "btnLayout", "") - warning_dialog.addItem( - rrGlobal.genUIType.closeButton, "Ok", "btnLayout") - warning_dialog.execute() - del warning_dialog - - def run_publish(self): - """Run publish process.""" - env = {"AVALON_PROJECT": str(self.context.get("project")), - "AVALON_ASSET": str(self.context.get("asset")), - "AVALON_TASK": str(self.context.get("task")), - # "AVALON_APP_NAME": str(self.context.get("app_name")) - } - - print(">>> setting environment:") - for k, v in env.items(): - print(" {}: {}".format(k, v)) - - publishing_paths = [os.path.join(self.job.imageDir, - os.path.dirname( - self.job.imageFileName))] - - # add additional channels - channel_idx = 0 - channel = self.job.channelFileName(channel_idx) - while channel: - channel_path = os.path.dirname( - os.path.join(self.job.imageDir, channel)) - if channel_path not in publishing_paths: - publishing_paths.append(channel_path) - channel_idx += 1 - channel = self.job.channelFileName(channel_idx) - - args = [os.path.join(self.openpype_root, self.openpype_executable), - 'publish', '-t', "rr_control", "--gui" - ] - - args += publishing_paths - - print(">>> running {}".format(" ".join(args))) - orig = os.environ.copy() - orig.update(env) - try: - subprocess.call(args, env=orig) - except subprocess.CalledProcessError as e: - self._show_rr_warning(" Publish failed [ {} ]".format( - e.returncode - )) - - -print("running selector") -selector = OpenPypeContextSelector() - -# try to set context from environment -selector.context["project"] = os.getenv("AVALON_PROJECT") -selector.context["asset"] = os.getenv("AVALON_ASSET") -selector.context["task"] = os.getenv("AVALON_TASK") -# selector.context["app_name"] = os.getenv("AVALON_APP_NAME") - -# if anything inside is None, scratch the whole thing and -# ask user for context. -for _, v in selector.context.items(): - if not v: - selector.context = {} - break - -selector.process_job() diff --git a/openpype/modules/royalrender/rr_root/render_apps/_config/E01__OpenPype__PublishJob.cfg b/openpype/modules/royalrender/rr_root/render_apps/_config/E01__OpenPype__PublishJob.cfg deleted file mode 100644 index 864eeaf15aa..00000000000 --- a/openpype/modules/royalrender/rr_root/render_apps/_config/E01__OpenPype__PublishJob.cfg +++ /dev/null @@ -1,71 +0,0 @@ -IconApp= E01__OpenPype.png -Name= OpenPype -rendererName= Once -Version= 1 -Version_Minor= 0 -Type=Execute -TYPEv9=Execute -ExecuteJobType=Once - - -################################# [Windows] [Linux] [Osx] ################################## - - -CommandLine=> - -CommandLine= - - -::win CommandLine= set "CUDA_VISIBLE_DEVICES=" -::lx CommandLine= setenv CUDA_VISIBLE_DEVICES -::osx CommandLine= setenv CUDA_VISIBLE_DEVICES - - -CommandLine= - - -CommandLine= - - -CommandLine= - - -CommandLine= "" --headless publish - --targets royalrender - --targets farm - - - -CommandLine= - - - - -################################## Render Settings ################################## - - - -################################## Submitter Settings ################################## -StartMultipleInstances= 0~0 -SceneFileExtension= *.json -AllowImageNameChange= 0 -AllowImageDirChange= 0 -SequenceDivide= 0~1 -PPSequenceCheck=0~0 -PPCreateSmallVideo=0~0 -PPCreateFullVideo=0~0 -AllowLocalRenderOut= 0~0 - - -################################## Client Settings ################################## - -IconApp=E01__OpenPype.png - -licenseFailLine= - -errorSearchLine= - -permanentErrorSearchLine = - -Frozen_MinCoreUsage=0.3 -Frozen_Minutes=30 diff --git a/openpype/modules/royalrender/rr_root/render_apps/_config/E01__OpenPype___global.inc b/openpype/modules/royalrender/rr_root/render_apps/_config/E01__OpenPype___global.inc deleted file mode 100644 index ba38337340d..00000000000 --- a/openpype/modules/royalrender/rr_root/render_apps/_config/E01__OpenPype___global.inc +++ /dev/null @@ -1,2 +0,0 @@ -IconApp= E01__OpenPype.png -Name= OpenPype diff --git a/openpype/modules/royalrender/rr_root/render_apps/_install_paths/OpenPype.cfg b/openpype/modules/royalrender/rr_root/render_apps/_install_paths/OpenPype.cfg deleted file mode 100644 index 07f7547d297..00000000000 --- a/openpype/modules/royalrender/rr_root/render_apps/_install_paths/OpenPype.cfg +++ /dev/null @@ -1,12 +0,0 @@ -[Windows] -Executable= openpype_console.exe -Path= OS; \OpenPype\*\openpype_console.exe -Path= 32; \OpenPype\*\openpype_console.exe - -[Linux] -Executable= openpype_console -Path= OS; /opt/openpype/*/openpype_console - -[Mac] -Executable= openpype_console -Path= OS; /Applications/OpenPype*/Content/MacOS/openpype_console From c71910f3d5a89b0cee6e99c51744f86f4ef81e1b Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 24 Jan 2024 12:47:48 +0100 Subject: [PATCH 05/42] OP-7687 - added Ayon specific config --- .../_config/{E01__OpenPype.png => E01__Ayon.png} | Bin .../render_apps/_config/E01__Ayon___global.inc | 2 ++ .../rr_root/render_apps/_install_paths/Ayon.cfg | 12 ++++++++++++ 3 files changed, 14 insertions(+) rename openpype/modules/royalrender/rr_root/render_apps/_config/{E01__OpenPype.png => E01__Ayon.png} (100%) create mode 100644 openpype/modules/royalrender/rr_root/render_apps/_config/E01__Ayon___global.inc create mode 100644 openpype/modules/royalrender/rr_root/render_apps/_install_paths/Ayon.cfg diff --git a/openpype/modules/royalrender/rr_root/render_apps/_config/E01__OpenPype.png b/openpype/modules/royalrender/rr_root/render_apps/_config/E01__Ayon.png similarity index 100% rename from openpype/modules/royalrender/rr_root/render_apps/_config/E01__OpenPype.png rename to openpype/modules/royalrender/rr_root/render_apps/_config/E01__Ayon.png diff --git a/openpype/modules/royalrender/rr_root/render_apps/_config/E01__Ayon___global.inc b/openpype/modules/royalrender/rr_root/render_apps/_config/E01__Ayon___global.inc new file mode 100644 index 00000000000..ba38337340d --- /dev/null +++ b/openpype/modules/royalrender/rr_root/render_apps/_config/E01__Ayon___global.inc @@ -0,0 +1,2 @@ +IconApp= E01__OpenPype.png +Name= OpenPype diff --git a/openpype/modules/royalrender/rr_root/render_apps/_install_paths/Ayon.cfg b/openpype/modules/royalrender/rr_root/render_apps/_install_paths/Ayon.cfg new file mode 100644 index 00000000000..534eb868990 --- /dev/null +++ b/openpype/modules/royalrender/rr_root/render_apps/_install_paths/Ayon.cfg @@ -0,0 +1,12 @@ +[Windows] +Executable= ayon_console.exe +Path= OS; \Ayon\*\ayon_console.exe +Path= 32; \Ayon\*\ayon_console.exe + +[Linux] +Executable= ayon_console +Path= OS; /opt/ayon/*/ayon_console + +[Mac] +Executable= ayon_console +Path= OS; /Applications/Ayon*/Content/MacOS/ayon_console From d95a580261624f80ff23720158fe925944ea1f43 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 24 Jan 2024 14:02:41 +0100 Subject: [PATCH 06/42] OP-7687 - added Ayon specific config --- .../perjob/ayon_inject_envvar.py | 4 +- .../_config/E01__Ayon___global.inc | 2 - .../_config/{E01__Ayon.png => E05__Ayon.png} | Bin .../_config/E05__Ayon__PublishJob.cfg | 71 ++++++++++++++++++ .../_config/E05__Ayon___global.inc | 2 + 5 files changed, 76 insertions(+), 3 deletions(-) delete mode 100644 openpype/modules/royalrender/rr_root/render_apps/_config/E01__Ayon___global.inc rename openpype/modules/royalrender/rr_root/render_apps/_config/{E01__Ayon.png => E05__Ayon.png} (100%) create mode 100644 openpype/modules/royalrender/rr_root/render_apps/_config/E05__Ayon__PublishJob.cfg create mode 100644 openpype/modules/royalrender/rr_root/render_apps/_config/E05__Ayon___global.inc diff --git a/openpype/modules/royalrender/rr_root/plugins/submitter_onsubmission/perjob/ayon_inject_envvar.py b/openpype/modules/royalrender/rr_root/plugins/submitter_onsubmission/perjob/ayon_inject_envvar.py index 3c7831f665b..3b55084dc46 100644 --- a/openpype/modules/royalrender/rr_root/plugins/submitter_onsubmission/perjob/ayon_inject_envvar.py +++ b/openpype/modules/royalrender/rr_root/plugins/submitter_onsubmission/perjob/ayon_inject_envvar.py @@ -21,6 +21,7 @@ class InjectEnvironment: by RR. Expected set environments on RR worker: + - AYON_SERVER_URL - AYON_API_KEY - API key to Ayon server, most likely from service account - AYON_EXECUTABLE_PATH - locally accessible path for `ayon_console` (could be removed if it would be possible to have it in renderApps config @@ -151,7 +152,7 @@ def _get_launch_environments(self, metadata_content): """ Enhances environemnt with required for Ayon to be launched.""" job_envs = self._get_job_environments(metadata_content) ayon_environment = { - "AYON_SERVER_URL": job_envs["AYON_SERVER_URL"], + "AYON_SERVER_URL": os.environ["AYON_SERVER_URL"], "AYON_API_KEY": os.environ["AYON_API_KEY"], "AYON_BUNDLE_NAME": job_envs["AYON_BUNDLE_NAME"], } @@ -200,6 +201,7 @@ def _set_rrEnv_to_job(self, rrEnv_path): if __name__ == "__main__": os.environ["AYON_API_KEY"] = "d84bde8c757540ec907e96edb58455bd" # TEMP os.environ["AYON_EXECUTABLE_PATH"] = "c:/Users/petrk/Documents/ayon/ayon-launcher/build/output/ayon_console.exe" # TEMP + os.environ["AYON_SERVER_URL"] = "http://localhost:5000" # TEMP try: injector = InjectEnvironment() diff --git a/openpype/modules/royalrender/rr_root/render_apps/_config/E01__Ayon___global.inc b/openpype/modules/royalrender/rr_root/render_apps/_config/E01__Ayon___global.inc deleted file mode 100644 index ba38337340d..00000000000 --- a/openpype/modules/royalrender/rr_root/render_apps/_config/E01__Ayon___global.inc +++ /dev/null @@ -1,2 +0,0 @@ -IconApp= E01__OpenPype.png -Name= OpenPype diff --git a/openpype/modules/royalrender/rr_root/render_apps/_config/E01__Ayon.png b/openpype/modules/royalrender/rr_root/render_apps/_config/E05__Ayon.png similarity index 100% rename from openpype/modules/royalrender/rr_root/render_apps/_config/E01__Ayon.png rename to openpype/modules/royalrender/rr_root/render_apps/_config/E05__Ayon.png diff --git a/openpype/modules/royalrender/rr_root/render_apps/_config/E05__Ayon__PublishJob.cfg b/openpype/modules/royalrender/rr_root/render_apps/_config/E05__Ayon__PublishJob.cfg new file mode 100644 index 00000000000..9604d3c468c --- /dev/null +++ b/openpype/modules/royalrender/rr_root/render_apps/_config/E05__Ayon__PublishJob.cfg @@ -0,0 +1,71 @@ +IconApp= E05__Ayon.png +Name= Ayon +rendererName= Once +Version= 1 +Version_Minor= 0 +Type=Execute +TYPEv9=Execute +ExecuteJobType=Once + + +################################# [Windows] [Linux] [Osx] ################################## + + +CommandLine=> + +CommandLine= + + +::win CommandLine= set "CUDA_VISIBLE_DEVICES=" +::lx CommandLine= setenv CUDA_VISIBLE_DEVICES +::osx CommandLine= setenv CUDA_VISIBLE_DEVICES + + +CommandLine= + + +CommandLine= + + +CommandLine= + + +CommandLine= "" --headless publish + --targets royalrender + --targets farm + + + +CommandLine= + + + + +################################## Render Settings ################################## + + + +################################## Submitter Settings ################################## +StartMultipleInstances= 0~0 +SceneFileExtension= *.json +AllowImageNameChange= 0 +AllowImageDirChange= 0 +SequenceDivide= 0~1 +PPSequenceCheck=0~0 +PPCreateSmallVideo=0~0 +PPCreateFullVideo=0~0 +AllowLocalRenderOut= 0~0 + + +################################## Client Settings ################################## + +IconApp=E05__Ayon.png + +licenseFailLine= + +errorSearchLine= + +permanentErrorSearchLine = + +Frozen_MinCoreUsage=0.3 +Frozen_Minutes=30 diff --git a/openpype/modules/royalrender/rr_root/render_apps/_config/E05__Ayon___global.inc b/openpype/modules/royalrender/rr_root/render_apps/_config/E05__Ayon___global.inc new file mode 100644 index 00000000000..1581eb26b11 --- /dev/null +++ b/openpype/modules/royalrender/rr_root/render_apps/_config/E05__Ayon___global.inc @@ -0,0 +1,2 @@ +IconApp= E05__Ayon.png +Name= Ayon From efd5e894e2a0f65e474a730fd7606d398ed4c262 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 24 Jan 2024 14:48:23 +0100 Subject: [PATCH 07/42] OP-7687 - update error message --- openpype/modules/royalrender/lib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/modules/royalrender/lib.py b/openpype/modules/royalrender/lib.py index 9c4221d9cd2..b37bf286028 100644 --- a/openpype/modules/royalrender/lib.py +++ b/openpype/modules/royalrender/lib.py @@ -110,7 +110,7 @@ def process(self, instance): if not self._rr_root: raise KnownPublishError( ("Missing RoyalRender root. " - "You need to configure RoyalRender module.")) + "Admin needs to configure RoyalRender module in Settings .")) self.rr_api = rrApi(self._rr_root) From bad37663bba5c8b9ef46b9e5eac2299b275ff148 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 24 Jan 2024 14:49:00 +0100 Subject: [PATCH 08/42] OP-7687 - add submitter parameter for env injection This should be added only to render jobs. --- openpype/modules/royalrender/lib.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/openpype/modules/royalrender/lib.py b/openpype/modules/royalrender/lib.py index b37bf286028..2ddeeb40a68 100644 --- a/openpype/modules/royalrender/lib.py +++ b/openpype/modules/royalrender/lib.py @@ -175,6 +175,10 @@ def get_job(self, instance, script_path, render_path, node_name): instance, render_path, start_frame, end_frame) instance.data["expectedFiles"].extend(expected_files) + submitter_parameters = [ + rrApi.SubmitterParameter("OSperjob_ayon_inject_envvar", "1~1") + ] + job = RRJob( Software="", Renderer="", @@ -197,7 +201,8 @@ def get_job(self, instance, script_path, render_path, node_name): CompanyProjectName=instance.context.data["projectName"], ImageWidth=instance.data["resolutionWidth"], ImageHeight=instance.data["resolutionHeight"], - CustomAttributes=custom_attributes + CustomAttributes=custom_attributes, + SubmitterParameters=submitter_parameters ) return job From 60d944dcbfafe3d61cea7f2cf14f810a38e6ccad Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 24 Jan 2024 14:57:31 +0100 Subject: [PATCH 09/42] OP-7687 - add submitter parameter for env injection This should be added only to render jobs. --- openpype/modules/royalrender/lib.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/openpype/modules/royalrender/lib.py b/openpype/modules/royalrender/lib.py index 2ddeeb40a68..9d8cd1de86b 100644 --- a/openpype/modules/royalrender/lib.py +++ b/openpype/modules/royalrender/lib.py @@ -18,6 +18,7 @@ RRJob, RREnvList, get_rr_platform, + SubmitterParameter ) from openpype.pipeline import OpenPypePyblishPluginMixin from openpype.pipeline.publish import KnownPublishError @@ -176,7 +177,7 @@ def get_job(self, instance, script_path, render_path, node_name): instance.data["expectedFiles"].extend(expected_files) submitter_parameters = [ - rrApi.SubmitterParameter("OSperjob_ayon_inject_envvar", "1~1") + SubmitterParameter("OSperjob_ayon_inject_envvar", "1~1") ] job = RRJob( From aa38c74c992ada623dc9a7fd1679ad78e169959f Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 24 Jan 2024 15:12:46 +0100 Subject: [PATCH 10/42] OP-7687 - removed obsolete implementation Was adding env vars from client not rendering worker. Limitation on 2k characters. --- openpype/modules/royalrender/lib.py | 65 ------------------- .../publish/create_maya_royalrender_job.py | 1 - .../publish/create_nuke_royalrender_job.py | 1 - 3 files changed, 67 deletions(-) diff --git a/openpype/modules/royalrender/lib.py b/openpype/modules/royalrender/lib.py index 9d8cd1de86b..4893a765ae0 100644 --- a/openpype/modules/royalrender/lib.py +++ b/openpype/modules/royalrender/lib.py @@ -313,68 +313,3 @@ def pad_file_name(self, path, first_frame): path = path.replace(first_frame, "#" * padding) return path - - def inject_environment(self, instance, job): - # type: (pyblish.api.Instance, RRJob) -> RRJob - """Inject environment variables for RR submission. - - This function mimics the behaviour of the Deadline - integration. It is just temporary solution until proper - runtime environment injection is implemented in RR. - - Args: - instance (pyblish.api.Instance): Publishing instance - job (RRJob): RRJob instance to be injected. - - Returns: - RRJob: Injected RRJob instance. - - Throws: - RuntimeError: If any of the required env vars is missing. - - """ - - temp_file_name = "{}_{}.json".format( - datetime.utcnow().strftime('%Y%m%d%H%M%S%f'), - str(uuid.uuid1()) - ) - - export_url = os.path.join(tempfile.gettempdir(), temp_file_name) - print(">>> Temporary path: {}".format(export_url)) - - args = [ - "--headless", - "extractenvironments", - export_url - ] - - anatomy_data = instance.context.data["anatomyData"] - - add_kwargs = { - "project": anatomy_data["project"]["name"], - "asset": instance.context.data["asset"], - "task": anatomy_data["task"]["name"], - "app": instance.context.data.get("appName"), - "envgroup": "farm" - } - - if os.getenv('IS_TEST'): - args.append("--automatic-tests") - - if not all(add_kwargs.values()): - raise RuntimeError(( - "Missing required env vars: AVALON_PROJECT, AVALON_ASSET," - " AVALON_TASK, AVALON_APP_NAME" - )) - - for key, value in add_kwargs.items(): - args.extend([f"--{key}", value]) - self.log.debug("Executing: {}".format(" ".join(args))) - run_openpype_process(*args, logger=self.log) - - self.log.debug("Loading file ...") - with open(export_url) as fp: - contents = json.load(fp) - - job.rrEnvList = RREnvList(contents).serialize() - return job diff --git a/openpype/modules/royalrender/plugins/publish/create_maya_royalrender_job.py b/openpype/modules/royalrender/plugins/publish/create_maya_royalrender_job.py index 775a2964fd4..36f3dcc99a6 100644 --- a/openpype/modules/royalrender/plugins/publish/create_maya_royalrender_job.py +++ b/openpype/modules/royalrender/plugins/publish/create_maya_royalrender_job.py @@ -38,6 +38,5 @@ def process(self, instance): job = self.get_job(instance, self.scene_path, first_file_path, layer_name) job = self.update_job_with_host_specific(instance, job) - job = self.inject_environment(instance, job) instance.data["rrJobs"].append(job) diff --git a/openpype/modules/royalrender/plugins/publish/create_nuke_royalrender_job.py b/openpype/modules/royalrender/plugins/publish/create_nuke_royalrender_job.py index 4f589e56f89..71daa6edf80 100644 --- a/openpype/modules/royalrender/plugins/publish/create_nuke_royalrender_job.py +++ b/openpype/modules/royalrender/plugins/publish/create_nuke_royalrender_job.py @@ -25,7 +25,6 @@ def process(self, instance): jobs = self.create_jobs(instance) for job in jobs: job = self.update_job_with_host_specific(instance, job) - job = self.inject_environment(instance, job) instance.data["rrJobs"].append(job) From 6ae8afea3d1a67f30198b4ff4dd75e7700499018 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 24 Jan 2024 15:16:34 +0100 Subject: [PATCH 11/42] OP-7687 - removed debug env vars --- .../submitter_onsubmission/perjob/ayon_inject_envvar.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/openpype/modules/royalrender/rr_root/plugins/submitter_onsubmission/perjob/ayon_inject_envvar.py b/openpype/modules/royalrender/rr_root/plugins/submitter_onsubmission/perjob/ayon_inject_envvar.py index 3b55084dc46..8d705a5cf99 100644 --- a/openpype/modules/royalrender/rr_root/plugins/submitter_onsubmission/perjob/ayon_inject_envvar.py +++ b/openpype/modules/royalrender/rr_root/plugins/submitter_onsubmission/perjob/ayon_inject_envvar.py @@ -199,9 +199,6 @@ def _set_rrEnv_to_job(self, rrEnv_path): if __name__ == "__main__": - os.environ["AYON_API_KEY"] = "d84bde8c757540ec907e96edb58455bd" # TEMP - os.environ["AYON_EXECUTABLE_PATH"] = "c:/Users/petrk/Documents/ayon/ayon-launcher/build/output/ayon_console.exe" # TEMP - os.environ["AYON_SERVER_URL"] = "http://localhost:5000" # TEMP try: injector = InjectEnvironment() From d2e40c27d71089e88afb57c6458ecd4a9460a1d5 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 24 Jan 2024 15:30:29 +0100 Subject: [PATCH 12/42] OP-7687 - changes to Ayon --- .../publish/create_publish_royalrender_job.py | 31 ++----------------- 1 file changed, 3 insertions(+), 28 deletions(-) diff --git a/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py b/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py index 95ebe82c149..2c5637a22df 100644 --- a/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py +++ b/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py @@ -73,7 +73,7 @@ class CreatePublishRoyalRenderJob(pyblish.api.InstancePlugin, "AVALON_APP_NAME", "OPENPYPE_USERNAME", "OPENPYPE_SG_USER", - "OPENPYPE_MONGO" + "AYON_BUNDLE_NAME" ] priority = 50 @@ -231,13 +231,13 @@ def get_job(self, instance, instances): ] job = RRJob( - Software="OpenPype", + Software="Ayon", Renderer="Once", SeqStart=1, SeqEnd=1, SeqStep=1, SeqFileOffset=0, - Version=self._sanitize_version(os.environ.get("OPENPYPE_VERSION")), + Version=os.environ["AYON_BUNDLE_NAME"], SceneName=abs_metadata_path, # command line arguments CustomAddCmdFlags=" ".join(args), @@ -264,28 +264,3 @@ def get_job(self, instance, instances): job.WaitForPreIDs += jobs_pre_ids return job - - def _sanitize_version(self, version): - """Returns version in format MAJOR.MINORPATCH - - 3.15.7-nightly.2 >> 3.157 - """ - if not version: - return - VERSION_REGEX = re.compile( - r"(?P0|[1-9]\d*)" - r"\.(?P0|[1-9]\d*)" - r"\.(?P0|[1-9]\d*)" - r"(?:-(?P[a-zA-Z\d\-.]*))?" - r"(?:\+(?P[a-zA-Z\d\-.]*))?" - ) - - valid_parts = VERSION_REGEX.findall(version) - if len(valid_parts) != 1: - # Return invalid version with filled 'origin' attribute - return version - - # Unpack found version - major, minor, patch, pre, post = valid_parts[0] - - return "{}.{}{}".format(major, minor, patch) From bf599f9b86670594bea7e870b1495da1fdae27c4 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 24 Jan 2024 15:31:34 +0100 Subject: [PATCH 13/42] OP-7687 - changes for Ayon Stored to rrEnvList just to be saved to .json from where it is parsed. Hopefully there would be a way how to parse it directly from RR job and not reaching to .json --- openpype/modules/royalrender/lib.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/openpype/modules/royalrender/lib.py b/openpype/modules/royalrender/lib.py index 4893a765ae0..9906512dae6 100644 --- a/openpype/modules/royalrender/lib.py +++ b/openpype/modules/royalrender/lib.py @@ -180,6 +180,14 @@ def get_job(self, instance, script_path, render_path, node_name): SubmitterParameter("OSperjob_ayon_inject_envvar", "1~1") ] + anatomy_data = instance.context.data["anatomyData"] + environment = RREnvList({ + "AVALON_PROJECT": anatomy_data["project"]["name"], + "AVALON_ASSET": instance.context.data["asset"], + "AVALON_TASK": anatomy_data["task"]["name"], + "AVALON_APP": instance.context.data.get("appName") + }) + job = RRJob( Software="", Renderer="", @@ -203,7 +211,8 @@ def get_job(self, instance, script_path, render_path, node_name): ImageWidth=instance.data["resolutionWidth"], ImageHeight=instance.data["resolutionHeight"], CustomAttributes=custom_attributes, - SubmitterParameters=submitter_parameters + SubmitterParameters=submitter_parameters, + rrEnvList=environment.serialize(), ) return job From cd8ab10738089ca071964a75513714c78006fc47 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 24 Jan 2024 15:49:53 +0100 Subject: [PATCH 14/42] OP-7687 - Hound --- .../submitter_onsubmission/perjob/ayon_inject_envvar.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/openpype/modules/royalrender/rr_root/plugins/submitter_onsubmission/perjob/ayon_inject_envvar.py b/openpype/modules/royalrender/rr_root/plugins/submitter_onsubmission/perjob/ayon_inject_envvar.py index 8d705a5cf99..c1f67a3c420 100644 --- a/openpype/modules/royalrender/rr_root/plugins/submitter_onsubmission/perjob/ayon_inject_envvar.py +++ b/openpype/modules/royalrender/rr_root/plugins/submitter_onsubmission/perjob/ayon_inject_envvar.py @@ -70,7 +70,6 @@ def _get_metadata_dir(self): image_dir = job.imageDir logs.append(f"_get_metadata_dir::{image_dir}") - # return "C:/projects_local/ayon_test/assets/characters/characterA/work/lookdev/renders/maya/at_characterA_lookdev_v019/Main" return image_dir def _get_metadata(self, meta_dir): @@ -85,7 +84,8 @@ def _get_metadata(self, meta_dir): def _is_required_environment(self): if (not os.environ.get("AYON_API_KEY") or - not os.path.exists(os.environ.get("AYON_EXECUTABLE_PATH", ""))): + not os.path.exists(os.environ.get("AYON_EXECUTABLE_PATH", "")) + ): msg = ("AYON_API_KEY and AYON_EXECUTABLE_PATHenv var must be set " "for Ayon jobs!") logs.append(msg) @@ -197,7 +197,6 @@ def _set_rrEnv_to_job(self, rrEnv_path): rr.setJob(job) - if __name__ == "__main__": try: From 1b59df377fdf6e548fba78f71e27bdefca687812 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Mon, 29 Jan 2024 18:23:22 +0100 Subject: [PATCH 15/42] OP-7687 - added check for AYON_RENDER_JOB to injection script --- .../perjob/ayon_inject_envvar.py | 41 ++++++++++--------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/openpype/modules/royalrender/rr_root/plugins/submitter_onsubmission/perjob/ayon_inject_envvar.py b/openpype/modules/royalrender/rr_root/plugins/submitter_onsubmission/perjob/ayon_inject_envvar.py index c1f67a3c420..d0ac4559b7f 100644 --- a/openpype/modules/royalrender/rr_root/plugins/submitter_onsubmission/perjob/ayon_inject_envvar.py +++ b/openpype/modules/royalrender/rr_root/plugins/submitter_onsubmission/perjob/ayon_inject_envvar.py @@ -26,6 +26,8 @@ class InjectEnvironment: - AYON_EXECUTABLE_PATH - locally accessible path for `ayon_console` (could be removed if it would be possible to have it in renderApps config and to be accessible from there as there it is required for publish jobs). + - AYON_FILTER_ENVIRONMENTS - potential black list of unwanted environment + variables (separated by ';') - will be filtered out from created .rrEnv. Ayon submission job must be adding this line to .xml submission file: OSperjob_ayon_inject_envvar=1~1 @@ -48,17 +50,21 @@ def inject(self): return self.meta_dir = meta_dir + envs = self._get_job_environments() + if not envs.get("AYON_RENDER_JOB"): + logs.append("Not a ayon render job, skipping.") + return + if not self._is_required_environment(): return - context = self._get_context(metadata) + context = self._get_context() logs.append("context {}".format(context)) executable = self._get_executable() logs.append("executable {}".format(executable)) - extracted_env = self._extract_environments(executable, context, - metadata) + extracted_env = self._extract_environments(executable, context) rrEnv_path = self._create_rrEnv(meta_dir, extracted_env) @@ -84,16 +90,15 @@ def _get_metadata(self, meta_dir): def _is_required_environment(self): if (not os.environ.get("AYON_API_KEY") or - not os.path.exists(os.environ.get("AYON_EXECUTABLE_PATH", "")) - ): + not os.path.exists(os.environ.get("AYON_EXECUTABLE_PATH", ""))): msg = ("AYON_API_KEY and AYON_EXECUTABLE_PATHenv var must be set " "for Ayon jobs!") logs.append(msg) return False return True - def _get_context(self, metadata_content): - envs = self._get_job_environments(metadata_content) + def _get_context(self): + envs = self._get_job_environments() return {"project": envs["AVALON_PROJECT"], "asset": envs["AVALON_ASSET"], @@ -101,14 +106,10 @@ def _get_context(self, metadata_content): "app": envs["AVALON_APP_NAME"], "envgroup": "farm"} - def _get_job_environments(self, metadata_content): - """Gets environments set on job. - - It seems that it is not possible to query "rrEnvList" on job directly, - it must be parsed from .json document. - """ - job = metadata_content["job"] - env_list = job["rrEnvList"] + def _get_job_environments(self): + """Gets environments set on job.""" + job = rr.getJob() + env_list = job.customData_Str('rrEnvList') envs = {} for env in env_list.split("~~~"): key, value = env.split("=") @@ -119,7 +120,7 @@ def _get_executable(self): # rr_python_utils.cache.get_rr_bin_folder() # TODO maybe useful return os.environ["AYON_EXECUTABLE_PATH"] - def _extract_environments(self, executable, context, metadata_content): + def _extract_environments(self, executable, context): # tempfile.TemporaryFile cannot be used because of locking export_url = self._get_export_url() @@ -134,7 +135,7 @@ def _extract_environments(self, executable, context, metadata_content): for key, value in context.items(): args.extend(["--{}".format(key), value]) - environments = self._get_launch_environments(metadata_content) + environments = self._get_launch_environments() logs.append("Running:: {}".format(args)) proc = subprocess.Popen(args, env=environments, @@ -148,9 +149,9 @@ def _extract_environments(self, executable, context, metadata_content): with open(export_url) as json_file: return json.load(json_file) - def _get_launch_environments(self, metadata_content): + def _get_launch_environments(self): """ Enhances environemnt with required for Ayon to be launched.""" - job_envs = self._get_job_environments(metadata_content) + job_envs = self._get_job_environments() ayon_environment = { "AYON_SERVER_URL": os.environ["AYON_SERVER_URL"], "AYON_API_KEY": os.environ["AYON_API_KEY"], @@ -206,5 +207,5 @@ def _set_rrEnv_to_job(self, rrEnv_path): logs.append(f"Error happened::{str(exp)}") log_path = os.path.join(injector.meta_dir, "log.txt") - with open(log_path, "w") as fp: + with open(log_path, "a") as fp: fp.writelines(s + '\n' for s in logs) From 14497bc9cd761d283f31c5b66dbb65cc125a40c4 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Mon, 29 Jan 2024 18:24:21 +0100 Subject: [PATCH 16/42] OP-7687 - OSperjob_ayon_inject_envvar must be on submission Doesn't work on each/separate job. Added some needed env vars. --- openpype/modules/royalrender/lib.py | 8 ++++---- .../plugins/publish/submit_jobs_to_royalrender.py | 3 ++- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/openpype/modules/royalrender/lib.py b/openpype/modules/royalrender/lib.py index 9906512dae6..05cd4c8648a 100644 --- a/openpype/modules/royalrender/lib.py +++ b/openpype/modules/royalrender/lib.py @@ -176,16 +176,16 @@ def get_job(self, instance, script_path, render_path, node_name): instance, render_path, start_frame, end_frame) instance.data["expectedFiles"].extend(expected_files) - submitter_parameters = [ - SubmitterParameter("OSperjob_ayon_inject_envvar", "1~1") - ] + submitter_parameters = [] anatomy_data = instance.context.data["anatomyData"] environment = RREnvList({ "AVALON_PROJECT": anatomy_data["project"]["name"], "AVALON_ASSET": instance.context.data["asset"], "AVALON_TASK": anatomy_data["task"]["name"], - "AVALON_APP": instance.context.data.get("appName") + "AVALON_APP_NAME": instance.context.data.get("appName"), + "AYON_RENDER_JOB": "1", + "AYON_BUNDLE_NAME": os.environ["AYON_BUNDLE_NAME"] }) job = RRJob( diff --git a/openpype/modules/royalrender/plugins/publish/submit_jobs_to_royalrender.py b/openpype/modules/royalrender/plugins/publish/submit_jobs_to_royalrender.py index 8fc8604b839..7ec54efe607 100644 --- a/openpype/modules/royalrender/plugins/publish/submit_jobs_to_royalrender.py +++ b/openpype/modules/royalrender/plugins/publish/submit_jobs_to_royalrender.py @@ -99,7 +99,8 @@ def create_file(self, name, ext, contents=None): return temp.name def get_submission_parameters(self): - return [SubmitterParameter("RequiredMemory", "0")] + return [SubmitterParameter("RequiredMemory", "0"), + SubmitterParameter("OSperjob_ayon_inject_envvar", "1~1")] @staticmethod def _resolve_rr_path(context, rr_path_name): From aca73ec2a494e492d1bfafd2a1327e5fd187c83a Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Tue, 30 Jan 2024 12:17:58 +0100 Subject: [PATCH 17/42] OP-7687 - added logging of Ayon launch environment --- .../submitter_onsubmission/perjob/ayon_inject_envvar.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/openpype/modules/royalrender/rr_root/plugins/submitter_onsubmission/perjob/ayon_inject_envvar.py b/openpype/modules/royalrender/rr_root/plugins/submitter_onsubmission/perjob/ayon_inject_envvar.py index d0ac4559b7f..a16d191c2f6 100644 --- a/openpype/modules/royalrender/rr_root/plugins/submitter_onsubmission/perjob/ayon_inject_envvar.py +++ b/openpype/modules/royalrender/rr_root/plugins/submitter_onsubmission/perjob/ayon_inject_envvar.py @@ -107,7 +107,11 @@ def _get_context(self): "envgroup": "farm"} def _get_job_environments(self): - """Gets environments set on job.""" + """Gets environments set on job. + + It seems that it is not possible to query "rrEnvList" on job directly, + it must be parsed from .json document. + """ job = rr.getJob() env_list = job.customData_Str('rrEnvList') envs = {} @@ -157,7 +161,7 @@ def _get_launch_environments(self): "AYON_API_KEY": os.environ["AYON_API_KEY"], "AYON_BUNDLE_NAME": job_envs["AYON_BUNDLE_NAME"], } - + logs.append("Ayon launch environments:: {}".format(ayon_environment)) environment = os.environ.copy() environment.update(ayon_environment) return environment From a1aaf6974256e097bc9ac82e22bf0018fa04cb58 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Tue, 23 Jan 2024 19:01:23 +0100 Subject: [PATCH 18/42] OP-7687 - not applicable for Ayon Maybe AYON_BUNDLE_NAME should be used instead of Version if it would be possible to parse it out. --- .../plugins/publish/create_publish_royalrender_job.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py b/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py index d4af1c2aeef..95ebe82c149 100644 --- a/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py +++ b/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py @@ -270,6 +270,8 @@ def _sanitize_version(self, version): 3.15.7-nightly.2 >> 3.157 """ + if not version: + return VERSION_REGEX = re.compile( r"(?P0|[1-9]\d*)" r"\.(?P0|[1-9]\d*)" From 995147e8e6de833e49dfead6b5941e52a80ee524 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Tue, 23 Jan 2024 19:05:04 +0100 Subject: [PATCH 19/42] OP-7687 - added Royal Render env var injection script for Ayon --- .../perjob/ayon_inject_envvar.py | 212 ++++++++++++++++++ 1 file changed, 212 insertions(+) create mode 100644 openpype/modules/royalrender/rr_root/plugins/submitter_onsubmission/perjob/ayon_inject_envvar.py diff --git a/openpype/modules/royalrender/rr_root/plugins/submitter_onsubmission/perjob/ayon_inject_envvar.py b/openpype/modules/royalrender/rr_root/plugins/submitter_onsubmission/perjob/ayon_inject_envvar.py new file mode 100644 index 00000000000..3c7831f665b --- /dev/null +++ b/openpype/modules/royalrender/rr_root/plugins/submitter_onsubmission/perjob/ayon_inject_envvar.py @@ -0,0 +1,212 @@ +import os +import json +import glob +from datetime import datetime +import tempfile +import subprocess +import uuid + +logs = [] + + +class InjectEnvironment: + """Creates rrEnv file and adds it to the rendered job. + + RR evnList has limitation on 2000 characters, which might not be enough. + This script should be triggered by render jobs that were published from + Ayon, it uses .json metadata to parse context and required Ayon launch + environments to generate environment variable file for particular context. + + This file is converted into rrEnv file and added to the job to be picked up + by RR. + + Expected set environments on RR worker: + - AYON_API_KEY - API key to Ayon server, most likely from service account + - AYON_EXECUTABLE_PATH - locally accessible path for `ayon_console` + (could be removed if it would be possible to have it in renderApps config + and to be accessible from there as there it is required for publish jobs). + + Ayon submission job must be adding this line to .xml submission file: + OSperjob_ayon_inject_envvar=1~1 + + Scripts logs into folder with metadata json - could be removed if there + is a way how to log into RR output. + + """ + + def __init__(self): + self.meta_dir = None + + def inject(self): + logs.append("InjectEnvironment starting") + meta_dir = self._get_metadata_dir() + + metadata = self._get_metadata(meta_dir) + if not metadata: + logs.append("No metadata json found {}".format(meta_dir)) + return + self.meta_dir = meta_dir + + if not self._is_required_environment(): + return + + context = self._get_context(metadata) + logs.append("context {}".format(context)) + + executable = self._get_executable() + logs.append("executable {}".format(executable)) + + extracted_env = self._extract_environments(executable, context, + metadata) + + rrEnv_path = self._create_rrEnv(meta_dir, extracted_env) + + self._set_rrEnv_to_job(rrEnv_path) + logs.append("InjectEnvironment ending") + + def _get_metadata_dir(self): + job = rr.getJob() + image_dir = job.imageDir + logs.append(f"_get_metadata_dir::{image_dir}") + + # return "C:/projects_local/ayon_test/assets/characters/characterA/work/lookdev/renders/maya/at_characterA_lookdev_v019/Main" + return image_dir + + def _get_metadata(self, meta_dir): + search_frm = "{}{}*_metadata.json".format(meta_dir, os.path.sep) + metadata_files = glob.glob(search_frm) + if not metadata_files: + return {} + + logs.append(f"_get_metadata::{metadata_files[0]}") + with open(metadata_files[0]) as json_file: + return json.load(json_file) + + def _is_required_environment(self): + if (not os.environ.get("AYON_API_KEY") or + not os.path.exists(os.environ.get("AYON_EXECUTABLE_PATH", ""))): + msg = ("AYON_API_KEY and AYON_EXECUTABLE_PATHenv var must be set " + "for Ayon jobs!") + logs.append(msg) + return False + return True + + def _get_context(self, metadata_content): + envs = self._get_job_environments(metadata_content) + + return {"project": envs["AVALON_PROJECT"], + "asset": envs["AVALON_ASSET"], + "task": envs["AVALON_TASK"], + "app": envs["AVALON_APP_NAME"], + "envgroup": "farm"} + + def _get_job_environments(self, metadata_content): + """Gets environments set on job. + + It seems that it is not possible to query "rrEnvList" on job directly, + it must be parsed from .json document. + """ + job = metadata_content["job"] + env_list = job["rrEnvList"] + envs = {} + for env in env_list.split("~~~"): + key, value = env.split("=") + envs[key] = value + return envs + + def _get_executable(self): + # rr_python_utils.cache.get_rr_bin_folder() # TODO maybe useful + return os.environ["AYON_EXECUTABLE_PATH"] + + def _extract_environments(self, executable, context, metadata_content): + # tempfile.TemporaryFile cannot be used because of locking + export_url = self._get_export_url() + + args = [ + executable, + "--headless", + "extractenvironments", + export_url + ] + + if all(context.values()): + for key, value in context.items(): + args.extend(["--{}".format(key), value]) + + environments = self._get_launch_environments(metadata_content) + + logs.append("Running:: {}".format(args)) + proc = subprocess.Popen(args, env=environments, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + output, error = proc.communicate() + + if not os.path.exists(export_url): + raise RuntimeError("Extract failed with {}".format(error)) + + with open(export_url) as json_file: + return json.load(json_file) + + def _get_launch_environments(self, metadata_content): + """ Enhances environemnt with required for Ayon to be launched.""" + job_envs = self._get_job_environments(metadata_content) + ayon_environment = { + "AYON_SERVER_URL": job_envs["AYON_SERVER_URL"], + "AYON_API_KEY": os.environ["AYON_API_KEY"], + "AYON_BUNDLE_NAME": job_envs["AYON_BUNDLE_NAME"], + } + + environment = os.environ.copy() + environment.update(ayon_environment) + return environment + + def _get_export_url(self): + temp_file_name = "{}_{}.json".format( + datetime.utcnow().strftime('%Y%m%d%H%M%S%f'), + str(uuid.uuid1()) + ) + export_url = os.path.join(tempfile.gettempdir(), temp_file_name) + return export_url + + def _create_rrEnv(self, meta_dir, extracted_env): + filter_out = os.environ.get("AYON_FILTER_ENVIRONMENTS") + filter_envs = set() + if filter_out: + filter_envs = set(filter_out.split(";")) + + lines = [] + for key, value in extracted_env.items(): + if key in filter_envs: + continue + + line = f"{key} = {value}" + lines.append(line) + + rrenv_path = os.path.join(meta_dir, "rrEnv.rrEnv") + with open(rrenv_path, "w") as fp: + fp.writelines(s + '\n' for s in lines) + + return rrenv_path + + def _set_rrEnv_to_job(self, rrEnv_path): + logs.append(f"_set_rrEnv_to_job::{rrEnv_path}") + job = rr.getJob() + + job.customDataSet_Str("rrEnvFile", rrEnv_path) + rr.setJob(job) + + + +if __name__ == "__main__": + os.environ["AYON_API_KEY"] = "d84bde8c757540ec907e96edb58455bd" # TEMP + os.environ["AYON_EXECUTABLE_PATH"] = "c:/Users/petrk/Documents/ayon/ayon-launcher/build/output/ayon_console.exe" # TEMP + + try: + injector = InjectEnvironment() + injector.inject() + except Exception as exp: + logs.append(f"Error happened::{str(exp)}") + + log_path = os.path.join(injector.meta_dir, "log.txt") + with open(log_path, "w") as fp: + fp.writelines(s + '\n' for s in logs) From fd3a8b1c293169284d77248af4bb9ac36553feda Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 24 Jan 2024 12:42:31 +0100 Subject: [PATCH 20/42] OP-7687 - removed obsolete notworking scripts --- .../_prepost_scripts/OpenPypeEnvironment.cfg | 11 ----------- .../_prepost_scripts/PreOpenPypeInjectEnvironments.py | 4 ---- 2 files changed, 15 deletions(-) delete mode 100644 openpype/modules/royalrender/rr_root/render_apps/_prepost_scripts/OpenPypeEnvironment.cfg delete mode 100644 openpype/modules/royalrender/rr_root/render_apps/_prepost_scripts/PreOpenPypeInjectEnvironments.py diff --git a/openpype/modules/royalrender/rr_root/render_apps/_prepost_scripts/OpenPypeEnvironment.cfg b/openpype/modules/royalrender/rr_root/render_apps/_prepost_scripts/OpenPypeEnvironment.cfg deleted file mode 100644 index 70f0bc2e244..00000000000 --- a/openpype/modules/royalrender/rr_root/render_apps/_prepost_scripts/OpenPypeEnvironment.cfg +++ /dev/null @@ -1,11 +0,0 @@ -PrePostType= pre -CommandLine= - -CommandLine= rrPythonconsole" > "render_apps/_prepost_scripts/PreOpenPypeInjectEnvironments.py" - -CommandLine= - - -CommandLine= "" -CommandLine= - diff --git a/openpype/modules/royalrender/rr_root/render_apps/_prepost_scripts/PreOpenPypeInjectEnvironments.py b/openpype/modules/royalrender/rr_root/render_apps/_prepost_scripts/PreOpenPypeInjectEnvironments.py deleted file mode 100644 index 891de9594c5..00000000000 --- a/openpype/modules/royalrender/rr_root/render_apps/_prepost_scripts/PreOpenPypeInjectEnvironments.py +++ /dev/null @@ -1,4 +0,0 @@ -# -*- coding: utf-8 -*- -import os - -os.environ["OPENYPYPE_TESTVAR"] = "OpenPype was here" From 68d57bbbf2292aeba327d2468fffdcdc0bc3c0aa Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 24 Jan 2024 12:47:03 +0100 Subject: [PATCH 21/42] OP-7687 - removed obsolete Openpype --- .../perjob/m50__openpype_publish_render.py | 197 ------------------ .../_config/E01__OpenPype__PublishJob.cfg | 71 ------- .../_config/E01__OpenPype___global.inc | 2 - .../render_apps/_install_paths/OpenPype.cfg | 12 -- 4 files changed, 282 deletions(-) delete mode 100644 openpype/modules/royalrender/rr_root/plugins/control_job/perjob/m50__openpype_publish_render.py delete mode 100644 openpype/modules/royalrender/rr_root/render_apps/_config/E01__OpenPype__PublishJob.cfg delete mode 100644 openpype/modules/royalrender/rr_root/render_apps/_config/E01__OpenPype___global.inc delete mode 100644 openpype/modules/royalrender/rr_root/render_apps/_install_paths/OpenPype.cfg diff --git a/openpype/modules/royalrender/rr_root/plugins/control_job/perjob/m50__openpype_publish_render.py b/openpype/modules/royalrender/rr_root/plugins/control_job/perjob/m50__openpype_publish_render.py deleted file mode 100644 index cdc37588cd0..00000000000 --- a/openpype/modules/royalrender/rr_root/plugins/control_job/perjob/m50__openpype_publish_render.py +++ /dev/null @@ -1,197 +0,0 @@ -# -*- coding: utf-8 -*- -"""This is RR control plugin that runs on the job by user interaction. - -It asks user for context to publish, getting it from OpenPype. In order to -run it needs `OPENPYPE_ROOT` to be set to know where to execute OpenPype. - -""" -import rr # noqa -import rrGlobal # noqa -import subprocess -import os -import glob -import platform -import tempfile -import json - - -class OpenPypeContextSelector: - """Class to handle publishing context determination in RR.""" - - def __init__(self): - self.job = rr.getJob() - self.context = {} - - self.openpype_executable = "openpype_gui" - if platform.system().lower() == "windows": - self.openpype_executable = "{}.exe".format( - self.openpype_executable) - - op_path = os.environ.get("OPENPYPE_ROOT") - print("initializing ... {}".format(op_path)) - if not op_path: - print("Warning: OpenPype root is not found.") - - if platform.system().lower() == "windows": - print(" * trying to find OpenPype on local computer.") - op_path = os.path.join( - os.environ.get("PROGRAMFILES"), - "OpenPype", "openpype_console.exe" - ) - if not os.path.exists(op_path): - # try to find in user local context - op_path = os.path.join( - os.environ.get("LOCALAPPDATA"), - "Programs", - "OpenPype", "openpype_console.exe" - ) - if not os.path.exists(op_path): - raise Exception("Error: OpenPype was not found.") - - op_path = os.path.dirname(op_path) - print(" - found OpenPype installation {}".format(op_path)) - - self.openpype_root = op_path - - def _process_metadata_file(self): - """Find and process metadata file. - - Try to find metadata json file in job folder to get context from. - - Returns: - dict: Context from metadata json file. - - """ - image_dir = self.job.imageDir - metadata_files = glob.glob( - "{}{}*_metadata.json".format(image_dir, os.path.sep)) - if not metadata_files: - return {} - - raise NotImplementedError( - "Processing existing metadata not implemented yet.") - - def process_job(self): - """Process selected job. - - This should process selected job. If context can be determined - automatically, no UI will be show and publishing will directly - proceed. - """ - if not self.context and not self.show(): - return - - self.context["user"] = self.job.userName - self.run_publish() - - def show(self): - """Show UI for context selection. - - Because of RR UI limitations, this must be done using OpenPype - itself. - - """ - tf = tempfile.TemporaryFile(delete=False) - context_file = tf.name - op_args = [os.path.join(self.openpype_root, self.openpype_executable), - "contextselection", tf.name] - - tf.close() - print(">>> running {}".format(" ".join(op_args))) - - subprocess.call(op_args) - - with open(context_file, "r") as cf: - self.context = json.load(cf) - - os.unlink(context_file) - print(">>> context: {}".format(self.context)) - - if not self.context or \ - not self.context.get("project") or \ - not self.context.get("asset") or \ - not self.context.get("task"): - self._show_rr_warning("Context selection failed.") - return False - - # self.context["app_name"] = self.job.renderer.name - # there should be mapping between OpenPype and Royal Render - # app names and versions, but since app_name is not used - # currently down the line (but it is required by OP publish command - # right now). - # self.context["app_name"] = "maya/2022" - return True - - @staticmethod - def _show_rr_warning(text): - warning_dialog = rrGlobal.getGenericUI() - warning_dialog.addItem(rrGlobal.genUIType.label, "infoLabel", "") - warning_dialog.setText("infoLabel", text) - warning_dialog.addItem( - rrGlobal.genUIType.layoutH, "btnLayout", "") - warning_dialog.addItem( - rrGlobal.genUIType.closeButton, "Ok", "btnLayout") - warning_dialog.execute() - del warning_dialog - - def run_publish(self): - """Run publish process.""" - env = {"AVALON_PROJECT": str(self.context.get("project")), - "AVALON_ASSET": str(self.context.get("asset")), - "AVALON_TASK": str(self.context.get("task")), - # "AVALON_APP_NAME": str(self.context.get("app_name")) - } - - print(">>> setting environment:") - for k, v in env.items(): - print(" {}: {}".format(k, v)) - - publishing_paths = [os.path.join(self.job.imageDir, - os.path.dirname( - self.job.imageFileName))] - - # add additional channels - channel_idx = 0 - channel = self.job.channelFileName(channel_idx) - while channel: - channel_path = os.path.dirname( - os.path.join(self.job.imageDir, channel)) - if channel_path not in publishing_paths: - publishing_paths.append(channel_path) - channel_idx += 1 - channel = self.job.channelFileName(channel_idx) - - args = [os.path.join(self.openpype_root, self.openpype_executable), - 'publish', '-t', "rr_control", "--gui" - ] - - args += publishing_paths - - print(">>> running {}".format(" ".join(args))) - orig = os.environ.copy() - orig.update(env) - try: - subprocess.call(args, env=orig) - except subprocess.CalledProcessError as e: - self._show_rr_warning(" Publish failed [ {} ]".format( - e.returncode - )) - - -print("running selector") -selector = OpenPypeContextSelector() - -# try to set context from environment -selector.context["project"] = os.getenv("AVALON_PROJECT") -selector.context["asset"] = os.getenv("AVALON_ASSET") -selector.context["task"] = os.getenv("AVALON_TASK") -# selector.context["app_name"] = os.getenv("AVALON_APP_NAME") - -# if anything inside is None, scratch the whole thing and -# ask user for context. -for _, v in selector.context.items(): - if not v: - selector.context = {} - break - -selector.process_job() diff --git a/openpype/modules/royalrender/rr_root/render_apps/_config/E01__OpenPype__PublishJob.cfg b/openpype/modules/royalrender/rr_root/render_apps/_config/E01__OpenPype__PublishJob.cfg deleted file mode 100644 index 864eeaf15aa..00000000000 --- a/openpype/modules/royalrender/rr_root/render_apps/_config/E01__OpenPype__PublishJob.cfg +++ /dev/null @@ -1,71 +0,0 @@ -IconApp= E01__OpenPype.png -Name= OpenPype -rendererName= Once -Version= 1 -Version_Minor= 0 -Type=Execute -TYPEv9=Execute -ExecuteJobType=Once - - -################################# [Windows] [Linux] [Osx] ################################## - - -CommandLine=> - -CommandLine= - - -::win CommandLine= set "CUDA_VISIBLE_DEVICES=" -::lx CommandLine= setenv CUDA_VISIBLE_DEVICES -::osx CommandLine= setenv CUDA_VISIBLE_DEVICES - - -CommandLine= - - -CommandLine= - - -CommandLine= - - -CommandLine= "" --headless publish - --targets royalrender - --targets farm - - - -CommandLine= - - - - -################################## Render Settings ################################## - - - -################################## Submitter Settings ################################## -StartMultipleInstances= 0~0 -SceneFileExtension= *.json -AllowImageNameChange= 0 -AllowImageDirChange= 0 -SequenceDivide= 0~1 -PPSequenceCheck=0~0 -PPCreateSmallVideo=0~0 -PPCreateFullVideo=0~0 -AllowLocalRenderOut= 0~0 - - -################################## Client Settings ################################## - -IconApp=E01__OpenPype.png - -licenseFailLine= - -errorSearchLine= - -permanentErrorSearchLine = - -Frozen_MinCoreUsage=0.3 -Frozen_Minutes=30 diff --git a/openpype/modules/royalrender/rr_root/render_apps/_config/E01__OpenPype___global.inc b/openpype/modules/royalrender/rr_root/render_apps/_config/E01__OpenPype___global.inc deleted file mode 100644 index ba38337340d..00000000000 --- a/openpype/modules/royalrender/rr_root/render_apps/_config/E01__OpenPype___global.inc +++ /dev/null @@ -1,2 +0,0 @@ -IconApp= E01__OpenPype.png -Name= OpenPype diff --git a/openpype/modules/royalrender/rr_root/render_apps/_install_paths/OpenPype.cfg b/openpype/modules/royalrender/rr_root/render_apps/_install_paths/OpenPype.cfg deleted file mode 100644 index 07f7547d297..00000000000 --- a/openpype/modules/royalrender/rr_root/render_apps/_install_paths/OpenPype.cfg +++ /dev/null @@ -1,12 +0,0 @@ -[Windows] -Executable= openpype_console.exe -Path= OS; \OpenPype\*\openpype_console.exe -Path= 32; \OpenPype\*\openpype_console.exe - -[Linux] -Executable= openpype_console -Path= OS; /opt/openpype/*/openpype_console - -[Mac] -Executable= openpype_console -Path= OS; /Applications/OpenPype*/Content/MacOS/openpype_console From ccfe4e23aa2f29a7c6d0903f2296c2715323e15b Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 24 Jan 2024 12:47:48 +0100 Subject: [PATCH 22/42] OP-7687 - added Ayon specific config --- .../_config/{E01__OpenPype.png => E01__Ayon.png} | Bin .../render_apps/_config/E01__Ayon___global.inc | 2 ++ .../rr_root/render_apps/_install_paths/Ayon.cfg | 12 ++++++++++++ 3 files changed, 14 insertions(+) rename openpype/modules/royalrender/rr_root/render_apps/_config/{E01__OpenPype.png => E01__Ayon.png} (100%) create mode 100644 openpype/modules/royalrender/rr_root/render_apps/_config/E01__Ayon___global.inc create mode 100644 openpype/modules/royalrender/rr_root/render_apps/_install_paths/Ayon.cfg diff --git a/openpype/modules/royalrender/rr_root/render_apps/_config/E01__OpenPype.png b/openpype/modules/royalrender/rr_root/render_apps/_config/E01__Ayon.png similarity index 100% rename from openpype/modules/royalrender/rr_root/render_apps/_config/E01__OpenPype.png rename to openpype/modules/royalrender/rr_root/render_apps/_config/E01__Ayon.png diff --git a/openpype/modules/royalrender/rr_root/render_apps/_config/E01__Ayon___global.inc b/openpype/modules/royalrender/rr_root/render_apps/_config/E01__Ayon___global.inc new file mode 100644 index 00000000000..ba38337340d --- /dev/null +++ b/openpype/modules/royalrender/rr_root/render_apps/_config/E01__Ayon___global.inc @@ -0,0 +1,2 @@ +IconApp= E01__OpenPype.png +Name= OpenPype diff --git a/openpype/modules/royalrender/rr_root/render_apps/_install_paths/Ayon.cfg b/openpype/modules/royalrender/rr_root/render_apps/_install_paths/Ayon.cfg new file mode 100644 index 00000000000..534eb868990 --- /dev/null +++ b/openpype/modules/royalrender/rr_root/render_apps/_install_paths/Ayon.cfg @@ -0,0 +1,12 @@ +[Windows] +Executable= ayon_console.exe +Path= OS; \Ayon\*\ayon_console.exe +Path= 32; \Ayon\*\ayon_console.exe + +[Linux] +Executable= ayon_console +Path= OS; /opt/ayon/*/ayon_console + +[Mac] +Executable= ayon_console +Path= OS; /Applications/Ayon*/Content/MacOS/ayon_console From abafb78411db5e91c548401eb44eaa300ea8f118 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 24 Jan 2024 14:02:41 +0100 Subject: [PATCH 23/42] OP-7687 - added Ayon specific config --- .../perjob/ayon_inject_envvar.py | 4 +- .../_config/E01__Ayon___global.inc | 2 - .../_config/{E01__Ayon.png => E05__Ayon.png} | Bin .../_config/E05__Ayon__PublishJob.cfg | 71 ++++++++++++++++++ .../_config/E05__Ayon___global.inc | 2 + 5 files changed, 76 insertions(+), 3 deletions(-) delete mode 100644 openpype/modules/royalrender/rr_root/render_apps/_config/E01__Ayon___global.inc rename openpype/modules/royalrender/rr_root/render_apps/_config/{E01__Ayon.png => E05__Ayon.png} (100%) create mode 100644 openpype/modules/royalrender/rr_root/render_apps/_config/E05__Ayon__PublishJob.cfg create mode 100644 openpype/modules/royalrender/rr_root/render_apps/_config/E05__Ayon___global.inc diff --git a/openpype/modules/royalrender/rr_root/plugins/submitter_onsubmission/perjob/ayon_inject_envvar.py b/openpype/modules/royalrender/rr_root/plugins/submitter_onsubmission/perjob/ayon_inject_envvar.py index 3c7831f665b..3b55084dc46 100644 --- a/openpype/modules/royalrender/rr_root/plugins/submitter_onsubmission/perjob/ayon_inject_envvar.py +++ b/openpype/modules/royalrender/rr_root/plugins/submitter_onsubmission/perjob/ayon_inject_envvar.py @@ -21,6 +21,7 @@ class InjectEnvironment: by RR. Expected set environments on RR worker: + - AYON_SERVER_URL - AYON_API_KEY - API key to Ayon server, most likely from service account - AYON_EXECUTABLE_PATH - locally accessible path for `ayon_console` (could be removed if it would be possible to have it in renderApps config @@ -151,7 +152,7 @@ def _get_launch_environments(self, metadata_content): """ Enhances environemnt with required for Ayon to be launched.""" job_envs = self._get_job_environments(metadata_content) ayon_environment = { - "AYON_SERVER_URL": job_envs["AYON_SERVER_URL"], + "AYON_SERVER_URL": os.environ["AYON_SERVER_URL"], "AYON_API_KEY": os.environ["AYON_API_KEY"], "AYON_BUNDLE_NAME": job_envs["AYON_BUNDLE_NAME"], } @@ -200,6 +201,7 @@ def _set_rrEnv_to_job(self, rrEnv_path): if __name__ == "__main__": os.environ["AYON_API_KEY"] = "d84bde8c757540ec907e96edb58455bd" # TEMP os.environ["AYON_EXECUTABLE_PATH"] = "c:/Users/petrk/Documents/ayon/ayon-launcher/build/output/ayon_console.exe" # TEMP + os.environ["AYON_SERVER_URL"] = "http://localhost:5000" # TEMP try: injector = InjectEnvironment() diff --git a/openpype/modules/royalrender/rr_root/render_apps/_config/E01__Ayon___global.inc b/openpype/modules/royalrender/rr_root/render_apps/_config/E01__Ayon___global.inc deleted file mode 100644 index ba38337340d..00000000000 --- a/openpype/modules/royalrender/rr_root/render_apps/_config/E01__Ayon___global.inc +++ /dev/null @@ -1,2 +0,0 @@ -IconApp= E01__OpenPype.png -Name= OpenPype diff --git a/openpype/modules/royalrender/rr_root/render_apps/_config/E01__Ayon.png b/openpype/modules/royalrender/rr_root/render_apps/_config/E05__Ayon.png similarity index 100% rename from openpype/modules/royalrender/rr_root/render_apps/_config/E01__Ayon.png rename to openpype/modules/royalrender/rr_root/render_apps/_config/E05__Ayon.png diff --git a/openpype/modules/royalrender/rr_root/render_apps/_config/E05__Ayon__PublishJob.cfg b/openpype/modules/royalrender/rr_root/render_apps/_config/E05__Ayon__PublishJob.cfg new file mode 100644 index 00000000000..9604d3c468c --- /dev/null +++ b/openpype/modules/royalrender/rr_root/render_apps/_config/E05__Ayon__PublishJob.cfg @@ -0,0 +1,71 @@ +IconApp= E05__Ayon.png +Name= Ayon +rendererName= Once +Version= 1 +Version_Minor= 0 +Type=Execute +TYPEv9=Execute +ExecuteJobType=Once + + +################################# [Windows] [Linux] [Osx] ################################## + + +CommandLine=> + +CommandLine= + + +::win CommandLine= set "CUDA_VISIBLE_DEVICES=" +::lx CommandLine= setenv CUDA_VISIBLE_DEVICES +::osx CommandLine= setenv CUDA_VISIBLE_DEVICES + + +CommandLine= + + +CommandLine= + + +CommandLine= + + +CommandLine= "" --headless publish + --targets royalrender + --targets farm + + + +CommandLine= + + + + +################################## Render Settings ################################## + + + +################################## Submitter Settings ################################## +StartMultipleInstances= 0~0 +SceneFileExtension= *.json +AllowImageNameChange= 0 +AllowImageDirChange= 0 +SequenceDivide= 0~1 +PPSequenceCheck=0~0 +PPCreateSmallVideo=0~0 +PPCreateFullVideo=0~0 +AllowLocalRenderOut= 0~0 + + +################################## Client Settings ################################## + +IconApp=E05__Ayon.png + +licenseFailLine= + +errorSearchLine= + +permanentErrorSearchLine = + +Frozen_MinCoreUsage=0.3 +Frozen_Minutes=30 diff --git a/openpype/modules/royalrender/rr_root/render_apps/_config/E05__Ayon___global.inc b/openpype/modules/royalrender/rr_root/render_apps/_config/E05__Ayon___global.inc new file mode 100644 index 00000000000..1581eb26b11 --- /dev/null +++ b/openpype/modules/royalrender/rr_root/render_apps/_config/E05__Ayon___global.inc @@ -0,0 +1,2 @@ +IconApp= E05__Ayon.png +Name= Ayon From ce5621309f475d9b4267bea73e518d378ab07858 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 24 Jan 2024 14:48:23 +0100 Subject: [PATCH 24/42] OP-7687 - update error message --- openpype/modules/royalrender/lib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/modules/royalrender/lib.py b/openpype/modules/royalrender/lib.py index 9c4221d9cd2..b37bf286028 100644 --- a/openpype/modules/royalrender/lib.py +++ b/openpype/modules/royalrender/lib.py @@ -110,7 +110,7 @@ def process(self, instance): if not self._rr_root: raise KnownPublishError( ("Missing RoyalRender root. " - "You need to configure RoyalRender module.")) + "Admin needs to configure RoyalRender module in Settings .")) self.rr_api = rrApi(self._rr_root) From 9e8409b193cab97f64baa8b6b24e06006300cacc Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 24 Jan 2024 14:49:00 +0100 Subject: [PATCH 25/42] OP-7687 - add submitter parameter for env injection This should be added only to render jobs. --- openpype/modules/royalrender/lib.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/openpype/modules/royalrender/lib.py b/openpype/modules/royalrender/lib.py index b37bf286028..2ddeeb40a68 100644 --- a/openpype/modules/royalrender/lib.py +++ b/openpype/modules/royalrender/lib.py @@ -175,6 +175,10 @@ def get_job(self, instance, script_path, render_path, node_name): instance, render_path, start_frame, end_frame) instance.data["expectedFiles"].extend(expected_files) + submitter_parameters = [ + rrApi.SubmitterParameter("OSperjob_ayon_inject_envvar", "1~1") + ] + job = RRJob( Software="", Renderer="", @@ -197,7 +201,8 @@ def get_job(self, instance, script_path, render_path, node_name): CompanyProjectName=instance.context.data["projectName"], ImageWidth=instance.data["resolutionWidth"], ImageHeight=instance.data["resolutionHeight"], - CustomAttributes=custom_attributes + CustomAttributes=custom_attributes, + SubmitterParameters=submitter_parameters ) return job From 35f80508751392ebeeeb17bf3dc7e87de816ee46 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 24 Jan 2024 14:57:31 +0100 Subject: [PATCH 26/42] OP-7687 - add submitter parameter for env injection This should be added only to render jobs. --- openpype/modules/royalrender/lib.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/openpype/modules/royalrender/lib.py b/openpype/modules/royalrender/lib.py index 2ddeeb40a68..9d8cd1de86b 100644 --- a/openpype/modules/royalrender/lib.py +++ b/openpype/modules/royalrender/lib.py @@ -18,6 +18,7 @@ RRJob, RREnvList, get_rr_platform, + SubmitterParameter ) from openpype.pipeline import OpenPypePyblishPluginMixin from openpype.pipeline.publish import KnownPublishError @@ -176,7 +177,7 @@ def get_job(self, instance, script_path, render_path, node_name): instance.data["expectedFiles"].extend(expected_files) submitter_parameters = [ - rrApi.SubmitterParameter("OSperjob_ayon_inject_envvar", "1~1") + SubmitterParameter("OSperjob_ayon_inject_envvar", "1~1") ] job = RRJob( From 64bb038ee05e3a9574981f1023fd24d2fa24d379 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 24 Jan 2024 15:12:46 +0100 Subject: [PATCH 27/42] OP-7687 - removed obsolete implementation Was adding env vars from client not rendering worker. Limitation on 2k characters. --- openpype/modules/royalrender/lib.py | 65 ------------------- .../publish/create_maya_royalrender_job.py | 1 - .../publish/create_nuke_royalrender_job.py | 1 - 3 files changed, 67 deletions(-) diff --git a/openpype/modules/royalrender/lib.py b/openpype/modules/royalrender/lib.py index 9d8cd1de86b..4893a765ae0 100644 --- a/openpype/modules/royalrender/lib.py +++ b/openpype/modules/royalrender/lib.py @@ -313,68 +313,3 @@ def pad_file_name(self, path, first_frame): path = path.replace(first_frame, "#" * padding) return path - - def inject_environment(self, instance, job): - # type: (pyblish.api.Instance, RRJob) -> RRJob - """Inject environment variables for RR submission. - - This function mimics the behaviour of the Deadline - integration. It is just temporary solution until proper - runtime environment injection is implemented in RR. - - Args: - instance (pyblish.api.Instance): Publishing instance - job (RRJob): RRJob instance to be injected. - - Returns: - RRJob: Injected RRJob instance. - - Throws: - RuntimeError: If any of the required env vars is missing. - - """ - - temp_file_name = "{}_{}.json".format( - datetime.utcnow().strftime('%Y%m%d%H%M%S%f'), - str(uuid.uuid1()) - ) - - export_url = os.path.join(tempfile.gettempdir(), temp_file_name) - print(">>> Temporary path: {}".format(export_url)) - - args = [ - "--headless", - "extractenvironments", - export_url - ] - - anatomy_data = instance.context.data["anatomyData"] - - add_kwargs = { - "project": anatomy_data["project"]["name"], - "asset": instance.context.data["asset"], - "task": anatomy_data["task"]["name"], - "app": instance.context.data.get("appName"), - "envgroup": "farm" - } - - if os.getenv('IS_TEST'): - args.append("--automatic-tests") - - if not all(add_kwargs.values()): - raise RuntimeError(( - "Missing required env vars: AVALON_PROJECT, AVALON_ASSET," - " AVALON_TASK, AVALON_APP_NAME" - )) - - for key, value in add_kwargs.items(): - args.extend([f"--{key}", value]) - self.log.debug("Executing: {}".format(" ".join(args))) - run_openpype_process(*args, logger=self.log) - - self.log.debug("Loading file ...") - with open(export_url) as fp: - contents = json.load(fp) - - job.rrEnvList = RREnvList(contents).serialize() - return job diff --git a/openpype/modules/royalrender/plugins/publish/create_maya_royalrender_job.py b/openpype/modules/royalrender/plugins/publish/create_maya_royalrender_job.py index 775a2964fd4..36f3dcc99a6 100644 --- a/openpype/modules/royalrender/plugins/publish/create_maya_royalrender_job.py +++ b/openpype/modules/royalrender/plugins/publish/create_maya_royalrender_job.py @@ -38,6 +38,5 @@ def process(self, instance): job = self.get_job(instance, self.scene_path, first_file_path, layer_name) job = self.update_job_with_host_specific(instance, job) - job = self.inject_environment(instance, job) instance.data["rrJobs"].append(job) diff --git a/openpype/modules/royalrender/plugins/publish/create_nuke_royalrender_job.py b/openpype/modules/royalrender/plugins/publish/create_nuke_royalrender_job.py index 4f589e56f89..71daa6edf80 100644 --- a/openpype/modules/royalrender/plugins/publish/create_nuke_royalrender_job.py +++ b/openpype/modules/royalrender/plugins/publish/create_nuke_royalrender_job.py @@ -25,7 +25,6 @@ def process(self, instance): jobs = self.create_jobs(instance) for job in jobs: job = self.update_job_with_host_specific(instance, job) - job = self.inject_environment(instance, job) instance.data["rrJobs"].append(job) From 52f513b8794ee9647e42bfa71c230bc89bb29ca9 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 24 Jan 2024 15:16:34 +0100 Subject: [PATCH 28/42] OP-7687 - removed debug env vars --- .../submitter_onsubmission/perjob/ayon_inject_envvar.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/openpype/modules/royalrender/rr_root/plugins/submitter_onsubmission/perjob/ayon_inject_envvar.py b/openpype/modules/royalrender/rr_root/plugins/submitter_onsubmission/perjob/ayon_inject_envvar.py index 3b55084dc46..8d705a5cf99 100644 --- a/openpype/modules/royalrender/rr_root/plugins/submitter_onsubmission/perjob/ayon_inject_envvar.py +++ b/openpype/modules/royalrender/rr_root/plugins/submitter_onsubmission/perjob/ayon_inject_envvar.py @@ -199,9 +199,6 @@ def _set_rrEnv_to_job(self, rrEnv_path): if __name__ == "__main__": - os.environ["AYON_API_KEY"] = "d84bde8c757540ec907e96edb58455bd" # TEMP - os.environ["AYON_EXECUTABLE_PATH"] = "c:/Users/petrk/Documents/ayon/ayon-launcher/build/output/ayon_console.exe" # TEMP - os.environ["AYON_SERVER_URL"] = "http://localhost:5000" # TEMP try: injector = InjectEnvironment() From 20dfda33ebf79edfed4150e519972e53ac2ff8b6 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 24 Jan 2024 15:30:29 +0100 Subject: [PATCH 29/42] OP-7687 - changes to Ayon --- .../publish/create_publish_royalrender_job.py | 31 ++----------------- 1 file changed, 3 insertions(+), 28 deletions(-) diff --git a/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py b/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py index 95ebe82c149..2c5637a22df 100644 --- a/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py +++ b/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py @@ -73,7 +73,7 @@ class CreatePublishRoyalRenderJob(pyblish.api.InstancePlugin, "AVALON_APP_NAME", "OPENPYPE_USERNAME", "OPENPYPE_SG_USER", - "OPENPYPE_MONGO" + "AYON_BUNDLE_NAME" ] priority = 50 @@ -231,13 +231,13 @@ def get_job(self, instance, instances): ] job = RRJob( - Software="OpenPype", + Software="Ayon", Renderer="Once", SeqStart=1, SeqEnd=1, SeqStep=1, SeqFileOffset=0, - Version=self._sanitize_version(os.environ.get("OPENPYPE_VERSION")), + Version=os.environ["AYON_BUNDLE_NAME"], SceneName=abs_metadata_path, # command line arguments CustomAddCmdFlags=" ".join(args), @@ -264,28 +264,3 @@ def get_job(self, instance, instances): job.WaitForPreIDs += jobs_pre_ids return job - - def _sanitize_version(self, version): - """Returns version in format MAJOR.MINORPATCH - - 3.15.7-nightly.2 >> 3.157 - """ - if not version: - return - VERSION_REGEX = re.compile( - r"(?P0|[1-9]\d*)" - r"\.(?P0|[1-9]\d*)" - r"\.(?P0|[1-9]\d*)" - r"(?:-(?P[a-zA-Z\d\-.]*))?" - r"(?:\+(?P[a-zA-Z\d\-.]*))?" - ) - - valid_parts = VERSION_REGEX.findall(version) - if len(valid_parts) != 1: - # Return invalid version with filled 'origin' attribute - return version - - # Unpack found version - major, minor, patch, pre, post = valid_parts[0] - - return "{}.{}{}".format(major, minor, patch) From 63adb504c87291dd19c254bb1aebf4e49706b07c Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 24 Jan 2024 15:31:34 +0100 Subject: [PATCH 30/42] OP-7687 - changes for Ayon Stored to rrEnvList just to be saved to .json from where it is parsed. Hopefully there would be a way how to parse it directly from RR job and not reaching to .json --- openpype/modules/royalrender/lib.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/openpype/modules/royalrender/lib.py b/openpype/modules/royalrender/lib.py index 4893a765ae0..9906512dae6 100644 --- a/openpype/modules/royalrender/lib.py +++ b/openpype/modules/royalrender/lib.py @@ -180,6 +180,14 @@ def get_job(self, instance, script_path, render_path, node_name): SubmitterParameter("OSperjob_ayon_inject_envvar", "1~1") ] + anatomy_data = instance.context.data["anatomyData"] + environment = RREnvList({ + "AVALON_PROJECT": anatomy_data["project"]["name"], + "AVALON_ASSET": instance.context.data["asset"], + "AVALON_TASK": anatomy_data["task"]["name"], + "AVALON_APP": instance.context.data.get("appName") + }) + job = RRJob( Software="", Renderer="", @@ -203,7 +211,8 @@ def get_job(self, instance, script_path, render_path, node_name): ImageWidth=instance.data["resolutionWidth"], ImageHeight=instance.data["resolutionHeight"], CustomAttributes=custom_attributes, - SubmitterParameters=submitter_parameters + SubmitterParameters=submitter_parameters, + rrEnvList=environment.serialize(), ) return job From d8330eb334da4b18abeeff9ebf45fe8bb5d08fd2 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 24 Jan 2024 15:49:53 +0100 Subject: [PATCH 31/42] OP-7687 - Hound --- .../submitter_onsubmission/perjob/ayon_inject_envvar.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/openpype/modules/royalrender/rr_root/plugins/submitter_onsubmission/perjob/ayon_inject_envvar.py b/openpype/modules/royalrender/rr_root/plugins/submitter_onsubmission/perjob/ayon_inject_envvar.py index 8d705a5cf99..c1f67a3c420 100644 --- a/openpype/modules/royalrender/rr_root/plugins/submitter_onsubmission/perjob/ayon_inject_envvar.py +++ b/openpype/modules/royalrender/rr_root/plugins/submitter_onsubmission/perjob/ayon_inject_envvar.py @@ -70,7 +70,6 @@ def _get_metadata_dir(self): image_dir = job.imageDir logs.append(f"_get_metadata_dir::{image_dir}") - # return "C:/projects_local/ayon_test/assets/characters/characterA/work/lookdev/renders/maya/at_characterA_lookdev_v019/Main" return image_dir def _get_metadata(self, meta_dir): @@ -85,7 +84,8 @@ def _get_metadata(self, meta_dir): def _is_required_environment(self): if (not os.environ.get("AYON_API_KEY") or - not os.path.exists(os.environ.get("AYON_EXECUTABLE_PATH", ""))): + not os.path.exists(os.environ.get("AYON_EXECUTABLE_PATH", "")) + ): msg = ("AYON_API_KEY and AYON_EXECUTABLE_PATHenv var must be set " "for Ayon jobs!") logs.append(msg) @@ -197,7 +197,6 @@ def _set_rrEnv_to_job(self, rrEnv_path): rr.setJob(job) - if __name__ == "__main__": try: From 5d8e5ec7fd3288e07c30be92dd9e41956e05c694 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Mon, 29 Jan 2024 18:23:22 +0100 Subject: [PATCH 32/42] OP-7687 - added check for AYON_RENDER_JOB to injection script --- .../perjob/ayon_inject_envvar.py | 41 ++++++++++--------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/openpype/modules/royalrender/rr_root/plugins/submitter_onsubmission/perjob/ayon_inject_envvar.py b/openpype/modules/royalrender/rr_root/plugins/submitter_onsubmission/perjob/ayon_inject_envvar.py index c1f67a3c420..d0ac4559b7f 100644 --- a/openpype/modules/royalrender/rr_root/plugins/submitter_onsubmission/perjob/ayon_inject_envvar.py +++ b/openpype/modules/royalrender/rr_root/plugins/submitter_onsubmission/perjob/ayon_inject_envvar.py @@ -26,6 +26,8 @@ class InjectEnvironment: - AYON_EXECUTABLE_PATH - locally accessible path for `ayon_console` (could be removed if it would be possible to have it in renderApps config and to be accessible from there as there it is required for publish jobs). + - AYON_FILTER_ENVIRONMENTS - potential black list of unwanted environment + variables (separated by ';') - will be filtered out from created .rrEnv. Ayon submission job must be adding this line to .xml submission file: OSperjob_ayon_inject_envvar=1~1 @@ -48,17 +50,21 @@ def inject(self): return self.meta_dir = meta_dir + envs = self._get_job_environments() + if not envs.get("AYON_RENDER_JOB"): + logs.append("Not a ayon render job, skipping.") + return + if not self._is_required_environment(): return - context = self._get_context(metadata) + context = self._get_context() logs.append("context {}".format(context)) executable = self._get_executable() logs.append("executable {}".format(executable)) - extracted_env = self._extract_environments(executable, context, - metadata) + extracted_env = self._extract_environments(executable, context) rrEnv_path = self._create_rrEnv(meta_dir, extracted_env) @@ -84,16 +90,15 @@ def _get_metadata(self, meta_dir): def _is_required_environment(self): if (not os.environ.get("AYON_API_KEY") or - not os.path.exists(os.environ.get("AYON_EXECUTABLE_PATH", "")) - ): + not os.path.exists(os.environ.get("AYON_EXECUTABLE_PATH", ""))): msg = ("AYON_API_KEY and AYON_EXECUTABLE_PATHenv var must be set " "for Ayon jobs!") logs.append(msg) return False return True - def _get_context(self, metadata_content): - envs = self._get_job_environments(metadata_content) + def _get_context(self): + envs = self._get_job_environments() return {"project": envs["AVALON_PROJECT"], "asset": envs["AVALON_ASSET"], @@ -101,14 +106,10 @@ def _get_context(self, metadata_content): "app": envs["AVALON_APP_NAME"], "envgroup": "farm"} - def _get_job_environments(self, metadata_content): - """Gets environments set on job. - - It seems that it is not possible to query "rrEnvList" on job directly, - it must be parsed from .json document. - """ - job = metadata_content["job"] - env_list = job["rrEnvList"] + def _get_job_environments(self): + """Gets environments set on job.""" + job = rr.getJob() + env_list = job.customData_Str('rrEnvList') envs = {} for env in env_list.split("~~~"): key, value = env.split("=") @@ -119,7 +120,7 @@ def _get_executable(self): # rr_python_utils.cache.get_rr_bin_folder() # TODO maybe useful return os.environ["AYON_EXECUTABLE_PATH"] - def _extract_environments(self, executable, context, metadata_content): + def _extract_environments(self, executable, context): # tempfile.TemporaryFile cannot be used because of locking export_url = self._get_export_url() @@ -134,7 +135,7 @@ def _extract_environments(self, executable, context, metadata_content): for key, value in context.items(): args.extend(["--{}".format(key), value]) - environments = self._get_launch_environments(metadata_content) + environments = self._get_launch_environments() logs.append("Running:: {}".format(args)) proc = subprocess.Popen(args, env=environments, @@ -148,9 +149,9 @@ def _extract_environments(self, executable, context, metadata_content): with open(export_url) as json_file: return json.load(json_file) - def _get_launch_environments(self, metadata_content): + def _get_launch_environments(self): """ Enhances environemnt with required for Ayon to be launched.""" - job_envs = self._get_job_environments(metadata_content) + job_envs = self._get_job_environments() ayon_environment = { "AYON_SERVER_URL": os.environ["AYON_SERVER_URL"], "AYON_API_KEY": os.environ["AYON_API_KEY"], @@ -206,5 +207,5 @@ def _set_rrEnv_to_job(self, rrEnv_path): logs.append(f"Error happened::{str(exp)}") log_path = os.path.join(injector.meta_dir, "log.txt") - with open(log_path, "w") as fp: + with open(log_path, "a") as fp: fp.writelines(s + '\n' for s in logs) From df85b44d069de9721ca2ff45e2af1ad34546548b Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Mon, 29 Jan 2024 18:24:21 +0100 Subject: [PATCH 33/42] OP-7687 - OSperjob_ayon_inject_envvar must be on submission Doesn't work on each/separate job. Added some needed env vars. --- openpype/modules/royalrender/lib.py | 8 ++++---- .../plugins/publish/submit_jobs_to_royalrender.py | 3 ++- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/openpype/modules/royalrender/lib.py b/openpype/modules/royalrender/lib.py index 9906512dae6..05cd4c8648a 100644 --- a/openpype/modules/royalrender/lib.py +++ b/openpype/modules/royalrender/lib.py @@ -176,16 +176,16 @@ def get_job(self, instance, script_path, render_path, node_name): instance, render_path, start_frame, end_frame) instance.data["expectedFiles"].extend(expected_files) - submitter_parameters = [ - SubmitterParameter("OSperjob_ayon_inject_envvar", "1~1") - ] + submitter_parameters = [] anatomy_data = instance.context.data["anatomyData"] environment = RREnvList({ "AVALON_PROJECT": anatomy_data["project"]["name"], "AVALON_ASSET": instance.context.data["asset"], "AVALON_TASK": anatomy_data["task"]["name"], - "AVALON_APP": instance.context.data.get("appName") + "AVALON_APP_NAME": instance.context.data.get("appName"), + "AYON_RENDER_JOB": "1", + "AYON_BUNDLE_NAME": os.environ["AYON_BUNDLE_NAME"] }) job = RRJob( diff --git a/openpype/modules/royalrender/plugins/publish/submit_jobs_to_royalrender.py b/openpype/modules/royalrender/plugins/publish/submit_jobs_to_royalrender.py index 8fc8604b839..7ec54efe607 100644 --- a/openpype/modules/royalrender/plugins/publish/submit_jobs_to_royalrender.py +++ b/openpype/modules/royalrender/plugins/publish/submit_jobs_to_royalrender.py @@ -99,7 +99,8 @@ def create_file(self, name, ext, contents=None): return temp.name def get_submission_parameters(self): - return [SubmitterParameter("RequiredMemory", "0")] + return [SubmitterParameter("RequiredMemory", "0"), + SubmitterParameter("OSperjob_ayon_inject_envvar", "1~1")] @staticmethod def _resolve_rr_path(context, rr_path_name): From aced1de58ac5a39f01fbbcb73ee820d80a7d6615 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Tue, 30 Jan 2024 12:17:58 +0100 Subject: [PATCH 34/42] OP-7687 - added logging of Ayon launch environment --- .../submitter_onsubmission/perjob/ayon_inject_envvar.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/openpype/modules/royalrender/rr_root/plugins/submitter_onsubmission/perjob/ayon_inject_envvar.py b/openpype/modules/royalrender/rr_root/plugins/submitter_onsubmission/perjob/ayon_inject_envvar.py index d0ac4559b7f..a16d191c2f6 100644 --- a/openpype/modules/royalrender/rr_root/plugins/submitter_onsubmission/perjob/ayon_inject_envvar.py +++ b/openpype/modules/royalrender/rr_root/plugins/submitter_onsubmission/perjob/ayon_inject_envvar.py @@ -107,7 +107,11 @@ def _get_context(self): "envgroup": "farm"} def _get_job_environments(self): - """Gets environments set on job.""" + """Gets environments set on job. + + It seems that it is not possible to query "rrEnvList" on job directly, + it must be parsed from .json document. + """ job = rr.getJob() env_list = job.customData_Str('rrEnvList') envs = {} @@ -157,7 +161,7 @@ def _get_launch_environments(self): "AYON_API_KEY": os.environ["AYON_API_KEY"], "AYON_BUNDLE_NAME": job_envs["AYON_BUNDLE_NAME"], } - + logs.append("Ayon launch environments:: {}".format(ayon_environment)) environment = os.environ.copy() environment.update(ayon_environment) return environment From 36daf44ccc0b58f2dee6759f56ba97ecddb1cdc9 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 1 Feb 2024 14:42:31 +0100 Subject: [PATCH 35/42] OP-7687 - disabling for legacy Openpype It was cleaned up for Ayon only --- openpype/modules/royalrender/royal_render_module.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/openpype/modules/royalrender/royal_render_module.py b/openpype/modules/royalrender/royal_render_module.py index 10d74d01d1a..8c21cce960d 100644 --- a/openpype/modules/royalrender/royal_render_module.py +++ b/openpype/modules/royalrender/royal_render_module.py @@ -3,6 +3,10 @@ import os import openpype.modules from openpype.modules import OpenPypeModule, IPluginPaths +from openpype import AYON_SERVER_ENABLED +from openpype.lib import Logger + +log = Logger.get_logger("SyncServer") class RoyalRenderModule(OpenPypeModule, IPluginPaths): @@ -31,6 +35,11 @@ def initialize(self, module_settings): self.enabled = rr_settings["enabled"] self.rr_paths = rr_settings.get("rr_paths") + # Ayon only + if not AYON_SERVER_ENABLED: + log.info("RoyalRender is not implemented for Openpype") + self.enabled = False + @staticmethod def get_plugin_paths(): # type: () -> dict From 9f8c189c0da864b355725c2e73b329be7c09ba95 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Mon, 5 Feb 2024 17:41:19 +0100 Subject: [PATCH 36/42] OP-7687 - changed injection to prescript It seems that previous implementation is triggered too early and on submission machine. Still bit WIP as `tcp.jobList_GetInfo` might not be available in Demo license. In that case this all will be unusable and should be scrapped. --- .../publish/submit_jobs_to_royalrender.py | 2 +- .../Ayon_ayon_inject_envvar.cfg | 42 +++++++++++++++++++ .../_prepost_scripts}/ayon_inject_envvar.py | 35 ++++++++++++++-- 3 files changed, 74 insertions(+), 5 deletions(-) create mode 100644 openpype/modules/royalrender/rr_root/render_apps/_prepost_scripts/Ayon_ayon_inject_envvar.cfg rename openpype/modules/royalrender/rr_root/{plugins/submitter_onsubmission/perjob => render_apps/_prepost_scripts}/ayon_inject_envvar.py (87%) diff --git a/openpype/modules/royalrender/plugins/publish/submit_jobs_to_royalrender.py b/openpype/modules/royalrender/plugins/publish/submit_jobs_to_royalrender.py index 7ec54efe607..0e809ee8ca1 100644 --- a/openpype/modules/royalrender/plugins/publish/submit_jobs_to_royalrender.py +++ b/openpype/modules/royalrender/plugins/publish/submit_jobs_to_royalrender.py @@ -100,7 +100,7 @@ def create_file(self, name, ext, contents=None): def get_submission_parameters(self): return [SubmitterParameter("RequiredMemory", "0"), - SubmitterParameter("OSperjob_ayon_inject_envvar", "1~1")] + SubmitterParameter("PPAyoninjectenvvar", "1~1")] @staticmethod def _resolve_rr_path(context, rr_path_name): diff --git a/openpype/modules/royalrender/rr_root/render_apps/_prepost_scripts/Ayon_ayon_inject_envvar.cfg b/openpype/modules/royalrender/rr_root/render_apps/_prepost_scripts/Ayon_ayon_inject_envvar.cfg new file mode 100644 index 00000000000..d54e83b237d --- /dev/null +++ b/openpype/modules/royalrender/rr_root/render_apps/_prepost_scripts/Ayon_ayon_inject_envvar.cfg @@ -0,0 +1,42 @@ +# config file format version 7.0 +# +# Author: Royal Render, Holger Schoenberger, Binary Alchemy +# +# Last change: v8.2.24 +# +# +# Deletes this job from the rrServer queue +# +# AuthStr is required in case anonymous does not have the right to delete jobs. +# Or if you have enabled "Authorization is required for all connections" +# AuthStr will not work via a router/remote connection +# +# +################################## Identify script ################################## +Name= Ayon inject env var + +PrePostType=Pre + +# Optional flags: The following lines are disabled and set to the default value +# AllowedForExecuteOnceJobs= false +# AllowedForSingleOutput = true +PrePostChecked= true +# ExecutePerChannel = false +# PrePostShowParamA= false +# PrePostShowParamB= false +# PrePostParamA= 100 +# PrePostParamB= 100 + + +################################## [Windows] [Linux] [OSX] ################################## + +CommandLine= + + + +CommandLine= rrPythonconsole" > "render_apps/_prepost_scripts/ayon_inject_envvar.py" -jid + + + +CommandLine= + diff --git a/openpype/modules/royalrender/rr_root/plugins/submitter_onsubmission/perjob/ayon_inject_envvar.py b/openpype/modules/royalrender/rr_root/render_apps/_prepost_scripts/ayon_inject_envvar.py similarity index 87% rename from openpype/modules/royalrender/rr_root/plugins/submitter_onsubmission/perjob/ayon_inject_envvar.py rename to openpype/modules/royalrender/rr_root/render_apps/_prepost_scripts/ayon_inject_envvar.py index a16d191c2f6..1d115a49125 100644 --- a/openpype/modules/royalrender/rr_root/plugins/submitter_onsubmission/perjob/ayon_inject_envvar.py +++ b/openpype/modules/royalrender/rr_root/render_apps/_prepost_scripts/ayon_inject_envvar.py @@ -1,10 +1,16 @@ import os +import sys import json import glob from datetime import datetime import tempfile import subprocess import uuid +import argparse + +modPath = rrGlobal.rrModPyrrPath() +sys.path.append(modPath) +import libpyRR39 as rr logs = [] @@ -30,7 +36,7 @@ class InjectEnvironment: variables (separated by ';') - will be filtered out from created .rrEnv. Ayon submission job must be adding this line to .xml submission file: - OSperjob_ayon_inject_envvar=1~1 + PPAyoninjectenvvar=1~1 Scripts logs into folder with metadata json - could be removed if there is a way how to log into RR output. @@ -72,7 +78,7 @@ def inject(self): logs.append("InjectEnvironment ending") def _get_metadata_dir(self): - job = rr.getJob() + job = self._get_job() image_dir = job.imageDir logs.append(f"_get_metadata_dir::{image_dir}") @@ -106,13 +112,32 @@ def _get_context(self): "app": envs["AVALON_APP_NAME"], "envgroup": "farm"} + def _get_job(self): + logs.append("get_jobs") + parser = argparse.ArgumentParser() + parser.add_argument("-jid") + parser.add_argument("-authStr") + args = parser.parse_args() + + # print("Set up server and login info") + tcp = rr._rrTCP("") + tcp.setServer(rrGlobal.rrServer(), 7773) + # tcp.setLogin(args.authStr, "") + tcp.setLogin("", "") + + if not tcp.jobList_GetInfo(int(args.jid)): + print("Error jobList_GetInfo: " + tcp.errorMessage()) + sys.exit() + + return tcp.jobs.getJobSend(int(args.jid)) + def _get_job_environments(self): """Gets environments set on job. It seems that it is not possible to query "rrEnvList" on job directly, it must be parsed from .json document. """ - job = rr.getJob() + job = self._get_job() env_list = job.customData_Str('rrEnvList') envs = {} for env in env_list.split("~~~"): @@ -148,6 +173,8 @@ def _extract_environments(self, executable, context): output, error = proc.communicate() if not os.path.exists(export_url): + logs.append("output::{}".format(output)) + logs.append("error::{}".format(error)) raise RuntimeError("Extract failed with {}".format(error)) with open(export_url) as json_file: @@ -196,7 +223,7 @@ def _create_rrEnv(self, meta_dir, extracted_env): def _set_rrEnv_to_job(self, rrEnv_path): logs.append(f"_set_rrEnv_to_job::{rrEnv_path}") - job = rr.getJob() + job = self._get_job() job.customDataSet_Str("rrEnvFile", rrEnv_path) rr.setJob(job) From a78cfb250c9e56cffb8ea9bbd6d24204bc5d039f Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 9 Feb 2024 11:56:40 +0100 Subject: [PATCH 37/42] Update openpype/modules/royalrender/royal_render_module.py Co-authored-by: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> --- openpype/modules/royalrender/royal_render_module.py | 1 - 1 file changed, 1 deletion(-) diff --git a/openpype/modules/royalrender/royal_render_module.py b/openpype/modules/royalrender/royal_render_module.py index 8c21cce960d..76843e781fd 100644 --- a/openpype/modules/royalrender/royal_render_module.py +++ b/openpype/modules/royalrender/royal_render_module.py @@ -6,7 +6,6 @@ from openpype import AYON_SERVER_ENABLED from openpype.lib import Logger -log = Logger.get_logger("SyncServer") class RoyalRenderModule(OpenPypeModule, IPluginPaths): From 9bc61b7f785a2c03be13e09bae680c59c9f87018 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 9 Feb 2024 11:56:47 +0100 Subject: [PATCH 38/42] Update openpype/modules/royalrender/royal_render_module.py Co-authored-by: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> --- openpype/modules/royalrender/royal_render_module.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/modules/royalrender/royal_render_module.py b/openpype/modules/royalrender/royal_render_module.py index 76843e781fd..f3ee040921b 100644 --- a/openpype/modules/royalrender/royal_render_module.py +++ b/openpype/modules/royalrender/royal_render_module.py @@ -36,7 +36,7 @@ def initialize(self, module_settings): # Ayon only if not AYON_SERVER_ENABLED: - log.info("RoyalRender is not implemented for Openpype") + self.log.info("RoyalRender is not implemented for Openpype") self.enabled = False @staticmethod From f4202ea7344fce9feab16f6e92905e181ddeb8f1 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Mon, 18 Mar 2024 11:55:19 +0100 Subject: [PATCH 39/42] Merge --- .../rr_root/render_apps/_prepost_scripts/ayon_inject_envvar.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/modules/royalrender/rr_root/render_apps/_prepost_scripts/ayon_inject_envvar.py b/openpype/modules/royalrender/rr_root/render_apps/_prepost_scripts/ayon_inject_envvar.py index 1d115a49125..ddd2977e193 100644 --- a/openpype/modules/royalrender/rr_root/render_apps/_prepost_scripts/ayon_inject_envvar.py +++ b/openpype/modules/royalrender/rr_root/render_apps/_prepost_scripts/ayon_inject_envvar.py @@ -239,4 +239,4 @@ def _set_rrEnv_to_job(self, rrEnv_path): log_path = os.path.join(injector.meta_dir, "log.txt") with open(log_path, "a") as fp: - fp.writelines(s + '\n' for s in logs) + fp.writelines(s.replace("\\r\\n", "\n") + '\n' for s in logs) From 5d6435faffc821d0df15a24b301957b2cb71e60c Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 5 Apr 2024 17:28:18 +0200 Subject: [PATCH 40/42] Added field for rrEnvFile Cannot attach in pre load script later, only fill non-existent file. --- openpype/modules/royalrender/lib.py | 4 +++- openpype/modules/royalrender/rr_job.py | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/openpype/modules/royalrender/lib.py b/openpype/modules/royalrender/lib.py index 05cd4c8648a..3897e85824c 100644 --- a/openpype/modules/royalrender/lib.py +++ b/openpype/modules/royalrender/lib.py @@ -188,6 +188,7 @@ def get_job(self, instance, script_path, render_path, node_name): "AYON_BUNDLE_NAME": os.environ["AYON_BUNDLE_NAME"] }) + render_dir = render_dir.replace("\\", "/") job = RRJob( Software="", Renderer="", @@ -198,7 +199,7 @@ def get_job(self, instance, script_path, render_path, node_name): Version=0, SceneName=script_path, IsActive=True, - ImageDir=render_dir.replace("\\", "/"), + ImageDir=render_dir, ImageFilename=file_name, ImageExtension=file_ext, ImagePreNumberLetter="", @@ -213,6 +214,7 @@ def get_job(self, instance, script_path, render_path, node_name): CustomAttributes=custom_attributes, SubmitterParameters=submitter_parameters, rrEnvList=environment.serialize(), + rrEnvFile=os.path.join(render_dir, "rrEnv.rrEenv"), ) return job diff --git a/openpype/modules/royalrender/rr_job.py b/openpype/modules/royalrender/rr_job.py index 62a82d45e85..10f09ed878f 100644 --- a/openpype/modules/royalrender/rr_job.py +++ b/openpype/modules/royalrender/rr_job.py @@ -173,6 +173,7 @@ class RRJob(object): # Environment # only used in RR 8.3 and newer rrEnvList = attr.ib(default=None, type=str) # type: str + rrEnvFile = attr.ib(default=None, type=str) # type: str class SubmitterParameter: From d97c35b75490c90cd15a6869c086e979269c5407 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 5 Apr 2024 17:30:19 +0200 Subject: [PATCH 41/42] Updates to pre load script Cannot figure out how to attach rrEnv file to existing job, submission already points to nonexistent file which gets filled by this cript. --- .../_prepost_scripts/ayon_inject_envvar.py | 85 ++++++++----------- 1 file changed, 34 insertions(+), 51 deletions(-) diff --git a/openpype/modules/royalrender/rr_root/render_apps/_prepost_scripts/ayon_inject_envvar.py b/openpype/modules/royalrender/rr_root/render_apps/_prepost_scripts/ayon_inject_envvar.py index ddd2977e193..9316410bc16 100644 --- a/openpype/modules/royalrender/rr_root/render_apps/_prepost_scripts/ayon_inject_envvar.py +++ b/openpype/modules/royalrender/rr_root/render_apps/_prepost_scripts/ayon_inject_envvar.py @@ -16,15 +16,18 @@ class InjectEnvironment: - """Creates rrEnv file and adds it to the rendered job. + """Creates rrEnv file. RR evnList has limitation on 2000 characters, which might not be enough. This script should be triggered by render jobs that were published from Ayon, it uses .json metadata to parse context and required Ayon launch environments to generate environment variable file for particular context. - This file is converted into rrEnv file and added to the job to be picked up - by RR. + This file is converted into rrEnv file. + + Render job already points to non-existent location which got filled only + by this process. (Couldn't figure way how to attach new file to existing + job.) Expected set environments on RR worker: - AYON_SERVER_URL @@ -49,11 +52,6 @@ def __init__(self): def inject(self): logs.append("InjectEnvironment starting") meta_dir = self._get_metadata_dir() - - metadata = self._get_metadata(meta_dir) - if not metadata: - logs.append("No metadata json found {}".format(meta_dir)) - return self.meta_dir = meta_dir envs = self._get_job_environments() @@ -74,26 +72,16 @@ def inject(self): rrEnv_path = self._create_rrEnv(meta_dir, extracted_env) - self._set_rrEnv_to_job(rrEnv_path) - logs.append("InjectEnvironment ending") + logs.append(f"InjectEnvironment ending, rrEnv file {rrEnv_path}") def _get_metadata_dir(self): + """Get folder where metadata.json and renders should be produced.""" job = self._get_job() image_dir = job.imageDir logs.append(f"_get_metadata_dir::{image_dir}") return image_dir - def _get_metadata(self, meta_dir): - search_frm = "{}{}*_metadata.json".format(meta_dir, os.path.sep) - metadata_files = glob.glob(search_frm) - if not metadata_files: - return {} - - logs.append(f"_get_metadata::{metadata_files[0]}") - with open(metadata_files[0]) as json_file: - return json.load(json_file) - def _is_required_environment(self): if (not os.environ.get("AYON_API_KEY") or not os.path.exists(os.environ.get("AYON_EXECUTABLE_PATH", ""))): @@ -124,7 +112,7 @@ def _get_job(self): tcp.setServer(rrGlobal.rrServer(), 7773) # tcp.setLogin(args.authStr, "") tcp.setLogin("", "") - + logs.append("jid:{}".format(int(args.jid))) if not tcp.jobList_GetInfo(int(args.jid)): print("Error jobList_GetInfo: " + tcp.errorMessage()) sys.exit() @@ -149,6 +137,28 @@ def _get_executable(self): # rr_python_utils.cache.get_rr_bin_folder() # TODO maybe useful return os.environ["AYON_EXECUTABLE_PATH"] + def _get_launch_environments(self): + """ Enhances environemnt with required for Ayon to be launched.""" + job_envs = self._get_job_environments() + ayon_environment = { + "AYON_SERVER_URL": os.environ["AYON_SERVER_URL"], + "AYON_API_KEY": os.environ["AYON_API_KEY"], + "AYON_BUNDLE_NAME": job_envs["AYON_BUNDLE_NAME"], + } + logs.append("Ayon launch environments:: {}".format(ayon_environment)) + environment = os.environ.copy() + environment.update(ayon_environment) + return environment + + def _get_export_url(self): + """Returns unique path with extracted env variables from Ayon.""" + temp_file_name = "{}_{}.json".format( + datetime.utcnow().strftime('%Y%m%d%H%M%S%f'), + str(uuid.uuid1()) + ) + export_url = os.path.join(tempfile.gettempdir(), temp_file_name) + return export_url + def _extract_environments(self, executable, context): # tempfile.TemporaryFile cannot be used because of locking export_url = self._get_export_url() @@ -180,28 +190,8 @@ def _extract_environments(self, executable, context): with open(export_url) as json_file: return json.load(json_file) - def _get_launch_environments(self): - """ Enhances environemnt with required for Ayon to be launched.""" - job_envs = self._get_job_environments() - ayon_environment = { - "AYON_SERVER_URL": os.environ["AYON_SERVER_URL"], - "AYON_API_KEY": os.environ["AYON_API_KEY"], - "AYON_BUNDLE_NAME": job_envs["AYON_BUNDLE_NAME"], - } - logs.append("Ayon launch environments:: {}".format(ayon_environment)) - environment = os.environ.copy() - environment.update(ayon_environment) - return environment - - def _get_export_url(self): - temp_file_name = "{}_{}.json".format( - datetime.utcnow().strftime('%Y%m%d%H%M%S%f'), - str(uuid.uuid1()) - ) - export_url = os.path.join(tempfile.gettempdir(), temp_file_name) - return export_url - def _create_rrEnv(self, meta_dir, extracted_env): + """Create rrEnv.rrEnv file in metadata folder that render job points""" filter_out = os.environ.get("AYON_FILTER_ENVIRONMENTS") filter_envs = set() if filter_out: @@ -221,22 +211,15 @@ def _create_rrEnv(self, meta_dir, extracted_env): return rrenv_path - def _set_rrEnv_to_job(self, rrEnv_path): - logs.append(f"_set_rrEnv_to_job::{rrEnv_path}") - job = self._get_job() - - job.customDataSet_Str("rrEnvFile", rrEnv_path) - rr.setJob(job) - if __name__ == "__main__": - try: injector = InjectEnvironment() injector.inject() except Exception as exp: logs.append(f"Error happened::{str(exp)}") - log_path = os.path.join(injector.meta_dir, "log.txt") + tmpdir = injector.meta_dir + log_path = os.path.join(tmpdir, "log.txt") with open(log_path, "a") as fp: fp.writelines(s.replace("\\r\\n", "\n") + '\n' for s in logs) From 57b1b1f01efbe1312b7e1e823747fc0a0ff3e37c Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Mon, 15 Apr 2024 15:39:21 +0200 Subject: [PATCH 42/42] Update openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py Co-authored-by: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> --- .../plugins/publish/create_publish_royalrender_job.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py b/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py index 2c5637a22df..b5390bff1f4 100644 --- a/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py +++ b/openpype/modules/royalrender/plugins/publish/create_publish_royalrender_job.py @@ -231,7 +231,7 @@ def get_job(self, instance, instances): ] job = RRJob( - Software="Ayon", + Software="AYON", Renderer="Once", SeqStart=1, SeqEnd=1,