diff --git a/example_data/agents.zip b/example_data/agents.zip new file mode 100644 index 0000000..574874e Binary files /dev/null and b/example_data/agents.zip differ diff --git a/example_data/batch_creation/calls_batch.zip b/example_data/batch_creation/calls_batch.zip new file mode 100644 index 0000000..c0be7c0 Binary files /dev/null and b/example_data/batch_creation/calls_batch.zip differ diff --git a/example_data/clients.zip b/example_data/clients.zip new file mode 100644 index 0000000..decb897 Binary files /dev/null and b/example_data/clients.zip differ diff --git a/example_data/conversation_generation/conversations.zip b/example_data/conversation_generation/conversations.zip new file mode 100644 index 0000000..040b95c Binary files /dev/null and b/example_data/conversation_generation/conversations.zip differ diff --git a/example_data/conversation_generation/ground_truths.zip b/example_data/conversation_generation/ground_truths.zip new file mode 100644 index 0000000..6fc0c28 Binary files /dev/null and b/example_data/conversation_generation/ground_truths.zip differ diff --git a/example_data/conversation_generation/metadata.zip b/example_data/conversation_generation/metadata.zip new file mode 100644 index 0000000..6c34d4a Binary files /dev/null and b/example_data/conversation_generation/metadata.zip differ diff --git a/example_data/text_to_audio/audio_files.zip b/example_data/text_to_audio/audio_files.zip new file mode 100644 index 0000000..d72cbc5 Binary files /dev/null and b/example_data/text_to_audio/audio_files.zip differ diff --git a/example_data/text_to_audio/dataframe.zip b/example_data/text_to_audio/dataframe.zip new file mode 100644 index 0000000..56c58bc Binary files /dev/null and b/example_data/text_to_audio/dataframe.zip differ diff --git a/src/calls_generation/__init__.py b/src/calls_generation/__init__.py index 84f1919..e331086 100644 --- a/src/calls_generation/__init__.py +++ b/src/calls_generation/__init__.py @@ -11,3 +11,4 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from .skip import skip_and_import_local_data diff --git a/src/calls_generation/skip.py b/src/calls_generation/skip.py index 9836747..5954c50 100644 --- a/src/calls_generation/skip.py +++ b/src/calls_generation/skip.py @@ -23,7 +23,7 @@ import mlrun -def log_example(): +def skip_and_import_local_data(): """ This function logs example data to the database and to the project. Call this function from the notebook in order to skip the calls generation workflow. @@ -42,14 +42,12 @@ def log_example(): print("- Initialized tables") # load agent and client data: - # import artifacts from exported files and convert to data item: agents = project.import_artifact( - item_path=str(example_data_dir / "agents.yaml") + item_path=str(example_data_dir / "agents.zip"), ).to_dataitem() clients = project.import_artifact( - item_path=str(example_data_dir / "clients.yaml") + item_path=str(example_data_dir / "clients.zip"), ).to_dataitem() - # get data in bytes and load as list of dictionaries: agents = yaml.load(agents.get(), Loader=yaml.FullLoader) clients = yaml.load(clients.get(), Loader=yaml.FullLoader) @@ -63,9 +61,9 @@ def log_example(): ("text-to-audio", example_data_dir / "text_to_audio"), ("batch-creation", example_data_dir / "batch_creation"), ]: + print(f"- logging step {step_name}") _import_artifacts( project=project, - step_name=step_name, artifact_directory=artifact_directory, ) @@ -85,22 +83,21 @@ def _insert_agents_and_clients_to_db(agents: list, clients: list): sess.execute(insert(Client), clients) -def _import_artifacts(project, step_name: str, artifact_directory: Path): - print(f"- logging step {step_name}") - +def _import_artifacts(project, artifact_directory: Path): # iterate over artifacts and log them: for artifact_file in artifact_directory.iterdir(): if artifact_file.is_file(): - artifact_key = f"{step_name}_{artifact_file.stem}" artifact = project.import_artifact( item_path=str(artifact_file), - new_key=artifact_key, ) print(f" - artifact {artifact.key} imported") def save_current_example_data(): project = mlrun.get_current_project() + export_dir = Path("example_data") + if not export_dir.exists(): + export_dir.mkdir(parents=True, exist_ok=True) for artifact_name, target_path in [ ("client-data-generator_clients", "clients.yaml"), @@ -112,6 +109,9 @@ def save_current_example_data(): ("text-to-audio_audio_files_dataframe", "text_to_audio/dataframe.zip"), ("batch-creation_calls_batch", "batch_creation/calls_batch.zip"), ]: + export_path = export_dir / target_path + if not export_path.exists(): + export_path.parent.mkdir(parents=True, exist_ok=True) project.get_artifact(artifact_name).export(f"example_data/{target_path}") print(f"- exported {artifact_name} to {target_path}") print("*** all artifacts exported successfully ***")