From c51b33074314ca85d42cd87dfa118c170b840dd2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Baldo?= Date: Thu, 4 Jul 2024 18:23:55 -0300 Subject: [PATCH 1/2] Adjust paths for Python scripts inside the deepspeed-trainer container in ilab-training-launcher. --- training/ilab-wrapper/ilab-training-launcher | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/training/ilab-wrapper/ilab-training-launcher b/training/ilab-wrapper/ilab-training-launcher index 2c14262..fb40cd8 100755 --- a/training/ilab-wrapper/ilab-training-launcher +++ b/training/ilab-wrapper/ilab-training-launcher @@ -28,15 +28,15 @@ PODMAN_COMMAND=("podman" "run" "--device" "${CONTAINER_DEVICE}" \ "${CONTAINER_NAME}") # Convert ilab generate output to match SDG output format for train and test data mkdir -p ${SDG_OUTPUT_PATH}/training -"${PODMAN_COMMAND[@]}" bash -c "python /training/ilab_to_sdg.py \"${TRAINING_DATA_PATH}\" train \"${DATASET_NAME}\"; mv sdg_out.jsonl /instructlab/training/train.jsonl" -"${PODMAN_COMMAND[@]}" bash -c "python /training/ilab_to_sdg.py \"${TESTING_DATA_PATH}\" test \"${DATASET_NAME}\"; mv sdg_out.jsonl /instructlab/training/test.jsonl" +"${PODMAN_COMMAND[@]}" bash -c "python /training/src/instructlab/training/ilab_to_sdg.py \"${TRAINING_DATA_PATH}\" train \"${DATASET_NAME}\"; mv sdg_out.jsonl /instructlab/training/train.jsonl" +"${PODMAN_COMMAND[@]}" bash -c "python /training/src/instructlab/training/ilab_to_sdg.py \"${TESTING_DATA_PATH}\" test \"${DATASET_NAME}\"; mv sdg_out.jsonl /instructlab/training/test.jsonl" # Add curated subset of taxonomy "${PODMAN_COMMAND[@]}" bash -c "cat /training/sample-data/train_all_pruned_SDG.jsonl >> /instructlab/training/train.jsonl" # Pre-process generated data before training "${PODMAN_COMMAND[@]}" bash -c \ -"python data_process.py --logging_level INFO \ +"python /training/src/instructlab/training/data_process.py --logging_level INFO \ --data_path /instructlab/training/train.jsonl \ --data_output_path=/instructlab/training \ --max_seq_len 4096 \ From 79368f1c8327de9ca7ad680ed6089bf73eead720 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Baldo?= Date: Tue, 9 Jul 2024 16:04:26 -0300 Subject: [PATCH 2/2] There was one missing Python script path correction. --- training/ilab-wrapper/ilab-training-launcher | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/training/ilab-wrapper/ilab-training-launcher b/training/ilab-wrapper/ilab-training-launcher index fb40cd8..416698d 100755 --- a/training/ilab-wrapper/ilab-training-launcher +++ b/training/ilab-wrapper/ilab-training-launcher @@ -55,7 +55,7 @@ torchrun \ --node_rank 0 \ --nproc_per_node ${NPROC_PER_NODE} \ --rdzv_id 101 \ ---rdzv_endpoint 0.0.0.0:8888 /training/main_ds.py \ +--rdzv_endpoint 0.0.0.0:8888 /training/src/instructlab/training/main_ds.py \ --model_name_or_path /instructlab/models/ibm/granite-7b-base \ --data_path /instructlab/training/data.jsonl \ --output_dir="/instructlab/training_output" \