Support rclone for gptj

mlcommons · Feb 2, 2024 · c2cf8a7 · c2cf8a7
1 parent c4085bf
commit c2cf8a7
Showing 1 changed file with 18 additions and 2 deletions.
diff --git a/cm-mlops/script/get-ml-model-gptj/_cm.json b/cm-mlops/script/get-ml-model-gptj/_cm.json
@@ -25,7 +25,7 @@
       "tags": "download-and-extract",
       "update_tags_from_env_with_prefix": {
         "_url.": [
-          "CM_PACKAGE_URL"
+          "CM_DOWNLOAD_URL"
         ]
       },
       "enable_if_env": {
@@ -84,7 +84,9 @@
         "CM_DOWNLOAD_FILENAME": "checkpoint.zip",
         "CM_UNZIP": "yes",
         "CM_DOWNLOAD_CHECKSUM_NOT_USED": "e677e28aaf03da84584bb3073b7ee315",
-        "CM_PACKAGE_URL": "https://cloud.mlcommons.org/index.php/s/QAZ2oM94MkFtbQx/download"
+        "CM_PACKAGE_URL": "https://cloud.mlcommons.org/index.php/s/QAZ2oM94MkFtbQx/download",
+        "CM_RCLONE_CONFIG": "rclone config create mlc-inference s3 provider=LyveCloud access_key_id=0LITLNQMHZALM5AK secret_access_key=YQKYTMBY23TMZHLOYFJKL5CHHS0CWYUC endpoint=s3.us-east-1.lyvecloud.seagate.com",
+        "CM_RCLONE_URL": "mlc-inference:mlcommons-inference-wg-s3/gpt-j"
       },
       "add_deps_recursive": {
         "dae": {
@@ -182,6 +184,20 @@
         "dae": {
           "tags": "_wget"
         }
+      },
+      "env": {
+        "CM_DOWNLOAD_URL": "<<<CM_PACKAGE_URL>>>"
+      }
+    },
+    "rclone": {
+      "group": "download-tool",
+      "add_deps_recursive": {
+        "dae": {
+          "tags": "_rclone"
+        }
+      },
+      "env": {
+        "CM_DOWNLOAD_URL": "<<<CM_RCLONE_URL>>>"
       }
     },
     "pytorch,int8,intel": {