don't embed video bytes

huggingface · Oct 28, 2024 · 9c1d3d4 · 9c1d3d4
1 parent d884ff8
commit 9c1d3d4
Show file tree

Hide file tree

Showing 14 changed files with 38 additions and 38 deletions.
diff --git a/front/admin_ui/poetry.lock b/front/admin_ui/poetry.lock
diff --git a/jobs/cache_maintenance/poetry.lock b/jobs/cache_maintenance/poetry.lock
diff --git a/jobs/mongodb_migration/poetry.lock b/jobs/mongodb_migration/poetry.lock
diff --git a/libs/libapi/poetry.lock b/libs/libapi/poetry.lock
diff --git a/libs/libcommon/poetry.lock b/libs/libcommon/poetry.lock
diff --git a/libs/libcommon/pyproject.toml b/libs/libcommon/pyproject.toml
@@ -9,7 +9,7 @@ license = "Apache-2.0"
 python = "3.9.18"
 appdirs = "^1.4.4"
 cryptography = "^43.0.1"
-datasets = {git = "https://github.com/huggingface/datasets.git", rev = "ff0149f3473078e5e287576a1c3a033081f5caa9", extras = ["audio", "vision"]}
+datasets = {git = "https://github.com/huggingface/datasets.git", rev = "f75f48926773f39798b85b6b5499e84c5e9e53b1", extras = ["audio", "vision"]}
 decord = "0.6.0"
 environs = "^9.5.0"
 fsspec = {version = "2024.3.1", extras = ["s3"]}

diff --git a/libs/libcommon/src/libcommon/constants.py b/libs/libcommon/src/libcommon/constants.py
@@ -33,7 +33,7 @@
 PROCESSING_STEP_CONFIG_PARQUET_AND_INFO_ROW_GROUP_SIZE_FOR_AUDIO_DATASETS = 100
 PROCESSING_STEP_CONFIG_PARQUET_AND_INFO_ROW_GROUP_SIZE_FOR_IMAGE_DATASETS = 100
 PROCESSING_STEP_CONFIG_PARQUET_AND_INFO_ROW_GROUP_SIZE_FOR_BINARY_DATASETS = 100
-PROCESSING_STEP_CONFIG_PARQUET_AND_INFO_ROW_GROUP_SIZE_FOR_VIDEO_DATASETS = 10
+PROCESSING_STEP_CONFIG_PARQUET_AND_INFO_ROW_GROUP_SIZE_FOR_VIDEO_DATASETS = 1000  # we store the URLs anyway, not the actual files
 PARQUET_REVISION = "refs/convert/parquet"
 
 TAG_NFAA_CONTENT = "not-for-all-audiences"

diff --git a/services/admin/poetry.lock b/services/admin/poetry.lock
diff --git a/services/api/poetry.lock b/services/api/poetry.lock
diff --git a/services/rows/poetry.lock b/services/rows/poetry.lock
diff --git a/services/search/poetry.lock b/services/search/poetry.lock
diff --git a/services/sse-api/poetry.lock b/services/sse-api/poetry.lock
diff --git a/services/webhook/poetry.lock b/services/webhook/poetry.lock
diff --git a/services/worker/poetry.lock b/services/worker/poetry.lock