From 734ade3c6a603863fa2f5c9145693012c9cad04f Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Thu, 20 Apr 2023 03:57:48 +0300 Subject: [PATCH] index: add cache --- dvc/repo/index.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/dvc/repo/index.py b/dvc/repo/index.py index 1a3227c991c..963ced955f9 100644 --- a/dvc/repo/index.py +++ b/dvc/repo/index.py @@ -141,6 +141,7 @@ def _load_data_from_outs(index, prefix, outs): def _load_storage_from_out(storage_map, key, out): from dvc.config import NoRemoteError from dvc_data.index import FileStorage, ObjectStorage + from fsspec.utils import tokenize if out.odb: storage_map.add_data(ObjectStorage(key, out.odb)) @@ -156,6 +157,15 @@ def _load_storage_from_out(storage_map, key, out): index=remote.index, ) ) + + token = f"{out.fs.protocol}-{tokenize(out.fs_path)}" + storage_map.add_cache( + FileStorage( + key=key, + fs=out.cache.fs, + path=out.cache.fs.path.join(out.cache.path, "fs", token), + ) + ) else: storage_map.add_remote(ObjectStorage(key, remote.odb, index=remote.index)) except NoRemoteError: @@ -163,7 +173,16 @@ def _load_storage_from_out(storage_map, key, out): if out.stage.is_import: dep = out.stage.deps[0] - storage_map.add_data(FileStorage(key, dep.fs, dep.fs_path)) + storage_map.add_remote(FileStorage(key, dep.fs, dep.fs_path)) + + token = f"{dep.fs.protocol}-{tokenize(dep.fs_path)}" + storage_map.add_cache( + FileStorage( + key=key, + fs=out.cache.fs, + path=out.cache.fs.path.join(out.cache.path, "fs", token), + ) + ) class Index: