From dd9f81dc37b1f01d286a61c164d05466cecb44da Mon Sep 17 00:00:00 2001 From: eikyuu Date: Wed, 11 Sep 2024 00:51:47 +0200 Subject: [PATCH 01/13] feat(trait): created EncryptedFilesystem trait holding all of EncryptedFs methods --- src/encryptedfs.rs | 69 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/src/encryptedfs.rs b/src/encryptedfs.rs index c491d69..c307cbf 100644 --- a/src/encryptedfs.rs +++ b/src/encryptedfs.rs @@ -9,6 +9,7 @@ use std::backtrace::Backtrace; use std::collections::{HashMap, HashSet, VecDeque}; use std::fmt::Debug; use std::fs::{DirEntry, File, OpenOptions, ReadDir}; +use std::future::Future; use std::io::{Read, Seek, SeekFrom, Write}; use std::num::{NonZeroUsize, ParseIntError}; use std::path::{Path, PathBuf}; @@ -576,6 +577,74 @@ pub struct EncryptedFs { read_only: bool, } +pub trait EncryptedFilesystem: Future> { + async fn new( + data_dir: PathBuf, + password_provider: Box, + cipher: Cipher, + read_only: bool, + ) -> FsResult>; + fn exists(&self, ino: u64) -> bool; + fn is_dir(&self, ino: u64) -> bool; + fn is_file(&self, ino: u64) -> bool; + async fn create( + &self, + parent: u64, + name: &SecretString, + create_attr: CreateFileAttr, + read: bool, + write: bool, + ) -> FsResult<(u64, FileAttr)>; + async fn find_by_name(&self, parent: u64, name: &SecretString) -> FsResult>; + fn len(&self, ino: u64) -> FsResult; + async fn remove_dir(&self, parent: u64, name: &SecretString) -> FsResult<()>; + async fn remove_file(&self, parent: u64, name: &SecretString) -> FsResult<()>; + fn exists_by_name(&self, parent: u64, name: &SecretString) -> FsResult; + async fn read_dir(&self, ino: u64) -> FsResult; + async fn read_dir_plus(&self, ino: u64) -> FsResult; + async fn get_attr(&self, ino: u64) -> FsResult; + async fn set_attr(&self, ino: u64, set_attr: SetFileAttr) -> FsResult<()>; + async fn read(&self, ino: u64, offset: u64, buf: &mut [u8], handle: u64) -> FsResult; + async fn release(&self, handle: u64) -> FsResult<()>; + async fn is_read_handle(&self, fh: u64) -> bool; + async fn is_write_handle(&self, fh: u64) -> bool; + async fn write(&self, ino: u64, offset: u64, buf: &[u8], handle: u64) -> FsResult; + async fn flush(&self, handle: u64) -> FsResult<()>; + async fn copy_file_range( + &self, + file_range_req: &CopyFileRangeReq, + size: usize, + ) -> FsResult; + async fn open(&self, ino: u64, read: bool, write: bool) -> FsResult; + async fn set_len(&self, ino: u64, size: u64) -> FsResult<()>; + async fn rename( + &self, + parent: u64, + name: &SecretString, + new_parent: u64, + new_name: &SecretString, + ) -> FsResult<()>; + async fn create_write( + &self, + file: W, + ) -> FsResult>; + async fn create_write_seek( + &self, + file: W, + ) -> FsResult>; + async fn create_read(&self, reader: R) -> FsResult>; + async fn create_read_seek( + &self, + reader: R, + ) -> FsResult>; + async fn passwd( + data_dir: &Path, + old_password: SecretString, + new_password: SecretString, + cipher: Cipher, + ) -> FsResult<()>; +} + impl EncryptedFs { #[allow(clippy::missing_panics_doc)] #[allow(clippy::missing_errors_doc)] From b3bc6655c534e7f3ba46a77b12bf005183dd7c24 Mon Sep 17 00:00:00 2001 From: eikyuu Date: Wed, 11 Sep 2024 22:49:26 +0200 Subject: [PATCH 02/13] fix(async-trait) --- src/encryptedfs.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/encryptedfs.rs b/src/encryptedfs.rs index c307cbf..121eea5 100644 --- a/src/encryptedfs.rs +++ b/src/encryptedfs.rs @@ -577,7 +577,8 @@ pub struct EncryptedFs { read_only: bool, } -pub trait EncryptedFilesystem: Future> { +#[async_trait] +pub trait EncryptedFilesystem { async fn new( data_dir: PathBuf, password_provider: Box, From d9fa4b6f338109b3a8711d11dbebeb87ee6784b4 Mon Sep 17 00:00:00 2001 From: eikyuu Date: Wed, 11 Sep 2024 23:29:40 +0200 Subject: [PATCH 03/13] refactor(trait implementation): moving trait methods outside of impl block for EncryptedFs into a trait implementation block --- src/encryptedfs.rs | 654 ++++++++++++++++++++++----------------------- 1 file changed, 322 insertions(+), 332 deletions(-) diff --git a/src/encryptedfs.rs b/src/encryptedfs.rs index 121eea5..d8f485b 100644 --- a/src/encryptedfs.rs +++ b/src/encryptedfs.rs @@ -646,10 +646,10 @@ pub trait EncryptedFilesystem { ) -> FsResult<()>; } -impl EncryptedFs { +impl EncryptedFilesystem for EncryptedFs { #[allow(clippy::missing_panics_doc)] #[allow(clippy::missing_errors_doc)] - pub async fn new( + async fn new( data_dir: PathBuf, password_provider: Box, cipher: Cipher, @@ -710,28 +710,23 @@ impl EncryptedFs { Ok(arc) } - pub fn exists(&self, ino: u64) -> bool { + fn exists(&self, ino: u64) -> bool { self.ino_file(ino).is_file() } - pub fn is_dir(&self, ino: u64) -> bool { + fn is_dir(&self, ino: u64) -> bool { self.contents_path(ino).is_dir() } - pub fn is_file(&self, ino: u64) -> bool { + fn is_file(&self, ino: u64) -> bool { self.contents_path(ino).is_file() } - #[allow(dead_code)] - async fn is_read_only(&self) -> bool { - self.read_only - } - /// Create a new node in the filesystem #[allow(clippy::missing_panics_doc)] #[allow(clippy::missing_errors_doc)] #[allow(clippy::too_many_lines)] - pub async fn create( + async fn create( &self, parent: u64, name: &SecretString, @@ -890,11 +885,7 @@ impl EncryptedFs { #[allow(clippy::missing_panics_doc)] #[allow(clippy::missing_errors_doc)] - pub async fn find_by_name( - &self, - parent: u64, - name: &SecretString, - ) -> FsResult> { + async fn find_by_name(&self, parent: u64, name: &SecretString) -> FsResult> { if !self.exists(parent) { return Err(FsError::InodeNotFound); } @@ -923,7 +914,7 @@ impl EncryptedFs { /// Count children of a directory. This **EXCLUDES** "." and "..". #[allow(clippy::missing_errors_doc)] - pub fn len(&self, ino: u64) -> FsResult { + fn len(&self, ino: u64) -> FsResult { if !self.is_dir(ino) { return Err(FsError::InvalidInodeType); } @@ -941,7 +932,7 @@ impl EncryptedFs { /// Delete a directory #[allow(clippy::missing_panics_doc)] #[allow(clippy::missing_errors_doc)] - pub async fn remove_dir(&self, parent: u64, name: &SecretString) -> FsResult<()> { + async fn remove_dir(&self, parent: u64, name: &SecretString) -> FsResult<()> { if !self.is_dir(parent) { return Err(FsError::InvalidInodeType); } @@ -1018,7 +1009,7 @@ impl EncryptedFs { /// Delete a file #[allow(clippy::missing_panics_doc)] #[allow(clippy::missing_errors_doc)] - pub async fn remove_file(&self, parent: u64, name: &SecretString) -> FsResult<()> { + async fn remove_file(&self, parent: u64, name: &SecretString) -> FsResult<()> { if !self.is_dir(parent) { return Err(FsError::InvalidInodeType); } @@ -1089,7 +1080,7 @@ impl EncryptedFs { #[allow(clippy::missing_panics_doc)] #[allow(clippy::missing_errors_doc)] - pub fn exists_by_name(&self, parent: u64, name: &SecretString) -> FsResult { + fn exists_by_name(&self, parent: u64, name: &SecretString) -> FsResult { if !self.exists(parent) { return Err(FsError::InodeNotFound); } @@ -1102,7 +1093,7 @@ impl EncryptedFs { } #[allow(clippy::missing_errors_doc)] - pub async fn read_dir(&self, ino: u64) -> FsResult { + async fn read_dir(&self, ino: u64) -> FsResult { if !self.is_dir(ino) { return Err(FsError::InvalidInodeType); } @@ -1118,7 +1109,7 @@ impl EncryptedFs { } /// Like [`EncryptedFs::read_dir`] but with [`FileAttr`] so we don't need to query again for those. - pub async fn read_dir_plus(&self, ino: u64) -> FsResult { + async fn read_dir_plus(&self, ino: u64) -> FsResult { if !self.is_dir(ino) { return Err(FsError::InvalidInodeType); } @@ -1133,203 +1124,9 @@ impl EncryptedFs { Ok(self.create_directory_entry_plus_iterator(iter).await) } - async fn create_directory_entry_plus( - &self, - entry: io::Result, - ) -> FsResult { - let entry = self.create_directory_entry(entry).await?; - let lock = self.serialize_inode_locks.clone(); - let lock_ino = lock.get_or_insert_with(entry.ino, || RwLock::new(false)); - let _ino_guard = lock_ino.read(); - let attr = self.get_inode_from_cache_or_storage(entry.ino).await?; - Ok(DirectoryEntryPlus { - ino: entry.ino, - name: entry.name, - kind: entry.kind, - attr, - }) - } - - async fn create_directory_entry_plus_iterator( - &self, - read_dir: ReadDir, - ) -> DirectoryEntryPlusIterator { - #[allow(clippy::cast_possible_truncation)] - let futures: Vec<_> = read_dir - .into_iter() - .map(|entry| { - let fs = { - self.self_weak - .lock() - .unwrap() - .as_ref() - .unwrap() - .upgrade() - .unwrap() - }; - DIR_ENTRIES_RT.spawn(async move { fs.create_directory_entry_plus(entry).await }) - }) - .collect(); - - // do these futures in parallel and return them - let mut res = VecDeque::with_capacity(futures.len()); - for f in futures { - res.push_back(f.await.unwrap()); - } - DirectoryEntryPlusIterator(res) - } - - async fn create_directory_entry( - &self, - entry: io::Result, - ) -> FsResult { - if entry.is_err() { - return Err(entry.err().unwrap().into()); - } - if let Err(e) = entry { - error!(err = %e, "reading directory entry"); - return Err(e.into()); - } - let entry = entry.unwrap(); - let name = entry.file_name().to_string_lossy().to_string(); - let name = { - if name == "$." { - SecretString::from_str(".").unwrap() - } else if name == "$.." { - SecretString::from_str("..").unwrap() - } else { - // try from cache - let lock = self.get_dir_entries_name_cache().await?; - let mut cache = lock.lock().await; - if let Some(name_cached) = cache.get(&name).cloned() { - name_cached - } else { - drop(cache); - if let Ok(decrypted_name) = - crypto::decrypt_file_name(&name, self.cipher, &*self.key.get().await?) - .map_err(|err| { - error!(err = %err, "decrypting file name"); - err - }) - { - lock.lock().await.put(name.clone(), decrypted_name.clone()); - decrypted_name - } else { - return Err(FsError::InvalidInput("invalid file name")); - } - } - } - }; - let file_path = entry.path().to_str().unwrap().to_string(); - // try from cache - let lock = self.dir_entries_meta_cache.get().await?; - let mut cache = lock.lock().await; - if let Some((ino, kind)) = cache.get(&file_path) { - return Ok(DirectoryEntry { - ino: *ino, - name, - kind: *kind, - }); - } - drop(cache); - let lock = self - .serialize_dir_entries_ls_locks - .get_or_insert_with(file_path.clone(), || RwLock::new(false)); - let guard = lock.read().await; - let file = File::open(entry.path())?; - let res: bincode::Result<(u64, FileType)> = bincode::deserialize_from(crypto::create_read( - file, - self.cipher, - &*self.key.get().await?, - )); - drop(guard); - if let Err(e) = res { - error!(err = %e, "deserializing directory entry"); - return Err(e.into()); - } - let (ino, kind): (u64, FileType) = res.unwrap(); - // add to cache - self.dir_entries_meta_cache - .get() - .await? - .lock() - .await - .put(file_path, (ino, kind)); - Ok(DirectoryEntry { ino, name, kind }) - } - - async fn get_dir_entries_name_cache( - &self, - ) -> FsResult>>> { - self.dir_entries_name_cache.get().await - } - - async fn create_directory_entry_iterator(&self, read_dir: ReadDir) -> DirectoryEntryIterator { - #[allow(clippy::cast_possible_truncation)] - let futures: Vec<_> = read_dir - .into_iter() - .map(|entry| { - let fs = { - self.self_weak - .lock() - .unwrap() - .as_ref() - .unwrap() - .upgrade() - .unwrap() - }; - DIR_ENTRIES_RT.spawn(async move { fs.create_directory_entry(entry).await }) - }) - .collect(); - - // do these futures in parallel and return them - let mut res = VecDeque::with_capacity(futures.len()); - for f in futures { - res.push_back(f.await.unwrap()); - } - DirectoryEntryIterator(res) - } - - #[allow(clippy::missing_errors_doc)] - async fn get_inode_from_storage(&self, ino: u64) -> FsResult { - let lock = self - .serialize_inode_locks - .get_or_insert_with(ino, || RwLock::new(false)); - let _guard = lock.read(); - - let path = self.ino_file(ino); - if !path.is_file() { - return Err(FsError::InodeNotFound); - } - let file = OpenOptions::new().read(true).open(path).map_err(|err| { - error!(err = %err, "opening file"); - FsError::InodeNotFound - })?; - Ok(bincode::deserialize_from(crypto::create_read( - file, - self.cipher, - &*self.key.get().await?, - ))?) - } - - async fn get_inode_from_cache_or_storage(&self, ino: u64) -> FsResult { - let lock = self.attr_cache.get().await?; - let mut guard = lock.write().await; - let attr = guard.get(&ino); - if let Some(attr) = attr { - Ok(*attr) - } else { - drop(guard); - let attr = self.get_inode_from_storage(ino).await?; - let mut guard = lock.write().await; - guard.put(ino, attr); - Ok(attr) - } - } - /// Get metadata #[allow(clippy::missing_errors_doc)] - pub async fn get_attr(&self, ino: u64) -> FsResult { + async fn get_attr(&self, ino: u64) -> FsResult { let mut attr = self.get_inode_from_cache_or_storage(ino).await?; // merge time info with any open read handles @@ -1364,56 +1161,13 @@ impl EncryptedFs { } /// Set metadata - pub async fn set_attr(&self, ino: u64, set_attr: SetFileAttr) -> FsResult<()> { + async fn set_attr(&self, ino: u64, set_attr: SetFileAttr) -> FsResult<()> { if self.read_only { return Err(FsError::ReadOnly); } self.set_attr2(ino, set_attr, false).await } - async fn set_attr2( - &self, - ino: u64, - set_attr: SetFileAttr, - overwrite_size: bool, - ) -> FsResult<()> { - let serialize_update_lock = self - .serialize_update_inode_locks - .get_or_insert_with(ino, || Mutex::new(false)); - let _serialize_update_guard = serialize_update_lock.lock().await; - - let mut attr = self.get_attr(ino).await?; - merge_attr(&mut attr, &set_attr, overwrite_size); - let now = SystemTime::now(); - attr.ctime = now; - attr.atime = now; - - self.write_inode_to_storage(&attr).await?; - - Ok(()) - } - - async fn write_inode_to_storage(&self, attr: &FileAttr) -> Result<(), FsError> { - let lock = self - .serialize_inode_locks - .get_or_insert_with(attr.ino, || RwLock::new(false)); - let guard = lock.write().await; - crypto::atomic_serialize_encrypt_into( - &self.ino_file(attr.ino), - attr, - self.cipher, - &*self.key.get().await?, - )?; - drop(guard); - // update cache also - { - let lock = self.attr_cache.get().await?; - let mut guard = lock.write().await; - guard.put(attr.ino, *attr); - } - Ok(()) - } - /// Read the contents from an `offset`. /// /// If we try to read outside of file size, we return zero bytes. @@ -1421,13 +1175,7 @@ impl EncryptedFs { #[instrument(skip(self, buf), fields(len = %buf.len()), ret(level = Level::DEBUG))] #[allow(clippy::missing_errors_doc)] #[allow(clippy::cast_possible_truncation)] - pub async fn read( - &self, - ino: u64, - offset: u64, - buf: &mut [u8], - handle: u64, - ) -> FsResult { + async fn read(&self, ino: u64, offset: u64, buf: &mut [u8], handle: u64) -> FsResult { if !self.exists(ino) { return Err(FsError::InodeNotFound); } @@ -1528,7 +1276,7 @@ impl EncryptedFs { #[allow(clippy::missing_panics_doc)] #[allow(clippy::too_many_lines)] - pub async fn release(&self, handle: u64) -> FsResult<()> { + async fn release(&self, handle: u64) -> FsResult<()> { if handle == 0 { // in the case of directory or if the file was crated // without being opened we don't use a handle @@ -1639,12 +1387,12 @@ impl EncryptedFs { } /// Check if a file is opened for reading with this handle. - pub async fn is_read_handle(&self, fh: u64) -> bool { + async fn is_read_handle(&self, fh: u64) -> bool { self.read_handles.read().await.contains_key(&fh) } /// Check if a file is opened for writing with this handle. - pub async fn is_write_handle(&self, fh: u64) -> bool { + async fn is_write_handle(&self, fh: u64) -> bool { self.write_handles.read().await.contains_key(&fh) } @@ -1654,7 +1402,7 @@ impl EncryptedFs { /// If the file is not opened for writing, /// it will return an error of type [FsError::InvalidFileHandle]. #[instrument(skip(self, buf), fields(len = %buf.len()), ret(level = Level::DEBUG))] - pub async fn write(&self, ino: u64, offset: u64, buf: &[u8], handle: u64) -> FsResult { + async fn write(&self, ino: u64, offset: u64, buf: &[u8], handle: u64) -> FsResult { if self.read_only { return Err(FsError::ReadOnly); } @@ -1762,7 +1510,7 @@ impl EncryptedFs { /// Flush the data to the underlying storage. #[allow(clippy::missing_panics_doc)] - pub async fn flush(&self, handle: u64) -> FsResult<()> { + async fn flush(&self, handle: u64) -> FsResult<()> { if handle == 0 { // in the case of directory or if the file was crated without being opened we don't use a handle return Ok(()); @@ -1797,7 +1545,7 @@ impl EncryptedFs { } /// Helpful when we want to copy just some portions of the file. - pub async fn copy_file_range( + async fn copy_file_range( &self, file_range_req: &CopyFileRangeReq, size: usize, @@ -1842,7 +1590,7 @@ impl EncryptedFs { /// Open a file. We can open multiple times for read but only one to write at a time. #[allow(clippy::missing_panics_doc)] - pub async fn open(&self, ino: u64, read: bool, write: bool) -> FsResult { + async fn open(&self, ino: u64, read: bool, write: bool) -> FsResult { if write && self.read_only { return Err(FsError::ReadOnly); } @@ -1909,7 +1657,7 @@ impl EncryptedFs { /// Truncates or extends the underlying file, updating the size of this file to become size. #[allow(clippy::missing_panics_doc)] #[allow(clippy::too_many_lines)] - pub async fn set_len(&self, ino: u64, size: u64) -> FsResult<()> { + async fn set_len(&self, ino: u64, size: u64) -> FsResult<()> { if self.read_only { return Err(FsError::ReadOnly); } @@ -1995,55 +1743,8 @@ impl EncryptedFs { Ok(()) } - /// This will write any dirty data to the file from all writers and reset them. - /// Timestamps and size will be updated to the storage. - /// > ⚠️ **Warning** - /// > Need to be called in a context with write lock on `self.read_write_inode.lock().await.get(ino)`. - /// > That is because we want to make sure caller is holding a lock while all writers flush and we can't - /// > lock here also as we would end-up in a deadlock. - async fn flush_and_reset_writers(&self, ino: u64) -> FsResult<()> { - if self.read_only { - return Err(FsError::ReadOnly); - } - let opened_files_for_write_guard = self.opened_files_for_write.read().await; - let handle = opened_files_for_write_guard.get(&ino); - if let Some(handle) = handle { - let write_handles_guard = self.write_handles.write().await; - let ctx = write_handles_guard.get(handle); - if let Some(lock) = ctx { - let mut ctx = lock.lock().await; - - let mut writer = ctx.writer.take().unwrap(); - let file = writer.finish()?; - file.sync_all()?; - File::open(self.contents_path(ctx.ino).parent().unwrap())?.sync_all()?; - let handle = *handle; - let set_attr: SetFileAttr = ctx.attr.clone().into(); - drop(ctx); - drop(opened_files_for_write_guard); - drop(write_handles_guard); - self.set_attr(ino, set_attr).await?; - self.reset_handles(ino, Some(handle), true).await?; - let write_handles_guard = self.write_handles.write().await; - let mut ctx = write_handles_guard.get(&handle).unwrap().lock().await; - let writer = self - .create_write_seek( - OpenOptions::new() - .read(true) - .write(true) - .open(self.contents_path(ino))?, - ) - .await?; - ctx.writer = Some(Box::new(writer)); - let attr = self.get_inode_from_storage(ino).await?; - ctx.attr = attr.into(); - } - } - Ok(()) - } - #[allow(clippy::missing_panics_doc)] - pub async fn rename( + async fn rename( &self, parent: u64, name: &SecretString, @@ -2135,7 +1836,7 @@ impl EncryptedFs { } /// Create a crypto writer using internal encryption info. - pub async fn create_write( + async fn create_write( &self, file: W, ) -> FsResult> { @@ -2147,7 +1848,7 @@ impl EncryptedFs { } /// Create a crypto writer with seek using internal encryption info. - pub async fn create_write_seek( + async fn create_write_seek( &self, file: W, ) -> FsResult> { @@ -2159,10 +1860,7 @@ impl EncryptedFs { } /// Create a crypto reader using internal encryption info. - pub async fn create_read( - &self, - reader: R, - ) -> FsResult> { + async fn create_read(&self, reader: R) -> FsResult> { Ok(crypto::create_read( reader, self.cipher, @@ -2171,7 +1869,7 @@ impl EncryptedFs { } /// Create a crypto reader with seek using internal encryption info. - pub async fn create_read_seek( + async fn create_read_seek( &self, reader: R, ) -> FsResult> { @@ -2183,7 +1881,7 @@ impl EncryptedFs { } /// Change the password of the filesystem used to access the encryption key. - pub async fn passwd( + async fn passwd( data_dir: &Path, old_password: SecretString, new_password: SecretString, @@ -2210,6 +1908,297 @@ impl EncryptedFs { )?; Ok(()) } +} + +impl EncryptedFs { + #[allow(dead_code)] + async fn is_read_only(&self) -> bool { + self.read_only + } + + async fn create_directory_entry_plus( + &self, + entry: io::Result, + ) -> FsResult { + let entry = self.create_directory_entry(entry).await?; + let lock = self.serialize_inode_locks.clone(); + let lock_ino = lock.get_or_insert_with(entry.ino, || RwLock::new(false)); + let _ino_guard = lock_ino.read(); + let attr = self.get_inode_from_cache_or_storage(entry.ino).await?; + Ok(DirectoryEntryPlus { + ino: entry.ino, + name: entry.name, + kind: entry.kind, + attr, + }) + } + + async fn create_directory_entry_plus_iterator( + &self, + read_dir: ReadDir, + ) -> DirectoryEntryPlusIterator { + #[allow(clippy::cast_possible_truncation)] + let futures: Vec<_> = read_dir + .into_iter() + .map(|entry| { + let fs = { + self.self_weak + .lock() + .unwrap() + .as_ref() + .unwrap() + .upgrade() + .unwrap() + }; + DIR_ENTRIES_RT.spawn(async move { fs.create_directory_entry_plus(entry).await }) + }) + .collect(); + + // do these futures in parallel and return them + let mut res = VecDeque::with_capacity(futures.len()); + for f in futures { + res.push_back(f.await.unwrap()); + } + DirectoryEntryPlusIterator(res) + } + + async fn create_directory_entry( + &self, + entry: io::Result, + ) -> FsResult { + if entry.is_err() { + return Err(entry.err().unwrap().into()); + } + if let Err(e) = entry { + error!(err = %e, "reading directory entry"); + return Err(e.into()); + } + let entry = entry.unwrap(); + let name = entry.file_name().to_string_lossy().to_string(); + let name = { + if name == "$." { + SecretString::from_str(".").unwrap() + } else if name == "$.." { + SecretString::from_str("..").unwrap() + } else { + // try from cache + let lock = self.get_dir_entries_name_cache().await?; + let mut cache = lock.lock().await; + if let Some(name_cached) = cache.get(&name).cloned() { + name_cached + } else { + drop(cache); + if let Ok(decrypted_name) = + crypto::decrypt_file_name(&name, self.cipher, &*self.key.get().await?) + .map_err(|err| { + error!(err = %err, "decrypting file name"); + err + }) + { + lock.lock().await.put(name.clone(), decrypted_name.clone()); + decrypted_name + } else { + return Err(FsError::InvalidInput("invalid file name")); + } + } + } + }; + let file_path = entry.path().to_str().unwrap().to_string(); + // try from cache + let lock = self.dir_entries_meta_cache.get().await?; + let mut cache = lock.lock().await; + if let Some((ino, kind)) = cache.get(&file_path) { + return Ok(DirectoryEntry { + ino: *ino, + name, + kind: *kind, + }); + } + drop(cache); + let lock = self + .serialize_dir_entries_ls_locks + .get_or_insert_with(file_path.clone(), || RwLock::new(false)); + let guard = lock.read().await; + let file = File::open(entry.path())?; + let res: bincode::Result<(u64, FileType)> = bincode::deserialize_from(crypto::create_read( + file, + self.cipher, + &*self.key.get().await?, + )); + drop(guard); + if let Err(e) = res { + error!(err = %e, "deserializing directory entry"); + return Err(e.into()); + } + let (ino, kind): (u64, FileType) = res.unwrap(); + // add to cache + self.dir_entries_meta_cache + .get() + .await? + .lock() + .await + .put(file_path, (ino, kind)); + Ok(DirectoryEntry { ino, name, kind }) + } + + async fn get_dir_entries_name_cache( + &self, + ) -> FsResult>>> { + self.dir_entries_name_cache.get().await + } + + async fn create_directory_entry_iterator(&self, read_dir: ReadDir) -> DirectoryEntryIterator { + #[allow(clippy::cast_possible_truncation)] + let futures: Vec<_> = read_dir + .into_iter() + .map(|entry| { + let fs = { + self.self_weak + .lock() + .unwrap() + .as_ref() + .unwrap() + .upgrade() + .unwrap() + }; + DIR_ENTRIES_RT.spawn(async move { fs.create_directory_entry(entry).await }) + }) + .collect(); + + // do these futures in parallel and return them + let mut res = VecDeque::with_capacity(futures.len()); + for f in futures { + res.push_back(f.await.unwrap()); + } + DirectoryEntryIterator(res) + } + + #[allow(clippy::missing_errors_doc)] + async fn get_inode_from_storage(&self, ino: u64) -> FsResult { + let lock = self + .serialize_inode_locks + .get_or_insert_with(ino, || RwLock::new(false)); + let _guard = lock.read(); + + let path = self.ino_file(ino); + if !path.is_file() { + return Err(FsError::InodeNotFound); + } + let file = OpenOptions::new().read(true).open(path).map_err(|err| { + error!(err = %err, "opening file"); + FsError::InodeNotFound + })?; + Ok(bincode::deserialize_from(crypto::create_read( + file, + self.cipher, + &*self.key.get().await?, + ))?) + } + + async fn get_inode_from_cache_or_storage(&self, ino: u64) -> FsResult { + let lock = self.attr_cache.get().await?; + let mut guard = lock.write().await; + let attr = guard.get(&ino); + if let Some(attr) = attr { + Ok(*attr) + } else { + drop(guard); + let attr = self.get_inode_from_storage(ino).await?; + let mut guard = lock.write().await; + guard.put(ino, attr); + Ok(attr) + } + } + + async fn set_attr2( + &self, + ino: u64, + set_attr: SetFileAttr, + overwrite_size: bool, + ) -> FsResult<()> { + let serialize_update_lock = self + .serialize_update_inode_locks + .get_or_insert_with(ino, || Mutex::new(false)); + let _serialize_update_guard = serialize_update_lock.lock().await; + + let mut attr = self.get_attr(ino).await?; + merge_attr(&mut attr, &set_attr, overwrite_size); + let now = SystemTime::now(); + attr.ctime = now; + attr.atime = now; + + self.write_inode_to_storage(&attr).await?; + + Ok(()) + } + + async fn write_inode_to_storage(&self, attr: &FileAttr) -> Result<(), FsError> { + let lock = self + .serialize_inode_locks + .get_or_insert_with(attr.ino, || RwLock::new(false)); + let guard = lock.write().await; + crypto::atomic_serialize_encrypt_into( + &self.ino_file(attr.ino), + attr, + self.cipher, + &*self.key.get().await?, + )?; + drop(guard); + // update cache also + { + let lock = self.attr_cache.get().await?; + let mut guard = lock.write().await; + guard.put(attr.ino, *attr); + } + Ok(()) + } + + /// This will write any dirty data to the file from all writers and reset them. + /// Timestamps and size will be updated to the storage. + /// > ⚠️ **Warning** + /// > Need to be called in a context with write lock on `self.read_write_inode.lock().await.get(ino)`. + /// > That is because we want to make sure caller is holding a lock while all writers flush and we can't + /// > lock here also as we would end-up in a deadlock. + async fn flush_and_reset_writers(&self, ino: u64) -> FsResult<()> { + if self.read_only { + return Err(FsError::ReadOnly); + } + let opened_files_for_write_guard = self.opened_files_for_write.read().await; + let handle = opened_files_for_write_guard.get(&ino); + if let Some(handle) = handle { + let write_handles_guard = self.write_handles.write().await; + let ctx = write_handles_guard.get(handle); + if let Some(lock) = ctx { + let mut ctx = lock.lock().await; + + let mut writer = ctx.writer.take().unwrap(); + let file = writer.finish()?; + file.sync_all()?; + File::open(self.contents_path(ctx.ino).parent().unwrap())?.sync_all()?; + let handle = *handle; + let set_attr: SetFileAttr = ctx.attr.clone().into(); + drop(ctx); + drop(opened_files_for_write_guard); + drop(write_handles_guard); + self.set_attr(ino, set_attr).await?; + self.reset_handles(ino, Some(handle), true).await?; + let write_handles_guard = self.write_handles.write().await; + let mut ctx = write_handles_guard.get(&handle).unwrap().lock().await; + let writer = self + .create_write_seek( + OpenOptions::new() + .read(true) + .write(true) + .open(self.contents_path(ino))?, + ) + .await?; + ctx.writer = Some(Box::new(writer)); + let attr = self.get_inode_from_storage(ino).await?; + ctx.attr = attr.into(); + } + } + Ok(()) + } fn next_handle(&self) -> u64 { self.current_handle @@ -2519,6 +2508,7 @@ impl EncryptedFs { } } } + pub struct CopyFileRangeReq { src_ino: u64, src_offset: u64, From 8d7050ac8bc3456c53551871aaafcfa9560be2cb Mon Sep 17 00:00:00 2001 From: eikyuu Date: Wed, 11 Sep 2024 23:38:15 +0200 Subject: [PATCH 04/13] fix(import): importing EncryptedFilesystem trait to solve E0599 --- src/mount/linux.rs | 1 + src/test_common.rs | 1 + 2 files changed, 2 insertions(+) diff --git a/src/mount/linux.rs b/src/mount/linux.rs index 3b4a834..7f1d8e1 100644 --- a/src/mount/linux.rs +++ b/src/mount/linux.rs @@ -30,6 +30,7 @@ use tracing::{debug, error, instrument, trace, warn}; use tracing::{info, Level}; use crate::crypto::Cipher; +use crate::encryptedfs::EncryptedFilesystem; use crate::encryptedfs::{ CopyFileRangeReq, CreateFileAttr, EncryptedFs, FileAttr, FileType, FsError, FsResult, PasswordProvider, SetFileAttr, diff --git a/src/test_common.rs b/src/test_common.rs index bbe2de4..514d0e6 100644 --- a/src/test_common.rs +++ b/src/test_common.rs @@ -11,6 +11,7 @@ use thread_local::ThreadLocal; use tokio::sync::Mutex; use crate::crypto::Cipher; +use crate::encryptedfs::EncryptedFilesystem; use crate::encryptedfs::{ CopyFileRangeReq, CreateFileAttr, EncryptedFs, FileType, PasswordProvider, }; From ad1fdad133254d9aa75d5fc2258773155ce26e09 Mon Sep 17 00:00:00 2001 From: eikyuu Date: Thu, 12 Sep 2024 00:56:14 +0200 Subject: [PATCH 05/13] fix(asyn-trait): adding async_trait to impl/for block --- src/encryptedfs.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/encryptedfs.rs b/src/encryptedfs.rs index d8f485b..671d1db 100644 --- a/src/encryptedfs.rs +++ b/src/encryptedfs.rs @@ -646,6 +646,7 @@ pub trait EncryptedFilesystem { ) -> FsResult<()>; } +#[async_trait] impl EncryptedFilesystem for EncryptedFs { #[allow(clippy::missing_panics_doc)] #[allow(clippy::missing_errors_doc)] From 65ed8dcc8c346149ac0e7aa8d20393c7cef8debf Mon Sep 17 00:00:00 2001 From: eikyuu Date: Fri, 13 Sep 2024 00:12:28 +0200 Subject: [PATCH 06/13] fix(dyn in FsResult): moving methods to trait implementation, impl in method return (as in FsResult) is not authorized anymore. Change is to move to Box. This change implied small changes in methods implementation as well as in other methods using the impacted methods. --- src/encryptedfs.rs | 48 ++++++++++++++++++++++++++-------------------- 1 file changed, 27 insertions(+), 21 deletions(-) diff --git a/src/encryptedfs.rs b/src/encryptedfs.rs index 671d1db..eb557ba 100644 --- a/src/encryptedfs.rs +++ b/src/encryptedfs.rs @@ -628,16 +628,19 @@ pub trait EncryptedFilesystem { async fn create_write( &self, file: W, - ) -> FsResult>; + ) -> FsResult>>; async fn create_write_seek( &self, file: W, - ) -> FsResult>; - async fn create_read(&self, reader: R) -> FsResult>; + ) -> FsResult>>; + async fn create_read( + &self, + reader: R, + ) -> FsResult>>; async fn create_read_seek( &self, reader: R, - ) -> FsResult>; + ) -> FsResult>>; async fn passwd( data_dir: &Path, old_password: SecretString, @@ -1840,45 +1843,48 @@ impl EncryptedFilesystem for EncryptedFs { async fn create_write( &self, file: W, - ) -> FsResult> { - Ok(crypto::create_write( + ) -> FsResult>> { + Ok(Box::new(crypto::create_write( file, self.cipher, &*self.key.get().await?, - )) + ))) } /// Create a crypto writer with seek using internal encryption info. async fn create_write_seek( &self, file: W, - ) -> FsResult> { - Ok(crypto::create_write_seek( + ) -> FsResult>> { + Ok(Box::new(crypto::create_write_seek( file, self.cipher, &*self.key.get().await?, - )) + ))) } /// Create a crypto reader using internal encryption info. - async fn create_read(&self, reader: R) -> FsResult> { - Ok(crypto::create_read( + async fn create_read( + &self, + reader: R, + ) -> FsResult>> { + Ok(Box::new(crypto::create_read( reader, self.cipher, &*self.key.get().await?, - )) + ))) } /// Create a crypto reader with seek using internal encryption info. async fn create_read_seek( &self, reader: R, - ) -> FsResult> { - Ok(crypto::create_read_seek( + ) -> FsResult>> { + Ok(Box::new(crypto::create_read_seek( reader, self.cipher, &*self.key.get().await?, - )) + ))) } /// Change the password of the filesystem used to access the encryption key. @@ -2193,7 +2199,7 @@ impl EncryptedFs { .open(self.contents_path(ino))?, ) .await?; - ctx.writer = Some(Box::new(writer)); + ctx.writer = Some(writer); let attr = self.get_inode_from_storage(ino).await?; ctx.attr = attr.into(); } @@ -2237,7 +2243,7 @@ impl EncryptedFs { let attr = self.get_inode_from_storage(ino).await?; let mut ctx = guard.get(handle).unwrap().lock().await; let reader = self.create_read_seek(File::open(&path)?).await?; - ctx.reader = Some(Box::new(reader)); + ctx.reader = Some(reader); ctx.attr = attr.into(); } } @@ -2270,7 +2276,7 @@ impl EncryptedFs { .create_write_seek(OpenOptions::new().read(true).write(true).open(&path)?) .await?; let mut ctx = lock.lock().await; - ctx.writer = Some(Box::new(writer)); + ctx.writer = Some(writer); let attr = self.get_inode_from_storage(ino).await?; ctx.attr = attr.into(); } @@ -2294,7 +2300,7 @@ impl EncryptedFs { let ctx = ReadHandleContext { ino, attr, - reader: Some(Box::new(reader)), + reader: Some(reader), }; self.read_handles .write() @@ -2327,7 +2333,7 @@ impl EncryptedFs { let ctx = WriteHandleContext { ino, attr, - writer: Some(Box::new(writer)), + writer: Some(writer), }; self.write_handles .write() From 7438129d366a17562e54032f6b139f3300a8a97c Mon Sep 17 00:00:00 2001 From: eikyuu Date: Fri, 13 Sep 2024 00:16:20 +0200 Subject: [PATCH 07/13] fix(lifetime): adding 'static lifetime to solve compiler error --- src/encryptedfs.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/encryptedfs.rs b/src/encryptedfs.rs index eb557ba..5fac26e 100644 --- a/src/encryptedfs.rs +++ b/src/encryptedfs.rs @@ -633,11 +633,11 @@ pub trait EncryptedFilesystem { &self, file: W, ) -> FsResult>>; - async fn create_read( + async fn create_read( &self, reader: R, ) -> FsResult>>; - async fn create_read_seek( + async fn create_read_seek( &self, reader: R, ) -> FsResult>>; @@ -1864,7 +1864,7 @@ impl EncryptedFilesystem for EncryptedFs { } /// Create a crypto reader using internal encryption info. - async fn create_read( + async fn create_read( &self, reader: R, ) -> FsResult>> { @@ -1876,7 +1876,7 @@ impl EncryptedFilesystem for EncryptedFs { } /// Create a crypto reader with seek using internal encryption info. - async fn create_read_seek( + async fn create_read_seek( &self, reader: R, ) -> FsResult>> { From 7ca5fa489a400909e46c18aef8e8bc3322d9064d Mon Sep 17 00:00:00 2001 From: eikyuu Date: Fri, 13 Sep 2024 00:17:43 +0200 Subject: [PATCH 08/13] fix(import) --- src/run.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/run.rs b/src/run.rs index 98789ac..7b39b48 100644 --- a/src/run.rs +++ b/src/run.rs @@ -17,6 +17,7 @@ use tracing::{error, info, warn, Level}; use crate::keyring; use rencfs::crypto::Cipher; +use rencfs::encryptedfs::EncryptedFilesystem; use rencfs::encryptedfs::{EncryptedFs, FsError, PasswordProvider}; use rencfs::mount::MountPoint; use rencfs::{log, mount}; From a02326ca0fa199cb1c2bcbd9a3e5d9c0eac7a3dd Mon Sep 17 00:00:00 2001 From: eikyuu Date: Fri, 13 Sep 2024 00:18:24 +0200 Subject: [PATCH 09/13] chore(cleaning) --- src/encryptedfs.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/encryptedfs.rs b/src/encryptedfs.rs index 5fac26e..70afade 100644 --- a/src/encryptedfs.rs +++ b/src/encryptedfs.rs @@ -9,7 +9,6 @@ use std::backtrace::Backtrace; use std::collections::{HashMap, HashSet, VecDeque}; use std::fmt::Debug; use std::fs::{DirEntry, File, OpenOptions, ReadDir}; -use std::future::Future; use std::io::{Read, Seek, SeekFrom, Write}; use std::num::{NonZeroUsize, ParseIntError}; use std::path::{Path, PathBuf}; From 5513396a9ee9dd1c65d69bc202b95b1112552642 Mon Sep 17 00:00:00 2001 From: eikyuu Date: Fri, 13 Sep 2024 00:23:22 +0200 Subject: [PATCH 10/13] fix(imports) --- examples/change_password.rs | 1 + examples/change_password_cli.rs | 1 + examples/encryptedfs.rs | 4 +++- src/encryptedfs/bench.rs | 2 ++ src/encryptedfs/test.rs | 1 + 5 files changed, 8 insertions(+), 1 deletion(-) diff --git a/examples/change_password.rs b/examples/change_password.rs index 707727d..d4838cb 100644 --- a/examples/change_password.rs +++ b/examples/change_password.rs @@ -1,5 +1,6 @@ use core::str::FromStr; use rencfs::crypto::Cipher; +use rencfs::encryptedfs::EncryptedFilesystem; use rencfs::encryptedfs::{EncryptedFs, FsError}; use secrecy::SecretString; use std::env::args; diff --git a/examples/change_password_cli.rs b/examples/change_password_cli.rs index ba13c11..4ba7f98 100644 --- a/examples/change_password_cli.rs +++ b/examples/change_password_cli.rs @@ -6,6 +6,7 @@ use rpassword::read_password; use secrecy::{ExposeSecret, SecretString}; use tracing::{error, info}; +use rencfs::encryptedfs::EncryptedFilesystem; use rencfs::encryptedfs::{EncryptedFs, FsError}; #[tokio::main] diff --git a/examples/encryptedfs.rs b/examples/encryptedfs.rs index 93770ac..05eb0d3 100644 --- a/examples/encryptedfs.rs +++ b/examples/encryptedfs.rs @@ -7,7 +7,9 @@ use secrecy::SecretString; use rencfs::crypto::Cipher; use rencfs::encryptedfs::write_all_string_to_fs; -use rencfs::encryptedfs::{CreateFileAttr, EncryptedFs, FileType, PasswordProvider}; +use rencfs::encryptedfs::{ + CreateFileAttr, EncryptedFilesystem, EncryptedFs, FileType, PasswordProvider, +}; const ROOT_INODE: u64 = 1; diff --git a/src/encryptedfs/bench.rs b/src/encryptedfs/bench.rs index d61f702..1d6d751 100644 --- a/src/encryptedfs/bench.rs +++ b/src/encryptedfs/bench.rs @@ -8,6 +8,8 @@ use rand::Rng; #[allow(unused_imports)] use secrecy::SecretString; +#[allow(unused_imports)] +use crate::encryptedfs::EncryptedFilesystem; #[allow(unused_imports)] use crate::encryptedfs::{DirectoryEntry, DirectoryEntryPlus, FileType, ROOT_INODE}; #[allow(unused_imports)] diff --git a/src/encryptedfs/test.rs b/src/encryptedfs/test.rs index 59e2ece..2fa2e3e 100644 --- a/src/encryptedfs/test.rs +++ b/src/encryptedfs/test.rs @@ -7,6 +7,7 @@ use tracing_test::traced_test; use crate::crypto::Cipher; use crate::encryptedfs::write_all_bytes_to_fs; +use crate::encryptedfs::EncryptedFilesystem; use crate::encryptedfs::INODES_DIR; use crate::encryptedfs::KEY_ENC_FILENAME; use crate::encryptedfs::KEY_SALT_FILENAME; From fdb7642a6bfd35c0beb6beebe82ac8da7336a21f Mon Sep 17 00:00:00 2001 From: eikyuu Date: Mon, 16 Sep 2024 23:09:49 +0200 Subject: [PATCH 11/13] fix(sync): removing Sync trait requirement in test_common::bench solve the issue induced by the previous changes --- src/test_common.rs | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/test_common.rs b/src/test_common.rs index 514d0e6..52f3f10 100644 --- a/src/test_common.rs +++ b/src/test_common.rs @@ -189,12 +189,7 @@ pub async fn read_exact(fs: &EncryptedFs, ino: u64, offset: u64, buf: &mut [u8], } #[allow(dead_code)] -pub fn bench( - key: &'static str, - worker_threads: usize, - read_only: bool, - f: F, -) { +pub fn bench(key: &'static str, worker_threads: usize, read_only: bool, f: F) { block_on( async { run_test(TestSetup { key, read_only }, f).await; From 51a28f6d63470d554dccc33bd4edca8f5e9253e8 Mon Sep 17 00:00:00 2001 From: eikyuu Date: Tue, 17 Sep 2024 22:34:16 +0200 Subject: [PATCH 12/13] fix(): fixed trait requirements; moved new() method from trait impl block to impl block --- src/encryptedfs.rs | 134 ++++++++++++++++++++++----------------------- src/test_common.rs | 7 ++- 2 files changed, 70 insertions(+), 71 deletions(-) diff --git a/src/encryptedfs.rs b/src/encryptedfs.rs index 102393c..af56e35 100644 --- a/src/encryptedfs.rs +++ b/src/encryptedfs.rs @@ -577,13 +577,7 @@ pub struct EncryptedFs { } #[async_trait] -pub trait EncryptedFilesystem { - async fn new( - data_dir: PathBuf, - password_provider: Box, - cipher: Cipher, - read_only: bool, - ) -> FsResult>; +pub trait EncryptedFilesystem: Send + Sync { fn exists(&self, ino: u64) -> bool; fn is_dir(&self, ino: u64) -> bool; fn is_file(&self, ino: u64) -> bool; @@ -650,69 +644,6 @@ pub trait EncryptedFilesystem { #[async_trait] impl EncryptedFilesystem for EncryptedFs { - #[allow(clippy::missing_panics_doc)] - #[allow(clippy::missing_errors_doc)] - async fn new( - data_dir: PathBuf, - password_provider: Box, - cipher: Cipher, - read_only: bool, - ) -> FsResult> { - let key_provider = KeyProvider { - key_path: data_dir.join(SECURITY_DIR).join(KEY_ENC_FILENAME), - salt_path: data_dir.join(SECURITY_DIR).join(KEY_SALT_FILENAME), - password_provider, - cipher, - }; - let key = ExpireValue::new(key_provider, Duration::from_secs(10 * 60)); - - ensure_structure_created(&data_dir.clone()).await?; - key.get().await?; // this will check the password - - let fs = Self { - data_dir, - write_handles: RwLock::new(HashMap::new()), - read_handles: RwLock::new(HashMap::new()), - current_handle: AtomicU64::new(1), - cipher, - opened_files_for_read: RwLock::new(HashMap::new()), - opened_files_for_write: RwLock::new(HashMap::new()), - serialize_inode_locks: Arc::new(ArcHashMap::default()), - serialize_update_inode_locks: ArcHashMap::default(), - serialize_dir_entries_ls_locks: Arc::new(ArcHashMap::default()), - serialize_dir_entries_hash_locks: Arc::new(ArcHashMap::default()), - key, - self_weak: std::sync::Mutex::new(None), - read_write_locks: ArcHashMap::default(), - // todo: take duration from param - attr_cache: ExpireValue::new(AttrCacheProvider {}, Duration::from_secs(10 * 60)), - // todo: take duration from param - dir_entries_name_cache: ExpireValue::new( - DirEntryNameCacheProvider {}, - Duration::from_secs(10 * 60), - ), - // todo: take duration from param - dir_entries_meta_cache: ExpireValue::new( - DirEntryMetaCacheProvider {}, - Duration::from_secs(10 * 60), - ), - sizes_write: Mutex::default(), - sizes_read: Mutex::default(), - requested_read: Mutex::default(), - read_only, - }; - - let arc = Arc::new(fs); - arc.self_weak - .lock() - .expect("cannot obtain lock") - .replace(Arc::downgrade(&arc)); - - arc.ensure_root_exists().await?; - - Ok(arc) - } - fn exists(&self, ino: u64) -> bool { self.ino_file(ino).is_file() } @@ -1917,6 +1848,69 @@ impl EncryptedFilesystem for EncryptedFs { } impl EncryptedFs { + #[allow(clippy::missing_panics_doc)] + #[allow(clippy::missing_errors_doc)] + pub async fn new( + data_dir: PathBuf, + password_provider: Box, + cipher: Cipher, + read_only: bool, + ) -> FsResult> { + let key_provider = KeyProvider { + key_path: data_dir.join(SECURITY_DIR).join(KEY_ENC_FILENAME), + salt_path: data_dir.join(SECURITY_DIR).join(KEY_SALT_FILENAME), + password_provider, + cipher, + }; + let key = ExpireValue::new(key_provider, Duration::from_secs(10 * 60)); + + ensure_structure_created(&data_dir.clone()).await?; + key.get().await?; // this will check the password + + let fs = Self { + data_dir, + write_handles: RwLock::new(HashMap::new()), + read_handles: RwLock::new(HashMap::new()), + current_handle: AtomicU64::new(1), + cipher, + opened_files_for_read: RwLock::new(HashMap::new()), + opened_files_for_write: RwLock::new(HashMap::new()), + serialize_inode_locks: Arc::new(ArcHashMap::default()), + serialize_update_inode_locks: ArcHashMap::default(), + serialize_dir_entries_ls_locks: Arc::new(ArcHashMap::default()), + serialize_dir_entries_hash_locks: Arc::new(ArcHashMap::default()), + key, + self_weak: std::sync::Mutex::new(None), + read_write_locks: ArcHashMap::default(), + // todo: take duration from param + attr_cache: ExpireValue::new(AttrCacheProvider {}, Duration::from_secs(10 * 60)), + // todo: take duration from param + dir_entries_name_cache: ExpireValue::new( + DirEntryNameCacheProvider {}, + Duration::from_secs(10 * 60), + ), + // todo: take duration from param + dir_entries_meta_cache: ExpireValue::new( + DirEntryMetaCacheProvider {}, + Duration::from_secs(10 * 60), + ), + sizes_write: Mutex::default(), + sizes_read: Mutex::default(), + requested_read: Mutex::default(), + read_only, + }; + + let arc = Arc::new(fs); + arc.self_weak + .lock() + .expect("cannot obtain lock") + .replace(Arc::downgrade(&arc)); + + arc.ensure_root_exists().await?; + + Ok(arc) + } + #[allow(dead_code)] async fn is_read_only(&self) -> bool { self.read_only diff --git a/src/test_common.rs b/src/test_common.rs index 7e5a25a..3883650 100644 --- a/src/test_common.rs +++ b/src/test_common.rs @@ -189,7 +189,12 @@ pub async fn read_exact(fs: &EncryptedFs, ino: u64, offset: u64, buf: &mut [u8], } #[allow(dead_code)] -pub fn bench(key: &'static str, worker_threads: usize, read_only: bool, f: F) { +pub fn bench( + key: &'static str, + worker_threads: usize, + read_only: bool, + f: F, +) { block_on( async { run_test(TestSetup { key, read_only }, f).await; From 1df6b3138c6075b6f44906de0e32eb9676f829f5 Mon Sep 17 00:00:00 2001 From: eikyuu Date: Tue, 17 Sep 2024 22:47:09 +0200 Subject: [PATCH 13/13] chore(docs) --- src/encryptedfs.rs | 84 +++++++++++++++++++++++----------------------- 1 file changed, 42 insertions(+), 42 deletions(-) diff --git a/src/encryptedfs.rs b/src/encryptedfs.rs index af56e35..494a249 100644 --- a/src/encryptedfs.rs +++ b/src/encryptedfs.rs @@ -581,6 +581,10 @@ pub trait EncryptedFilesystem: Send + Sync { fn exists(&self, ino: u64) -> bool; fn is_dir(&self, ino: u64) -> bool; fn is_file(&self, ino: u64) -> bool; + /// Create a new node in the filesystem + #[allow(clippy::missing_panics_doc)] + #[allow(clippy::missing_errors_doc)] + #[allow(clippy::too_many_lines)] async fn create( &self, parent: u64, @@ -590,26 +594,59 @@ pub trait EncryptedFilesystem: Send + Sync { write: bool, ) -> FsResult<(u64, FileAttr)>; async fn find_by_name(&self, parent: u64, name: &SecretString) -> FsResult>; + /// Count children of a directory. This **EXCLUDES** "." and "..". + #[allow(clippy::missing_errors_doc)] fn len(&self, ino: u64) -> FsResult; + /// Delete a directory + #[allow(clippy::missing_panics_doc)] + #[allow(clippy::missing_errors_doc)] async fn remove_dir(&self, parent: u64, name: &SecretString) -> FsResult<()>; + /// Delete a file + #[allow(clippy::missing_panics_doc)] + #[allow(clippy::missing_errors_doc)] async fn remove_file(&self, parent: u64, name: &SecretString) -> FsResult<()>; fn exists_by_name(&self, parent: u64, name: &SecretString) -> FsResult; async fn read_dir(&self, ino: u64) -> FsResult; + /// Like [`EncryptedFs::read_dir`] but with [`FileAttr`] so we don't need to query again for those. async fn read_dir_plus(&self, ino: u64) -> FsResult; + /// Get metadata + #[allow(clippy::missing_errors_doc)] async fn get_attr(&self, ino: u64) -> FsResult; + /// Set metadata async fn set_attr(&self, ino: u64, set_attr: SetFileAttr) -> FsResult<()>; + /// Read the contents from an `offset`. + /// + /// If we try to read outside of file size, we return zero bytes. + /// If the file is not opened for read, it will return an error of type [FsError::InvalidFileHandle]. + #[allow(clippy::missing_errors_doc)] + #[allow(clippy::cast_possible_truncation)] async fn read(&self, ino: u64, offset: u64, buf: &mut [u8], handle: u64) -> FsResult; async fn release(&self, handle: u64) -> FsResult<()>; + /// Check if a file is opened for reading with this handle. async fn is_read_handle(&self, fh: u64) -> bool; + /// Check if a file is opened for writing with this handle. async fn is_write_handle(&self, fh: u64) -> bool; + /// Writes the contents of `buf` to the file with `ino` starting at `offset`. + /// + /// If we write outside file size, we fill up with zeros until the `offset`. + /// If the file is not opened for writing, + /// it will return an error of type [FsError::InvalidFileHandle]. async fn write(&self, ino: u64, offset: u64, buf: &[u8], handle: u64) -> FsResult; + /// Flush the data to the underlying storage. + #[allow(clippy::missing_panics_doc)] async fn flush(&self, handle: u64) -> FsResult<()>; + /// Helpful when we want to copy just some portions of the file. async fn copy_file_range( &self, file_range_req: &CopyFileRangeReq, size: usize, ) -> FsResult; + /// Open a file. We can open multiple times for read but only one to write at a time. + #[allow(clippy::missing_panics_doc)] async fn open(&self, ino: u64, read: bool, write: bool) -> FsResult; + /// Truncates or extends the underlying file, updating the size of this file to become size. + #[allow(clippy::missing_panics_doc)] + #[allow(clippy::too_many_lines)] async fn set_len(&self, ino: u64, size: u64) -> FsResult<()>; async fn rename( &self, @@ -618,22 +655,27 @@ pub trait EncryptedFilesystem: Send + Sync { new_parent: u64, new_name: &SecretString, ) -> FsResult<()>; + /// Create a crypto writer using internal encryption info. async fn create_write( &self, file: W, ) -> FsResult>>; + /// Create a crypto writer with seek using internal encryption info. async fn create_write_seek( &self, file: W, ) -> FsResult>>; + /// Create a crypto reader using internal encryption info. async fn create_read( &self, reader: R, ) -> FsResult>>; + /// Create a crypto reader with seek using internal encryption info. async fn create_read_seek( &self, reader: R, ) -> FsResult>>; + /// Change the password of the filesystem used to access the encryption key. async fn passwd( data_dir: &Path, old_password: SecretString, @@ -656,10 +698,6 @@ impl EncryptedFilesystem for EncryptedFs { self.contents_path(ino).is_file() } - /// Create a new node in the filesystem - #[allow(clippy::missing_panics_doc)] - #[allow(clippy::missing_errors_doc)] - #[allow(clippy::too_many_lines)] async fn create( &self, parent: u64, @@ -846,8 +884,6 @@ impl EncryptedFilesystem for EncryptedFs { self.get_inode_from_cache_or_storage(ino).await.map(Some) } - /// Count children of a directory. This **EXCLUDES** "." and "..". - #[allow(clippy::missing_errors_doc)] fn len(&self, ino: u64) -> FsResult { if !self.is_dir(ino) { return Err(FsError::InvalidInodeType); @@ -863,9 +899,6 @@ impl EncryptedFilesystem for EncryptedFs { Ok(count) } - /// Delete a directory - #[allow(clippy::missing_panics_doc)] - #[allow(clippy::missing_errors_doc)] async fn remove_dir(&self, parent: u64, name: &SecretString) -> FsResult<()> { if !self.is_dir(parent) { return Err(FsError::InvalidInodeType); @@ -940,9 +973,6 @@ impl EncryptedFilesystem for EncryptedFs { .await? } - /// Delete a file - #[allow(clippy::missing_panics_doc)] - #[allow(clippy::missing_errors_doc)] async fn remove_file(&self, parent: u64, name: &SecretString) -> FsResult<()> { if !self.is_dir(parent) { return Err(FsError::InvalidInodeType); @@ -1042,7 +1072,6 @@ impl EncryptedFilesystem for EncryptedFs { Ok(self.create_directory_entry_iterator(iter).await) } - /// Like [`EncryptedFs::read_dir`] but with [`FileAttr`] so we don't need to query again for those. async fn read_dir_plus(&self, ino: u64) -> FsResult { if !self.is_dir(ino) { return Err(FsError::InvalidInodeType); @@ -1058,8 +1087,6 @@ impl EncryptedFilesystem for EncryptedFs { Ok(self.create_directory_entry_plus_iterator(iter).await) } - /// Get metadata - #[allow(clippy::missing_errors_doc)] async fn get_attr(&self, ino: u64) -> FsResult { let mut attr = self.get_inode_from_cache_or_storage(ino).await?; @@ -1094,7 +1121,6 @@ impl EncryptedFilesystem for EncryptedFs { Ok(attr) } - /// Set metadata async fn set_attr(&self, ino: u64, set_attr: SetFileAttr) -> FsResult<()> { if self.read_only { return Err(FsError::ReadOnly); @@ -1102,13 +1128,7 @@ impl EncryptedFilesystem for EncryptedFs { self.set_attr2(ino, set_attr, false).await } - /// Read the contents from an `offset`. - /// - /// If we try to read outside of file size, we return zero bytes. - /// If the file is not opened for read, it will return an error of type [FsError::InvalidFileHandle]. #[instrument(skip(self, buf), fields(len = %buf.len()), ret(level = Level::DEBUG))] - #[allow(clippy::missing_errors_doc)] - #[allow(clippy::cast_possible_truncation)] async fn read(&self, ino: u64, offset: u64, buf: &mut [u8], handle: u64) -> FsResult { if !self.exists(ino) { return Err(FsError::InodeNotFound); @@ -1320,21 +1340,14 @@ impl EncryptedFilesystem for EncryptedFs { Ok(()) } - /// Check if a file is opened for reading with this handle. async fn is_read_handle(&self, fh: u64) -> bool { self.read_handles.read().await.contains_key(&fh) } - /// Check if a file is opened for writing with this handle. async fn is_write_handle(&self, fh: u64) -> bool { self.write_handles.read().await.contains_key(&fh) } - /// Writes the contents of `buf` to the file with `ino` starting at `offset`. - /// - /// If we write outside file size, we fill up with zeros until the `offset`. - /// If the file is not opened for writing, - /// it will return an error of type [FsError::InvalidFileHandle]. #[instrument(skip(self, buf), fields(len = %buf.len()), ret(level = Level::DEBUG))] async fn write(&self, ino: u64, offset: u64, buf: &[u8], handle: u64) -> FsResult { if self.read_only { @@ -1442,8 +1455,6 @@ impl EncryptedFilesystem for EncryptedFs { Ok(len) } - /// Flush the data to the underlying storage. - #[allow(clippy::missing_panics_doc)] async fn flush(&self, handle: u64) -> FsResult<()> { if handle == 0 { // in the case of directory or if the file was crated without being opened we don't use a handle @@ -1478,7 +1489,6 @@ impl EncryptedFilesystem for EncryptedFs { Ok(()) } - /// Helpful when we want to copy just some portions of the file. async fn copy_file_range( &self, file_range_req: &CopyFileRangeReq, @@ -1522,8 +1532,6 @@ impl EncryptedFilesystem for EncryptedFs { Ok(len) } - /// Open a file. We can open multiple times for read but only one to write at a time. - #[allow(clippy::missing_panics_doc)] async fn open(&self, ino: u64, read: bool, write: bool) -> FsResult { if write && self.read_only { return Err(FsError::ReadOnly); @@ -1588,9 +1596,6 @@ impl EncryptedFilesystem for EncryptedFs { Ok(fh) } - /// Truncates or extends the underlying file, updating the size of this file to become size. - #[allow(clippy::missing_panics_doc)] - #[allow(clippy::too_many_lines)] async fn set_len(&self, ino: u64, size: u64) -> FsResult<()> { if self.read_only { return Err(FsError::ReadOnly); @@ -1769,7 +1774,6 @@ impl EncryptedFilesystem for EncryptedFs { Ok(()) } - /// Create a crypto writer using internal encryption info. async fn create_write( &self, file: W, @@ -1781,7 +1785,6 @@ impl EncryptedFilesystem for EncryptedFs { ))) } - /// Create a crypto writer with seek using internal encryption info. async fn create_write_seek( &self, file: W, @@ -1793,7 +1796,6 @@ impl EncryptedFilesystem for EncryptedFs { ))) } - /// Create a crypto reader using internal encryption info. async fn create_read( &self, reader: R, @@ -1805,7 +1807,6 @@ impl EncryptedFilesystem for EncryptedFs { ))) } - /// Create a crypto reader with seek using internal encryption info. async fn create_read_seek( &self, reader: R, @@ -1817,7 +1818,6 @@ impl EncryptedFilesystem for EncryptedFs { ))) } - /// Change the password of the filesystem used to access the encryption key. async fn passwd( data_dir: &Path, old_password: SecretBox,