Skip to content

Commit

Permalink
Small refactoring to simplify method calls
Browse files Browse the repository at this point in the history
  • Loading branch information
@abdolence
abdolence committed Aug 31, 2024
1 parent 5379981 commit 6ad5b00
Show file tree
Hide file tree
Showing 4 changed files with 30 additions and 33 deletions.
2 changes: 1 addition & 1 deletion Cargo.lock
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion Cargo.toml
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[package]
name = "redacter"
version = "0.11.0"
version = "0.11.1"
edition = "2021"
authors = ["Abdulla Abdurakhmanov <[email protected]>"]
license = "Apache-2.0"
Expand Down
17 changes: 6 additions & 11 deletions src/commands/copy_command.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ use crate::file_converters::FileConverters;
use crate::file_systems::{DetectFileSystem, FileSystemConnection, FileSystemRef};
use crate::file_tools::{FileMatcher, FileMatcherResult, FileMimeOverride};
use crate::redacters::{
Redacter, RedacterBaseOptions, RedacterOptions, RedacterThrottler, Redacters, StreamRedacter,
RedacterBaseOptions, RedacterOptions, RedacterThrottler, Redacters, StreamRedacter,
};
use crate::reporter::AppReporter;
use crate::AppResult;
Expand Down Expand Up @@ -252,7 +252,7 @@ async fn transfer_and_redact_file<
source_fs: &mut SFS,
destination_fs: &mut DFS,
options: &CopyCommandOptions,
redacter: &Option<(RedacterBaseOptions, Vec<impl Redacter>)>,
redacter: &Option<(RedacterBaseOptions, Vec<Redacters<'a>>)>,
file_converters: &FileConverters<'a>,
redacter_throttler: &mut Option<RedacterThrottler>,
) -> AppResult<TransferFileResult> {
Expand Down Expand Up @@ -352,7 +352,7 @@ async fn redact_upload_file<
source_reader: S,
dest_file_ref: &FileSystemRef,
options: &CopyCommandOptions,
redacter_with_options: &(RedacterBaseOptions, Vec<impl Redacter>),
redacter_with_options: &(RedacterBaseOptions, Vec<Redacters<'a>>),
file_converters: &FileConverters<'a>,
redacter_throttler: &mut Option<RedacterThrottler>,
) -> AppResult<TransferFileResult> {
Expand All @@ -363,11 +363,11 @@ async fn redact_upload_file<
.file_mime_override
.override_for_file_ref(dest_file_ref.clone());

let (redact_plan, supported_redacters) = stream_redacter
let redact_plan = stream_redacter
.create_redact_plan(redacters, &dest_file_ref_overridden)
.await?;

if !supported_redacters.is_empty() {
if !redact_plan.supported_redacters.is_empty() {
if let Some(ref mut throttler) = redacter_throttler {
*throttler = throttler.update(Instant::now());
let delay = throttler.delay();
Expand All @@ -386,12 +386,7 @@ async fn redact_upload_file<
}
}
match stream_redacter
.redact_stream(
source_reader,
redact_plan,
&supported_redacters,
&dest_file_ref_overridden,
)
.redact_stream(source_reader, redact_plan, &dest_file_ref_overridden)
.await
{
Ok(redacted_result)
Expand Down
42 changes: 22 additions & 20 deletions src/redacters/stream_redacter.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,10 +24,11 @@ pub struct StreamRedacter<'a> {
bar: &'a ProgressBar,
}

pub struct StreamRedactPlan {
pub struct StreamRedactPlan<'a> {
pub apply_pdf_image_converter: bool,
pub apply_ocr: bool,
pub leave_data_table_as_text: bool,
pub supported_redacters: Vec<&'a Redacters<'a>>,
}

impl<'a> StreamRedacter<'a> {
Expand All @@ -45,24 +46,24 @@ impl<'a> StreamRedacter<'a> {

pub async fn create_redact_plan(
&'a self,
redacters: &'a Vec<impl Redacter>,
redacters: &'a Vec<Redacters<'a>>,
file_ref: &FileSystemRef,
) -> AppResult<(StreamRedactPlan, Vec<&'a impl Redacter>)> {
) -> AppResult<StreamRedactPlan> {
let mut stream_redact_plan = StreamRedactPlan {
apply_pdf_image_converter: false,
apply_ocr: false,
leave_data_table_as_text: false,
supported_redacters: vec![],
};
// Supports natively
let mut supported_redacters = Vec::new();
for redacter in redacters {
let supported_options = redacter.redact_support(file_ref).await?;
if supported_options == RedactSupport::Supported {
supported_redacters.push(redacter);
stream_redact_plan.supported_redacters.push(redacter);
}
}

if supported_redacters.is_empty() {
if stream_redact_plan.supported_redacters.is_empty() {
match &file_ref.media_type {
Some(file_ref_media) => {
// Supports with conversion
Expand All @@ -75,10 +76,10 @@ impl<'a> StreamRedacter<'a> {
})
.await?;
if supported_options == RedactSupport::Supported {
supported_redacters.push(redacter);
stream_redact_plan.supported_redacters.push(redacter);
}
}
if !supported_redacters.is_empty() {
if !stream_redact_plan.supported_redacters.is_empty() {
stream_redact_plan.leave_data_table_as_text = true;
}
} else if self.file_converters.pdf_image_converter.is_some()
Expand All @@ -92,15 +93,17 @@ impl<'a> StreamRedacter<'a> {
})
.await?;
if supported_options == RedactSupport::Supported {
supported_redacters.push(redacter);
stream_redact_plan.supported_redacters.push(redacter);
}
}

if !supported_redacters.is_empty() {
if !stream_redact_plan.supported_redacters.is_empty() {
stream_redact_plan.apply_pdf_image_converter = true;
}

if supported_redacters.is_empty() && self.file_converters.ocr.is_some() {
if stream_redact_plan.supported_redacters.is_empty()
&& self.file_converters.ocr.is_some()
{
for redacter in redacters {
let supported_options = redacter
.redact_support(&FileSystemRef {
Expand All @@ -109,10 +112,10 @@ impl<'a> StreamRedacter<'a> {
})
.await?;
if supported_options == RedactSupport::Supported {
supported_redacters.push(redacter);
stream_redact_plan.supported_redacters.push(redacter);
}
}
if !supported_redacters.is_empty() {
if !stream_redact_plan.supported_redacters.is_empty() {
stream_redact_plan.apply_pdf_image_converter = true;
stream_redact_plan.apply_ocr = true;
}
Expand All @@ -128,10 +131,10 @@ impl<'a> StreamRedacter<'a> {
})
.await?;
if supported_options == RedactSupport::Supported {
supported_redacters.push(redacter);
stream_redact_plan.supported_redacters.push(redacter);
}
}
if !supported_redacters.is_empty() {
if !stream_redact_plan.supported_redacters.is_empty() {
stream_redact_plan.apply_ocr = true;
}
}
Expand All @@ -140,24 +143,23 @@ impl<'a> StreamRedacter<'a> {
}
}

Ok((stream_redact_plan, supported_redacters))
Ok(stream_redact_plan)
}

pub async fn redact_stream<
S: Stream<Item = AppResult<bytes::Bytes>> + Send + Unpin + Sync + 'static,
>(
&'a self,
input: S,
redact_plan: StreamRedactPlan,
redacters: &[&'a impl Redacter],
redact_plan: StreamRedactPlan<'a>,
file_ref: &FileSystemRef,
) -> AppResult<RedactStreamResult> {
let mut redacted = self
.stream_to_redact_item(self.redacter_base_options, input, file_ref, &redact_plan)
.await?;
let mut number_of_redactions = 0;

for (index, redacter) in redacters.iter().enumerate() {
for (index, redacter) in redact_plan.supported_redacters.iter().enumerate() {
let width = " ".repeat(index);
if redact_plan.apply_pdf_image_converter {
match (
Expand Down Expand Up @@ -272,7 +274,7 @@ impl<'a> StreamRedacter<'a> {
redacter_base_options: &RedacterBaseOptions,
input: S,
file_ref: &FileSystemRef,
redact_plan: &StreamRedactPlan,
redact_plan: &StreamRedactPlan<'a>,
) -> AppResult<RedacterDataItem> {
match file_ref.media_type {
Some(ref mime)
Expand Down

0 comments on commit 6ad5b00

Please sign in to comment.