From b9aea820dbc28d784a739c18dcbe8d2bace5f608 Mon Sep 17 00:00:00 2001 From: ayushpatnaikgit Date: Tue, 4 Jul 2023 09:57:05 +0530 Subject: [PATCH] Reduce requirements for interpolation and bias correction. --- src/data_cleaning/bias_correction.jl | 2 +- src/data_cleaning/interpolation.jl | 2 +- src/data_cleaning/outlier_removal.jl | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/data_cleaning/bias_correction.jl b/src/data_cleaning/bias_correction.jl index ce349cf..bc27896 100644 --- a/src/data_cleaning/bias_correction.jl +++ b/src/data_cleaning/bias_correction.jl @@ -36,7 +36,7 @@ bias_PSTT2021(radiance_datacube, ncfobs_datacube) function bias_PSTT2021(radiance_datacube, ncfobs_datacube, mask=ones(Int8, (size(radiance_datacube)[1],size(radiance_datacube)[2]))) for i in 1:size(radiance_datacube)[1] for j in 1:size(radiance_datacube)[2] - if count(i->(ismissing(i)),radiance_datacube[i, j, :])/length(radiance_datacube[i, j, :]) > 0.50 + if count(i->(ismissing(i)),radiance_datacube[i, j, :])/length(radiance_datacube[i, j, :]) > 0.90 continue end if ismissing(mask[i, j]) diff --git a/src/data_cleaning/interpolation.jl b/src/data_cleaning/interpolation.jl index 7090f95..1c434ac 100644 --- a/src/data_cleaning/interpolation.jl +++ b/src/data_cleaning/interpolation.jl @@ -10,7 +10,7 @@ na_interp_linear(x) ``` """ function na_interp_linear(timeseries) - if count(i->(ismissing(i)), timeseries) > length(timeseries) *1/2 + if count(i->(ismissing(i)), timeseries) > length(timeseries) *0.9 return zero(1:length(timeseries)) end data = copy(timeseries) diff --git a/src/data_cleaning/outlier_removal.jl b/src/data_cleaning/outlier_removal.jl index cea07f5..4680f07 100644 --- a/src/data_cleaning/outlier_removal.jl +++ b/src/data_cleaning/outlier_removal.jl @@ -19,7 +19,7 @@ function outlier_variance(radiance_datacube, mask=ones(Int8, (size(radiance_data if ismissing(mask[i, j]) continue end - if count(i->(ismissing(i)),radiance_datacube[i, j, :])/length(radiance_datacube[i, j, :]) > 0.50 # Don't do anything if there are too many missings + if count(i->(ismissing(i)),radiance_datacube[i, j, :])/length(radiance_datacube[i, j, :]) > 0.90 # Don't do anything if there are too many missings continue end stds[i, j] = std(detrend_ts(filter(x -> !ismissing(x), radiance_datacube[i, j, :])))