From 81d93cc9001176cfea7532571fc50dde6c035c34 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Thu, 28 Nov 2024 13:27:28 +0100 Subject: [PATCH 01/28] this should work --- .github/workflows/ci.yml | 3 ++ .gitignore | 3 ++ src/DataWrangling/ECCO/ECCO_metadata.jl | 45 +++++++++++++++++++++++-- 3 files changed, 49 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ae713c5b..7aebb4ad 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,6 +19,9 @@ jobs: arch: - x64 include: + - os: windows-latest + arch: x86 + version: '1.10' - os: macOS-latest arch: arm64 version: '1.10' diff --git a/.gitignore b/.gitignore index 997ed814..3c3ca28d 100644 --- a/.gitignore +++ b/.gitignore @@ -28,6 +28,9 @@ docs/src/literated/ *.svg *.gif +# Password files +*.netrc + # File generated by Pkg, the package manager, based on a corresponding Project.toml # It records a fixed state of all packages used by the project. As such, it should not be # committed for packages, but should be committed for applications that require a static diff --git a/src/DataWrangling/ECCO/ECCO_metadata.jl b/src/DataWrangling/ECCO/ECCO_metadata.jl index 161c73af..0ab464d7 100644 --- a/src/DataWrangling/ECCO/ECCO_metadata.jl +++ b/src/DataWrangling/ECCO/ECCO_metadata.jl @@ -5,6 +5,7 @@ using ClimaOcean.DataWrangling import Dates: year, month, day using Base: @propagate_inbounds +using Downloads import Oceananigans.Fields: set!, location import Base @@ -237,10 +238,50 @@ function download_dataset(metadata::ECCOMetadata; url = urls(metadata)) throw(ArgumentError(msg)) end - cmd = `wget --http-user=$(username) --http-passwd=$(password) --directory-prefix=$dir $fileurl` - run(cmd) + downloader = ECCO_downloader(username, password, dir) + Downloads.download(fileurl, filepath; downloader, verbose=true) end end + # Always remove the file after downloading + # to avoid storing the password in the netrc file + remove_netrc!(dir) + return nothing end + +# ECCO downloader +function ECCO_downloader(username, password, dir) + filepath = update_ECCO_netrc!(username, password, dir) + downloader = Downloads.Downloader() + easy_hook = (easy, _) -> Downloads.Curl.setopt(easy, Downloads.Curl.CURLOPT_NETRC_FILE, filepath) + + downloader.easy_hook = easy_hook + return downloader +end + +# Code snippet adapted from https://github.com/evetion/SpaceLiDAR.jl/blob/master/src/utils.jl#L150 +function update_ECCO_netrc!(username, password, dir) + if Sys.iswindows() + filepath = joinpath(dir, "ECCO_netrc") + else + filepath = joinpath(dir, "ECCO.netrc") + end + + open(filepath, "a") do f + write(f, "\n") + write(f, "machine ecco.jpl.nasa.gov login $username password $password\n") + end + + return filepath +end + +function remove_netrc!(dir) + if Sys.iswindows() + filepath = joinpath(dir, "ECCO_netrc") + else + filepath = joinpath(dir, "ECCO.netrc") + end + + rm(filepath; force = true) +end \ No newline at end of file From 867da9cc5b58108d0b8fb08f887764b734aa7012 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Thu, 28 Nov 2024 13:29:45 +0100 Subject: [PATCH 02/28] better naming --- src/DataWrangling/ECCO/ECCO_metadata.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/DataWrangling/ECCO/ECCO_metadata.jl b/src/DataWrangling/ECCO/ECCO_metadata.jl index 0ab464d7..63794c01 100644 --- a/src/DataWrangling/ECCO/ECCO_metadata.jl +++ b/src/DataWrangling/ECCO/ECCO_metadata.jl @@ -252,16 +252,16 @@ end # ECCO downloader function ECCO_downloader(username, password, dir) - filepath = update_ECCO_netrc!(username, password, dir) + netrc_file = ECCO_netrc!(username, password, dir) downloader = Downloads.Downloader() - easy_hook = (easy, _) -> Downloads.Curl.setopt(easy, Downloads.Curl.CURLOPT_NETRC_FILE, filepath) + easy_hook = (easy, _) -> Downloads.Curl.setopt(easy, Downloads.Curl.CURLOPT_NETRC_FILE, netrc_file) downloader.easy_hook = easy_hook return downloader end # Code snippet adapted from https://github.com/evetion/SpaceLiDAR.jl/blob/master/src/utils.jl#L150 -function update_ECCO_netrc!(username, password, dir) +function ECCO_netrc!(username, password, dir) if Sys.iswindows() filepath = joinpath(dir, "ECCO_netrc") else From 71f2b1933fd8df6cf82f20221d39e028b4ea5d34 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Thu, 28 Nov 2024 13:31:08 +0100 Subject: [PATCH 03/28] only one download --- src/DataWrangling/ECCO/ECCO_metadata.jl | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/DataWrangling/ECCO/ECCO_metadata.jl b/src/DataWrangling/ECCO/ECCO_metadata.jl index 63794c01..0fe710dc 100644 --- a/src/DataWrangling/ECCO/ECCO_metadata.jl +++ b/src/DataWrangling/ECCO/ECCO_metadata.jl @@ -219,6 +219,9 @@ function download_dataset(metadata::ECCOMetadata; url = urls(metadata)) password = get(ENV, "ECCO_PASSWORD", nothing) dir = metadata.dir + # Write down the username and password in a .netrc file + downloader = ECCO_downloader(username, password, dir) + @distribute for metadatum in metadata # Distribute the download among ranks if MPI is initialized fileurl = metadata_url(url, metadatum) @@ -238,13 +241,11 @@ function download_dataset(metadata::ECCOMetadata; url = urls(metadata)) throw(ArgumentError(msg)) end - downloader = ECCO_downloader(username, password, dir) Downloads.download(fileurl, filepath; downloader, verbose=true) end end - # Always remove the file after downloading - # to avoid storing the password in the netrc file + # Remove the .netrc file after downloading to avoid storing the credentials remove_netrc!(dir) return nothing From b36f60fc60d4432f0b43000b486d7af905651480 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Thu, 28 Nov 2024 13:36:56 +0100 Subject: [PATCH 04/28] add download test --- test/runtests.jl | 64 ++++++++++++++++++++-------------------- test/test_downloading.jl | 8 +++++ 2 files changed, 40 insertions(+), 32 deletions(-) diff --git a/test/runtests.jl b/test/runtests.jl index cdeec98d..c8d9a1a9 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -9,9 +9,9 @@ test_group = Symbol(test_group) using ClimaOcean.ECCO: download_dataset if test_group == :init || test_group == :all - using CUDA - CUDA.set_runtime_version!(v"12.6"; local_toolkit = true) - CUDA.precompile_runtime() + # using CUDA + # CUDA.set_runtime_version!(v"12.6"; local_toolkit = true) + # CUDA.precompile_runtime() #### #### Download bathymetry data @@ -19,48 +19,48 @@ if test_group == :init || test_group == :all download_bathymetry() - #### - #### Download JRA55 data - #### + # #### + # #### Download JRA55 data + # #### - atmosphere = JRA55_prescribed_atmosphere() + # atmosphere = JRA55_prescribed_atmosphere() - #### - #### Download ECCO data - #### + # #### + # #### Download ECCO data + # #### - download_dataset(temperature_metadata) - download_dataset(salinity_metadata) + # download_dataset(temperature_metadata) + # download_dataset(salinity_metadata) end -# Tests JRA55 utilities, plus some DataWrangling utilities -if test_group == :jra55 || test_group == :all - include("test_jra55.jl") -end +# # Tests JRA55 utilities, plus some DataWrangling utilities +# if test_group == :jra55 || test_group == :all +# include("test_jra55.jl") +# end -if test_group == :ecco || test_group == :all - include("test_ecco.jl") -end +# if test_group == :ecco || test_group == :all +# include("test_ecco.jl") +# end # Tests that we can download JRA55 utilities if test_group == :downloading || test_group == :all include("test_downloading.jl") end -if test_group == :fluxes || test_group == :all - include("test_surface_fluxes.jl") -end +# if test_group == :fluxes || test_group == :all +# include("test_surface_fluxes.jl") +# end -if test_group == :bathymetry || test_group == :all - include("test_bathymetry.jl") -end +# if test_group == :bathymetry || test_group == :all +# include("test_bathymetry.jl") +# end -if test_group == :simulations || test_group == :all - CUDA.set_runtime_version!(v"12.2", local_toolkit = true) # Seems to help in finding the correct CUDA version - include("test_simulations.jl") -end +# if test_group == :simulations || test_group == :all +# CUDA.set_runtime_version!(v"12.2", local_toolkit = true) # Seems to help in finding the correct CUDA version +# include("test_simulations.jl") +# end -if test_group == :distributed || test_group == :all - include("test_distributed_utils.jl") -end +# if test_group == :distributed || test_group == :all +# include("test_distributed_utils.jl") +# end diff --git a/test/test_downloading.jl b/test/test_downloading.jl index 7bc53331..4fc77961 100644 --- a/test/test_downloading.jl +++ b/test/test_downloading.jl @@ -6,3 +6,11 @@ include("runtests_setup.jl") fts = ClimaOcean.JRA55.JRA55_field_time_series(name; time_indices=2:3) end end + +@testset "Availability of ECCO data" begin + @info "Testing that we can download ECCO data..." + for variable in keys(ClimaOcean.ECCO.ECCO4_short_names) + metadata = ECCOMetadata(variable) + ClimaOcean.ECCO.download_dataset(metadata) + end +end \ No newline at end of file From 31fb896449b9817878b9336801ce1849e8468bfc Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Thu, 28 Nov 2024 13:54:51 +0100 Subject: [PATCH 05/28] joinpath does not work on windows --- src/DataWrangling/ECCO/ECCO_metadata.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/DataWrangling/ECCO/ECCO_metadata.jl b/src/DataWrangling/ECCO/ECCO_metadata.jl index 0fe710dc..bf79471e 100644 --- a/src/DataWrangling/ECCO/ECCO_metadata.jl +++ b/src/DataWrangling/ECCO/ECCO_metadata.jl @@ -145,12 +145,12 @@ short_name(data::ECCOMetadata{<:Any, <:ECCO2Daily}) = ECCO2_short_names[data.n short_name(data::ECCOMetadata{<:Any, <:ECCO2Monthly}) = ECCO2_short_names[data.name] short_name(data::ECCOMetadata{<:Any, <:ECCO4Monthly}) = ECCO4_short_names[data.name] -metadata_url(prefix, m::ECCOMetadata{<:Any, <:ECCO2Daily}) = joinpath(prefix, short_name(m), metadata_filename(m)) -metadata_url(prefix, m::ECCOMetadata{<:Any, <:ECCO2Monthly}) = joinpath(prefix, short_name(m), metadata_filename(m)) +metadata_url(prefix, m::ECCOMetadata{<:Any, <:ECCO2Daily}) = prefix * "/" * short_name(m) * "/" * metadata_filename(m) +metadata_url(prefix, m::ECCOMetadata{<:Any, <:ECCO2Monthly}) = prefix * "/" * short_name(m) * "/" * metadata_filename(m) function metadata_url(prefix, m::ECCOMetadata{<:Any, <:ECCO4Monthly}) year = string(Dates.year(m.dates)) - return joinpath(prefix, short_name(m), year, metadata_filename(m)) + return prefix * "/" * short_name(m) * "/" * year * "/" * metadata_filename(m) end location(data::ECCOMetadata) = ECCO_location[data.name] From 5b229db8e79da38efe4c71568db21e662f9ce481 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Thu, 28 Nov 2024 14:28:40 +0100 Subject: [PATCH 06/28] test also downloading the bathymetry --- test/runtests.jl | 2 +- test/test_downloading.jl | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/test/runtests.jl b/test/runtests.jl index c8d9a1a9..8d03677f 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -17,7 +17,7 @@ if test_group == :init || test_group == :all #### Download bathymetry data #### - download_bathymetry() + # download_bathymetry() # #### # #### Download JRA55 data diff --git a/test/test_downloading.jl b/test/test_downloading.jl index 4fc77961..66a58124 100644 --- a/test/test_downloading.jl +++ b/test/test_downloading.jl @@ -13,4 +13,9 @@ end metadata = ECCOMetadata(variable) ClimaOcean.ECCO.download_dataset(metadata) end +end + +@testset "Availability of the Bathymetry" begin + @info "Testing that we can download the bathymetry..." + download_bathymetry() end \ No newline at end of file From 3ee4eaa79bcc0bd17225cf44d4a042b3e72b7525 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Thu, 28 Nov 2024 15:01:48 +0100 Subject: [PATCH 07/28] test dowloading bathymetry --- src/Bathymetry.jl | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/src/Bathymetry.jl b/src/Bathymetry.jl index 970b49f1..44188591 100644 --- a/src/Bathymetry.jl +++ b/src/Bathymetry.jl @@ -93,17 +93,10 @@ function regrid_bathymetry(target_grid; major_basins = Inf) # Allow an `Inf` number of ``lakes'' filepath = joinpath(dir, filename) - fileurl = joinpath(url, filename) - - @root begin # perform all this only on rank 0, aka the "root" rank - if !isfile(filepath) - try - Downloads.download(fileurl, filepath; progress=download_progress, verbose=true) - catch - cmd = `wget --no-check-certificate -O $filepath $fileurl` - @root run(cmd) - end - end + fileurl = url * "/" * filename # joinpath on windows creates the wrong url + + @root if !isfile(filepath) # perform all this only on rank 0, aka the "root" rank + Downloads.download(fileurl, filepath; progress=download_progress, verbose=true) end dataset = Dataset(filepath) From cba1991176d2cd23b99987e5fd2ce63b194aaa31 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Thu, 28 Nov 2024 15:30:28 +0100 Subject: [PATCH 08/28] restore tests --- test/runtests.jl | 72 ++++++++++++++++++++++++------------------------ 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/test/runtests.jl b/test/runtests.jl index 8d03677f..0c71ec17 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -9,58 +9,58 @@ test_group = Symbol(test_group) using ClimaOcean.ECCO: download_dataset if test_group == :init || test_group == :all - # using CUDA - # CUDA.set_runtime_version!(v"12.6"; local_toolkit = true) - # CUDA.precompile_runtime() + using CUDA + CUDA.set_runtime_version!(v"12.6"; local_toolkit = true) + CUDA.precompile_runtime() - #### - #### Download bathymetry data - #### + ### + ### Download bathymetry data + ### - # download_bathymetry() + download_bathymetry() - # #### - # #### Download JRA55 data - # #### + #### + #### Download JRA55 data + #### - # atmosphere = JRA55_prescribed_atmosphere() + atmosphere = JRA55_prescribed_atmosphere() - # #### - # #### Download ECCO data - # #### + #### + #### Download ECCO data + #### - # download_dataset(temperature_metadata) - # download_dataset(salinity_metadata) + download_dataset(temperature_metadata) + download_dataset(salinity_metadata) end -# # Tests JRA55 utilities, plus some DataWrangling utilities -# if test_group == :jra55 || test_group == :all -# include("test_jra55.jl") -# end +# Tests JRA55 utilities, plus some DataWrangling utilities +if test_group == :jra55 || test_group == :all + include("test_jra55.jl") +end -# if test_group == :ecco || test_group == :all -# include("test_ecco.jl") -# end +if test_group == :ecco || test_group == :all + include("test_ecco.jl") +end # Tests that we can download JRA55 utilities if test_group == :downloading || test_group == :all include("test_downloading.jl") end -# if test_group == :fluxes || test_group == :all -# include("test_surface_fluxes.jl") -# end +if test_group == :fluxes || test_group == :all + include("test_surface_fluxes.jl") +end -# if test_group == :bathymetry || test_group == :all -# include("test_bathymetry.jl") -# end +if test_group == :bathymetry || test_group == :all + include("test_bathymetry.jl") +end -# if test_group == :simulations || test_group == :all -# CUDA.set_runtime_version!(v"12.2", local_toolkit = true) # Seems to help in finding the correct CUDA version -# include("test_simulations.jl") -# end +if test_group == :simulations || test_group == :all + CUDA.set_runtime_version!(v"12.2", local_toolkit = true) # Seems to help in finding the correct CUDA version + include("test_simulations.jl") +end -# if test_group == :distributed || test_group == :all -# include("test_distributed_utils.jl") -# end +if test_group == :distributed || test_group == :all + include("test_distributed_utils.jl") +end From 51aff06944473d9229f2016ca4a7d2f9bbdd0263 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Thu, 28 Nov 2024 16:10:53 +0100 Subject: [PATCH 09/28] gracefull downloading --- src/Bathymetry.jl | 5 +++-- src/DataWrangling/DataWrangling.jl | 23 ++++++++++++++++------- src/DataWrangling/ECCO/ECCO.jl | 2 +- src/DataWrangling/ECCO/ECCO_metadata.jl | 4 ++-- src/DataWrangling/JRA55.jl | 7 +++++-- 5 files changed, 27 insertions(+), 14 deletions(-) diff --git a/src/Bathymetry.jl b/src/Bathymetry.jl index 44188591..6d161225 100644 --- a/src/Bathymetry.jl +++ b/src/Bathymetry.jl @@ -3,7 +3,7 @@ module Bathymetry export regrid_bathymetry, retrieve_bathymetry using ImageMorphology -using ..DataWrangling: download_progress +using ..DataWrangling: DownloadProgress using Oceananigans using Oceananigans.Architectures: architecture, on_architecture @@ -96,7 +96,8 @@ function regrid_bathymetry(target_grid; fileurl = url * "/" * filename # joinpath on windows creates the wrong url @root if !isfile(filepath) # perform all this only on rank 0, aka the "root" rank - Downloads.download(fileurl, filepath; progress=download_progress, verbose=true) + progress = DownloadProgress(filename) + Downloads.download(fileurl, filepath; progress) end dataset = Dataset(filepath) diff --git a/src/DataWrangling/DataWrangling.jl b/src/DataWrangling/DataWrangling.jl index 62fc00c7..520665a1 100644 --- a/src/DataWrangling/DataWrangling.jl +++ b/src/DataWrangling/DataWrangling.jl @@ -13,20 +13,30 @@ using Oceananigans: pretty_filesize, location using Oceananigans.Utils: launch! using KernelAbstractions: @kernel, @index -next_fraction = Ref(0.0) -download_start_time = Ref(time_ns()) +struct DownloadProgress <: Function + fileurl :: String + messages :: Int + next_fraction :: Ref{Float64} + download_start_time :: Ref{UInt64} +end + +DownloadProgress(fileurl) = DownloadProgress(fileurl, 10, Ref(0.0), Ref(time_ns())) """ - download_progress(total, now; filename="") + DowloadProgress(total, now; filename="") + +a graceful progres for downloading files """ -function download_progress(total, now; filename="") - messages = 10 +function (d::DownloadProgress)(total, now; filename="") + messages = d.messages + next_fraction = d.next_fraction + download_start_time = d.download_start_time if total > 0 fraction = now / total if fraction < 1 / messages && next_fraction[] == 0 - @info @sprintf("Downloading %s (size: %s)...", filename, pretty_filesize(total)) + @info @sprintf("Downloading %s (size: %s)...", d.fileurl, pretty_filesize(total)) next_fraction[] = 1 / messages download_start_time[] = time_ns() end @@ -40,7 +50,6 @@ function download_progress(total, now; filename="") end else if now > 0 && next_fraction[] == 0 - @info "Downloading $filename..." next_fraction[] = 1 / messages download_start_time[] = time_ns() end diff --git a/src/DataWrangling/ECCO/ECCO.jl b/src/DataWrangling/ECCO/ECCO.jl index bdef42ce..b6168131 100644 --- a/src/DataWrangling/ECCO/ECCO.jl +++ b/src/DataWrangling/ECCO/ECCO.jl @@ -6,7 +6,7 @@ export ECCORestoring, LinearlyTaperedPolarMask using ClimaOcean using ClimaOcean.DataWrangling -using ClimaOcean.DataWrangling: inpaint_mask!, NearestNeighborInpainting +using ClimaOcean.DataWrangling: inpaint_mask!, NearestNeighborInpainting, DownloadProgress using ClimaOcean.InitialConditions: three_dimensional_regrid!, interpolate! using Oceananigans diff --git a/src/DataWrangling/ECCO/ECCO_metadata.jl b/src/DataWrangling/ECCO/ECCO_metadata.jl index bf79471e..51a140ed 100644 --- a/src/DataWrangling/ECCO/ECCO_metadata.jl +++ b/src/DataWrangling/ECCO/ECCO_metadata.jl @@ -240,8 +240,8 @@ function download_dataset(metadata::ECCOMetadata; url = urls(metadata)) and setting your ECCO_USERNAME and ECCO_PASSWORD." * instructions_msg throw(ArgumentError(msg)) end - - Downloads.download(fileurl, filepath; downloader, verbose=true) + progress = DownloadProgress("$(summary(metadata))") + Downloads.download(fileurl, filepath; downloader, progress) end end diff --git a/src/DataWrangling/JRA55.jl b/src/DataWrangling/JRA55.jl index ce8daa0f..bd9ca7e3 100644 --- a/src/DataWrangling/JRA55.jl +++ b/src/DataWrangling/JRA55.jl @@ -13,7 +13,7 @@ using Oceananigans.Fields: interpolate! using Oceananigans.OutputReaders: Cyclical, TotallyInMemory, AbstractInMemoryBackend, FlavorOfFTS, time_indices using ClimaOcean -using ClimaOcean.DataWrangling: download_progress +using ClimaOcean.DataWrangling: DownloadProgress using ClimaOcean.OceanSeaIceModels: PrescribedAtmosphere, @@ -391,7 +391,10 @@ function JRA55_field_time_series(variable_name; # Note, we don't re-use existing jld2 files. @root begin - isfile(filepath) || download(url, filepath) + if isfile(filepath) + progress = DownloadProgress("JRA55 $variable_name data", 50) + Downloads.download(url, filepath; progress) + end isfile(jld2_filepath) && rm(jld2_filepath) end From d22edc924d7b16bcfc9af7cf97357a4818fedc73 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Thu, 28 Nov 2024 16:43:22 +0100 Subject: [PATCH 10/28] try it now --- src/Bathymetry.jl | 5 ++--- src/DataWrangling/DataWrangling.jl | 23 +++++++---------------- src/DataWrangling/ECCO/ECCO.jl | 2 +- src/DataWrangling/ECCO/ECCO_metadata.jl | 4 ++-- src/DataWrangling/JRA55.jl | 7 ++----- 5 files changed, 14 insertions(+), 27 deletions(-) diff --git a/src/Bathymetry.jl b/src/Bathymetry.jl index 6d161225..ea9a358a 100644 --- a/src/Bathymetry.jl +++ b/src/Bathymetry.jl @@ -3,7 +3,7 @@ module Bathymetry export regrid_bathymetry, retrieve_bathymetry using ImageMorphology -using ..DataWrangling: DownloadProgress +using ..DataWrangling: download_progress using Oceananigans using Oceananigans.Architectures: architecture, on_architecture @@ -96,8 +96,7 @@ function regrid_bathymetry(target_grid; fileurl = url * "/" * filename # joinpath on windows creates the wrong url @root if !isfile(filepath) # perform all this only on rank 0, aka the "root" rank - progress = DownloadProgress(filename) - Downloads.download(fileurl, filepath; progress) + Downloads.download(fileurl, filepath; progress=download_progress) end dataset = Dataset(filepath) diff --git a/src/DataWrangling/DataWrangling.jl b/src/DataWrangling/DataWrangling.jl index 520665a1..62fc00c7 100644 --- a/src/DataWrangling/DataWrangling.jl +++ b/src/DataWrangling/DataWrangling.jl @@ -13,30 +13,20 @@ using Oceananigans: pretty_filesize, location using Oceananigans.Utils: launch! using KernelAbstractions: @kernel, @index -struct DownloadProgress <: Function - fileurl :: String - messages :: Int - next_fraction :: Ref{Float64} - download_start_time :: Ref{UInt64} -end - -DownloadProgress(fileurl) = DownloadProgress(fileurl, 10, Ref(0.0), Ref(time_ns())) +next_fraction = Ref(0.0) +download_start_time = Ref(time_ns()) """ - DowloadProgress(total, now; filename="") - -a graceful progres for downloading files + download_progress(total, now; filename="") """ -function (d::DownloadProgress)(total, now; filename="") - messages = d.messages - next_fraction = d.next_fraction - download_start_time = d.download_start_time +function download_progress(total, now; filename="") + messages = 10 if total > 0 fraction = now / total if fraction < 1 / messages && next_fraction[] == 0 - @info @sprintf("Downloading %s (size: %s)...", d.fileurl, pretty_filesize(total)) + @info @sprintf("Downloading %s (size: %s)...", filename, pretty_filesize(total)) next_fraction[] = 1 / messages download_start_time[] = time_ns() end @@ -50,6 +40,7 @@ function (d::DownloadProgress)(total, now; filename="") end else if now > 0 && next_fraction[] == 0 + @info "Downloading $filename..." next_fraction[] = 1 / messages download_start_time[] = time_ns() end diff --git a/src/DataWrangling/ECCO/ECCO.jl b/src/DataWrangling/ECCO/ECCO.jl index b6168131..37d37bfc 100644 --- a/src/DataWrangling/ECCO/ECCO.jl +++ b/src/DataWrangling/ECCO/ECCO.jl @@ -6,7 +6,7 @@ export ECCORestoring, LinearlyTaperedPolarMask using ClimaOcean using ClimaOcean.DataWrangling -using ClimaOcean.DataWrangling: inpaint_mask!, NearestNeighborInpainting, DownloadProgress +using ClimaOcean.DataWrangling: inpaint_mask!, NearestNeighborInpainting, download_progress using ClimaOcean.InitialConditions: three_dimensional_regrid!, interpolate! using Oceananigans diff --git a/src/DataWrangling/ECCO/ECCO_metadata.jl b/src/DataWrangling/ECCO/ECCO_metadata.jl index 51a140ed..d0c9821c 100644 --- a/src/DataWrangling/ECCO/ECCO_metadata.jl +++ b/src/DataWrangling/ECCO/ECCO_metadata.jl @@ -240,8 +240,8 @@ function download_dataset(metadata::ECCOMetadata; url = urls(metadata)) and setting your ECCO_USERNAME and ECCO_PASSWORD." * instructions_msg throw(ArgumentError(msg)) end - progress = DownloadProgress("$(summary(metadata))") - Downloads.download(fileurl, filepath; downloader, progress) + + Downloads.download(fileurl, filepath; downloader, progerss=download_progress) end end diff --git a/src/DataWrangling/JRA55.jl b/src/DataWrangling/JRA55.jl index bd9ca7e3..6586a727 100644 --- a/src/DataWrangling/JRA55.jl +++ b/src/DataWrangling/JRA55.jl @@ -13,7 +13,7 @@ using Oceananigans.Fields: interpolate! using Oceananigans.OutputReaders: Cyclical, TotallyInMemory, AbstractInMemoryBackend, FlavorOfFTS, time_indices using ClimaOcean -using ClimaOcean.DataWrangling: DownloadProgress +using ClimaOcean.DataWrangling: download_progress using ClimaOcean.OceanSeaIceModels: PrescribedAtmosphere, @@ -391,10 +391,7 @@ function JRA55_field_time_series(variable_name; # Note, we don't re-use existing jld2 files. @root begin - if isfile(filepath) - progress = DownloadProgress("JRA55 $variable_name data", 50) - Downloads.download(url, filepath; progress) - end + isfile(filepath) || download(url, filepath; progress=download_progress) isfile(jld2_filepath) && rm(jld2_filepath) end From ebaa07df236150130408c150656a9838d64253b2 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Thu, 28 Nov 2024 17:40:29 +0100 Subject: [PATCH 11/28] fix typo --- src/DataWrangling/ECCO/ECCO_metadata.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/DataWrangling/ECCO/ECCO_metadata.jl b/src/DataWrangling/ECCO/ECCO_metadata.jl index d0c9821c..9bcb2c02 100644 --- a/src/DataWrangling/ECCO/ECCO_metadata.jl +++ b/src/DataWrangling/ECCO/ECCO_metadata.jl @@ -241,7 +241,7 @@ function download_dataset(metadata::ECCOMetadata; url = urls(metadata)) throw(ArgumentError(msg)) end - Downloads.download(fileurl, filepath; downloader, progerss=download_progress) + Downloads.download(fileurl, filepath; downloader, progress=download_progress) end end From 879d611865ad6296334f210908061139da338128 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Thu, 28 Nov 2024 18:08:04 +0100 Subject: [PATCH 12/28] make sure we delete the previous data before testing the download --- test/runtests_setup.jl | 7 +++++-- test/test_downloading.jl | 11 ++++++++++- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/test/runtests_setup.jl b/test/runtests_setup.jl index e815da71..f8d83ebd 100644 --- a/test/runtests_setup.jl +++ b/test/runtests_setup.jl @@ -12,6 +12,7 @@ using Oceananigans.Architectures: architecture, on_architecture using Oceananigans.OutputReaders: interpolate! using ClimaOcean +using ClimaOcean.Bathymetry: download_bathymetry_cache using CFTime using Dates @@ -28,13 +29,15 @@ temperature_metadata = ECCOMetadata(:temperature, dates) salinity_metadata = ECCOMetadata(:salinity, dates) # Fictitious grid that triggers bathymetry download -function download_bathymetry() +function download_bathymetry(; dir = download_bathymetry_cache, + filename = "ETOPO_2022_v1_60s_N90W180_surface.nc") + grid = LatitudeLongitudeGrid(size = (10, 10, 1), longitude = (0, 100), latitude = (0, 50), z = (-6000, 0)) - bottom = regrid_bathymetry(grid) + bottom = regrid_bathymetry(grid; dir, filename) return nothing end diff --git a/test/test_downloading.jl b/test/test_downloading.jl index 66a58124..02eb5590 100644 --- a/test/test_downloading.jl +++ b/test/test_downloading.jl @@ -1,8 +1,11 @@ include("runtests_setup.jl") +using ClimaOcean.ECCO: metadata_path + @testset "Availability of JRA55 data" begin @info "Testing that we can download all the JRA55 data..." for name in ClimaOcean.DataWrangling.JRA55.JRA55_variable_names + fts = ClimaOcean.JRA55.JRA55_field_time_series(name; time_indices=2:3) end end @@ -11,11 +14,17 @@ end @info "Testing that we can download ECCO data..." for variable in keys(ClimaOcean.ECCO.ECCO4_short_names) metadata = ECCOMetadata(variable) + filepath = metadata_path(metadata) + isfile(filepath) && rm(filepath; force=true) ClimaOcean.ECCO.download_dataset(metadata) end end @testset "Availability of the Bathymetry" begin @info "Testing that we can download the bathymetry..." - download_bathymetry() + dir="./" + filename="ETOPO_2022_v1_60s_N90W180_surface.nc" + filepath=joinpath(dir, filename) + isfile(filepath) && rm(filepath; force=true) + download_bathymetry(; dir, filename) end \ No newline at end of file From 0c38ce126af2d16142001a38fdbd585e4697a680 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Mon, 2 Dec 2024 14:25:21 +0100 Subject: [PATCH 13/28] should work --- src/DataWrangling/ECCO/ECCO_metadata.jl | 72 +++++++++++-------------- test/test_downloading.jl | 2 +- 2 files changed, 33 insertions(+), 41 deletions(-) diff --git a/src/DataWrangling/ECCO/ECCO_metadata.jl b/src/DataWrangling/ECCO/ECCO_metadata.jl index 9bcb2c02..a2673bf0 100644 --- a/src/DataWrangling/ECCO/ECCO_metadata.jl +++ b/src/DataWrangling/ECCO/ECCO_metadata.jl @@ -218,42 +218,44 @@ function download_dataset(metadata::ECCOMetadata; url = urls(metadata)) username = get(ENV, "ECCO_USERNAME", nothing) password = get(ENV, "ECCO_PASSWORD", nothing) dir = metadata.dir - - # Write down the username and password in a .netrc file - downloader = ECCO_downloader(username, password, dir) - - @distribute for metadatum in metadata # Distribute the download among ranks if MPI is initialized - - fileurl = metadata_url(url, metadatum) - filepath = metadata_path(metadatum) - - if !isfile(filepath) - instructions_msg = "\n See ClimaOcean.jl/src/ECCO/README.md for instructions." - if isnothing(username) - msg = "Could not find the ECCO_PASSWORD environment variable. \ - See ClimaOcean.jl/src/ECCO/README.md for instructions on obtaining \ - and setting your ECCO_USERNAME and ECCO_PASSWORD." * instructions_msg - throw(ArgumentError(msg)) - elseif isnothing(password) - msg = "Could not find the ECCO_PASSWORD environment variable. \ - See ClimaOcean.jl/src/ECCO/README.md for instructions on obtaining \ - and setting your ECCO_USERNAME and ECCO_PASSWORD." * instructions_msg - throw(ArgumentError(msg)) + + # Create a temporary directory to store the .netrc file + # The directory will be deleted after the download is complete + mktempdir(dir) do tmp + + # Write down the username and password in a .netrc file + downloader = ECCO_downloader(username, password, tmp) + + @distribute for metadatum in metadata # Distribute the download among ranks if MPI is initialized + + fileurl = metadata_url(url, metadatum) + filepath = metadata_path(metadatum) + + if !isfile(filepath) + instructions_msg = "\n See ClimaOcean.jl/src/ECCO/README.md for instructions." + if isnothing(username) + msg = "Could not find the ECCO_PASSWORD environment variable. \ + See ClimaOcean.jl/src/ECCO/README.md for instructions on obtaining \ + and setting your ECCO_USERNAME and ECCO_PASSWORD." * instructions_msg + throw(ArgumentError(msg)) + elseif isnothing(password) + msg = "Could not find the ECCO_PASSWORD environment variable. \ + See ClimaOcean.jl/src/ECCO/README.md for instructions on obtaining \ + and setting your ECCO_USERNAME and ECCO_PASSWORD." * instructions_msg + throw(ArgumentError(msg)) + end + + Downloads.download(fileurl, filepath; downloader, progress=download_progress) end - - Downloads.download(fileurl, filepath; downloader, progress=download_progress) end end - - # Remove the .netrc file after downloading to avoid storing the credentials - remove_netrc!(dir) - + return nothing end -# ECCO downloader +# netcr-based downloader function ECCO_downloader(username, password, dir) - netrc_file = ECCO_netrc!(username, password, dir) + netrc_file = ECCO_netrc(username, password, dir) downloader = Downloads.Downloader() easy_hook = (easy, _) -> Downloads.Curl.setopt(easy, Downloads.Curl.CURLOPT_NETRC_FILE, netrc_file) @@ -262,7 +264,7 @@ function ECCO_downloader(username, password, dir) end # Code snippet adapted from https://github.com/evetion/SpaceLiDAR.jl/blob/master/src/utils.jl#L150 -function ECCO_netrc!(username, password, dir) +function ECCO_netrc(username, password, dir) if Sys.iswindows() filepath = joinpath(dir, "ECCO_netrc") else @@ -276,13 +278,3 @@ function ECCO_netrc!(username, password, dir) return filepath end - -function remove_netrc!(dir) - if Sys.iswindows() - filepath = joinpath(dir, "ECCO_netrc") - else - filepath = joinpath(dir, "ECCO.netrc") - end - - rm(filepath; force = true) -end \ No newline at end of file diff --git a/test/test_downloading.jl b/test/test_downloading.jl index 02eb5590..9f12846a 100644 --- a/test/test_downloading.jl +++ b/test/test_downloading.jl @@ -27,4 +27,4 @@ end filepath=joinpath(dir, filename) isfile(filepath) && rm(filepath; force=true) download_bathymetry(; dir, filename) -end \ No newline at end of file +end From c0892b91d2098b01eca757b2a5ee2fff83ae0996 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Mon, 2 Dec 2024 14:33:17 +0100 Subject: [PATCH 14/28] test distributed downloading --- test/test_distributed_utils.jl | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/test/test_distributed_utils.jl b/test/test_distributed_utils.jl index a6a20075..a0ac2a09 100644 --- a/test/test_distributed_utils.jl +++ b/test/test_distributed_utils.jl @@ -3,6 +3,10 @@ include("runtests_setup.jl") using MPI MPI.Init() +using ClimaOcean.ECCO: download_dataset, metadata_path +using CFTime +using Dates + @testset begin rank = MPI.Comm_rank(MPI.COMM_WORLD) @@ -47,4 +51,13 @@ MPI.Init() @onrank 3, begin @test a == [4, 8] end -end \ No newline at end of file +end + +@testset "Distributed ECCO download" begin + metadata = ECCOMetadata(:temperature, dates=Date(1992, 1, 1):Month(1):Date(1992, 4, 1)) + download_dataset(metadata) + + @root for metadatum in metadata + @test isfile(metadata_path(metadatum)) + end +end \ No newline at end of file From 6d73e98fed95d1f3dfae53d7b015cea393438683 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Mon, 2 Dec 2024 15:05:49 +0100 Subject: [PATCH 15/28] Update test_distributed_utils.jl --- test/test_distributed_utils.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_distributed_utils.jl b/test/test_distributed_utils.jl index a0ac2a09..1eb6090a 100644 --- a/test/test_distributed_utils.jl +++ b/test/test_distributed_utils.jl @@ -60,4 +60,4 @@ end @root for metadatum in metadata @test isfile(metadata_path(metadatum)) end -end \ No newline at end of file +end From 3fc3bd547cbdc7b03ba389914b7bc5830a85949d Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Mon, 2 Dec 2024 16:31:41 +0100 Subject: [PATCH 16/28] fix the download --- test/test_distributed_utils.jl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/test_distributed_utils.jl b/test/test_distributed_utils.jl index 1eb6090a..3ca8b06a 100644 --- a/test/test_distributed_utils.jl +++ b/test/test_distributed_utils.jl @@ -54,7 +54,8 @@ using Dates end @testset "Distributed ECCO download" begin - metadata = ECCOMetadata(:temperature, dates=Date(1992, 1, 1):Month(1):Date(1992, 4, 1)) + dates = DateTimeProlepticGregorian(1992, 1, 1) : Month(1) : DateTimeProlepticGregorian(1994, 4, 1) + metadata = ECCOMetadata(:u_velocity; dates) download_dataset(metadata) @root for metadatum in metadata From f3dec5c4f3ba6d6d6f0a224c584313f5c7d6a08a Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Mon, 2 Dec 2024 17:31:46 +0100 Subject: [PATCH 17/28] generalize the downloader --- src/DataWrangling/DataWrangling.jl | 37 +++++++++++++++++++++++++ src/DataWrangling/ECCO/ECCO_metadata.jl | 31 ++------------------- 2 files changed, 40 insertions(+), 28 deletions(-) diff --git a/src/DataWrangling/DataWrangling.jl b/src/DataWrangling/DataWrangling.jl index 62fc00c7..a8efdbe3 100644 --- a/src/DataWrangling/DataWrangling.jl +++ b/src/DataWrangling/DataWrangling.jl @@ -49,6 +49,43 @@ function download_progress(total, now; filename="") return nothing end +##### +##### Downloading utilities +##### + +# netrc-based downloader +# This downlader writes down a netrc file with the username and password for the given machine. +# To avoid storing passwords in plain text, it is recommended to use a temporary directory. +# For example: +# mktempdir(parent) do dir +# netrc_downloader(username, password, machine, dir) +# ... download files ... +# end +function netrc_downloader(username, password, machine, dir) + netrc_file = netrc_permission_file(username, password, machine, dir) + downloader = Downloads.Downloader() + easy_hook = (easy, _) -> Downloads.Curl.setopt(easy, Downloads.Curl.CURLOPT_NETRC_FILE, netrc_file) + + downloader.easy_hook = easy_hook + return downloader +end + +# Code snippet adapted from https://github.com/evetion/SpaceLiDAR.jl/blob/master/src/utils.jl#L150 +function netrc_permission_file(username, password, machine, dir) + if Sys.iswindows() + filepath = joinpath(dir, "ECCO_netrc") + else + filepath = joinpath(dir, "ECCO.netrc") + end + + open(filepath, "a") do f + write(f, "\n") + write(f, "machine $machine login $username password $password\n") + end + + return filepath +end + ##### ##### FieldTimeSeries utilities ##### diff --git a/src/DataWrangling/ECCO/ECCO_metadata.jl b/src/DataWrangling/ECCO/ECCO_metadata.jl index a2673bf0..05b87c31 100644 --- a/src/DataWrangling/ECCO/ECCO_metadata.jl +++ b/src/DataWrangling/ECCO/ECCO_metadata.jl @@ -1,6 +1,7 @@ using CFTime using Dates using ClimaOcean.DataWrangling +using ClimaOcean.DataWrangling: netrc_downloader import Dates: year, month, day @@ -222,9 +223,9 @@ function download_dataset(metadata::ECCOMetadata; url = urls(metadata)) # Create a temporary directory to store the .netrc file # The directory will be deleted after the download is complete mktempdir(dir) do tmp - + # Write down the username and password in a .netrc file - downloader = ECCO_downloader(username, password, tmp) + downloader = netrc_downloader(username, password, "ecco.jpl.nasa.gov", tmp) @distribute for metadatum in metadata # Distribute the download among ranks if MPI is initialized @@ -252,29 +253,3 @@ function download_dataset(metadata::ECCOMetadata; url = urls(metadata)) return nothing end - -# netcr-based downloader -function ECCO_downloader(username, password, dir) - netrc_file = ECCO_netrc(username, password, dir) - downloader = Downloads.Downloader() - easy_hook = (easy, _) -> Downloads.Curl.setopt(easy, Downloads.Curl.CURLOPT_NETRC_FILE, netrc_file) - - downloader.easy_hook = easy_hook - return downloader -end - -# Code snippet adapted from https://github.com/evetion/SpaceLiDAR.jl/blob/master/src/utils.jl#L150 -function ECCO_netrc(username, password, dir) - if Sys.iswindows() - filepath = joinpath(dir, "ECCO_netrc") - else - filepath = joinpath(dir, "ECCO.netrc") - end - - open(filepath, "a") do f - write(f, "\n") - write(f, "machine ecco.jpl.nasa.gov login $username password $password\n") - end - - return filepath -end From 3525b76b7b2d8dfb18de9810e61e4efb99032a28 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Mon, 2 Dec 2024 17:35:51 +0100 Subject: [PATCH 18/28] generalize more --- src/DataWrangling/DataWrangling.jl | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/src/DataWrangling/DataWrangling.jl b/src/DataWrangling/DataWrangling.jl index a8efdbe3..8cc1f4c3 100644 --- a/src/DataWrangling/DataWrangling.jl +++ b/src/DataWrangling/DataWrangling.jl @@ -53,14 +53,24 @@ end ##### Downloading utilities ##### -# netrc-based downloader -# This downlader writes down a netrc file with the username and password for the given machine. -# To avoid storing passwords in plain text, it is recommended to use a temporary directory. -# For example: -# mktempdir(parent) do dir -# netrc_downloader(username, password, machine, dir) -# ... download files ... -# end +""" + netrc_downloader(username, password, machine, dir) + +Create a downloader that uses a netrc file to authenticate with the given machine. +This downlader writes down a netrc file with the username and password for the given machine in the directory `dir`. +To avoid storing passwords in plain text, it is recommended to initialize the downloader in a temporary directory. + +For example: + +``` +mktempdir(dir) do tmp + + dowloader = netrc_downloader(username, password, machine, tmp) + ... download files ... + +end +``` +""" function netrc_downloader(username, password, machine, dir) netrc_file = netrc_permission_file(username, password, machine, dir) downloader = Downloads.Downloader() @@ -79,7 +89,6 @@ function netrc_permission_file(username, password, machine, dir) end open(filepath, "a") do f - write(f, "\n") write(f, "machine $machine login $username password $password\n") end From 6a3fa7dbb548764caaf8ba014a2fefb89b6b92aa Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Mon, 2 Dec 2024 17:39:34 +0100 Subject: [PATCH 19/28] generalize filename --- src/DataWrangling/DataWrangling.jl | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/DataWrangling/DataWrangling.jl b/src/DataWrangling/DataWrangling.jl index 8cc1f4c3..1d1996c3 100644 --- a/src/DataWrangling/DataWrangling.jl +++ b/src/DataWrangling/DataWrangling.jl @@ -57,8 +57,9 @@ end netrc_downloader(username, password, machine, dir) Create a downloader that uses a netrc file to authenticate with the given machine. -This downlader writes down a netrc file with the username and password for the given machine in the directory `dir`. -To avoid storing passwords in plain text, it is recommended to initialize the downloader in a temporary directory. +This downlader writes the username and password in a file named `auth.netrc` (for Unix) and `auth_netrc` (for Windows) +which is located in the directory `dir`. To avoid storing passwords in plain text, +it is recommended to initialize the downloader in a temporary directory. For example: @@ -83,9 +84,9 @@ end # Code snippet adapted from https://github.com/evetion/SpaceLiDAR.jl/blob/master/src/utils.jl#L150 function netrc_permission_file(username, password, machine, dir) if Sys.iswindows() - filepath = joinpath(dir, "ECCO_netrc") + filepath = joinpath(dir, "auth_netrc") else - filepath = joinpath(dir, "ECCO.netrc") + filepath = joinpath(dir, "auth.netrc") end open(filepath, "a") do f From 91be18816526bec9685923f0026307f98e694d37 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Mon, 2 Dec 2024 17:40:05 +0100 Subject: [PATCH 20/28] download_progress is part of the downloading utilities --- src/DataWrangling/DataWrangling.jl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/DataWrangling/DataWrangling.jl b/src/DataWrangling/DataWrangling.jl index 1d1996c3..85339b0d 100644 --- a/src/DataWrangling/DataWrangling.jl +++ b/src/DataWrangling/DataWrangling.jl @@ -13,6 +13,10 @@ using Oceananigans: pretty_filesize, location using Oceananigans.Utils: launch! using KernelAbstractions: @kernel, @index +##### +##### Downloading utilities +##### + next_fraction = Ref(0.0) download_start_time = Ref(time_ns()) @@ -49,10 +53,6 @@ function download_progress(total, now; filename="") return nothing end -##### -##### Downloading utilities -##### - """ netrc_downloader(username, password, machine, dir) From ab2563da948330b840d3b4e0a9b24179faed3a3f Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Mon, 2 Dec 2024 17:41:16 +0100 Subject: [PATCH 21/28] better docstring --- src/DataWrangling/DataWrangling.jl | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/DataWrangling/DataWrangling.jl b/src/DataWrangling/DataWrangling.jl index 85339b0d..dff85615 100644 --- a/src/DataWrangling/DataWrangling.jl +++ b/src/DataWrangling/DataWrangling.jl @@ -65,10 +65,8 @@ For example: ``` mktempdir(dir) do tmp - dowloader = netrc_downloader(username, password, machine, tmp) - ... download files ... - + Downloads.download(fileurl, filepath; downloader) end ``` """ From 25b42cde1862cf14ea486b5f704ca3d2804ea143 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Mon, 2 Dec 2024 17:42:09 +0100 Subject: [PATCH 22/28] better docstring --- src/DataWrangling/DataWrangling.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/DataWrangling/DataWrangling.jl b/src/DataWrangling/DataWrangling.jl index dff85615..2bc531fb 100644 --- a/src/DataWrangling/DataWrangling.jl +++ b/src/DataWrangling/DataWrangling.jl @@ -57,8 +57,8 @@ end netrc_downloader(username, password, machine, dir) Create a downloader that uses a netrc file to authenticate with the given machine. -This downlader writes the username and password in a file named `auth.netrc` (for Unix) and `auth_netrc` (for Windows) -which is located in the directory `dir`. To avoid storing passwords in plain text, +This downlader writes the username and password in a file named `auth.netrc` (for Unix) and +`auth_netrc` (for Windows), located in the directory `dir`. To avoid storing passwords in plain text, it is recommended to initialize the downloader in a temporary directory. For example: From a77cebb5c5a40fb6c66cb9498f988d524d295e2a Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Mon, 2 Dec 2024 18:24:45 +0100 Subject: [PATCH 23/28] change docstring --- src/DataWrangling/DataWrangling.jl | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/DataWrangling/DataWrangling.jl b/src/DataWrangling/DataWrangling.jl index 2bc531fb..792c07b3 100644 --- a/src/DataWrangling/DataWrangling.jl +++ b/src/DataWrangling/DataWrangling.jl @@ -58,8 +58,10 @@ end Create a downloader that uses a netrc file to authenticate with the given machine. This downlader writes the username and password in a file named `auth.netrc` (for Unix) and -`auth_netrc` (for Windows), located in the directory `dir`. To avoid storing passwords in plain text, -it is recommended to initialize the downloader in a temporary directory. +`auth_netrc` (for Windows), located in the directory `dir`. +To avoid leaving the password on disk after the downloader has been used, +it is recommended to initialize the downloader in a temporary directory, which will be removed +after the download is complete. For example: From c8b6e97d51a64efa9d8f4694868a760e3042eaee Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 4 Dec 2024 18:43:35 +0100 Subject: [PATCH 24/28] fix tests --- src/distributed_utils.jl | 107 ++++++++++++++++++++++++++------------- 1 file changed, 72 insertions(+), 35 deletions(-) diff --git a/src/distributed_utils.jl b/src/distributed_utils.jl index 089962d8..9cdea1cd 100644 --- a/src/distributed_utils.jl +++ b/src/distributed_utils.jl @@ -6,25 +6,27 @@ using MPI ##### # Utilities to make the macro work importing only ClimaOcean and not MPI -mpi_initialized() = MPI.Initialized() -mpi_rank() = MPI.Comm_rank(MPI.COMM_WORLD) -mpi_size() = MPI.Comm_size(MPI.COMM_WORLD) -global_barrier() = mpi_initialized() ? MPI.Barrier(MPI.COMM_WORLD) : nothing +mpi_initialized() = MPI.Initialized() +mpi_rank(comm) = MPI.Comm_rank(comm) +mpi_size(comm) = MPI.Comm_size(comm) +global_barrier(comm) = MPI.Barrier(comm) +global_communicator() = MPI.COMM_WORLD """ - @root exs... + @root communicator exs... -Perform `exs` only on rank 0, otherwise know as "root" rank. -Other ranks will wait for the root rank to finish before continuing +Perform `exs` only on rank 0 in communicator, otherwise known as the "root" rank. +Other ranks will wait for the root rank to finish before continuing. +If `communicator` is not provided, `MPI.COMM_WORLD` is used. """ -macro root(exp) +macro root(communicator, exp) command = quote if ClimaOcean.mpi_initialized() - rank = ClimaOcean.mpi_rank() + rank = ClimaOcean.mpi_rank($communicator) if rank == 0 $exp end - ClimaOcean.global_barrier() + ClimaOcean.global_barrier($communicator) else $exp end @@ -32,40 +34,54 @@ macro root(exp) return esc(command) end +macro root(exp) + command = quote + @root MPI.COMM_WORLD $exp + end + return esc(command) +end + """ - @onrank rank, exs... + @onrank communicator rank exs... -Perform `exp` only on rank `rank` -Other ranks will wait for the root rank to finish before continuing. -The expression is run anyways if MPI in not initialized +Perform `exp` only on rank `rank` (0-based index) in `communicator`. +Other ranks will wait for rank `rank` to finish before continuing. +The expression is run anyways if MPI in not initialized. +If `communicator` is not provided, `MPI.COMM_WORLD` is used. """ -macro onrank(exp_with_rank) - on_rank = exp_with_rank.args[1] - exp = exp_with_rank.args[2] +macro onrank(communicator, on_rank, exp) command = quote mpi_initialized = ClimaOcean.mpi_initialized() - rank = ClimaOcean.mpi_rank() if !mpi_initialized $exp else + rank = ClimaOcean.mpi_rank($communicator) if rank == $on_rank $exp end - ClimaOcean.global_barrier() + ClimaOcean.global_barrier($communicator) end end return esc(command) end +macro onrank(rank, exp) + command = quote + @onrank ClimaOcean.global_communicator() $rank $exp + end + return esc(command) +end + """ - @distribute for i in iterable + @distribute communicator for i in iterable ... end -Distribute a `for` loop among different ranks +Distribute a `for` loop among different ranks in `communicator`. +If `communicator` is not provided, `MPI.COMM_WORLD` is used. """ -macro distribute(exp) +macro distribute(communicator, exp) if exp.head != :for error("The `@distribute` macro expects a `for` loop") end @@ -74,46 +90,67 @@ macro distribute(exp) variable = exp.args[1].args[1] forbody = exp.args[2] + # Safety net if the iterable variable has the same name as the + # reserved variable names (nprocs, counter, rank) + nprocs = ifelse(variable == :nprocs, :othernprocs, :nprocs) + counter = ifelse(variable == :counter, :othercounter, :counter) + rank = ifelse(variable == :rank, :otherrank, :rank) + new_loop = quote mpi_initialized = ClimaOcean.mpi_initialized() if !mpi_initialized $exp else - rank = ClimaOcean.mpi_rank() - nprocs = ClimaOcean.mpi_size() - for (counter, $variable) in enumerate($iterable) - if (counter - 1) % nprocs == rank + $rank = ClimaOcean.mpi_rank($communicator) + $nprocs = ClimaOcean.mpi_size($communicator) + for ($counter, $variable) in enumerate($iterable) + if ($counter - 1) % $nprocs == $rank $forbody end end - ClimaOcean.global_barrier() + ClimaOcean.global_barrier($communicator) end end return esc(new_loop) end +macro distribute(exp) + command = quote + @distribute ClimaOcean.global_communicator() $exp + end + return esc(command) +end + """ - @handshake exs... + @handshake communicator exs... -perform `exs` on all ranks, but only one rank at a time, where -ranks `r2 > r1` wait for rank `r1` to finish before executing `exs` +perform `exs` on all ranks in `communicator`, but only one rank at a time, where +ranks `r2 > r1` wait for rank `r1` to finish before executing `exs`. +If `communicator` is not provided, `MPI.COMM_WORLD` is used. """ -macro handshake(exp) +macro handshake(communicator, exp) command = quote mpi_initialized = ClimaOcean.mpi_initialized() if !mpi_initialized $exp else - rank = ClimaOcean.mpi_rank() - nprocs = ClimaOcean.mpi_size() + rank = ClimaOcean.mpi_rank($communicator) + nprocs = ClimaOcean.mpi_size($communicator) for r in 0 : nprocs -1 if rank == r $exp end - ClimaOcean.global_barrier() + ClimaOcean.global_barrier($communicator) end end end return esc(command) -end \ No newline at end of file +end + +macro handshake(exp) + command = quote + @handshake ClimaOcean.global_communicator() $exp + end + return esc(command) +end From ad64272b4b0b3dbb4104b6b41d21f27e5ede745f Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Thu, 5 Dec 2024 15:36:33 +0100 Subject: [PATCH 25/28] distribute among tasks --- .../os_papa_surface_temperature.jl | 299 ++++++++++++++++++ src/DataWrangling/ECCO/ECCO_metadata.jl | 4 +- 2 files changed, 301 insertions(+), 2 deletions(-) create mode 100644 experiments/single_column_experiments/os_papa_surface_temperature.jl diff --git a/experiments/single_column_experiments/os_papa_surface_temperature.jl b/experiments/single_column_experiments/os_papa_surface_temperature.jl new file mode 100644 index 00000000..c6ec92a1 --- /dev/null +++ b/experiments/single_column_experiments/os_papa_surface_temperature.jl @@ -0,0 +1,299 @@ +# This simulation tests the difference between using a +# `BulkTemperature` and a `SkinTemperature` + +using ClimaOcean +using ClimaOcean.OceanSeaIceModels.CrossRealmFluxes: SimilarityTheoryTurbulentFluxes, SkinTemperature +using Oceananigans +using Oceananigans.Units +using Oceananigans.BuoyancyModels: buoyancy_frequency +using Oceananigans.Units: Time +using GLMakie +using Printf + +# Ocean station papa location +location_name = "ocean_station_papa" +λ★, φ★ = 35.1, 50.1 + +grid = RectilinearGrid(size = 200, + x = λ★, + y = φ★, + z = (-400, 0), + topology = (Flat, Flat, Bounded)) + +ocean1 = ocean_simulation(grid; Δt=10minutes, coriolis=FPlane(latitude = φ★)) +ocean2 = ocean_simulation(grid; Δt=10minutes, coriolis=FPlane(latitude = φ★)) + +# We set initial conditions from ECCO: +set!(ocean1.model.tracers.T, ECCOMetadata(:temperature)) +set!(ocean1.model.tracers.S, ECCOMetadata(:salinity)) +set!(ocean2.model.tracers.T, ECCOMetadata(:temperature)) +set!(ocean2.model.tracers.S, ECCOMetadata(:salinity)) + +simulation_days = 31 +snapshots_per_day = 8 # corresponding to JRA55's 3-hour frequency +last_time = simulation_days * snapshots_per_day +atmosphere = JRA55_prescribed_atmosphere(1:last_time; + longitude = λ★, + latitude = φ★, + backend = InMemory()) + +# We continue constructing a simulation. + +radiation = Radiation() +coupled_model_prescribed = OceanSeaIceModel(ocean1; atmosphere, radiation) + +similarity_theory = SimilarityTheoryTurbulentFluxes(grid; surface_temperature_type = SkinTemperature(; κ = 0.5)) +coupled_model_diagnostic = OceanSeaIceModel(ocean2; atmosphere, radiation, similarity_theory) + +for (coupled_model, suffix) in zip([coupled_model_diagnostic, coupled_model_prescribed], + ["diagnostic", "prescribed"]) + + simulation = Simulation(coupled_model, Δt=ocean1.Δt, stop_time=10days) + + wall_clock = Ref(time_ns()) + + function progress(sim) + msg = "Ocean Station Papa" + msg *= string(", iter: ", iteration(sim), ", time: ", prettytime(sim)) + + elapsed = 1e-9 * (time_ns() - wall_clock[]) + msg *= string(", wall time: ", prettytime(elapsed)) + wall_clock[] = time_ns() + + u, v, w = sim.model.ocean.model.velocities + msg *= @sprintf(", max|u|: (%.2e, %.2e)", maximum(abs, u), maximum(abs, v)) + + T = sim.model.ocean.model.tracers.T + S = sim.model.ocean.model.tracers.S + e = sim.model.ocean.model.tracers.e + + τx = first(sim.model.fluxes.total.ocean.momentum.u) + τy = first(sim.model.fluxes.total.ocean.momentum.v) + Q = first(sim.model.fluxes.total.ocean.heat) + + u★ = sqrt(sqrt(τx^2 + τy^2)) + Ts = first(interior(sim.model.fluxes.turbulent.fields.T_surface, 1, 1, 1)) + + Nz = size(T, 3) + msg *= @sprintf(", u★: %.2f m s⁻¹", u★) + msg *= @sprintf(", Q: %.2f W m⁻²", Q) + msg *= @sprintf(", T₀: %.2f ᵒC", Ts) + msg *= @sprintf(", S₀: %.2f g/kg", first(interior(S, 1, 1, Nz))) + msg *= @sprintf(", e₀: %.2e m² s⁻²", first(interior(e, 1, 1, Nz))) + + @info msg + end + + simulation.callbacks[:progress] = Callback(progress, IterationInterval(100)) + + # Build flux outputs + τx = coupled_model.fluxes.total.ocean.momentum.u + τy = coupled_model.fluxes.total.ocean.momentum.v + JT = coupled_model.fluxes.total.ocean.tracers.T + Js = coupled_model.fluxes.total.ocean.tracers.S + E = coupled_model.fluxes.turbulent.fields.water_vapor + Qc = coupled_model.fluxes.turbulent.fields.sensible_heat + Qv = coupled_model.fluxes.turbulent.fields.latent_heat + Ts = coupled_model.fluxes.turbulent.fields.T_surface + ρₒ = coupled_model.fluxes.ocean_reference_density + cₚ = coupled_model.fluxes.ocean_heat_capacity + + Q = ρₒ * cₚ * JT + ρτx = ρₒ * τx + ρτy = ρₒ * τy + N² = buoyancy_frequency(coupled_model.ocean.model) + κc = coupled_model.ocean.model.diffusivity_fields.κc + + fluxes = (; ρτx, ρτy, E, Js, Qv, Qc, Ts) + auxiliary_fields = (; N², κc) + fields = merge(coupled_model.ocean.model.velocities, coupled_model.ocean.model.tracers, auxiliary_fields) + + # Slice fields at the surface + outputs = merge(fields, fluxes) + + filename = "single_column_omip_$(location_name)_$(suffix)" + + simulation.output_writers[:jld2] = JLD2OutputWriter(coupled_model.ocean.model, outputs; filename, + schedule = TimeInterval(3hours), + overwrite_existing = true) + + run!(simulation) +end + +##### +##### Visualization +##### + +filename_prescribed = "single_column_omip_$(location_name)_prescribed.jld2" +filename_diagnostic = "single_column_omip_$(location_name)_diagnostic.jld2" + +# Diagnosed +ud = FieldTimeSeries(filename_diagnostic, "u") +vd = FieldTimeSeries(filename_diagnostic, "v") +Td = FieldTimeSeries(filename_diagnostic, "v") +ed = FieldTimeSeries(filename_diagnostic, "T") +Sd = FieldTimeSeries(filename_diagnostic, "S") +Nd² = FieldTimeSeries(filename_diagnostic, "N²") +κd = FieldTimeSeries(filename_diagnostic, "κc") + +Tsd = FieldTimeSeries(filename_diagnostic, "Ts") +Qvd = FieldTimeSeries(filename_diagnostic, "Qv") +Qcd = FieldTimeSeries(filename_diagnostic, "Qc") +Jsd = FieldTimeSeries(filename_diagnostic, "Js") +Evd = FieldTimeSeries(filename_diagnostic, "E") +ρτxd = FieldTimeSeries(filename_diagnostic, "ρτx") +ρτyd = FieldTimeSeries(filename_diagnostic, "ρτy") + +# Prescribed +up = FieldTimeSeries(filename_prescribed, "u") +vp = FieldTimeSeries(filename_prescribed, "v") +Tp = FieldTimeSeries(filename_prescribed, "T") +Sp = FieldTimeSeries(filename_prescribed, "S") +ep = FieldTimeSeries(filename_prescribed, "e") +Np² = FieldTimeSeries(filename_prescribed, "N²") +κp = FieldTimeSeries(filename_prescribed, "κc") + +Tsp = FieldTimeSeries(filename_prescribed, "Ts") +Qvp = FieldTimeSeries(filename_prescribed, "Qv") +Qcp = FieldTimeSeries(filename_prescribed, "Qc") +Jsp = FieldTimeSeries(filename_prescribed, "Js") +Evp = FieldTimeSeries(filename_prescribed, "E") +ρτxp = FieldTimeSeries(filename_prescribed, "ρτx") +ρτyp = FieldTimeSeries(filename_prescribed, "ρτy") + +Nz = size(ud, 3) +times = Qcd.times + +fig = Figure(size=(1800, 1800)) + +axτ = Axis(fig[1, 1:3], xlabel="Days since Oct 1 1992", ylabel="Wind stress (N m⁻²)") +axQ = Axis(fig[1, 4:6], xlabel="Days since Oct 1 1992", ylabel="Heat flux (W m⁻²)") +axu = Axis(fig[2, 1:3], xlabel="Days since Oct 1 1992", ylabel="Velocities (m s⁻¹)") +axT = Axis(fig[2, 4:6], xlabel="Days since Oct 1 1992", ylabel="Surface temperature (ᵒC)") +axF = Axis(fig[3, 1:3], xlabel="Days since Oct 1 1992", ylabel="Freshwater volume flux (m s⁻¹)") +axS = Axis(fig[3, 4:6], xlabel="Days since Oct 1 1992", ylabel="Surface salinity (g kg⁻¹)") + +axuz = Axis(fig[4:5, 1:2], xlabel="Velocities (m s⁻¹)", ylabel="z (m)") +axTz = Axis(fig[4:5, 3:4], xlabel="Temperature (ᵒC)", ylabel="z (m)") +axSz = Axis(fig[4:5, 5:6], xlabel="Salinity (g kg⁻¹)", ylabel="z (m)") +axNz = Axis(fig[6:7, 1:2], xlabel="Buoyancy frequency (s⁻²)", ylabel="z (m)") +axκz = Axis(fig[6:7, 3:4], xlabel="Eddy diffusivity (m² s⁻¹)", ylabel="z (m)", xscale=log10) +axez = Axis(fig[6:7, 5:6], xlabel="Turbulent kinetic energy (m² s⁻²)", ylabel="z (m)", xscale=log10) + +title = @sprintf("Single-column simulation at %.2f, %.2f", φ★, λ★) +Label(fig[0, 1:6], title) + +n = Observable(1) + +times = (times .- times[1]) ./days +Nt = length(times) +tn = @lift times[$n] + +colors = Makie.wong_colors() + +ρₒ = coupled_model_prescribed.fluxes.ocean_reference_density +τxp = interior(ρτxp, 1, 1, 1, :) ./ ρₒ +τyp = interior(ρτyp, 1, 1, 1, :) ./ ρₒ +up★ = @. (τxp^2 + τyp^2)^(1/4) +τxd = interior(ρτxd, 1, 1, 1, :) ./ ρₒ +τyd = interior(ρτyd, 1, 1, 1, :) ./ ρₒ +ud★ = @. (τxd^2 + τyd^2)^(1/4) + +lines!(axu, times, interior(ud, 1, 1, Nz, :), color=colors[1], label="Zonal") +lines!(axu, times, interior(vd, 1, 1, Nz, :), color=colors[2], label="Meridional") +lines!(axu, times, interior(up, 1, 1, Nz, :), color=colors[1], linestyle = :dash, label="Zonal") +lines!(axu, times, interior(vp, 1, 1, Nz, :), color=colors[2], linestyle = :dash, label="Meridional") +lines!(axu, times, ud★, color=colors[3], label="Ocean-side u★") +lines!(axu, times, up★, color=colors[3], label="Ocean-side u★", linestyle = :dash) +vlines!(axu, tn, linewidth=4, color=(:black, 0.5)) +axislegend(axu) + +lines!(axτ, times, interior(ρτxd, 1, 1, 1, :), label="Zonal") +lines!(axτ, times, interior(ρτyd, 1, 1, 1, :), label="Meridional") +lines!(axτ, times, interior(ρτxp, 1, 1, 1, :), linestyle=:dash, label="Zonal") +lines!(axτ, times, interior(ρτyp, 1, 1, 1, :), linestyle=:dash, label="Meridional") +vlines!(axτ, tn, linewidth=4, color=(:black, 0.5)) +axislegend(axτ) + +lines!(axT, times, interior(Tsd, 1, 1, 1, :), color=colors[2], linewidth=4, label="Ocean surface temperature") +lines!(axT, times, interior(Tsp, 1, 1, 1, :), color=colors[2], linewidth=4, linestyle = :dash, label="Ocean surface temperature") +vlines!(axT, tn, linewidth=4, color=(:black, 0.5)) +axislegend(axT) + +lines!(axQ, times, interior(Qvd, 1, 1, 1, 1:Nt), color=colors[2], label="Sensible", linewidth=2) +lines!(axQ, times, interior(Qcd, 1, 1, 1, 1:Nt), color=colors[3], label="Latent", linewidth=2) +lines!(axQ, times, interior(Qvp, 1, 1, 1, 1:Nt), color=colors[2], linestyle=:dash, label="Sensible", linewidth=2) +lines!(axQ, times, interior(Qcp, 1, 1, 1, 1:Nt), color=colors[3], linestyle=:dash, label="Latent", linewidth=2) +vlines!(axQ, tn, linewidth=4, color=(:black, 0.5)) +axislegend(axQ) + +lines!(axF, times, - interior(Evd, 1, 1, 1, 1:Nt), label="Evaporation") +lines!(axF, times, - interior(Evd, 1, 1, 1, 1:Nt), linestyle=:dash, label="Evaporation") +vlines!(axF, tn, linewidth=4, color=(:black, 0.5)) +axislegend(axF) + +lines!(axS, times, interior(Sd, 1, 1, Nz, :)) +lines!(axS, times, interior(Sp, 1, 1, Nz, :), linestyle=:dash) +vlines!(axS, tn, linewidth=4, color=(:black, 0.5)) + +zc = znodes(Tp) +zf = znodes(κp) +upn = @lift interior(up[$n], 1, 1, :) +vpn = @lift interior(vp[$n], 1, 1, :) +Tpn = @lift interior(Tp[$n], 1, 1, :) +Spn = @lift interior(Sp[$n], 1, 1, :) +κpn = @lift interior(κp[$n], 1, 1, :) +epn = @lift interior(ep[$n], 1, 1, :) +Np²n = @lift interior(Np²[$n], 1, 1, :) + +udn = @lift interior(ud[$n], 1, 1, :) +vdn = @lift interior(vd[$n], 1, 1, :) +Tdn = @lift interior(Td[$n], 1, 1, :) +Sdn = @lift interior(Sd[$n], 1, 1, :) +κdn = @lift interior(κd[$n], 1, 1, :) +edn = @lift interior(ed[$n], 1, 1, :) +Nd²n = @lift interior(Nd²[$n], 1, 1, :) + +scatterlines!(axuz, udn, zc, label="u") +scatterlines!(axuz, vdn, zc, label="v") +scatterlines!(axTz, Tdn, zc) +scatterlines!(axSz, Sdn, zc) +scatterlines!(axez, edn, zc) +scatterlines!(axNz, Nd²n, zf) +scatterlines!(axκz, κdn, zf) +scatterlines!(axuz, upn, zc, linestyle=:dash, label="u") +scatterlines!(axuz, vpn, zc, linestyle=:dash, label="v") +scatterlines!(axTz, Tpn, zc, linestyle=:dash) +scatterlines!(axSz, Spn, zc, linestyle=:dash) +scatterlines!(axez, epn, zc, linestyle=:dash) +scatterlines!(axNz, Np²n, zf, linestyle=:dash) +scatterlines!(axκz, κpn, zf, linestyle=:dash) + +axislegend(axuz) + +ulim = max(maximum(abs, up), maximum(abs, vp)) +xlims!(axuz, -ulim, ulim) + +Tmax = maximum(interior(Tp)) +Tmin = minimum(interior(Tp)) +xlims!(axTz, Tmin - 0.1, Tmax + 0.1) + +Nmax = maximum(interior(Np²)) +xlims!(axNz, -Nmax/10, Nmax * 1.05) + +κmax = maximum(interior(κp)) +xlims!(axκz, 1e-9, κmax * 1.1) + +emax = maximum(interior(ep)) +xlims!(axez, 1e-11, emax * 1.1) + +Smax = maximum(interior(Sp)) +Smin = minimum(interior(Sp)) +xlims!(axSz, Smin - 0.2, Smax + 0.2) + +record(fig, "single_column_profiles.mp4", 1:Nt, framerate=24) do nn + @info "Drawing frame $nn of $Nt..." + n[] = nn +end +nothing #hide +# ![](single_column_profiles.mp4) diff --git a/src/DataWrangling/ECCO/ECCO_metadata.jl b/src/DataWrangling/ECCO/ECCO_metadata.jl index 05b87c31..656724de 100644 --- a/src/DataWrangling/ECCO/ECCO_metadata.jl +++ b/src/DataWrangling/ECCO/ECCO_metadata.jl @@ -222,12 +222,12 @@ function download_dataset(metadata::ECCOMetadata; url = urls(metadata)) # Create a temporary directory to store the .netrc file # The directory will be deleted after the download is complete - mktempdir(dir) do tmp + @root mktempdir(dir) do tmp # Write down the username and password in a .netrc file downloader = netrc_downloader(username, password, "ecco.jpl.nasa.gov", tmp) - @distribute for metadatum in metadata # Distribute the download among ranks if MPI is initialized + asyncmap(metadata, ntasks=10) do metadatum # Distribute the download among tasks fileurl = metadata_url(url, metadatum) filepath = metadata_path(metadatum) From 5c380291fa0e8406b821225a467989496eb49de0 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Thu, 5 Dec 2024 15:37:17 +0100 Subject: [PATCH 26/28] whoops added wrong file --- .../os_papa_surface_temperature.jl | 299 ------------------ 1 file changed, 299 deletions(-) delete mode 100644 experiments/single_column_experiments/os_papa_surface_temperature.jl diff --git a/experiments/single_column_experiments/os_papa_surface_temperature.jl b/experiments/single_column_experiments/os_papa_surface_temperature.jl deleted file mode 100644 index c6ec92a1..00000000 --- a/experiments/single_column_experiments/os_papa_surface_temperature.jl +++ /dev/null @@ -1,299 +0,0 @@ -# This simulation tests the difference between using a -# `BulkTemperature` and a `SkinTemperature` - -using ClimaOcean -using ClimaOcean.OceanSeaIceModels.CrossRealmFluxes: SimilarityTheoryTurbulentFluxes, SkinTemperature -using Oceananigans -using Oceananigans.Units -using Oceananigans.BuoyancyModels: buoyancy_frequency -using Oceananigans.Units: Time -using GLMakie -using Printf - -# Ocean station papa location -location_name = "ocean_station_papa" -λ★, φ★ = 35.1, 50.1 - -grid = RectilinearGrid(size = 200, - x = λ★, - y = φ★, - z = (-400, 0), - topology = (Flat, Flat, Bounded)) - -ocean1 = ocean_simulation(grid; Δt=10minutes, coriolis=FPlane(latitude = φ★)) -ocean2 = ocean_simulation(grid; Δt=10minutes, coriolis=FPlane(latitude = φ★)) - -# We set initial conditions from ECCO: -set!(ocean1.model.tracers.T, ECCOMetadata(:temperature)) -set!(ocean1.model.tracers.S, ECCOMetadata(:salinity)) -set!(ocean2.model.tracers.T, ECCOMetadata(:temperature)) -set!(ocean2.model.tracers.S, ECCOMetadata(:salinity)) - -simulation_days = 31 -snapshots_per_day = 8 # corresponding to JRA55's 3-hour frequency -last_time = simulation_days * snapshots_per_day -atmosphere = JRA55_prescribed_atmosphere(1:last_time; - longitude = λ★, - latitude = φ★, - backend = InMemory()) - -# We continue constructing a simulation. - -radiation = Radiation() -coupled_model_prescribed = OceanSeaIceModel(ocean1; atmosphere, radiation) - -similarity_theory = SimilarityTheoryTurbulentFluxes(grid; surface_temperature_type = SkinTemperature(; κ = 0.5)) -coupled_model_diagnostic = OceanSeaIceModel(ocean2; atmosphere, radiation, similarity_theory) - -for (coupled_model, suffix) in zip([coupled_model_diagnostic, coupled_model_prescribed], - ["diagnostic", "prescribed"]) - - simulation = Simulation(coupled_model, Δt=ocean1.Δt, stop_time=10days) - - wall_clock = Ref(time_ns()) - - function progress(sim) - msg = "Ocean Station Papa" - msg *= string(", iter: ", iteration(sim), ", time: ", prettytime(sim)) - - elapsed = 1e-9 * (time_ns() - wall_clock[]) - msg *= string(", wall time: ", prettytime(elapsed)) - wall_clock[] = time_ns() - - u, v, w = sim.model.ocean.model.velocities - msg *= @sprintf(", max|u|: (%.2e, %.2e)", maximum(abs, u), maximum(abs, v)) - - T = sim.model.ocean.model.tracers.T - S = sim.model.ocean.model.tracers.S - e = sim.model.ocean.model.tracers.e - - τx = first(sim.model.fluxes.total.ocean.momentum.u) - τy = first(sim.model.fluxes.total.ocean.momentum.v) - Q = first(sim.model.fluxes.total.ocean.heat) - - u★ = sqrt(sqrt(τx^2 + τy^2)) - Ts = first(interior(sim.model.fluxes.turbulent.fields.T_surface, 1, 1, 1)) - - Nz = size(T, 3) - msg *= @sprintf(", u★: %.2f m s⁻¹", u★) - msg *= @sprintf(", Q: %.2f W m⁻²", Q) - msg *= @sprintf(", T₀: %.2f ᵒC", Ts) - msg *= @sprintf(", S₀: %.2f g/kg", first(interior(S, 1, 1, Nz))) - msg *= @sprintf(", e₀: %.2e m² s⁻²", first(interior(e, 1, 1, Nz))) - - @info msg - end - - simulation.callbacks[:progress] = Callback(progress, IterationInterval(100)) - - # Build flux outputs - τx = coupled_model.fluxes.total.ocean.momentum.u - τy = coupled_model.fluxes.total.ocean.momentum.v - JT = coupled_model.fluxes.total.ocean.tracers.T - Js = coupled_model.fluxes.total.ocean.tracers.S - E = coupled_model.fluxes.turbulent.fields.water_vapor - Qc = coupled_model.fluxes.turbulent.fields.sensible_heat - Qv = coupled_model.fluxes.turbulent.fields.latent_heat - Ts = coupled_model.fluxes.turbulent.fields.T_surface - ρₒ = coupled_model.fluxes.ocean_reference_density - cₚ = coupled_model.fluxes.ocean_heat_capacity - - Q = ρₒ * cₚ * JT - ρτx = ρₒ * τx - ρτy = ρₒ * τy - N² = buoyancy_frequency(coupled_model.ocean.model) - κc = coupled_model.ocean.model.diffusivity_fields.κc - - fluxes = (; ρτx, ρτy, E, Js, Qv, Qc, Ts) - auxiliary_fields = (; N², κc) - fields = merge(coupled_model.ocean.model.velocities, coupled_model.ocean.model.tracers, auxiliary_fields) - - # Slice fields at the surface - outputs = merge(fields, fluxes) - - filename = "single_column_omip_$(location_name)_$(suffix)" - - simulation.output_writers[:jld2] = JLD2OutputWriter(coupled_model.ocean.model, outputs; filename, - schedule = TimeInterval(3hours), - overwrite_existing = true) - - run!(simulation) -end - -##### -##### Visualization -##### - -filename_prescribed = "single_column_omip_$(location_name)_prescribed.jld2" -filename_diagnostic = "single_column_omip_$(location_name)_diagnostic.jld2" - -# Diagnosed -ud = FieldTimeSeries(filename_diagnostic, "u") -vd = FieldTimeSeries(filename_diagnostic, "v") -Td = FieldTimeSeries(filename_diagnostic, "v") -ed = FieldTimeSeries(filename_diagnostic, "T") -Sd = FieldTimeSeries(filename_diagnostic, "S") -Nd² = FieldTimeSeries(filename_diagnostic, "N²") -κd = FieldTimeSeries(filename_diagnostic, "κc") - -Tsd = FieldTimeSeries(filename_diagnostic, "Ts") -Qvd = FieldTimeSeries(filename_diagnostic, "Qv") -Qcd = FieldTimeSeries(filename_diagnostic, "Qc") -Jsd = FieldTimeSeries(filename_diagnostic, "Js") -Evd = FieldTimeSeries(filename_diagnostic, "E") -ρτxd = FieldTimeSeries(filename_diagnostic, "ρτx") -ρτyd = FieldTimeSeries(filename_diagnostic, "ρτy") - -# Prescribed -up = FieldTimeSeries(filename_prescribed, "u") -vp = FieldTimeSeries(filename_prescribed, "v") -Tp = FieldTimeSeries(filename_prescribed, "T") -Sp = FieldTimeSeries(filename_prescribed, "S") -ep = FieldTimeSeries(filename_prescribed, "e") -Np² = FieldTimeSeries(filename_prescribed, "N²") -κp = FieldTimeSeries(filename_prescribed, "κc") - -Tsp = FieldTimeSeries(filename_prescribed, "Ts") -Qvp = FieldTimeSeries(filename_prescribed, "Qv") -Qcp = FieldTimeSeries(filename_prescribed, "Qc") -Jsp = FieldTimeSeries(filename_prescribed, "Js") -Evp = FieldTimeSeries(filename_prescribed, "E") -ρτxp = FieldTimeSeries(filename_prescribed, "ρτx") -ρτyp = FieldTimeSeries(filename_prescribed, "ρτy") - -Nz = size(ud, 3) -times = Qcd.times - -fig = Figure(size=(1800, 1800)) - -axτ = Axis(fig[1, 1:3], xlabel="Days since Oct 1 1992", ylabel="Wind stress (N m⁻²)") -axQ = Axis(fig[1, 4:6], xlabel="Days since Oct 1 1992", ylabel="Heat flux (W m⁻²)") -axu = Axis(fig[2, 1:3], xlabel="Days since Oct 1 1992", ylabel="Velocities (m s⁻¹)") -axT = Axis(fig[2, 4:6], xlabel="Days since Oct 1 1992", ylabel="Surface temperature (ᵒC)") -axF = Axis(fig[3, 1:3], xlabel="Days since Oct 1 1992", ylabel="Freshwater volume flux (m s⁻¹)") -axS = Axis(fig[3, 4:6], xlabel="Days since Oct 1 1992", ylabel="Surface salinity (g kg⁻¹)") - -axuz = Axis(fig[4:5, 1:2], xlabel="Velocities (m s⁻¹)", ylabel="z (m)") -axTz = Axis(fig[4:5, 3:4], xlabel="Temperature (ᵒC)", ylabel="z (m)") -axSz = Axis(fig[4:5, 5:6], xlabel="Salinity (g kg⁻¹)", ylabel="z (m)") -axNz = Axis(fig[6:7, 1:2], xlabel="Buoyancy frequency (s⁻²)", ylabel="z (m)") -axκz = Axis(fig[6:7, 3:4], xlabel="Eddy diffusivity (m² s⁻¹)", ylabel="z (m)", xscale=log10) -axez = Axis(fig[6:7, 5:6], xlabel="Turbulent kinetic energy (m² s⁻²)", ylabel="z (m)", xscale=log10) - -title = @sprintf("Single-column simulation at %.2f, %.2f", φ★, λ★) -Label(fig[0, 1:6], title) - -n = Observable(1) - -times = (times .- times[1]) ./days -Nt = length(times) -tn = @lift times[$n] - -colors = Makie.wong_colors() - -ρₒ = coupled_model_prescribed.fluxes.ocean_reference_density -τxp = interior(ρτxp, 1, 1, 1, :) ./ ρₒ -τyp = interior(ρτyp, 1, 1, 1, :) ./ ρₒ -up★ = @. (τxp^2 + τyp^2)^(1/4) -τxd = interior(ρτxd, 1, 1, 1, :) ./ ρₒ -τyd = interior(ρτyd, 1, 1, 1, :) ./ ρₒ -ud★ = @. (τxd^2 + τyd^2)^(1/4) - -lines!(axu, times, interior(ud, 1, 1, Nz, :), color=colors[1], label="Zonal") -lines!(axu, times, interior(vd, 1, 1, Nz, :), color=colors[2], label="Meridional") -lines!(axu, times, interior(up, 1, 1, Nz, :), color=colors[1], linestyle = :dash, label="Zonal") -lines!(axu, times, interior(vp, 1, 1, Nz, :), color=colors[2], linestyle = :dash, label="Meridional") -lines!(axu, times, ud★, color=colors[3], label="Ocean-side u★") -lines!(axu, times, up★, color=colors[3], label="Ocean-side u★", linestyle = :dash) -vlines!(axu, tn, linewidth=4, color=(:black, 0.5)) -axislegend(axu) - -lines!(axτ, times, interior(ρτxd, 1, 1, 1, :), label="Zonal") -lines!(axτ, times, interior(ρτyd, 1, 1, 1, :), label="Meridional") -lines!(axτ, times, interior(ρτxp, 1, 1, 1, :), linestyle=:dash, label="Zonal") -lines!(axτ, times, interior(ρτyp, 1, 1, 1, :), linestyle=:dash, label="Meridional") -vlines!(axτ, tn, linewidth=4, color=(:black, 0.5)) -axislegend(axτ) - -lines!(axT, times, interior(Tsd, 1, 1, 1, :), color=colors[2], linewidth=4, label="Ocean surface temperature") -lines!(axT, times, interior(Tsp, 1, 1, 1, :), color=colors[2], linewidth=4, linestyle = :dash, label="Ocean surface temperature") -vlines!(axT, tn, linewidth=4, color=(:black, 0.5)) -axislegend(axT) - -lines!(axQ, times, interior(Qvd, 1, 1, 1, 1:Nt), color=colors[2], label="Sensible", linewidth=2) -lines!(axQ, times, interior(Qcd, 1, 1, 1, 1:Nt), color=colors[3], label="Latent", linewidth=2) -lines!(axQ, times, interior(Qvp, 1, 1, 1, 1:Nt), color=colors[2], linestyle=:dash, label="Sensible", linewidth=2) -lines!(axQ, times, interior(Qcp, 1, 1, 1, 1:Nt), color=colors[3], linestyle=:dash, label="Latent", linewidth=2) -vlines!(axQ, tn, linewidth=4, color=(:black, 0.5)) -axislegend(axQ) - -lines!(axF, times, - interior(Evd, 1, 1, 1, 1:Nt), label="Evaporation") -lines!(axF, times, - interior(Evd, 1, 1, 1, 1:Nt), linestyle=:dash, label="Evaporation") -vlines!(axF, tn, linewidth=4, color=(:black, 0.5)) -axislegend(axF) - -lines!(axS, times, interior(Sd, 1, 1, Nz, :)) -lines!(axS, times, interior(Sp, 1, 1, Nz, :), linestyle=:dash) -vlines!(axS, tn, linewidth=4, color=(:black, 0.5)) - -zc = znodes(Tp) -zf = znodes(κp) -upn = @lift interior(up[$n], 1, 1, :) -vpn = @lift interior(vp[$n], 1, 1, :) -Tpn = @lift interior(Tp[$n], 1, 1, :) -Spn = @lift interior(Sp[$n], 1, 1, :) -κpn = @lift interior(κp[$n], 1, 1, :) -epn = @lift interior(ep[$n], 1, 1, :) -Np²n = @lift interior(Np²[$n], 1, 1, :) - -udn = @lift interior(ud[$n], 1, 1, :) -vdn = @lift interior(vd[$n], 1, 1, :) -Tdn = @lift interior(Td[$n], 1, 1, :) -Sdn = @lift interior(Sd[$n], 1, 1, :) -κdn = @lift interior(κd[$n], 1, 1, :) -edn = @lift interior(ed[$n], 1, 1, :) -Nd²n = @lift interior(Nd²[$n], 1, 1, :) - -scatterlines!(axuz, udn, zc, label="u") -scatterlines!(axuz, vdn, zc, label="v") -scatterlines!(axTz, Tdn, zc) -scatterlines!(axSz, Sdn, zc) -scatterlines!(axez, edn, zc) -scatterlines!(axNz, Nd²n, zf) -scatterlines!(axκz, κdn, zf) -scatterlines!(axuz, upn, zc, linestyle=:dash, label="u") -scatterlines!(axuz, vpn, zc, linestyle=:dash, label="v") -scatterlines!(axTz, Tpn, zc, linestyle=:dash) -scatterlines!(axSz, Spn, zc, linestyle=:dash) -scatterlines!(axez, epn, zc, linestyle=:dash) -scatterlines!(axNz, Np²n, zf, linestyle=:dash) -scatterlines!(axκz, κpn, zf, linestyle=:dash) - -axislegend(axuz) - -ulim = max(maximum(abs, up), maximum(abs, vp)) -xlims!(axuz, -ulim, ulim) - -Tmax = maximum(interior(Tp)) -Tmin = minimum(interior(Tp)) -xlims!(axTz, Tmin - 0.1, Tmax + 0.1) - -Nmax = maximum(interior(Np²)) -xlims!(axNz, -Nmax/10, Nmax * 1.05) - -κmax = maximum(interior(κp)) -xlims!(axκz, 1e-9, κmax * 1.1) - -emax = maximum(interior(ep)) -xlims!(axez, 1e-11, emax * 1.1) - -Smax = maximum(interior(Sp)) -Smin = minimum(interior(Sp)) -xlims!(axSz, Smin - 0.2, Smax + 0.2) - -record(fig, "single_column_profiles.mp4", 1:Nt, framerate=24) do nn - @info "Drawing frame $nn of $Nt..." - n[] = nn -end -nothing #hide -# ![](single_column_profiles.mp4) From 926394fa6d3778bfba37677e64b36801bddde0a4 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Mon, 9 Dec 2024 17:25:01 +0100 Subject: [PATCH 27/28] correct looping --- src/DataWrangling/ECCO/ECCO_metadata.jl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/DataWrangling/ECCO/ECCO_metadata.jl b/src/DataWrangling/ECCO/ECCO_metadata.jl index 656724de..b9f78ddb 100644 --- a/src/DataWrangling/ECCO/ECCO_metadata.jl +++ b/src/DataWrangling/ECCO/ECCO_metadata.jl @@ -81,7 +81,6 @@ ECCOMetadata(name::Symbol, date, version=ECCO4Monthly(); dir=download_ECCO_cache ECCOMetadata(name, date, version, dir) # Treat ECCOMetadata as an array to allow iteration over the dates. -Base.length(metadata::ECCOMetadata) = length(metadata.dates) Base.eltype(metadata::ECCOMetadata) = Base.eltype(metadata.dates) @propagate_inbounds Base.getindex(m::ECCOMetadata, i::Int) = ECCOMetadata(m.name, m.dates[i], m.version, m.dir) @@ -102,10 +101,12 @@ Base.last(metadata::ECCOMetadata{<:AbstractCFDateTime}) = metadata Base.iterate(metadata::ECCOMetadata{<:AbstractCFDateTime}) = (metadata, nothing) Base.iterate(::ECCOMetadata{<:AbstractCFDateTime}, ::Any) = nothing +Base.length(metadata::ECCOMetadata) = length(metadata.dates) Base.size(data::ECCOMetadata{<:Any, <:ECCO2Daily}) = (1440, 720, 50, length(data.dates)) Base.size(data::ECCOMetadata{<:Any, <:ECCO2Monthly}) = (1440, 720, 50, length(data.dates)) Base.size(data::ECCOMetadata{<:Any, <:ECCO4Monthly}) = (720, 360, 50, length(data.dates)) +Base.length(metadata::ECCOMetadata{<:AbstractCFDateTime}) = 1 Base.size(::ECCOMetadata{<:AbstractCFDateTime, <:ECCO2Daily}) = (1440, 720, 50, 1) Base.size(::ECCOMetadata{<:AbstractCFDateTime, <:ECCO2Monthly}) = (1440, 720, 50, 1) Base.size(::ECCOMetadata{<:AbstractCFDateTime, <:ECCO4Monthly}) = (720, 360, 50, 1) From a3254de1a6c67448b900b9a96270b0a5af7e567c Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Tue, 10 Dec 2024 14:45:52 +0100 Subject: [PATCH 28/28] bugfix --- src/distributed_utils.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/distributed_utils.jl b/src/distributed_utils.jl index 9cdea1cd..a05612b5 100644 --- a/src/distributed_utils.jl +++ b/src/distributed_utils.jl @@ -36,7 +36,7 @@ end macro root(exp) command = quote - @root MPI.COMM_WORLD $exp + @root ClimaOcean.global_communicator() $exp end return esc(command) end