Download ECCO files using Downloads and .netrc files

.github/workflows/ci.yml

@@ -19,6 +19,9 @@ jobs:
         arch:
           - x64
         include:
+          - os: windows-latest
+            arch: x86
+            version: '1.10'
           - os: macOS-latest
             arch: arm64
             version: '1.10'

.gitignore

@@ -28,6 +28,9 @@ docs/src/literated/
 *.svg
 *.gif
 
+# Password files
+*.netrc
+
 # File generated by Pkg, the package manager, based on a corresponding Project.toml
 # It records a fixed state of all packages used by the project. As such, it should not be
 # committed for packages, but should be committed for applications that require a static

src/Bathymetry.jl

@@ -93,17 +93,10 @@ function regrid_bathymetry(target_grid;
                            major_basins = Inf) # Allow an `Inf` number of ``lakes''
 
     filepath = joinpath(dir, filename)
-    fileurl  = joinpath(url, filename)
-
-    @root begin # perform all this only on rank 0, aka the "root" rank
-        if !isfile(filepath)
-            try 
-                Downloads.download(fileurl, filepath; progress=download_progress, verbose=true)
-            catch 
-                cmd = `wget --no-check-certificate -O $filepath $fileurl`
-                @root run(cmd)
-            end
-        end
+    fileurl  = url * "/" * filename # joinpath on windows creates the wrong url
+
+    @root if !isfile(filepath) # perform all this only on rank 0, aka the "root" rank
+        Downloads.download(fileurl, filepath; progress=download_progress)
     end
 
     dataset = Dataset(filepath)

src/DataWrangling/ECCO/ECCO.jl

@@ -6,7 +6,7 @@ export ECCORestoring, LinearlyTaperedPolarMask
 
 using ClimaOcean
 using ClimaOcean.DataWrangling
-using ClimaOcean.DataWrangling: inpaint_mask!, NearestNeighborInpainting
+using ClimaOcean.DataWrangling: inpaint_mask!, NearestNeighborInpainting, download_progress
 using ClimaOcean.InitialConditions: three_dimensional_regrid!, interpolate!
 
 using Oceananigans

src/DataWrangling/ECCO/ECCO_metadata.jl

@@ -5,6 +5,7 @@ using ClimaOcean.DataWrangling
 import Dates: year, month, day
 
 using Base: @propagate_inbounds
+using Downloads
 
 import Oceananigans.Fields: set!, location
 import Base
@@ -144,12 +145,12 @@ short_name(data::ECCOMetadata{<:Any, <:ECCO2Daily})   = ECCO2_short_names[data.n
 short_name(data::ECCOMetadata{<:Any, <:ECCO2Monthly}) = ECCO2_short_names[data.name]
 short_name(data::ECCOMetadata{<:Any, <:ECCO4Monthly}) = ECCO4_short_names[data.name]
 
-metadata_url(prefix, m::ECCOMetadata{<:Any, <:ECCO2Daily}) = joinpath(prefix, short_name(m), metadata_filename(m))
-metadata_url(prefix, m::ECCOMetadata{<:Any, <:ECCO2Monthly}) = joinpath(prefix, short_name(m), metadata_filename(m))
+metadata_url(prefix, m::ECCOMetadata{<:Any, <:ECCO2Daily}) = prefix * "/" * short_name(m) * "/" * metadata_filename(m)
+metadata_url(prefix, m::ECCOMetadata{<:Any, <:ECCO2Monthly}) = prefix * "/" * short_name(m) * "/" * metadata_filename(m)
 
 function metadata_url(prefix, m::ECCOMetadata{<:Any, <:ECCO4Monthly})
     year = string(Dates.year(m.dates))
-    return joinpath(prefix, short_name(m), year, metadata_filename(m))
+    return prefix * "/" * short_name(m) * "/" * year * "/" * metadata_filename(m)
 end
 
 location(data::ECCOMetadata) = ECCO_location[data.name]
@@ -218,6 +219,9 @@ function download_dataset(metadata::ECCOMetadata; url = urls(metadata))
     password = get(ENV, "ECCO_PASSWORD", nothing)
     dir = metadata.dir
 
+    # Write down the username and password in a .netrc file
+    downloader = ECCO_downloader(username, password, dir)
+
     @distribute for metadatum in metadata # Distribute the download among ranks if MPI is initialized
 
         fileurl  = metadata_url(url, metadatum) 
@@ -237,10 +241,48 @@ function download_dataset(metadata::ECCOMetadata; url = urls(metadata))
                 throw(ArgumentError(msg))
             end
 
-            cmd = `wget --http-user=$(username) --http-passwd=$(password) --directory-prefix=$dir $fileurl`
-            run(cmd)
+            Downloads.download(fileurl, filepath; downloader, progress=download_progress)
         end
     end
 
+    # Remove the .netrc file after downloading to avoid storing the credentials
+    remove_netrc!(dir)
+
     return nothing
 end
+
+# ECCO downloader 
+function ECCO_downloader(username, password, dir)
+    netrc_file = ECCO_netrc!(username, password, dir)
+    downloader = Downloads.Downloader()
+    easy_hook  = (easy, _) -> Downloads.Curl.setopt(easy, Downloads.Curl.CURLOPT_NETRC_FILE, netrc_file)
+
+    downloader.easy_hook = easy_hook
+    return downloader
+end
+
+# Code snippet adapted from https://github.com/evetion/SpaceLiDAR.jl/blob/master/src/utils.jl#L150
+function ECCO_netrc!(username, password, dir)
+    if Sys.iswindows()
+        filepath = joinpath(dir, "ECCO_netrc")
+    else
+        filepath = joinpath(dir, "ECCO.netrc")
+    end
+
+    open(filepath, "a") do f
+        write(f, "\n")
+        write(f, "machine ecco.jpl.nasa.gov login $username password $password\n")
+    end
+
+    return filepath
+end
+
+function remove_netrc!(dir)
+    if Sys.iswindows()
+        filepath = joinpath(dir, "ECCO_netrc")
+    else
+        filepath = joinpath(dir, "ECCO.netrc")
+    end
+
+    rm(filepath; force = true)
+end

src/DataWrangling/JRA55.jl

@@ -391,7 +391,7 @@ function JRA55_field_time_series(variable_name;
 
     # Note, we don't re-use existing jld2 files.
     @root begin
-        isfile(filepath) || download(url, filepath)
+        isfile(filepath) || download(url, filepath; progress=download_progress)
         isfile(jld2_filepath) && rm(jld2_filepath)
     end
 

test/runtests.jl

@@ -13,9 +13,9 @@ if test_group == :init || test_group == :all
     CUDA.set_runtime_version!(v"12.6"; local_toolkit = true)
     CUDA.precompile_runtime()
 
-    ####
-    #### Download bathymetry data
-    ####
+    ###
+    ### Download bathymetry data
+    ###
 
     download_bathymetry() 
 

test/runtests_setup.jl

@@ -12,6 +12,7 @@ using Oceananigans.Architectures: architecture, on_architecture
 using Oceananigans.OutputReaders: interpolate!
 
 using ClimaOcean
+using ClimaOcean.Bathymetry: download_bathymetry_cache
 using CFTime
 using Dates 
 
@@ -28,13 +29,15 @@ temperature_metadata = ECCOMetadata(:temperature, dates)
 salinity_metadata    = ECCOMetadata(:salinity, dates)
 
 # Fictitious grid that triggers bathymetry download
-function download_bathymetry()
+function download_bathymetry(; dir = download_bathymetry_cache, 
+                          filename = "ETOPO_2022_v1_60s_N90W180_surface.nc")
+
     grid = LatitudeLongitudeGrid(size = (10, 10, 1), 
                                  longitude = (0, 100), 
                                  latitude = (0, 50),
                                  z = (-6000, 0))
 
-    bottom = regrid_bathymetry(grid)
+    bottom = regrid_bathymetry(grid; dir, filename)
 
     return nothing
 end

test/test_downloading.jl

@@ -1,8 +1,30 @@
 include("runtests_setup.jl")
 
+using ClimaOcean.ECCO: metadata_path
+
 @testset "Availability of JRA55 data" begin
     @info "Testing that we can download all the JRA55 data..."
     for name in ClimaOcean.DataWrangling.JRA55.JRA55_variable_names
+
         fts = ClimaOcean.JRA55.JRA55_field_time_series(name; time_indices=2:3)
     end
 end
+
+@testset "Availability of ECCO data" begin
+    @info "Testing that we can download ECCO data..."
+    for variable in keys(ClimaOcean.ECCO.ECCO4_short_names)
+        metadata = ECCOMetadata(variable)
+        filepath = metadata_path(metadata)
+        isfile(filepath) && rm(filepath; force=true)
+        ClimaOcean.ECCO.download_dataset(metadata)
+    end
+end
+
+@testset "Availability of the Bathymetry" begin
+    @info "Testing that we can download the bathymetry..."
+    dir="./"
+    filename="ETOPO_2022_v1_60s_N90W180_surface.nc"
+    filepath=joinpath(dir, filename)
+    isfile(filepath) && rm(filepath; force=true)
+    download_bathymetry(; dir, filename)
+end