From 8500d479a53fdf9bca79136213a8c13fd9bd8dae Mon Sep 17 00:00:00 2001 From: Min RK Date: Fri, 9 Jun 2023 12:50:32 +0200 Subject: [PATCH 01/12] Stop using MRAN - remove cutoff for picking RSPM, always use it - remove now-redundant r4.0-mran test - shift base R 3.6 snapshot date to after October 2017, when rspm snapshots seem to have started - update docs to point to RSPM --- docs/source/config_files.rst | 4 ++-- docs/source/howto/languages.rst | 7 ++----- repo2docker/buildpacks/r.py | 22 +--------------------- tests/r/r3.6-mran/runtime.txt | 1 - tests/r/{r3.6-mran => r3.6-rspm}/install.R | 0 tests/r/r3.6-rspm/runtime.txt | 1 + tests/r/{r3.6-mran => r3.6-rspm}/verify | 4 ++-- tests/r/r4.0-mran/install.R | 1 - tests/r/r4.0-mran/runtime.txt | 1 - tests/r/r4.0-mran/verify | 13 ------------- tests/unit/test_r.py | 22 +--------------------- 11 files changed, 9 insertions(+), 67 deletions(-) delete mode 100644 tests/r/r3.6-mran/runtime.txt rename tests/r/{r3.6-mran => r3.6-rspm}/install.R (100%) create mode 100644 tests/r/r3.6-rspm/runtime.txt rename tests/r/{r3.6-mran => r3.6-rspm}/verify (63%) delete mode 100644 tests/r/r4.0-mran/install.R delete mode 100644 tests/r/r4.0-mran/runtime.txt delete mode 100755 tests/r/r4.0-mran/verify diff --git a/docs/source/config_files.rst b/docs/source/config_files.rst index 1c2cca45b..457bfa207 100644 --- a/docs/source/config_files.rst +++ b/docs/source/config_files.rst @@ -118,7 +118,7 @@ with ``REQUIRE`` and ``environment.yml``, visit ================================================ This is used to install R libraries pinned to a specific snapshot on -`MRAN `_. +`Posit Package Manager `_. To set the date of the snapshot add a runtime.txt_. For an example ``install.R`` file, visit our `example install.R file `_. @@ -207,7 +207,7 @@ For these cases, we have a special file, ``runtime.txt``. Have ``python-x.y`` in ``runtime.txt`` to run the repository with Python version x.y. See our `Python2 example repository `_. -Have ``r----
`` in ``runtime.txt`` to run the repository with R version RVERSION and libraries from a YYYY-MM-DD snapshot of `MRAN `_. +Have ``r----
`` in ``runtime.txt`` to run the repository with R version RVERSION and libraries from a YYYY-MM-DD snapshot of `Posit Package Manager `__. RVERSION can be set to 3.4, 3.5, 3.6, or to patch releases for the 3.5 and 3.6 series. If you do not specify a version, the latest release will be used (currently R 3.6). See our `R example repository `_. diff --git a/docs/source/howto/languages.rst b/docs/source/howto/languages.rst index 75ea7e537..af51a6fdf 100644 --- a/docs/source/howto/languages.rst +++ b/docs/source/howto/languages.rst @@ -65,17 +65,14 @@ to install libraries from on the given date. You can install more R packages fro by adding a :ref:`install.R` file to your repo. RStudio and IRKernel are installed by default for all R versions. -If you request R 4.1 or later, or specify a snapshot date newer than -``2022-01-01``, `packagemanager.posit.co `_ +`packagemanager.posit.co `_ will be used to provide much faster installations via `binary packages `_. For *some* packages, this might require you install underlying system libraries using :ref:`apt.txt` - look at the page for the CRAN package you are interested in at `packagemanager.posit.co `_ to find a list. -For older R versions with an older snapshot date, `MRAN `_ -is used as source of packages. This purely provides source packages, and you should -migrate away from this if possible. +repo2docker stopped using the Microsoft mirror MRAN for older R versions after its shutdown in July, 2023. Julia diff --git a/repo2docker/buildpacks/r.py b/repo2docker/buildpacks/r.py index 464587d6a..c02be42ea 100644 --- a/repo2docker/buildpacks/r.py +++ b/repo2docker/buildpacks/r.py @@ -229,30 +229,10 @@ def get_rspm_snapshot_url(self, snapshot_date, max_days_prior=7): ) ) - @lru_cache() - def get_mran_snapshot_url(self, snapshot_date, max_days_prior=7): - for i in range(max_days_prior): - try_date = snapshot_date - datetime.timedelta(days=i) - # Fall back to MRAN if packagemanager.posit.co doesn't have it - url = f"https://mran.microsoft.com/snapshot/{try_date.isoformat()}" - r = requests.head(url) - if r.ok: - return url - raise ValueError( - "No snapshot found for {} or {} days prior in mran.microsoft.com".format( - snapshot_date.strftime("%Y-%m-%d"), max_days_prior - ) - ) - @lru_cache() def get_cran_mirror_url(self, snapshot_date): # Date after which we will use rspm + binary packages instead of MRAN + source packages - rspm_cutoff_date = datetime.date(2022, 1, 1) - - if snapshot_date >= rspm_cutoff_date or self.r_version >= V("4.1"): - return self.get_rspm_snapshot_url(snapshot_date) - else: - return self.get_mran_snapshot_url(snapshot_date) + return self.get_rspm_snapshot_url(snapshot_date) @lru_cache() def get_devtools_snapshot_url(self): diff --git a/tests/r/r3.6-mran/runtime.txt b/tests/r/r3.6-mran/runtime.txt deleted file mode 100644 index 308424ab2..000000000 --- a/tests/r/r3.6-mran/runtime.txt +++ /dev/null @@ -1 +0,0 @@ -r-3.6-2016-01-03 diff --git a/tests/r/r3.6-mran/install.R b/tests/r/r3.6-rspm/install.R similarity index 100% rename from tests/r/r3.6-mran/install.R rename to tests/r/r3.6-rspm/install.R diff --git a/tests/r/r3.6-rspm/runtime.txt b/tests/r/r3.6-rspm/runtime.txt new file mode 100644 index 000000000..cc18726fd --- /dev/null +++ b/tests/r/r3.6-rspm/runtime.txt @@ -0,0 +1 @@ +r-3.6-2017-11-01 diff --git a/tests/r/r3.6-mran/verify b/tests/r/r3.6-rspm/verify similarity index 63% rename from tests/r/r3.6-mran/verify rename to tests/r/r3.6-rspm/verify index 26f561028..475dd91f8 100755 --- a/tests/r/r3.6-mran/verify +++ b/tests/r/r3.6-rspm/verify @@ -7,7 +7,7 @@ if (!(version$major == "3" && as.double(version$minor) >= 6 && as.double(version quit("yes", 1) } -# Fail if MRAN isn't the configured CRAN mirror -if (!(startsWith(options()$repos["CRAN"], "https://mran.microsoft.com"))) { +# Fail if RSPM isn't the configured CRAN mirror +if (!(startsWith(options()$repos["CRAN"], "https://packagemanager.posit.co"))) { quit("yes", 1) } diff --git a/tests/r/r4.0-mran/install.R b/tests/r/r4.0-mran/install.R deleted file mode 100644 index f609959ef..000000000 --- a/tests/r/r4.0-mran/install.R +++ /dev/null @@ -1 +0,0 @@ -install.packages("digest") diff --git a/tests/r/r4.0-mran/runtime.txt b/tests/r/r4.0-mran/runtime.txt deleted file mode 100644 index b051a4326..000000000 --- a/tests/r/r4.0-mran/runtime.txt +++ /dev/null @@ -1 +0,0 @@ -r-4.0-2021-07-07 diff --git a/tests/r/r4.0-mran/verify b/tests/r/r4.0-mran/verify deleted file mode 100755 index b821ec6cc..000000000 --- a/tests/r/r4.0-mran/verify +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env Rscript -library('digest') - -# Fail if version is not 4.0 -print(version) -if (!(version$major == "4" && as.double(version$minor) >= 0 && as.double(version$minor) < 1)) { - quit("yes", 1) -} - -# The date we have chosen should give us an MRAN mirror -if (!(startsWith(options()$repos["CRAN"], "https://mran.microsoft.com"))) { - quit("yes", 1) -} \ No newline at end of file diff --git a/tests/unit/test_r.py b/tests/unit/test_r.py index f30d542a8..b565e814a 100644 --- a/tests/unit/test_r.py +++ b/tests/unit/test_r.py @@ -40,7 +40,7 @@ def test_version_completion(tmpdir, base_image): ("r-3.5-2019-01-01", (2019, 1, 1)), ], ) -def test_mran_date(tmpdir, runtime, expected, base_image): +def test_cran_date(tmpdir, runtime, expected, base_image): tmpdir.chdir() with open("runtime.txt", "w") as f: @@ -72,23 +72,3 @@ def test_snapshot_rspm_date(base_image): with pytest.raises(ValueError): r.get_rspm_snapshot_url(date(1691, 9, 5)) - - -@pytest.mark.parametrize("expected", [date(2019, 12, 29), date(2019, 12, 26)]) -@pytest.mark.parametrize("requested", [date(2019, 12, 31)]) -def test_snapshot_mran_date(requested, expected, base_image): - def mock_request_head(url): - r = Response() - if url == "https://mran.microsoft.com/snapshot/" + expected.isoformat(): - r.status_code = 200 - else: - r.status_code = 404 - r.reason = "Mock MRAN no snapshot" - return r - - with patch("requests.head", side_effect=mock_request_head): - r = buildpacks.RBuildPack(base_image) - assert ( - r.get_mran_snapshot_url(requested) - == f"https://mran.microsoft.com/snapshot/{expected.isoformat()}" - ) From 12240843457ddabc44cebed8353472d4025ad327 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Fri, 9 Jun 2023 17:56:42 +0530 Subject: [PATCH 02/12] Bump 3.6-rspm snapshot to after IRkernel was introduced --- tests/r/r3.6-rspm/runtime.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/r/r3.6-rspm/runtime.txt b/tests/r/r3.6-rspm/runtime.txt index cc18726fd..009200f36 100644 --- a/tests/r/r3.6-rspm/runtime.txt +++ b/tests/r/r3.6-rspm/runtime.txt @@ -1 +1 @@ -r-3.6-2017-11-01 +r-3.6-2018-11-20 From 5572e37a7d94951c7ea47162915f1d4b856112a2 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Fri, 9 Jun 2023 18:27:37 +0530 Subject: [PATCH 03/12] Bump up the R snapshot date some more --- tests/r/r3.6-rspm/runtime.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/r/r3.6-rspm/runtime.txt b/tests/r/r3.6-rspm/runtime.txt index 009200f36..b43a0768c 100644 --- a/tests/r/r3.6-rspm/runtime.txt +++ b/tests/r/r3.6-rspm/runtime.txt @@ -1 +1 @@ -r-3.6-2018-11-20 +r-3.6-2018-12-20 From 0a756e72dd810403f7d5cbaf709ed5fc5ae19577 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Fri, 9 Jun 2023 19:54:19 +0530 Subject: [PATCH 04/12] Try a slightly older snapshot date --- tests/r/r3.6-rspm/runtime.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/r/r3.6-rspm/runtime.txt b/tests/r/r3.6-rspm/runtime.txt index b43a0768c..613f8971f 100644 --- a/tests/r/r3.6-rspm/runtime.txt +++ b/tests/r/r3.6-rspm/runtime.txt @@ -1 +1 @@ -r-3.6-2018-12-20 +r-3.6-2018-11-31 From 3fb91c7f56b86f0283d44928fdfed9f6d339ee9e Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Fri, 9 Jun 2023 23:44:54 +0530 Subject: [PATCH 05/12] Try a slightly newer date --- tests/r/r3.6-rspm/runtime.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/r/r3.6-rspm/runtime.txt b/tests/r/r3.6-rspm/runtime.txt index 613f8971f..ef918a220 100644 --- a/tests/r/r3.6-rspm/runtime.txt +++ b/tests/r/r3.6-rspm/runtime.txt @@ -1 +1 @@ -r-3.6-2018-11-31 +r-3.6-2018-12-07 From 6c719bd6717b1e57e99e6c1937b4045523d2224c Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Sun, 11 Jun 2023 12:55:34 +0530 Subject: [PATCH 06/12] Remove unnecessary method call --- repo2docker/buildpacks/r.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/repo2docker/buildpacks/r.py b/repo2docker/buildpacks/r.py index c02be42ea..3bbcb00a7 100644 --- a/repo2docker/buildpacks/r.py +++ b/repo2docker/buildpacks/r.py @@ -229,11 +229,6 @@ def get_rspm_snapshot_url(self, snapshot_date, max_days_prior=7): ) ) - @lru_cache() - def get_cran_mirror_url(self, snapshot_date): - # Date after which we will use rspm + binary packages instead of MRAN + source packages - return self.get_rspm_snapshot_url(snapshot_date) - @lru_cache() def get_devtools_snapshot_url(self): """ @@ -272,7 +267,7 @@ def get_build_scripts(self): contents of runtime.txt. """ - cran_mirror_url = self.get_cran_mirror_url(self.checkpoint_date) + cran_mirror_url = self.get_rspm_snapshot_url(self.checkpoint_date) if self.platform != "linux/amd64": raise RuntimeError( From a0fd8743b221b21a2fcb8e59ca56bbb366c4c44e Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Sun, 11 Jun 2023 12:56:53 +0530 Subject: [PATCH 07/12] Remove mentions of MRAN --- docs/source/config_files.rst | 2 +- repo2docker/buildpacks/r.py | 3 +-- tests/r/README.md | 12 ------------ 3 files changed, 2 insertions(+), 15 deletions(-) diff --git a/docs/source/config_files.rst b/docs/source/config_files.rst index 457bfa207..eabbdb56b 100644 --- a/docs/source/config_files.rst +++ b/docs/source/config_files.rst @@ -118,7 +118,7 @@ with ``REQUIRE`` and ``environment.yml``, visit ================================================ This is used to install R libraries pinned to a specific snapshot on -`Posit Package Manager `_. +`Posit Package Manager `_. To set the date of the snapshot add a runtime.txt_. For an example ``install.R`` file, visit our `example install.R file `_. diff --git a/repo2docker/buildpacks/r.py b/repo2docker/buildpacks/r.py index 3bbcb00a7..73d08d77a 100644 --- a/repo2docker/buildpacks/r.py +++ b/repo2docker/buildpacks/r.py @@ -22,8 +22,7 @@ class RBuildPack(PythonBuildPack): Where 'year', 'month' and 'date' refer to a specific date whose CRAN snapshot we will use to fetch packages. - Uses https://packagemanager.posit.co, or MRAN if no snapshot - is found on packagemanager.posit.co + Uses https://packagemanager.posit.co. 2. A `DESCRIPTION` file signaling an R package diff --git a/tests/r/README.md b/tests/r/README.md index 61df6db48..51488e41f 100644 --- a/tests/r/README.md +++ b/tests/r/README.md @@ -24,18 +24,6 @@ `runtime.txt` is omitted and a recent enough snapshot date is assumed a RSPM snapshot of CRAN to be used. -### r3.6-mran - -- Test setup of a R 3.6 environment by specifying `r-3.6-...` in `runtime.txt`, - where the date provided in `runtime.txt` is old enough for a MRAN snapshot of - CRAN to be used. - -### r4.0-mran - -- Test setup of a R 4.0 environment by specifying `r-4.0-...` in `runtime.txt`, - where the date provided in `runtime.txt` is old enough for a MRAN snapshot of - CRAN to be used. - ### r4.0-rspm - Test setup of a R 4.0 environment by specifying `r-4.0-...` in `runtime.txt`, From 0dfce7677511785b9b4931bad44d814ea2baa569 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Sun, 11 Jun 2023 13:04:28 +0530 Subject: [PATCH 08/12] Throw a useful error message when snapshot is too old --- repo2docker/buildpacks/r.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/repo2docker/buildpacks/r.py b/repo2docker/buildpacks/r.py index 73d08d77a..5f44afe6e 100644 --- a/repo2docker/buildpacks/r.py +++ b/repo2docker/buildpacks/r.py @@ -9,6 +9,9 @@ from ._r_base import rstudio_base_scripts from .python import PythonBuildPack +# Aproximately the first snapshot on RSPM (Posit package manager) +# that seems to have a working IRKernel. +RSPM_CUTOFF_DATE = datetime.datetime(2018, 12, 7) class RBuildPack(PythonBuildPack): """ @@ -265,6 +268,12 @@ def get_build_scripts(self): We set the snapshot date used to install R libraries from based on the contents of runtime.txt. """ + if self.checkpoint_date < RSPM_CUTOFF_DATE: + raise RuntimeError( + f'Microsoft killed MRAN, the source of R package snapshots before {RSPM_CUTOFF_DATE.strptime("%Y-%m-%d")}. '\ + f'This repo has a snapshot date of {self.checkpoint_date.strftime("%Y-%m-%d")} specified in runtime.txt. '\ + 'Please use a newer snapshot date' + ) cran_mirror_url = self.get_rspm_snapshot_url(self.checkpoint_date) From 9f1a6406b69cbe402565dac156ab8cef7c710381 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Sun, 11 Jun 2023 13:09:40 +0530 Subject: [PATCH 09/12] Use correct type to represent cutoff date --- repo2docker/buildpacks/r.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repo2docker/buildpacks/r.py b/repo2docker/buildpacks/r.py index 5f44afe6e..ff0336911 100644 --- a/repo2docker/buildpacks/r.py +++ b/repo2docker/buildpacks/r.py @@ -11,7 +11,7 @@ # Aproximately the first snapshot on RSPM (Posit package manager) # that seems to have a working IRKernel. -RSPM_CUTOFF_DATE = datetime.datetime(2018, 12, 7) +RSPM_CUTOFF_DATE = datetime.date(2018, 12, 7) class RBuildPack(PythonBuildPack): """ From 5947b756bab340d2b226df0a0ceed382358f24f1 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Sun, 11 Jun 2023 13:09:48 +0530 Subject: [PATCH 10/12] Use correct method to format datetime --- repo2docker/buildpacks/r.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repo2docker/buildpacks/r.py b/repo2docker/buildpacks/r.py index ff0336911..8f92e0f01 100644 --- a/repo2docker/buildpacks/r.py +++ b/repo2docker/buildpacks/r.py @@ -270,7 +270,7 @@ def get_build_scripts(self): """ if self.checkpoint_date < RSPM_CUTOFF_DATE: raise RuntimeError( - f'Microsoft killed MRAN, the source of R package snapshots before {RSPM_CUTOFF_DATE.strptime("%Y-%m-%d")}. '\ + f'Microsoft killed MRAN, the source of R package snapshots before {RSPM_CUTOFF_DATE.strftime("%Y-%m-%d")}. '\ f'This repo has a snapshot date of {self.checkpoint_date.strftime("%Y-%m-%d")} specified in runtime.txt. '\ 'Please use a newer snapshot date' ) From 49bdc6c048079bd25c23f2600eed2d862490b305 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Sun, 11 Jun 2023 13:10:02 +0530 Subject: [PATCH 11/12] Add unit test for MRAN death --- tests/unit/test_r.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/unit/test_r.py b/tests/unit/test_r.py index b565e814a..d50295951 100644 --- a/tests/unit/test_r.py +++ b/tests/unit/test_r.py @@ -72,3 +72,14 @@ def test_snapshot_rspm_date(base_image): with pytest.raises(ValueError): r.get_rspm_snapshot_url(date(1691, 9, 5)) + + +def test_mran_dead(tmpdir, base_image): + tmpdir.chdir() + + with open("runtime.txt", "w") as f: + f.write('r-3.6-2017-06-04') + + r = buildpacks.RBuildPack(base_image) + with pytest.raises(RuntimeError, match=r'^Microsoft killed MRAN, the source of R package snapshots before 2018-12-07.*'): + r.get_build_scripts() \ No newline at end of file From 18e76f9d38b141a4b328478f4eb6d1026e5576c8 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 11 Jun 2023 07:40:57 +0000 Subject: [PATCH 12/12] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- repo2docker/buildpacks/r.py | 7 ++++--- tests/unit/test_r.py | 9 ++++++--- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/repo2docker/buildpacks/r.py b/repo2docker/buildpacks/r.py index 8f92e0f01..7dd724e04 100644 --- a/repo2docker/buildpacks/r.py +++ b/repo2docker/buildpacks/r.py @@ -13,6 +13,7 @@ # that seems to have a working IRKernel. RSPM_CUTOFF_DATE = datetime.date(2018, 12, 7) + class RBuildPack(PythonBuildPack): """ Setup R for use with a repository @@ -270,9 +271,9 @@ def get_build_scripts(self): """ if self.checkpoint_date < RSPM_CUTOFF_DATE: raise RuntimeError( - f'Microsoft killed MRAN, the source of R package snapshots before {RSPM_CUTOFF_DATE.strftime("%Y-%m-%d")}. '\ - f'This repo has a snapshot date of {self.checkpoint_date.strftime("%Y-%m-%d")} specified in runtime.txt. '\ - 'Please use a newer snapshot date' + f'Microsoft killed MRAN, the source of R package snapshots before {RSPM_CUTOFF_DATE.strftime("%Y-%m-%d")}. ' + f'This repo has a snapshot date of {self.checkpoint_date.strftime("%Y-%m-%d")} specified in runtime.txt. ' + "Please use a newer snapshot date" ) cran_mirror_url = self.get_rspm_snapshot_url(self.checkpoint_date) diff --git a/tests/unit/test_r.py b/tests/unit/test_r.py index d50295951..a7028d829 100644 --- a/tests/unit/test_r.py +++ b/tests/unit/test_r.py @@ -78,8 +78,11 @@ def test_mran_dead(tmpdir, base_image): tmpdir.chdir() with open("runtime.txt", "w") as f: - f.write('r-3.6-2017-06-04') + f.write("r-3.6-2017-06-04") r = buildpacks.RBuildPack(base_image) - with pytest.raises(RuntimeError, match=r'^Microsoft killed MRAN, the source of R package snapshots before 2018-12-07.*'): - r.get_build_scripts() \ No newline at end of file + with pytest.raises( + RuntimeError, + match=r"^Microsoft killed MRAN, the source of R package snapshots before 2018-12-07.*", + ): + r.get_build_scripts()