From 3514b342efd1f288c3e63bcbd41c476299a03ee8 Mon Sep 17 00:00:00 2001 From: Vanessa Sochat Date: Fri, 31 Aug 2018 14:20:37 -0400 Subject: [PATCH 1/6] test for fixing pull --- spython/main/build.py | 11 +++++++++-- spython/main/pull.py | 22 ++++++++++++++++++++-- 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/spython/main/build.py b/spython/main/build.py index 33391560..7c238612 100644 --- a/spython/main/build.py +++ b/spython/main/build.py @@ -25,6 +25,7 @@ def build(self, recipe=None, image=None, isolated=False, + isolated_root=None, sandbox=False, writable=False, build_folder=None, @@ -46,6 +47,7 @@ def build(self, recipe=None, defined, we look for "Singularity" file in $PWD image: the image to build (if None, will use arbitary name isolated: if True, run build with --isolated flag + isolated_root: if isolated is True, allowed to set a root sandbox: if True, create a writable sandbox writable: if True, use writable ext3 (sandbox takes preference) build_folder: where the container should be built. @@ -76,19 +78,24 @@ def build(self, recipe=None, if re.search('(docker|shub)://', recipe) and robot_name is False: image = self._get_filename(recipe, ext) else: - image = "%s.%s" %(self.RobotNamer.generate(),ext) + image = "%s.%s" %(self.RobotNamer.generate(), ext) # Does the user want a custom build folder? if build_folder is not None: if not os.path.exists(build_folder): - bot.exit('%s does not exist!' %build_folder) + bot.exit('%s does not exist!' % build_folder) image = "%s/%s" %(build_folder, image) + # The user wants to run an isolated build if isolated is True: cmd.append('--isolated') + # And change the default root folder for isolation + if isolated_root is not None: + cmd = cmd + ['--isolated-root', isolated_root ] + if sandbox is True: cmd.append('--sandbox') elif sandbox is True: diff --git a/spython/main/pull.py b/spython/main/pull.py index af56b3e1..ec723273 100644 --- a/spython/main/pull.py +++ b/spython/main/pull.py @@ -30,6 +30,8 @@ def pull(self, ext="simg", force=False, capture=False, + name_by_commit=False, + name_by_hash=False, stream=False): '''pull will pull a singularity hub or Docker image @@ -68,10 +70,17 @@ def pull(self, self.setenv('SINGULARITY_PULLFOLDER', pull_folder) # If we still don't have a custom name, base off of image uri. + # Determine how to tell client to name the image, preference to hash + if name is None: name = self._get_filename(image, ext) + cmd = cmd + ["--name", name] + + elif name_by_hash is True: + cmd.append('--hash') - cmd = cmd + ["--name", name] + elif name_by_commit is True: + cmd.append('--commit') if force is True: cmd = cmd + ["--force"] @@ -80,8 +89,17 @@ def pull(self, bot.info(' '.join(cmd)) final_image = os.path.join(pull_folder, name) - if stream is False: + + # Option 1: For hash or commit, need return value to get final_image + if name_by_commit or name_by_hash: + output = self._run_command(cmd, capture=True) + final_image = output.split('Container is at:')[-1].strip('\n').strip() + + # Option 2: Streaming we just run to show user + elif stream is False: self._run_command(cmd, capture=capture) + + # Option 3: A custom name we can predict (not commit/hash) and can also show else: return final_image, stream_command(cmd, sudo=False) From b3015563fcb4f893d4a72e4bd1f75350d98a45f1 Mon Sep 17 00:00:00 2001 From: Vanessa Sochat Date: Fri, 31 Aug 2018 14:21:30 -0400 Subject: [PATCH 2/6] updating changelog --- CHANGELOG.md | 1 + spython/version.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4e3f0836..23335842 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ The client here will eventually be released as "spython" (and eventually to singularity on pypi), and the versions here will coincide with these releases. ## [master](https://github.com/singularityhub/singularity-cli/tree/master) + - adding name_by_commit and name_by_hash to pull (0.0.40) - remove uri function should only right strip to support relative paths (0.0.39) - adjusting container build to use correct Github branch (vault/release-2.5) - adding support and documentation for container instances (0.0.38) diff --git a/spython/version.py b/spython/version.py index 9c678fc2..1582ccac 100644 --- a/spython/version.py +++ b/spython/version.py @@ -18,7 +18,7 @@ -__version__ = "0.0.39" +__version__ = "0.0.40" AUTHOR = 'Vanessa Sochat' AUTHOR_EMAIL = 'vsochat@stanford.edu' NAME = 'spython' From fe06e0fa3a615481d47b317aaf17bba08b428d76 Mon Sep 17 00:00:00 2001 From: Vanessa Sochat Date: Fri, 31 Aug 2018 14:24:00 -0400 Subject: [PATCH 3/6] removing isolated_root --- spython/main/build.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/spython/main/build.py b/spython/main/build.py index 7c238612..13c8cc65 100644 --- a/spython/main/build.py +++ b/spython/main/build.py @@ -25,7 +25,6 @@ def build(self, recipe=None, image=None, isolated=False, - isolated_root=None, sandbox=False, writable=False, build_folder=None, @@ -47,7 +46,6 @@ def build(self, recipe=None, defined, we look for "Singularity" file in $PWD image: the image to build (if None, will use arbitary name isolated: if True, run build with --isolated flag - isolated_root: if isolated is True, allowed to set a root sandbox: if True, create a writable sandbox writable: if True, use writable ext3 (sandbox takes preference) build_folder: where the container should be built. @@ -92,10 +90,6 @@ def build(self, recipe=None, if isolated is True: cmd.append('--isolated') - # And change the default root folder for isolation - if isolated_root is not None: - cmd = cmd + ['--isolated-root', isolated_root ] - if sandbox is True: cmd.append('--sandbox') elif sandbox is True: From 3cf68e2c3464adbca8931e43620e42d5ea1d73d8 Mon Sep 17 00:00:00 2001 From: Vanessa Sochat Date: Fri, 31 Aug 2018 14:27:58 -0400 Subject: [PATCH 4/6] fixing logic --- spython/main/pull.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/spython/main/pull.py b/spython/main/pull.py index ec723273..c4e53883 100644 --- a/spython/main/pull.py +++ b/spython/main/pull.py @@ -72,16 +72,19 @@ def pull(self, # If we still don't have a custom name, base off of image uri. # Determine how to tell client to name the image, preference to hash - if name is None: - name = self._get_filename(image, ext) - cmd = cmd + ["--name", name] - - elif name_by_hash is True: + if name_by_hash is True: cmd.append('--hash') elif name_by_commit is True: cmd.append('--commit') + elif name is None: + name = self._get_filename(image, ext) + + # Only add name if we aren't naming by hash or commit + if not name_by_commit and not name_by_hash: + cmd = cmd + ["--name", name] + if force is True: cmd = cmd + ["--force"] From b7e4ec1399f5d2eb4ce2d356a58ba7bfe360cb74 Mon Sep 17 00:00:00 2001 From: Vanessa Sochat Date: Fri, 7 Sep 2018 14:58:18 -0400 Subject: [PATCH 5/6] fixing None bug --- spython/main/pull.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/spython/main/pull.py b/spython/main/pull.py index c4e53883..8592f445 100644 --- a/spython/main/pull.py +++ b/spython/main/pull.py @@ -91,6 +91,10 @@ def pull(self, cmd.append(image) bot.info(' '.join(cmd)) + # If name is still None, make empty string + if name is None: + name = '' + final_image = os.path.join(pull_folder, name) # Option 1: For hash or commit, need return value to get final_image From 6d0d25d646936b73a8666b25c1b12760cf7524d9 Mon Sep 17 00:00:00 2001 From: Vanessa Sochat Date: Sun, 9 Sep 2018 17:14:10 -0400 Subject: [PATCH 6/6] trying workaround for pull --- spython/main/pull.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/spython/main/pull.py b/spython/main/pull.py index 8592f445..fea9e632 100644 --- a/spython/main/pull.py +++ b/spython/main/pull.py @@ -21,7 +21,9 @@ from spython.utils import stream_command import os import re +import shutil import sys +import tempfile def pull(self, image=None, @@ -99,8 +101,20 @@ def pull(self, # Option 1: For hash or commit, need return value to get final_image if name_by_commit or name_by_hash: - output = self._run_command(cmd, capture=True) - final_image = output.split('Container is at:')[-1].strip('\n').strip() + + # Set pull to temporary location + tmp_folder = tempfile.mkdtemp() + self.setenv('SINGULARITY_PULLFOLDER', tmp_folder) + self._run_command(cmd, capture=capture) + + try: + tmp_image = os.path.join(tmp_folder, os.listdir(tmp_folder)[0]) + final_image = os.path.join(pull_folder, os.path.basename(tmp_image)) + shutil.move(tmp_image, final_image) + shutil.rmtree(tmp_folder) + + except: + bot.error('Issue pulling image with commit or hash, try without?') # Option 2: Streaming we just run to show user elif stream is False: