From 38a28b04e670bac6b57fd2f05463f5d81d0d5c50 Mon Sep 17 00:00:00 2001 From: vsoch Date: Wed, 9 Jun 2021 09:06:00 -0600 Subject: [PATCH] updates to podman containers to pull digest and then tag and modulefiles Signed-off-by: vsoch --- docs/getting_started/user-guide.rst | 11 ++++++----- shpc/client/__init__.py | 10 ++++++++++ shpc/client/config.py | 5 +++++ shpc/main/container/docker.py | 17 ++++++++++++++--- shpc/main/modules/templates/docker.lua | 6 +++--- shpc/main/modules/templates/docker.tcl | 12 ++++++------ shpc/main/settings.py | 16 +++++++++++++--- 7 files changed, 57 insertions(+), 20 deletions(-) diff --git a/docs/getting_started/user-guide.rst b/docs/getting_started/user-guide.rst index f0889a183..664342d74 100644 --- a/docs/getting_started/user-guide.rst +++ b/docs/getting_started/user-guide.rst @@ -13,16 +13,17 @@ you should do that first. Why shpc? ========= -While the library is currently focused on Singularity containers (hence -the name) it's created to be modular, meaning that if another container technology -is wanted, it can be added. The module name would still be appropriate, as -singularity does imply a single entity that is "one library to rule them all!" +Singularity Registry HPC is created to be modular, meaning that we support a distinct +set of container technologies and module systems. The name of the library "Singularity +Registry HPC" does not refer specifically to the container technology "Singularity," +but more generally implies the same spirit -- a single entity that is "one library to rule them all!" + What is a registry? =================== A registry consists of a database of local containers configuration files, ``container.yaml`` -files organized in the root of the shpc install in the ``registry`` folder. The namespace +files organized in the root of the shpc install in one of the ``registry`` folders. The namespace is organized by Docker unique resources identifiers. When you install an identifier as we saw above, the container binaries and customized module files are added to the ``module_dir`` defined in your settings, which defaults to ``modules`` in the diff --git a/shpc/client/__init__.py b/shpc/client/__init__.py index 45fc0359a..b9624788b 100644 --- a/shpc/client/__init__.py +++ b/shpc/client/__init__.py @@ -142,6 +142,16 @@ def get_parser(): help="update configuration settings. Use set or get to see or set information.", formatter_class=argparse.RawTextHelpFormatter, ) + + config.add_argument( + "--central", + "-c", + dest="central", + help="make edits to the central config file.", + default=False, + action="store_true", + ) + config.add_argument( "params", nargs="*", diff --git a/shpc/client/config.py b/shpc/client/config.py index 27cb2c73a..bef91fe3b 100644 --- a/shpc/client/config.py +++ b/shpc/client/config.py @@ -2,6 +2,7 @@ __copyright__ = "Copyright 2021, Vanessa Sochat" __license__ = "MPL 2.0" +import shpc.defaults as defaults from shpc.logger import logger import sys @@ -18,6 +19,10 @@ def main(args, parser, extra, subparser): # The first "param" is either set of get command = args.params.pop(0) + # If the user wants the central config file + if args.central: + args.settings_file = defaults.default_settings_file + validate = True if not command == "edit" else False cli = get_client( quiet=args.quiet, settings_file=args.settings_file, validate=validate diff --git a/shpc/main/container/docker.py b/shpc/main/container/docker.py index b65da498c..0fdb0b2b0 100644 --- a/shpc/main/container/docker.py +++ b/shpc/main/container/docker.py @@ -60,9 +60,11 @@ def registry_pull(self, module_dir, container_dir, config, tag): if pull_type != "docker": logger.exit("%s only supports Docker (oci registry) pulls." % self.command) - # Podman doesn't keep a record of digest->tag, so we use tag - uri = "%s:%s" % (self.add_registry(config.docker), tag.name) - return self.pull(uri) + tag_uri = "%s:%s" % (self.add_registry(config.docker), tag.name) + tag_digest = "%s@%s" % (self.add_registry(config.docker), tag.digest) + self.pull(tag_digest) + # Podman doesn't keep a record of digest->tag, so we tag after + return self.tag(tag_digest, tag_uri) def pull(self, uri): """ @@ -73,6 +75,15 @@ def pull(self, uri): logger.exit("There was an issue pulling %s" % uri) return uri + def tag(self, image, tag_as): + """ + Given a container URI, tag as something else. + """ + res = shpc.utils.run_command([self.command, "tag", image, tag_as]) + if res["return_code"] != 0: + logger.exit("There was an issue tagging %s as %s" % (image, tag_as)) + return tag_as + def inspect(self, image): """ Inspect an image diff --git a/shpc/main/modules/templates/docker.lua b/shpc/main/modules/templates/docker.lua index db6551417..6095c8bba 100644 --- a/shpc/main/modules/templates/docker.lua +++ b/shpc/main/modules/templates/docker.lua @@ -16,11 +16,11 @@ Container: Commands include: - {|module_name|}-run: - {{ command }} run -i{% if tty %}t{% endif %} -u `id -u`:`id -g` --rm {% if envfile %}--env-file {{ module_dir }}/{{ envfile }}{% endif %} {% if bindpaths %}-v {{ bindpaths }} -v ${PWD} -w ${PWD} {% endif %} "$@" + {{ command }} run -i{% if tty %}t{% endif %} -u `id -u`:`id -g` --rm {% if envfile %}--env-file {{ module_dir }}/{{ envfile }} {% endif %} {% if bindpaths %}-v {{ bindpaths }} {% endif %}-v ${PWD} -w ${PWD} "$@" - {|module_name|}-shell: - {{ command }} run -i{% if tty %}t{% endif %} -u `id -u`:`id -g` --rm {% if envfile %}--env-file {{ module_dir }}/{{ envfile }}{% endif %} {% if bindpaths %}-v {{ bindpaths }} --entrypoint {{ shell }} -v ${PWD} -w ${PWD} {% endif %} + {{ command }} run -i{% if tty %}t{% endif %} -u `id -u`:`id -g` --rm {% if envfile %}--env-file {{ module_dir }}/{{ envfile }} {% endif %} {% if bindpaths %}-v {{ bindpaths }} --entrypoint {{ shell }} {% endif %}-v ${PWD} -w ${PWD} - {|module_name|}-exec: - {{ command }} run -i{% if tty %}t{% endif %} -u `id -u`:`id -g` --rm --entrypoint "" {% if envfile %}--env-file {{ module_dir }}/{{ envfile }}{% endif %} {% if bindpaths %}-v {{ bindpaths }} {% endif %} -v ${PWD} -w ${PWD} "$@" + {{ command }} run -i{% if tty %}t{% endif %} -u `id -u`:`id -g` --rm --entrypoint "" {% if envfile %}--env-file {{ module_dir }}/{{ envfile }} {% endif %} {% if bindpaths %}-v {{ bindpaths }} {% endif %} -v ${PWD} -w ${PWD} "$@" - {|module_name|}-inspect: {{ command }} inspect diff --git a/shpc/main/modules/templates/docker.tcl b/shpc/main/modules/templates/docker.tcl index a950cfe5e..ce68c6b0d 100644 --- a/shpc/main/modules/templates/docker.tcl +++ b/shpc/main/modules/templates/docker.tcl @@ -15,15 +15,15 @@ proc ModulesHelp { } { puts stderr " - {{ image }}" puts stderr "Commands include:" puts stderr " - {|module_name|}-run:" - puts stderr " {{ command }} run -i{% if tty %}t{% endif %} -u `id -u`:`id -g` --rm {% if envfile %}--env-file {{ module_dir }}/{{ envfile }}{% endif %} {% if bindpaths %}-v {{ bindpaths }} {% endif %} -v . -w . " + puts stderr " {{ command }} run -i{% if tty %}t{% endif %} -u `id -u`:`id -g` --rm {% if envfile %}--env-file {{ module_dir }}/{{ envfile }} {% endif %} {% if bindpaths %}-v {{ bindpaths }} {% endif %} -v . -w . " puts stderr " - {|module_name|}-shell:" - puts stderr " {{ command }} run -i{% if tty %}t{% endif %} -u `id -u`:`id -g` --rm --entrypoint {{ shell }}{% if envfile %} --env-file {{ module_dir }}/{{ envfile }}{% endif %} {% if bindpaths %}-v {{ bindpaths }} {% endif %} -v . -w . " + puts stderr " {{ command }} run -i{% if tty %}t{% endif %} -u `id -u`:`id -g` --rm --entrypoint \"\"{% if envfile %} --env-file {{ module_dir }}/{{ envfile }} {% endif %} {% if bindpaths %}-v {{ bindpaths }} {% endif %} -v . -w . " puts stderr " - {|module_name|}-exec:" - puts stderr " {{ command }} run -i{% if tty %}t{% endif %} -u `id -u`:`id -g` --rm --entrypoint \"\" {% if envfile %} --env-file {{ module_dir }}/{{ envfile }}{% endif %} {% if bindpaths %}-v {{ bindpaths }} {% endif %} -v . -w . $*" + puts stderr " {{ command }} run -i{% if tty %}t{% endif %} -u `id -u`:`id -g` --rm --entrypoint \"\" {% if envfile %} --env-file {{ module_dir }}/{{ envfile }} {% endif %} {% if bindpaths %}-v {{ bindpaths }} {% endif %} -v . -w . $*" puts stderr " - {|module_name|}-inspect:" puts stderr " {{ command }} inspect " {% if aliases %}{% for alias in aliases %} puts stderr " - {{ alias.name }}:" - puts stderr " {{ command }} run -i{% if tty %}t{% endif %} -u `id -u`:`id -g` --entrypoint {{ alias.entrypoint }} {% if envfile %}--envfile {{ module_dir }}/{{ envfile }} {% endif %}{% if bindpaths %}-v {{ bindpaths }} {% endif %}{% if alias.options %}{{ alias.options }} {% endif %} -v . -w . {{ alias.args }}" + puts stderr " {{ command }} run -i{% if tty %}t{% endif %} --rm -u `id -u`:`id -g` --entrypoint {{ alias.entrypoint }} {% if envfile %}--envfile {{ module_dir }}/{{ envfile }} {% endif %}{% if bindpaths %}-v {{ bindpaths }} {% endif %}{% if alias.options %}{{ alias.options }} {% endif %} -v . -w . {{ alias.args }}" {% endfor %}{% endif %} puts stderr "For each of the above, you can export:" @@ -59,7 +59,7 @@ conflict {{ name }} set shellCmd "{{ command }} \${PODMAN_OPTS} run \${PODMAN_COMMAND_OPTS} -u `id -u`:`id -g` --rm -i{% if tty %}t{% endif %} --entrypoint {{ shell }} {% if envfile %}--env-file {{ module_dir }}/{{ envfile }}{% endif %} {% if bindpaths %}-v {{ bindpaths }} {% endif %} -v $workdir -w $workdir ${containerPath}" # execCmd needs entrypoint to be the executor -set execCmd "{{ command }} \${PODMAN_OPTS} run -i{% if tty %}t{% endif %} \${PODMAN_COMMAND_OPTS} -u `id -u`:`id -g` --rm {% if envfile %} --env-file {{ module_dir }}/{{ envfile }}{% endif %} {% if bindpaths %}-v {{ bindpaths }} -v $workdir -w $workdir {% endif %} " +set execCmd "{{ command }} \${PODMAN_OPTS} run -i{% if tty %}t{% endif %} \${PODMAN_COMMAND_OPTS} -u `id -u`:`id -g` --rm {% if envfile %} --env-file {{ module_dir }}/{{ envfile }}{% endif %} {% if bindpaths %}-v {{ bindpaths }}{% endif %} -v $workdir -w $workdir" set runCmd "{{ command }} \${PODMAN_OPTS} run -i{% if tty %}t{% endif %} \${PODMAN_COMMAND_OPTS} -u `id -u`:`id -g` --rm {% if envfile %}--env-file {{ module_dir }}/{{ envfile }}{% endif %} {% if bindpaths %}-v {{ bindpaths }} {% endif %} -v $workdir -w $workdir ${containerPath}" set inspectCmd "{{ command }} \${PODMAN_OPTS} inspect ${containerPath}" @@ -68,7 +68,7 @@ set-alias {|module_name|}-shell "${shellCmd}" # exec functions to provide "alias" to module commands {% if aliases %}{% for alias in aliases %} -set-alias {{ alias.name }} "${execCmd} {% if alias.options %} {{ alias.options }} {% endif %} --entrypoint {{ alias.entrypoint }} ${containerPath} {{ alias.command }}" +set-alias {{ alias.name }} "${execCmd} {% if alias.options %} {{ alias.options }} {% endif %} --entrypoint {{ alias.entrypoint }} ${containerPath} {{ alias.args }}" {% endfor %}{% endif %} # A customizable exec function diff --git a/shpc/main/settings.py b/shpc/main/settings.py index bb3f7d5c8..9c7b99272 100644 --- a/shpc/main/settings.py +++ b/shpc/main/settings.py @@ -121,7 +121,8 @@ def add(self, key, value): current = self._settings.get(key) if current and not isinstance(current, list): logger.exit("You cannot only add to a list variable.") - current.append(value) + # Add to the beginning of the list + current = current + [value] value = list(set(current)) self._settings[key] = value self.change_validate(key, value) @@ -197,12 +198,21 @@ def delete(self, key): del self._settings[key] def save(self, filename=None): + """ + Save settings, but do not change order of anything. + """ filename = filename or self.settings_file if not filename: logger.exit("A filename is required to save to.") yaml = YAML() - with open(filename, "w") as fd: - yaml.dump(self._settings, fd) + + # This requires Python 3.7 support + try: + with open(filename, "w") as fd: + yaml.dump(self._settings, fd, sort_keys=False) + except: + with open(filename, "w") as fd: + yaml.dump(self._settings, fd) def __iter__(self): for key, value in self.__dict__.items():