atlas

#!/bin/bash
# getoptions generates code that fails this check, so we need to disable it
# shellcheck disable=SC2004

# Atlas uses semantic versioning (https://semver.org/). To cut a new release, checkout the README:
#    - https://github.com/openedx/openedx-atlas#releasing-a-new-version
_ATLAS_VERSION="unreleased"  # release.yml GitHub Action will fill in the tag version

# @getoptions
parser_definition() {
    _ATLAS_USAGE_HELP="Atlas is a CLI tool that has essentially one command: \`atlas pull\`

Configuration file:

    Atlas defaults to using a configuration file named \`atlas.yml\` placed
    in the root directory. Configuration file:

    pull:
      repository: <organization-name>/<repository-name>
      revision: <git-revision>
      directory: <repo-directory-name>:<local-dir-name> ...
      filter: <pattern> ...
      expand_glob: 0

    Atlas can also use a configuration file in a different path using the \`--config\` flag
    after \`atlas\`: \`atlas pull --config config.yml\`.

    Atlas can also be used without a configuration file by using the flags below after
    \`atlas pull\`.

Positional arguments DIRECTORY MAPPINGS ...

   One or more directory map pair separated by a colon (:) e.g. FROM_DIR:TO_DIR.

   The first directory (FROM_DIR) represents a directory in the git repository.
   The second directory (TO_DIR) represents a local directory to copy files to.

   At least one directory pair is required:

     $ atlas pull frontend-app-learning/messages:learning-app frontend-lib-test/messages:test-lib

   This syntax is inspired by the \`docker --volume from_dir:to_dir\` mounting syntax.

Options:

    \`-r\` or \`--repository\`:
        slug of the GitHub repository to pull from. Defaults 'openedx/openedx-translations'.

    \`-n\` or \`--revision\`:
        Git revision to pull from. Currently only branches and tags are supported. Defaults to 'main'.

        This option name used to be \`-b\` or \`--branch\`. The deprecated name will be removed in a future release.

    \`-f\` or \`--filter\`:
       A comma-separated (or space-separated) list of patterns match files and sub-directories.
       This is mainly useful to filter specific languages to download.

       The same filter is applied to all DIRECTORY MAPPINGS arguments.

       \`--filter=fr_CA,ar,es_419\` will match both directories named 'es_419' and
       files named 'es_419.json' among others

   \`-g\` or \`--expand-glob\`:
       Expand glob pattern e.g. 'atlas pull translations/*/done' to 'atlas pull translations/DoneXBlock/done'
       if it exists.

Example:

    $ cd frontend-app-learning/src/i18n/messages
    $ atlas pull --filter=fr_CA,ar,es_419 \\
            translations/frontend-app-learning/src/i18n/messages:frontend-app-learning \\
            translations/frontend-component-header/src/i18n/messages:frontend-component-header

    Will result in the following tree:

      ├── frontend-app-learning
      │   ├── ar.json
      │   ├── es_419.json
      │   └── fr_CA.json
      └── frontend-component-header
          ├── ar.json
          ├── es_419.json
          └── fr_CA.json
"

	setup       REST help:usage -- "${_ATLAS_USAGE_HELP}" ''
	msg 	 -- '' 'Commands:'
	cmd pull -- "pull"
	disp    :usage  -h --help
	disp    _ATLAS_VERSION  --version
}

parser_definition_pull() {
	setup   PULL_REST help:usage_pull -- \
		"Usage: atlas pull [options...] [directory mappings...]"
	msg -- 'Options:'
	param		CONFIG			--config			        -- "path to alternative atlas.yaml configuration file"
	param		REPOSITORY	    -r	--repository		    -- "The repository containing translation files"
	param		REVISION		-n --revision -b --branch   -- "A git revision of the repository to pull from. The name --branch is deprecated in favor of --revision."
	param		FILTER	        -f	--filter			    -- "List of patterns to select which files and sub-directories to checkout."
	flag		EXPAND_GLOB     -g	--expand-glob           -- "Expand glob pattern e.g. 'atlas pull translations/*/done' to 'atlas pull translations/DoneXBlock/done' if it exists."
	flag		VERBOSE		    -v	--verbose			    -- "verbose output to terminal"
	flag		SILENT		    -s	--silent			    -- "no output to terminal"
	disp    :usage_pull  -h --help
}
# @end

# @gengetoptions parser -i parser_definition parse
# Generated by getoptions (BEGIN)
# URL: https://github.com/ko1nksm/getoptions (v3.3.2)
REST=''
parse() {
  OPTIND=$(($#+1))
  while OPTARG= && [ "${REST}" != x ] && [ $# -gt 0 ]; do
    case $1 in
      --?*=*) OPTARG=$1; shift
        eval 'set -- "${OPTARG%%\=*}" "${OPTARG#*\=}"' ${1+'"$@"'}
        ;;
      --no-*|--without-*) unset OPTARG ;;
      -[h]?*) OPTARG=$1; shift
        eval 'set -- "${OPTARG%"${OPTARG#??}"}" -"${OPTARG#??}"' ${1+'"$@"'}
        case $2 in --*) set -- "$1" unknown "$2" && REST=x; esac;OPTARG= ;;
    esac
    case $1 in
      '-h'|'--help')
        usage
        exit 0 ;;
      '--version')
        echo "${_ATLAS_VERSION}"
        exit 0 ;;
      --)
        while [ $# -gt 0 ]; do
          REST="${REST} \"\${$(($OPTIND-$#))}\""
          shift
        done
        break ;;
      [-]?*) set "unknown" "$1"; break ;;
      *)
        case $1 in 'pull') ;;
          *) set "notcmd" "$1"; break
        esac
        while [ $# -gt 0 ]; do
          REST="${REST} \"\${$(($OPTIND-$#))}\""
          shift
        done
        break ;;
    esac
    shift
  done
  [ $# -eq 0 ] && { OPTIND=1; unset OPTARG; return 0; }
  case $1 in
    unknown) set "Unrecognized option: $2" "$@" ;;
    noarg) set "Does not allow an argument: $2" "$@" ;;
    required) set "Requires an argument: $2" "$@" ;;
    pattern:*) set "Does not match the pattern (${1#*:}): $2" "$@" ;;
    notcmd) set "Not a command: $2" "$@" ;;
    *) set "Validation error ($1): $2" "$@"
  esac
  echo "$1" >&2
  exit 1
}
usage() {
cat<<'GETOPTIONSHERE'
Atlas is a CLI tool that has essentially one command: `atlas pull`

Configuration file:

    Atlas defaults to using a configuration file named `atlas.yml` placed
    in the root directory. Configuration file:

    pull:
      repository: <organization-name>/<repository-name>
      revision: <git-revision>
      directory: <repo-directory-name>:<local-dir-name> ...
      filter: <pattern> ...
      expand_glob: 0

    Atlas can also use a configuration file in a different path using the `--config` flag
    after `atlas`: `atlas pull --config config.yml`.

    Atlas can also be used without a configuration file by using the flags below after
    `atlas pull`.

Positional arguments DIRECTORY MAPPINGS ...

   One or more directory map pair separated by a colon (:) e.g. FROM_DIR:TO_DIR.

   The first directory (FROM_DIR) represents a directory in the git repository.
   The second directory (TO_DIR) represents a local directory to copy files to.

   At least one directory pair is required:

     $ atlas pull frontend-app-learning/messages:learning-app frontend-lib-test/messages:test-lib

   This syntax is inspired by the `docker --volume from_dir:to_dir` mounting syntax.

Options:

    `-r` or `--repository`:
        slug of the GitHub repository to pull from. Defaults 'openedx/openedx-translations'.

    `-n` or `--revision`:
        Git revision to pull from. Currently only branches and tags are supported. Defaults to 'main'.

        This option name used to be `-b` or `--branch`. The deprecated name will be removed in a future release.

    `-f` or `--filter`:
       A comma-separated (or space-separated) list of patterns match files and sub-directories.
       This is mainly useful to filter specific languages to download.

       The same filter is applied to all DIRECTORY MAPPINGS arguments.

       `--filter=fr_CA,ar,es_419` will match both directories named 'es_419' and
       files named 'es_419.json' among others

   `-g` or `--expand-glob`:
       Expand glob pattern e.g. 'atlas pull translations/*/done' to 'atlas pull translations/DoneXBlock/done'
       if it exists.

Example:

    $ cd frontend-app-learning/src/i18n/messages
    $ atlas pull --filter=fr_CA,ar,es_419 \
            translations/frontend-app-learning/src/i18n/messages:frontend-app-learning \
            translations/frontend-component-header/src/i18n/messages:frontend-component-header

    Will result in the following tree:

      ├── frontend-app-learning
      │   ├── ar.json
      │   ├── es_419.json
      │   └── fr_CA.json
      └── frontend-component-header
          ├── ar.json
          ├── es_419.json
          └── fr_CA.json



Commands:
  pull      pull
  -h, --help                  
      --version               
GETOPTIONSHERE
}
# Generated by getoptions (END)
# @end

# @gengetoptions parser -i parser_definition_pull parse_pull
# Generated by getoptions (BEGIN)
# URL: https://github.com/ko1nksm/getoptions (v3.3.2)
CONFIG=''
REPOSITORY=''
REVISION=''
FILTER=''
EXPAND_GLOB=''
VERBOSE=''
SILENT=''
PULL_REST=''
parse_pull() {
  OPTIND=$(($#+1))
  while OPTARG= && [ "${PULL_REST}" != x ] && [ $# -gt 0 ]; do
    case $1 in
      --?*=*) OPTARG=$1; shift
        eval 'set -- "${OPTARG%%\=*}" "${OPTARG#*\=}"' ${1+'"$@"'}
        ;;
      --no-*|--without-*) unset OPTARG ;;
      -[rnbf]?*) OPTARG=$1; shift
        eval 'set -- "${OPTARG%"${OPTARG#??}"}" "${OPTARG#??}"' ${1+'"$@"'}
        ;;
      -[gvsh]?*) OPTARG=$1; shift
        eval 'set -- "${OPTARG%"${OPTARG#??}"}" -"${OPTARG#??}"' ${1+'"$@"'}
        case $2 in --*) set -- "$1" unknown "$2" && PULL_REST=x; esac;OPTARG= ;;
    esac
    case $1 in
      '--config')
        [ $# -le 1 ] && set "required" "$1" && break
        OPTARG=$2
        CONFIG="$OPTARG"
        shift ;;
      '-r'|'--repository')
        [ $# -le 1 ] && set "required" "$1" && break
        OPTARG=$2
        REPOSITORY="$OPTARG"
        shift ;;
      '-n'|'--revision'|'-b'|'--branch')
        [ $# -le 1 ] && set "required" "$1" && break
        OPTARG=$2
        REVISION="$OPTARG"
        shift ;;
      '-f'|'--filter')
        [ $# -le 1 ] && set "required" "$1" && break
        OPTARG=$2
        FILTER="$OPTARG"
        shift ;;
      '-g'|'--expand-glob')
        [ "${OPTARG:-}" ] && OPTARG=${OPTARG#*\=} && set "noarg" "$1" && break
        eval '[ ${OPTARG+x} ] &&:' && OPTARG='1' || OPTARG=''
        EXPAND_GLOB="$OPTARG"
        ;;
      '-v'|'--verbose')
        [ "${OPTARG:-}" ] && OPTARG=${OPTARG#*\=} && set "noarg" "$1" && break
        eval '[ ${OPTARG+x} ] &&:' && OPTARG='1' || OPTARG=''
        VERBOSE="$OPTARG"
        ;;
      '-s'|'--silent')
        [ "${OPTARG:-}" ] && OPTARG=${OPTARG#*\=} && set "noarg" "$1" && break
        eval '[ ${OPTARG+x} ] &&:' && OPTARG='1' || OPTARG=''
        SILENT="$OPTARG"
        ;;
      '-h'|'--help')
        usage_pull
        exit 0 ;;
      --)
        shift
        while [ $# -gt 0 ]; do
          PULL_REST="${PULL_REST} \"\${$(($OPTIND-$#))}\""
          shift
        done
        break ;;
      [-]?*) set "unknown" "$1"; break ;;
      *)
        PULL_REST="${PULL_REST} \"\${$(($OPTIND-$#))}\""
    esac
    shift
  done
  [ $# -eq 0 ] && { OPTIND=1; unset OPTARG; return 0; }
  case $1 in
    unknown) set "Unrecognized option: $2" "$@" ;;
    noarg) set "Does not allow an argument: $2" "$@" ;;
    required) set "Requires an argument: $2" "$@" ;;
    pattern:*) set "Does not match the pattern (${1#*:}): $2" "$@" ;;
    notcmd) set "Not a command: $2" "$@" ;;
    *) set "Validation error ($1): $2" "$@"
  esac
  echo "$1" >&2
  exit 1
}
usage_pull() {
cat<<'GETOPTIONSHERE'
Usage: atlas pull [options...] [directory mappings...]
Options:
      --config CONFIG         path to alternative atlas.yaml configuration file
  -r, --repository REPOSITORY The repository containing translation files
  -n, --revision, -b, --branch REVISION 
                              A git revision of the repository to pull from. The name --branch is deprecated in favor of --revision.
  -f, --filter FILTER         List of patterns to select which files and sub-directories to checkout.
  -g, --expand-glob           Expand glob pattern e.g. 'atlas pull translations/*/done' to 'atlas pull translations/DoneXBlock/done' if it exists.
  -v, --verbose               verbose output to terminal
  -s, --silent                no output to terminal
  -h, --help                  
GETOPTIONSHERE
}
# Generated by getoptions (END)
# @end

parse_yaml() {
    # adapted from https://gist.github.com/pkuczynski/8665367
    local prefix=$2
    local s='[[:space:]]*' w='[a-zA-Z0-9_]*'
    local fs
    fs=$(echo @|tr @ '\034')
    sed -ne "s|^\($s\)\($w\)$s:$s\"\(.*\)\"$s\$|\1$fs\2$fs\3|p" \
        -e "s|^\($s\)\($w\)$s:$s\(.*\)$s\$|\1$fs\2$fs\3|p"  "$1" |
    awk -F"$fs" '{
        indent = length($1)/2;
        vname[indent] = $2;
        for (i in vname) {if (i > indent) {delete vname[i]}}
        if (length($3) > 0) {
            vn=""; for (i=0; i<indent; i++) {vn=(vn)(vname[i])("_")}
            printf("%s%s%s=\"%s\"\n", "'"$prefix"'",vn, $2, $3);
        }
    }'
}

set_pull_params() {
  # Set vars from config yaml
  if [ "$CONFIG" ];
  then
    if [ -f "$CONFIG" ]; then
      eval "$(parse_yaml "$CONFIG")"
    else
      echo "$CONFIG does not exist."
      exit
    fi
  else
    if [ -f atlas.yml ]; then
      eval "$(parse_yaml atlas.yml)"
    else
      pull_directory=""
      pull_repository="openedx/openedx-translations"
      pull_revision="main"
      pull_filter=""
      pull_expand_glob=""
    fi
  fi

  # Override vars based on args
  if [ $# -gt 0 ];
  then
    # Due to sh/dash compatibility, arrays are not supported, therefore directories with spaces aren't supported.
    pull_directory="$*"
  fi

  if [ "$REPOSITORY" ];
  then
      pull_repository=$REPOSITORY
  fi

  if [ "$REVISION" ];
  then
      pull_revision=$REVISION
  fi

  if [ "$FILTER" ];
  then
      pull_filter="$FILTER"
  fi

  pull_filter="$(echo "$pull_filter" | tr ',' ' ')"  # Accept comma and/or space separated directories list


  if [ "$EXPAND_GLOB" ];
  then
      pull_expand_glob="$EXPAND_GLOB"
  fi
}

contains_substring() {
  # Checks if $1 string contains $2.
  printf '%s' "$1" | grep --fixed-strings --silent -e "$2"
  return $?
}

directory_exists() {
  # Checks if directory ($1) exists
  # This is mainly used to be mocked in tests
  test -d "$1"
  return $?
}

display_pull_params() {
  # Output configured vars to user
  echo "Pulling translation files"
  echo " - directory: ${pull_directory:-Not Specified}"
  echo " - repository: ${pull_repository:-Not Specified}"
  echo " - revision: ${pull_revision:-Not Specified}"
  echo " - filter: ${pull_filter:-Not Specified}"
  echo " - expand-glob: ${pull_expand_glob:-Not Specified}"
}

check_git_version() {
  MINIMUM_GIT_VERSION="2.25.0"  # The `git sparse-checkout` subcommand was introduced in 2.25.0
  GIT_VERSION=$(git --version | grep 'git version' | sed -e 's/ .Apple Git-.*//gI' | sed -e 's/git version //g')

  # Check if GIT_VERSION is a valid version number
  if ! echo "$GIT_VERSION" | grep -Eq '^[0-9]+(\.[0-9]+){1,3}$';
  then
    echo "Unable to determine git version. Please ensure git is installed and available on your PATH."
    return 1
  fi

  if [ "$(printf '%s\n' "$MINIMUM_GIT_VERSION" "$GIT_VERSION" | sort --version-sort | head -n1)" != "$MINIMUM_GIT_VERSION" ];
  then
    echo "Git version ${GIT_VERSION} is not supported. Please upgrade to ${MINIMUM_GIT_VERSION} or higher."
    return 1
  fi
}

git_sparse_checkout_set() {
    # Call `git sparse-checkout set --no-cone` with the list of include/exclude files as arguments
    # as a replacement of `sparse-checkout add` that isn't available in git 2.25.0
    xargs git sparse-checkout set
}

expand_glob_pattern() {
    # Expands glob pattern.
    # An alternative to `compgen -G` that isn't available in non-Bash shells.
    # Call: `expand_glob_pattern "translations/*/done"` to expand to `translations/DoneXBlock/done` if the file exists.

    # Shellcheck rule is disabled because we _want_ to expand the glob pattern.
    # shellcheck disable=SC2086
    echo $1;
}

pull_translations() {
  check_git_version || return 1

  if [ -z "$pull_directory" ];
  then
      echo "Missing positional argument:" >&2
      echo "  At least one DIRECTORY map is required as a positional argument:" >&2
      echo "  $ atlas pull DIRECTORY_FROM1:DIRECTORY_TO1 DIRECTORY_FROM2:DIRECTORY_TO2 ..." >&2
      return 1
  fi

  if [ -z "$SILENT" ];
  then
    echo "Creating a temporary Git repository to pull translations into \"./translations_TEMP\"..."
  fi

  # use git in quiet mode when verbose isn't set
  if [ "$VERBOSE" ];
  then
    quiet=""
  else
    quiet="--quiet"
  fi

  if [ "$pull_revision" ];
  then
    pull_revision_argument="--branch=${pull_revision}"
  else
    pull_revision_argument=""
  fi

  # Creating a shallow clone of the repo without without pulling the files yet
  # shellcheck disable=SC2086
  git clone $quiet $pull_revision_argument --filter=blob:none --no-checkout --depth=1 \
            "https://github.com/${pull_repository}.git" translations_TEMP || exit
  cd translations_TEMP || exit

  # finished "Creating a temporary Git repository to pull translations into <temp dir>..." step
  if [ -z "$SILENT" ];
  then
    echo "Done."

    echo "Setting git sparse-checkout rules..."
  fi

  # Set git sparse-checkout list of include/exclude files
  # Tells sparse checkout to ignore all files, except when otherwise noted
  # See https://www.git-scm.com/docs/git-sparse-checkout for detailed implementation
  git_sparse_checkout_rules="'--no-cone' '!*'"

  for directory_from_to in $pull_directory;
  do
    directory_from=$(echo "${directory_from_to}" | cut -f1 -d ':')
    directory_to=$(echo "${directory_from_to}" | cut -f2 -d ':')

    if [ "$pull_filter" ];
    then
      for one_filter in $pull_filter;
      do
        # Include directories that matches the language pattern e.g. `/ar/`
        git_sparse_checkout_rules="${git_sparse_checkout_rules} '${directory_from}/**/${one_filter}/**'"
        # Include files that matches the language pattern e.g. `ar.*`
        git_sparse_checkout_rules="${git_sparse_checkout_rules} '${directory_from}/**/${one_filter}.*'"
      done
    else
      # Include all files within provided DIRECTORY if no filter is specified.
      git_sparse_checkout_rules="${git_sparse_checkout_rules} '${directory_from}/**'"
    fi
  done

  echo "$git_sparse_checkout_rules" | git_sparse_checkout_set

  # finished "Creating a temporary Git repository to pull translations into <temp dir>..." step
  if [ -z "$SILENT" ];
  then
    echo "Done."

    echo "Pulling translation files from the repository..."
  fi

  # Retrieve translation files from the repo
  git checkout HEAD

  # Remove .git directory
  rm -rf .git

  # Leave the temp dir
  cd ..

  # finished "Pulling translations into <temp dir>..." step

  # Copy translation files out of the temp dir
  for directory_from_to in $pull_directory;
  do
    directory_from="$(echo "${directory_from_to}" | cut -f1 -d ':')"
    directory_from="./translations_TEMP/${directory_from}"

    if [ -n "$pull_expand_glob" ];
    then
      # Expand glob pattern e.g. `./translations_TEMP/*/done` to `./translations_TEMP/DoneXBlock/done`
      directory_from="$(expand_glob_pattern "$directory_from")"
    fi

    # If no colon is provided then directory_to=. (aka working directory)
    if contains_substring "$directory_from_to" ":";
    then
      directory_to="$(echo "${directory_from_to}" | cut -f2 -d ':')"
    else
      directory_to=.
    fi

    mkdir -p "$directory_to"
    if [ -z "$SILENT" ];
    then
      echo "Done."
      echo "Copying translations from \"${directory_from}\" to \"./${directory_to}\"..."
    fi

    if directory_exists "$directory_from";
    then
      cp -r "${directory_from}"/* "${directory_to}"/ || exit 1
    else
      if [ -z "$SILENT" ];
      then
        echo "Skipped copying \"${directory_from}\" because it was not found in the repository."
      fi
    fi
  done

  # finished "Copying translations from <temp dir> to <dest dir>..." step
  if [ -z "$SILENT" ];
  then
    echo "Done."

    echo "Removing temporary directory..."
  fi

  # Remove the temp dir
  rm -rf translations_TEMP

  # finshed "Removing temporary directory..." step
  if [ -z "$SILENT" ];
  then
    echo "Done."

    # finished pulling translations!
    echo ""
    echo "Translations pulled successfully!"
  fi
}

# used for shellspec mocking
test || __() { :; }

parse "$@"
eval "set -- $REST"

if [ $# -gt 0 ]; then
	cmd=$1
	shift
	case $cmd in
		pull)
			parse_pull "$@"
            eval "set -- $PULL_REST"
            set_pull_params "$@" # Make the arguments available to `set_pull_params`
            if [ -z "$SILENT" ];
            then
              display_pull_params
            fi
            # allow mocking pull_translations
            __ begin_pull_translations_mock __
            pull_translations || exit 1
            __ end_pull_translations_mock __
            ;;
		--) # no subcommand, arguments only
	esac
fi