From 5afa57bb0350f0988ff5536766ded4a85390cf21 Mon Sep 17 00:00:00 2001 From: Lianmin Zheng Date: Sat, 23 Jul 2022 15:37:25 -0400 Subject: [PATCH] [CI/CD] Store version number in alpa._version__ (#626) --- .github/workflows/release_alpa.yml | 31 ----- GENVER | 130 ------------------ VERSION | 1 - alpa/__init__.py | 1 + alpa/version.py | 3 + docker/scripts/build_alpa.sh | 2 +- docs/conf.py | 19 ++- setup.py | 18 ++- update_version.py | 203 +++++++++++++++++++++++++++++ 9 files changed, 238 insertions(+), 170 deletions(-) delete mode 100644 GENVER delete mode 100644 VERSION create mode 100644 alpa/version.py create mode 100644 update_version.py diff --git a/.github/workflows/release_alpa.yml b/.github/workflows/release_alpa.yml index 8252df216..bb7a4a920 100644 --- a/.github/workflows/release_alpa.yml +++ b/.github/workflows/release_alpa.yml @@ -61,34 +61,3 @@ jobs: echo "Publish to PyPI" ls -ltr dist/ python -m twine upload --verbose dist/* - - update-version: - runs-on: [ubuntu-latest] - needs: [release-alpa] - steps: - - uses: actions/checkout@v3 - - - name: Bump VERSION and commit - run: | - git config --local user.email "action@github.com" - git config --local user.name "GitHub Action" - git checkout -b actions/version-bump - cat VERSION | awk -F. -v OFS=. 'NF==1{print ++$NF}; NF>1{$NF=sprintf("%0*d", length($NF), ($NF+1)); print}' > VERSION.tmp && mv VERSION.tmp VERSION - git add VERSION - git commit -m "Bump candidate version [skip ci]" - - - name: Push branch - uses: ad-m/github-push-action@master - with: - github_token: ${{ secrets.PAT_TOKEN }} - branch: actions/version-bump - force: true - - - name: Create version-bump PR - uses: repo-sync/pull-request@v2 - with: - github_token: ${{ secrets.PAT_TOKEN }} - source_branch: "actions/version-bump" - destination_branch: "main" - pr_title: "Bumping VERSION file due to recent release" - pr_body: "*Automated PR*: This should update the candidate version. Please delete source branch upon merge" diff --git a/GENVER b/GENVER deleted file mode 100644 index 8fb613451..000000000 --- a/GENVER +++ /dev/null @@ -1,130 +0,0 @@ -#!/bin/bash - -# GENVER returns a semantic version of the current branch. -# -# It generates the semantic version based on the tag references on -# the repo. -# -# Args: -# Can pass "--pep440" to generate PEP-440 compliant version -# -# Assumptions: -# - Will match only tags matching "v[0-9]*" (e.g., v0.0.1) -# - Repo owners tag accordingly per release -# -# Example: -# -# 1. If the latest tag (v0.0.1) is at HEAD~4 and HEAD is at commit 60f987bf, -# running -# -# ./GENVER --pep440 -# -# will return: 0.0.1.dev4+g60f987bfd -# -# NOTE: PyPI will not allow these local versions to be pushed: "Can't use -# PEP 440 local versions. See https://packaging.python.org/specifications/core-metadata -# for more information." -# -# 2. If "--pep440" is not passed into example (1), it would return 0.0.1-4.g60f987bfd -# -# 3. If the latest tag (v0.0.2) is at HEAD of branch (60f987bf), it'll -# treat it as a released state, so running: -# -# ./GENVER --pep440 -# -# will return: 0.0.2 -# - - -set -e - -# Generates a PEP 440 compatible version instead of a semantic version. -if test "$1" = "--pep440"; then - GENVER_PEP440="true" -fi - -# Dependency checks. -dependencies="cat expr git" -for cmd in ${dependencies}; do - if ! command -v "${cmd}" >/dev/null 2>&1; then - >&2 echo "Command ${cmd} is unavailable." - exit 1 - fi -done - -# Validate the candidate version. -ROOT=$(git rev-parse --show-toplevel) -VERSION_FILE="${ROOT}/VERSION" - -if test -f "${VERSION_FILE}"; then - can_ver=$(cat "${VERSION_FILE}") -else - >&2 echo "Error: Failed to find the VERSION file, please refer to RELEASE.md" \ - "as we reference VERSION file for the base candidate version." - exit 1 -fi - -# Get the "git describe" version. -# Normal case, the versioning tags looks like "v1.0.0", "v1.0.1",... -if ! git_ver=$(git describe --tags --abbrev=9 --match "v[0-9]*" HEAD 2>/dev/null) -then - >&2 echo "Error: Can't get any versioning tag. Are tags pushed?" -fi - -# Strip the leading 'v' from the git describe version. -git_ver=$(expr "${git_ver}" : v*'\(.*\)') - -if expr "${git_ver}" : '.*-[0-9]*-g.*' >/dev/null -then - # Branch is in the developing state (e.g., there have been commits - # since the last tag). - - # Extract each part of the "git describe" version. - tag_ver=$(expr "$git_ver" : '\(.*\)-[0-9]*-g.*') - commit_count=$(expr "$git_ver" : '.*-\([0-9]*\)-g.*') - commit_hash=$(expr "$git_ver" : '.*-[0-9]*-\(g.*\)') - - if test "$can_ver" = "$tag_ver" - then - >&2 echo "Error: The candidate version on this branch is still at" \ - "what has been released. You can pull in latest VERSION file" \ - "to be up-to-date" - exit 1 - fi - - if test "${GENVER_PEP440:-}" = "true" - then - sem_ver="${tag_ver}.dev${commit_count}+${commit_hash}" - else - # Generate the semantic version for pre-release. - # Case 1 (candidate version contains hyphen): - # candidate version(VERSION file) : 1.0.0-p1 - # ======== - # git describe returns : 1.0.0-2-gde2198c - # ~~~~~~~~~~~ - # semantic version(generated) : 1.0.0-p1.2.gde2198c - # ========~~~~~~~~~~~ - # ^ ^ - # Case 2: - # candidate version(VERSION file) : 1.0.1 - # ===== - # git describe returns : 1.0.0-2-gde2198c - # ~~~~~~~~~~~ - # semantic version(generated) : 1.0.1-2.gde2198c - # =====~~~~~~~~~~~ - # ^ - if expr "$tag_ver" : ".*-" >/dev/null - then - sem_ver="${tag_ver}.${commit_count}.${commit_hash}" - else - sem_ver="${tag_ver}-${commit_count}.${commit_hash}" - fi - fi - -else - # Branch is in the released state. - # Use the version taken from tag as-is - sem_ver="${git_ver}" -fi - -echo "${sem_ver}" diff --git a/VERSION b/VERSION deleted file mode 100644 index 9faa1b7a7..000000000 --- a/VERSION +++ /dev/null @@ -1 +0,0 @@ -0.1.5 diff --git a/alpa/__init__.py b/alpa/__init__.py index 6172ac547..c95b30510 100644 --- a/alpa/__init__.py +++ b/alpa/__init__.py @@ -26,6 +26,7 @@ from alpa.shard_parallel.auto_sharding import AutoShardingOption from alpa.serialization import save_checkpoint, restore_checkpoint from alpa.timer import timers +from alpa.version import __version__ from . import api from . import collective diff --git a/alpa/version.py b/alpa/version.py new file mode 100644 index 000000000..f02dc58d8 --- /dev/null +++ b/alpa/version.py @@ -0,0 +1,3 @@ +"""Version information.""" + +__version__ = "1.0.0.dev0" diff --git a/docker/scripts/build_alpa.sh b/docker/scripts/build_alpa.sh index 09ea27c77..266267eae 100644 --- a/docker/scripts/build_alpa.sh +++ b/docker/scripts/build_alpa.sh @@ -50,7 +50,7 @@ git fetch origin +${ALPA_BRANCH} git checkout -qf FETCH_HEAD # install jaxlib and jax -export VERSION=$(bash GENVER --pep440) +python3 update_version.py --git-describe python3 setup.py bdist_wheel if ! python3 -m auditwheel show dist/alpa-*.whl | egrep 'platform tag: "(manylinux2014_x86_64|manylinux_2_17_x86_64)"' > /dev/null; then diff --git a/docs/conf.py b/docs/conf.py index ee60f4c10..806dae038 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -16,11 +16,22 @@ # -- Project information ----------------------------------------------------- project = 'Alpa' -#copyright = '2022, ' -#author = '' +author = 'Alpa Developers' +copyright = f'2022, {author}' -# The full version, including alpha/beta/rc tags -release = '0.1.4' + +def git_describe_version(): + """Get git describe version.""" + ver_py = os.path.join("..", "update_version.py") + libver = {"__file__": ver_py} + exec(compile(open(ver_py, "rb").read(), ver_py, "exec"), libver, libver) + gd_version, _ = libver["git_describe_version"]() + return gd_version + + +import alpa +version = git_describe_version() +release = version # -- General configuration --------------------------------------------------- diff --git a/setup.py b/setup.py index b9f737498..bcbb98c35 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,6 @@ import glob import os +import re import shutil import subprocess import sys @@ -7,6 +8,7 @@ from setuptools import setup, find_packages IS_WINDOWS = sys.platform == "win32" +ROOT_DIR = os.path.dirname(__file__) def get_cuda_version(cuda_home): @@ -98,6 +100,16 @@ def get_cuda_version_str(no_dot=False): ] +def get_alpa_version(): + with open(os.path.join(ROOT_DIR, "alpa", "version.py")) as fp: + version_match = re.search( + r"^__version__ = ['\"]([^'\"]*)['\"]", fp.read(), re.M + ) + if version_match: + return version_match.group(1) + raise RuntimeError("Unable to find version string.") + + def build(): """Build the custom pipeline marker API.""" # Check cuda version @@ -169,13 +181,13 @@ def finalize_options(self): if self.distribution.has_ext_modules(): self.install_lib = self.install_platlib - with open(os.path.join("README.md"), encoding="utf-8") as f: + with open("README.md", encoding="utf-8") as f: long_description = f.read() setup( name="alpa", - version=os.environ.get("VERSION"), - author="Alpa team", + version=get_alpa_version(), + author="Alpa Developers", author_email="", description= "Alpa automatically parallelizes large tensor computation graphs and " diff --git a/update_version.py b/update_version.py new file mode 100644 index 000000000..46d3c5eb1 --- /dev/null +++ b/update_version.py @@ -0,0 +1,203 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +This is the global script that set the version information of Alpa. +This script runs and update all the locations that related to versions + +List of affected files: +- root/python/alpa/version.py +""" +import os +import re +import argparse +import logging +import subprocess + +# Modify the following value during release +# --------------------------------------------------- +# Current version: +# We use the version of the incoming release for code +# that is under development. +# +# It is also fallback version to be used when --git-describe +# is not invoked, or when the repository does not present the +# git tags in a format that this script can use. +# +# Two tag formats are supported: +# - vMAJ.MIN.PATCH (e.g. v0.8.0) or +# - vMAJ.MIN.devN (e.g. v0.8.dev0) +__version__ = "v0.2.dev0" + +# --------------------------------------------------- + +PROJ_ROOT = os.path.dirname(os.path.abspath(os.path.expanduser(__file__))) + + +def py_str(cstr): + return cstr.decode("utf-8") + + +def git_describe_version(): + """Get PEP-440 compatible public and local version using git describe. + + Returns + ------- + pub_ver: str + Public version. + + local_ver: str + Local version (with additional label appended to pub_ver). + + Notes + ----- + - We follow PEP 440's convention of public version + and local versions. + - Only tags conforming to vMAJOR.MINOR.REV (e.g. "v0.7.0") + are considered in order to generate the version string. + See the use of `--match` in the `git` command below. + + Here are some examples: + + - pub_ver = '0.7.0', local_ver = '0.7.0': + We are at the 0.7.0 release. + - pub_ver = '0.8.dev94', local_ver = '0.8.dev94+g0d07a329e': + We are at the the 0.8 development cycle. + The current source contains 94 additional commits + after the most recent tag(v0.7.0), + the git short hash tag of the current commit is 0d07a329e. + """ + cmd = [ + "git", + "describe", + "--tags", + "--match", + "v[0-9]*.[0-9]*.[0-9]*", + "--match", + "v[0-9]*.[0-9]*.dev[0-9]*", + ] + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=PROJ_ROOT) + (out, _) = proc.communicate() + + if proc.returncode != 0: + msg = py_str(out) + if msg.find("not a git repository") != -1: + return __version__, __version__ + logging.warning("git describe: %s, use %s", msg, __version__) + return __version__, __version__ + describe = py_str(out).strip() + arr_info = describe.split("-") + + # Remove the v prefix, mainly to be robust + # to the case where v is not presented as well. + if arr_info[0].startswith("v"): + arr_info[0] = arr_info[0][1:] + + # hit the exact tag + if len(arr_info) == 1: + return arr_info[0], arr_info[0] + + if len(arr_info) != 3: + logging.warning("Invalid output from git describe %s", describe) + return __version__, __version__ + + dev_pos = arr_info[0].find(".dev") + + # Development versions: + # The code will reach this point in case it can't match a full release version, such as v0.7.0. + # + # 1. in case the last known label looks like vMAJ.MIN.devN e.g. v0.8.dev0, we use + # the current behaviour of just using vMAJ.MIN.devNNNN+gGIT_REV + if dev_pos != -1: + dev_version = arr_info[0][: arr_info[0].find(".dev")] + # 2. in case the last known label looks like vMAJ.MIN.PATCH e.g. v0.8.0 + # then we just carry on with a similar version to what git describe provides, which is + # vMAJ.MIN.PATCH.devNNNN+gGIT_REV + else: + dev_version = arr_info[0] + + pub_ver = "%s.dev%s" % (dev_version, arr_info[1]) + local_ver = "%s+%s" % (pub_ver, arr_info[2]) + return pub_ver, local_ver + + +# Implementations +def update(file_name, pattern, repl, dry_run=False): + update = [] + hit_counter = 0 + need_update = False + with open(file_name) as file: + for l in file: + result = re.findall(pattern, l) + if result: + assert len(result) == 1 + hit_counter += 1 + if result[0] != repl: + l = re.sub(pattern, repl, l) + need_update = True + print("%s: %s -> %s" % (file_name, result[0], repl)) + else: + print("%s: version is already %s" % (file_name, repl)) + + update.append(l) + if hit_counter != 1: + raise RuntimeError("Cannot find version in %s" % file_name) + + if need_update and not dry_run: + with open(file_name, "w") as output_file: + for l in update: + output_file.write(l) + + +def sync_version(pub_ver, local_ver, dry_run): + """Synchronize version.""" + # python uses the PEP-440: local version + update( + os.path.join(PROJ_ROOT, "alpa", "version.py"), + r"(?<=__version__ = \")[.0-9a-z\+]+", + local_ver, + dry_run, + ) + + +def main(): + logging.basicConfig(level=logging.INFO) + parser = argparse.ArgumentParser(description="Detect and synchronize version.") + parser.add_argument( + "--print-version", + action="store_true", + help="Print version to the command line. No changes is applied to files.", + ) + parser.add_argument( + "--git-describe", + action="store_true", + help="Use git describe to generate development version.", + ) + parser.add_argument("--dry-run", action="store_true") + + opt = parser.parse_args() + pub_ver, local_ver = __version__, __version__ + if opt.git_describe: + pub_ver, local_ver = git_describe_version() + if opt.print_version: + print(local_ver) + else: + sync_version(pub_ver, local_ver, opt.dry_run) + + +if __name__ == "__main__": + main()