Implement breakage tests

.github/workflows/breakage.yml

@@ -0,0 +1,62 @@
+name: Breakage
+on:
+  pull_request:
+    types: [opened, synchronize, reopened]
+
+jobs:
+  break:
+    name: Breakage of ${{ matrix.pkg }}
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        pkg: [CaNNOLeS, DCI, JSOSolvers, Percival]
+        pkgversion: ["master", "stable"]
+    steps:
+      - uses: actions/checkout@v2
+      - uses: julia-actions/setup-julia@v1
+        with:
+          version: 1
+          arch: x64
+      - uses: actions/cache@v1
+        env:
+          cache-name: cache-artifacts
+        with:
+          path: ~/.julia/artifacts
+          key: ${{ runner.os }}-test-${{ env.cache-name }}-${{ hashFiles('**/Project.toml') }}
+          restore-keys: |
+            ${{ runner.os }}-test-${{ env.cache-name }}-
+            ${{ runner.os }}-test-
+            ${{ runner.os }}-
+      - uses: julia-actions/julia-buildpkg@v1
+      - name: Breakage test
+        env:
+          PKG: ${{ matrix.pkg }}
+          VERSION: ${{ matrix.pkgversion }}
+          GITHUB_AUTH: ${{ secrets.GITHUB_TOKEN }}
+        run: julia -e 'using Pkg; pkg"add Git, JSON"; include("test/test-breakage.jl")'
+  deploy:
+    needs: break
+    name: Deploying breakage information
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: julia-actions/setup-julia@v1
+        with:
+          version: 1
+          arch: x64
+      - uses: actions/cache@v1
+        env:
+          cache-name: cache-artifacts
+        with:
+          path: ~/.julia/artifacts
+          key: ${{ runner.os }}-test-${{ env.cache-name }}-${{ hashFiles('**/Project.toml') }}
+          restore-keys: |
+            ${{ runner.os }}-test-${{ env.cache-name }}-
+            ${{ runner.os }}-test-
+            ${{ runner.os }}-
+      - uses: julia-actions/julia-buildpkg@v1
+      - name: Breakage test
+        env:
+          GITHUB_AUTH: ${{ secrets.GITHUB_TOKEN }}
+        run: julia -e 'using Pkg; pkg"add Git, GitHub, JSON"; include("test/test-breakage-deploy.jl")'

test/test-breakage-deploy.jl

@@ -0,0 +1,66 @@
+using Git, GitHub, JSON, Pkg
+
+"""
+    test_breakage_deploy()
+
+Read files from breakage-info and publish to the PR
+"""
+function test_breakage_deploy()
+  if get(ENV, "CI", "false") == "false"
+    error("Only run on CI")
+  end
+  if get(ENV, "GITHUB_AUTH", nothing) === nothing
+    error("GITHUB_AUTH not found")
+  end
+  key = ENV["GITHUB_AUTH"]
+  repo = "JuliaSmoothOptimizers/Krylov.jl"
+  user = split(repo, "/")[1]
+  upstream = "https://$user:$key@github.com/$repo"
+  Git.run(`config user.email "[email protected]"`)
+  Git.run(`config user.name "Abel Soares Siqueira"`)
+  Git.run(`remote add upstream $upstream`)
+  Git.run(`fetch upstream`)
+  Git.run(`checkout -f breakage-info`)
+
+  badge_pass(x) = "![](https://img.shields.io/badge/$x-Pass-green)"
+  badge_fail(x) = "![](https://img.shields.io/badge/$x-Fail-red)"
+  badge(tf, x) = tf ? badge_pass(x) : badge_fail(x)
+
+  packages = ["CaNNOLeS", "DCI", "JSOSolvers", "Percival"]
+
+  output = ":robot: Testing breakage of this pull request\n\n"
+  output *= "| Package Name | master | stable |\n"
+  output *= "|--|--|--|\n"
+  for package in packages
+    output *= "| $package | "
+
+    for version in ["master", "stable"]
+      info = JSON.parse(open("$package-$version"))
+      bdg = badge(info["pass"], info["tag"])
+      joburl = info["joburl"]
+      output *= "[$bdg]($joburl) | "
+    end
+    output *= "\n"
+  end
+
+  @debug(output)
+
+  myauth = GitHub.authenticate(key)
+  myrepo = GitHub.repo(repo, auth=myauth) # "JuliaSmoothOptimizers/Krylov.jl"
+  prs = pull_requests(myrepo, auth=myauth)
+  pr = nothing
+  prnumber = ENV["GITHUB_REF"]
+  @debug("PR NUMBER: $prnumber")
+  prnumber = split(prnumber, "/")[3]
+
+  for p in prs[1]
+    if p.number == Meta.parse(prnumber)
+      pr = p
+    end
+  end
+  @assert pr != nothing
+
+  GitHub.create_comment(GitHub.DEFAULT_API, myrepo, pr, :pr, auth=myauth, params=Dict(:body => output))
+end
+
+test_breakage_deploy()

test/test-breakage.jl

@@ -0,0 +1,98 @@
+using Dates, Git, JSON, Pkg
+
+"""
+    test_breakage()
+
+Downloads a package (given by ENV["PKG"]) and check if its tests pass using the pull request version of Krylov.jl.
+Packages are checked in two situations:
+
+- stable version (ENV["stable"])
+- master branch  (ENV["master"])
+
+If any tagged version breaks then the next version should be a major update or
+the broken package has bugs.
+"""
+function test_breakage()
+  if get(ENV, "CI", "false") == "false"
+    error("Only run on CI")
+  end
+  if get(ENV, "GITHUB_AUTH", nothing) === nothing
+    error("GITHUB_AUTH not found")
+  end
+  key = ENV["GITHUB_AUTH"]
+
+  Git.run(`config user.email "[email protected]"`)
+  Git.run(`config user.name "Abel Soares Siqueira"`)
+
+  package = get(ENV, "PKG", nothing)
+  version = get(ENV, "VERSION", nothing)
+  package === nothing || version === nothing && error("No environment variable PKG defined")
+  tag = version
+
+  passing = false
+  thispath = pwd()
+  mkpath("test-breakage")
+  try
+    @info "#"^100
+    @info "  Testing package $package"
+    @info "#"^100
+
+    cd(joinpath(thispath, "test-breakage"))
+    url = "https://github.com/JuliaSmoothOptimizers/$package.jl"
+    Git.run(`clone $url`)
+    cd("$package.jl")
+    if version == "stable"
+      tag = "v" * string(sort(VersionNumber.(split(Git.readstring(`tag`))))[end])
+      Git.run(`checkout $tag`)
+    end
+    pkg"activate ."
+    pkg"instantiate"
+    pkg"dev ../.."
+    pkg"build"
+    pkg"test"
+    passing = true
+  catch e
+    @error e
+  end
+
+  # Enter branch breakage-info and commit file $package-$version
+  info = Dict(:pass => passing,
+              :pr => ENV["GITHUB_REF"],
+              :joburl => joinpath(ENV["GITHUB_SERVER_URL"], ENV["GITHUB_REPOSITORY"], "actions/runs", ENV["GITHUB_RUN_ID"]),
+              :tag => tag,
+              :datetime => Dates.now()
+             )
+  cd(thispath)
+  repo = "JuliaSmoothOptimizers/Krylov.jl"
+  user = split(repo, "/")[1]
+  upstream = "https://$user:$key@github.com/$repo"
+  Git.run(`remote add upstream $upstream`)
+  Git.run(`fetch upstream`)
+  if !success(`git checkout -f -b breakage-info upstream/breakage-info`)
+    Git.run(`checkout --orphan breakage-info`)
+    Git.run(`reset --hard`)
+    Git.run(`commit --allow-empty -m "Initial commit"`)
+  end
+  open("$package-$version", "w") do io
+    JSON.print(io, info, 2)
+  end
+  Git.run(`add $package-$version`)
+  try
+    Git.run(`commit -m ":robot: test-breakage of $package-$version"`)
+    tries = 0
+    while !Git.success(`push upstream breakage-info`)
+      if tries > 10
+        error("Too many failures in pushing")
+      end
+      @warn("push failed, trying again")
+      sleep(5)
+      Git.run(`fetch upstream`)
+      Git.run(`merge upstream/breakage-info`)
+      tries += 1
+    end
+  catch ex
+    @error ex
+  end
+end
+
+test_breakage()