Update repo #47339
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Update repo | |
| on: | |
| push: | |
| branches: | |
| - master | |
| paths: | |
| - '.github/workflows/repo-update.yml' | |
| schedule: | |
| - cron: '0,30 0-23 * * *' | |
| concurrency: | |
| group: ${{ github.workflow }} | |
| cancel-in-progress: true | |
| jobs: | |
| check-repo: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| target_repo: [main@aarch64, main@arm, main@i686, main@x86_64, x11@aarch64, x11@arm, x11@i686, x11@x86_64, root@aarch64, root@arm, root@i686, root@x86_64] | |
| fail-fast: false | |
| steps: | |
| - name: Clone check-tools | |
| run: git clone "https://termux-pacman-bot:${{ secrets.TOKEN }}@github.com/termux-pacman/check-tools" | |
| - name: Import key of Termux-Pacman | |
| run: gpg --keyserver keyserver.ubuntu.com --recv-keys 998de27318e867ea976ba877389ceed64573dfca | |
| - name: Clone termux-packages | |
| run: | | |
| cd check-tools | |
| bash ./clone_repos.sh | |
| - name: Running a scan | |
| run: | | |
| target_repo='${{ matrix.target_repo }}' | |
| REPO="${target_repo%%@*}" | |
| ARCH="${target_repo#*@}" | |
| TOTAL="RESULT.${REPO}.${ARCH}" | |
| DIR_REPO="termux-packages-repo/" | |
| case "$REPO" in | |
| "main") DIR_REPO+="packages";; | |
| "x11") DIR_REPO+="x11-packages";; | |
| "root") DIR_REPO+="root-packages";; | |
| *) | |
| echo "Error: repo '${REPO}' is unknown" | |
| exit 0;; | |
| esac | |
| cd check-tools | |
| if [ ! -d "$DIR_REPO" ]; then | |
| echo "Error: '$DIR_REPO' not found." | |
| exit 0 | |
| fi | |
| bash ./check_pkgs_by_repo.sh "$REPO" "$ARCH" "$DIR_REPO" | |
| # Removing packages... | |
| DELETED_FILE="./output/$(date +%s)-deleted_termux-${REPO}_packages.txt" | |
| for reason in NOTSUPPORTARCH UNKNOWN; do | |
| infofile="./output/${reason}.${REPO}.${ARCH}" | |
| if [ -f "${infofile}" ]; then | |
| cat "${infofile}" >> "${DELETED_FILE}" | |
| fi | |
| done | |
| if [ -f "${DELETED_FILE}" ]; then | |
| # Install aws-cli | |
| git clone https://github.com/termux-pacman/aws-cli-action | |
| ./aws-cli-action/setup.sh '${{ secrets.AWS_ACCESS_KEY_ID }}' '${{ secrets.AWS_ACCESS_KEY }}' '${{ secrets.AWS_REGION }}' | |
| # Import gpg key | |
| mkdir gk | |
| echo "${{ secrets.SF_GPG_BOT }}" > gk/termux-pacman-bot.gpg | |
| gpg --pinentry-mode=loopback --passphrase '${{ secrets.PW_GPG_BOT }}' --import gk/termux-pacman-bot.gpg | |
| rm -fr gk | |
| # Upload file | |
| echo "$(sed '/^[[:space:]]*$/d' ${DELETED_FILE} | sort | uniq)" > "${DELETED_FILE}" | |
| gpg --batch --pinentry-mode=loopback --passphrase '${{ secrets.PW_GPG_BOT }}' --detach-sign --use-agent -u '${{ secrets.KEY_GPG_BOT }}' --no-armor "${DELETED_FILE}" | |
| for format_file in "" ".sig"; do | |
| aws s3 cp "${DELETED_FILE}${format_file}" s3://'${{ secrets.SFPU }}'/${REPO}/${ARCH}/"${DELETED_FILE##*/}${format_file}" | |
| done | |
| rm "${DELETED_FILE}.sig" | |
| fi | |
| mkdir total | |
| touch ./total/${TOTAL} | |
| for RESULT in "./output/"{NOTFOUND,DOESNOTMATCH}".${REPO}.${ARCH}"; do | |
| if [ -f ${RESULT} ]; then | |
| cat ${RESULT} | awk '{printf $1 "\n"}' >> ./total/${TOTAL} | |
| fi | |
| done | |
| tar cf total-${{ matrix.target_repo }}-${{ github.sha }}.tar.gz ./total | |
| - name: Store RESULT.* files | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: total-${{ matrix.target_repo }}-${{ github.sha }} | |
| path: ./check-tools/total-${{ matrix.target_repo }}-${{ github.sha }}.tar.gz | |
| if-no-files-found: error | |
| update-repo: | |
| needs: check-repo | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Clone repository | |
| run: git clone https://github.com/termux-pacman/termux-packages.git termux-packages-pacman | |
| - name: Get RESULT.* files | |
| uses: actions/[email protected] | |
| with: | |
| path: ./ | |
| - name: Creating a REBUILD-LIST-BUILD | |
| run: | | |
| for i in ./total-*/total-*.tar.gz; do | |
| tar xf $i -C ./ | |
| done | |
| rm -fr ./total-*/ | |
| touch ./REBUILD-LIST-BUILD | |
| cd termux-packages-pacman | |
| for i in $(sort ../total/RESULT.* | uniq); do | |
| BUILD_SOURCE=$(ls */${i}/build.sh 2> /dev/null) || true | |
| if [ -z "${BUILD_SOURCE}" ]; then | |
| BUILD_SOURCE=$(ls */*/${i}.subpackage.sh 2> /dev/null) || true | |
| if [ -n "${BUILD_SOURCE}" ]; then | |
| BUILD_SOURCE="$(dirname ${BUILD_SOURCE})/build.sh" | |
| else | |
| echo "Source package '${i}' was not found, skip" | |
| continue | |
| fi | |
| fi | |
| if ! $(grep -q '^'${BUILD_SOURCE}'$' ../REBUILD-LIST-BUILD) && [ $(($(expr $(expr $(date +%s) - $(git log -n 1 --date=format:'%s' --format=%cd -- $(dirname ${BUILD_SOURCE}))) / 3600) > 8)) = 1 ]; then | |
| echo "${BUILD_SOURCE}" >> ../REBUILD-LIST-BUILD | |
| fi | |
| done | |
| cd .. | |
| - name: Clone root repository | |
| run: git clone https://github.com/termux/termux-packages.git | |
| - name: Update repo | |
| run: | | |
| info() { | |
| echo "==> $1" | |
| } | |
| commit() { | |
| echo "-> $1" | |
| } | |
| install_list_issues() { | |
| if [ -f ../LIST-ISSUES ]; then | |
| rm ../LIST-ISSUES | |
| fi | |
| curl --header "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -s -X GET -G \ | |
| "https://api.github.com/repos/${{ github.repository }}/issues" -o ../LIST-ISSUES | |
| } | |
| # Edit | |
| info "Edit" | |
| cd termux-packages-pacman | |
| rm -fr */ | |
| cd ../termux-packages | |
| cp -r * ../termux-packages-pacman | |
| # Get list pkg | |
| info "Get list pkg" | |
| cd ../termux-packages-pacman | |
| list_files=" " | |
| for i in $(git status -s packages root-packages x11-packages | awk '{print $2}'); do | |
| dir_sp=(${i//// }) | |
| if ! grep -q " ${dir_sp[0]}/${dir_sp[1]} " <<< "$list_files"; then | |
| list_files+="${dir_sp[0]}/${dir_sp[1]} " | |
| source_build="${dir_sp[0]}/${dir_sp[1]}/build.sh" | |
| if $(grep -q '^'${source_build}'$' ../REBUILD-LIST-BUILD); then | |
| sed -i "/^${source_build////'\/'}$/d" ../REBUILD-LIST-BUILD | |
| fi | |
| fi | |
| done | |
| list_files=($list_files) | |
| # Sort list pkg | |
| info "Sort list pkg" | |
| cd ../termux-packages | |
| declare -A list_values | |
| list_sort="" | |
| list_sha="" | |
| for i in ${list_files[@]}; do | |
| sha_file=$(git log -n 1 --pretty=format:%H -- $i) | |
| value="$(git rev-list --count $sha_file)" | |
| list_values[${value}]=$((${list_values[${value}]}+1)) | |
| list_sort+="${value}-${list_values[${value}]} " | |
| list_sha+="${sha_file} " | |
| done | |
| list_sort=($list_sort) | |
| list_sha=($list_sha) | |
| copy_list_sort=($(sort -n <(printf "%s\n" "${list_sort[@]}"))) | |
| search_index() { | |
| for i in "${!list_sort[@]}"; do | |
| if [[ "${list_sort[$i]}" = "${1}" ]]; then | |
| echo "${i}"; | |
| fi | |
| done | |
| } | |
| sort_list_files="" | |
| for i in ${copy_list_sort[@]}; do | |
| index=$(search_index "$i") | |
| if [[ "${i#*-}" = "1" ]]; then | |
| sort_list_files+=" ${list_sha[$index]}###" | |
| else | |
| sort_list_files+="&&" | |
| fi | |
| sort_list_files+="${list_files[$index]}" | |
| done | |
| # Start upload | |
| info "Start upload" | |
| cd ../termux-packages-pacman | |
| needbuild=false | |
| if $(git log -1 --pretty=%B | grep -q '%needbuild'); then | |
| needbuild=true | |
| fi | |
| git config --global user.name "termux-pacman-bot" | |
| git config --global user.email "[email protected]" | |
| git remote set-url origin "https://termux-pacman-bot:${{ secrets.TOKEN }}@github.com/termux-pacman/termux-packages.git" | |
| commit "Update system repo" | |
| git add . | |
| git reset packages root-packages x11-packages README.md SECURITY.md repo.json | |
| { | |
| git commit -m "Update repo" | |
| git push origin master | |
| } || true | |
| commit "Update repo packages" | |
| install_list_issues | |
| for i in $sort_list_files; do | |
| i_sp=(${i//###/ }) | |
| commit " ${i_sp[1]}" | |
| cd ../termux-packages | |
| commit_file="$(git show -s --format=%B ${i_sp[0]})" | |
| cd ../termux-packages-pacman | |
| git add $(sed 's/&&/ /g' <<< "${i_sp[1]}") | |
| { | |
| if $needbuild; then | |
| git commit -m "$(sed 's/%ci:no-build//g; s/\[skip ci\]//g' <<< $commit_file)" | |
| else | |
| git commit -m "$commit_file" | |
| fi | |
| git push origin master | |
| } || true | |
| for j in $(sed 's/&&/ /g' <<< "${i_sp[1]}"); do | |
| while read LINE; do | |
| USER=$(awk -F ' // ' '{printf $1}' <<< "$LINE") | |
| STATE=$(awk -F ' // ' '{printf $2}' <<< "$LINE") | |
| TITLE=$(awk -F ' // ' '{printf $4}' <<< "$LINE") | |
| if [ $STATE != "open" ] || [ $USER != "termux-pacman-bot" ] || [[ $TITLE != "[AUTO-CHECK]: failed to recompile ${j}" ]]; then | |
| continue | |
| fi | |
| NUMBER=$(awk -F ' // ' '{printf $3}' <<< "$LINE") | |
| GH_TOKEN="${{ secrets.TOKEN }}" gh issue close ${NUMBER} \ | |
| -c "The package has been updated - https://github.com/termux-pacman/termux-packages/commit/$(git log -n 1 --format=%H -- ${j})." | |
| install_list_issues | |
| break | |
| done <<< $(jq -r '.[] | "\(.user.login) \("//") \(.state) \("//") \(.number) \("//") \(.title)"' ../LIST-ISSUES) | |
| done | |
| done | |
| # Recompile packages if necessary | |
| if [ -n "$(cat ../REBUILD-LIST-BUILD)" ]; then | |
| info "Rebuild packages" | |
| listchanged="" | |
| for i in $(cat ../REBUILD-LIST-BUILD); do | |
| # It is necessary to create an issue because the package has | |
| # a compilation error even after recompilation. | |
| if [[ $(git log -n 1 --format=%B -- "${i}") = "rebuild($(dirname ${i})): auto check by @termux-pacman-bot" ]] && \ | |
| [[ $(git log -n 1 --format=%an -- "${i}") = "termux-pacman-bot" ]] && \ | |
| [[ $(git log -n 1 --format=%ae -- "${i}") = "[email protected]" ]]; then | |
| if ! $(jq -r '.[] | "\(.user.login) \("/") \(.state) \("/") \(.title)"' ../LIST-ISSUES | grep -q "^termux-pacman-bot / open / \[AUTO-CHECK\]:.*$(dirname ${i})$"); then | |
| commit "Create an issue on $(dirname ${i})" | |
| GH_TOKEN="${{ secrets.TOKEN }}" gh issue create \ | |
| --title "[AUTO-CHECK]: failed to recompile $(dirname ${i})" \ | |
| --body "$(echo -e "This issue was automatically generated because the '$(dirname ${i})' package was not recompiled successfully.\n\nCommit by recompilation - https://github.com/termux-pacman/termux-packages/commit/$(git log -n 1 --format=%H -- ${i})")" \ | |
| --label "auto recompilation failed" \ | |
| --assignee Maxython | |
| fi | |
| continue | |
| fi | |
| echo "" >> ./${i} | |
| git add ./${i} | |
| listchanged+=" ${i}" | |
| done | |
| if [ -n "${listchanged}" ]; then | |
| git commit -m "$(echo -e "termux-pacman/auto-check-repo: preparing to recompile packages\n[skip ci]")" | |
| git push origin master | |
| for i in ${listchanged}; do | |
| sed -i '$d' ./${i} | |
| git add ./${i} | |
| git commit -m "rebuild($(dirname ${i})): auto check by @termux-pacman-bot" | |
| git push origin master | |
| done | |
| fi | |
| fi |