🐧🧹 HealthChecks 🖳🗑️ #1668
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
name: 🐧🧹 HealthChecks 🖳🗑️ | |
#MAX_RUNTIME: 02 Minutes */10 * * * * | |
on: | |
#push: | |
workflow_dispatch: | |
schedule: | |
# - cron: "45 03 * * *" # 03:45 AM UTC --> 09:30 AM Morning NPT | |
- cron: "0 */12 * * *" # Every 12 Hrs | |
env: | |
GITHUB_TOKEN: "${{ secrets.TOOLPACKS }}" | |
RCLONE_CF_R2_PUB: "${{ secrets.RCLONE_CF_R2_PUB }}" | |
R2_PUB_REPO: "https://pub.ajam.dev/repos/Azathothas/Toolpacks" | |
jobs: | |
#------------------------------------------------------------------------------------# | |
check-post: | |
runs-on: ubuntu-latest | |
permissions: | |
contents: write | |
steps: | |
- name: Debloat Runner | |
run: | | |
#Presets | |
set +x ; set +e | |
#--------------# | |
bash <(curl -qfsSL "https://pub.ajam.dev/repos/Azathothas/Arsenal/misc/Github/Runners/Ubuntu/debloat.sh") | |
continue-on-error: true | |
- name: Checkout repository | |
uses: actions/checkout@v4 | |
with: | |
path: main | |
filter: "blob:none" #https://github.blog/2020-12-21-get-up-to-speed-with-partial-clone-and-shallow-clone/ | |
- name: Setup Env | |
run: | | |
#presets | |
set +x ; set +e | |
#tmp | |
SYSTMP="$(dirname $(mktemp -u))" && export SYSTMP="$SYSTMP" | |
#GH ENV | |
echo "SYSTMP=$SYSTMP" >> "$GITHUB_ENV" | |
#GH Dirs | |
mkdir -p "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux" | |
mkdir -p "$GITHUB_WORKSPACE/main/arm64_v8a_Android" | |
mkdir -p "$GITHUB_WORKSPACE/main/x64_Windows" | |
mkdir -p "$GITHUB_WORKSPACE/main/x86_64_Linux" | |
#-------------# | |
mkdir -p "$HOME/bin" | |
##Setup rClone | |
mkdir -p "$HOME/.config/rclone" | |
echo "${{ secrets.RCLONE_CF_R2_PUB }}" > "$HOME/.config/rclone/rclone.conf" | |
export RCLONE_STATS="120s" ; echo "RCLONE_STATS=$RCLONE_STATS" >> "$GITHUB_ENV" | |
##User-Agent | |
USER_AGENT="$(curl -qfsSL 'https://pub.ajam.dev/repos/Azathothas/Wordlists/Misc/User-Agents/ua_chrome_macos_latest.txt')" && export USER_AGENT="$USER_AGENT" | |
echo "USER_AGENT=$USER_AGENT" >> "$GITHUB_ENV" | |
continue-on-error: true | |
- name: Install Addons | |
run: | | |
#presets | |
set +x ; set +e | |
#-------------# | |
bash <(curl -qfsSL "https://pub.ajam.dev/repos/Azathothas/Arsenal/misc/Linux/install_dev_tools.sh") | |
continue-on-error: true | |
- name: Install Golang | |
run: | | |
#presets | |
set +x ; set +e | |
#-------------# | |
pushd "$(mktemp -d)" >/dev/null | |
echo "yes" | bash <(curl -qfsSL "https://git.io/go-installer") | |
popd >/dev/null | |
continue-on-error: true | |
- name: Install NIX | |
run: | | |
#presets | |
set +x ; set +e | |
#-------------# | |
"/nix/nix-installer" uninstall --no-confirm 2>/dev/null | |
curl -qfsSL "https://install.determinate.systems/nix" | bash -s -- install linux --init none --no-confirm | |
source "/nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh" | |
sudo chown --recursive "runner" "/nix" | |
echo "root ALL=(ALL:ALL) ALL" | sudo tee -a "/etc/sudoers" | |
nix --version && nix-channel --list && nix-channel --update | |
continue-on-error: true | |
- name: Install Rust | |
run: | | |
#presets | |
set +x ; set +e | |
#-------------# | |
bash <(curl -qfsSL "https://sh.rustup.rs") -y | |
continue-on-error: true | |
- name: Install Zig | |
run: | | |
#presets | |
set +x ; set +e | |
#-------------# | |
sudo rm "/usr/local/zig" -rf 2>/dev/null ; sudo rm "/usr/local/zig" -rf 2>/dev/null | |
pushd "$(mktemp -d)" >/dev/null 2>&1 && curl -qfSLJO $(curl -qfsSL "https://ziglang.org/download/index.json" | jq -r '.master | ."x86_64-linux".tarball') | |
find . -type f -name '*.tar*' -exec tar -xf {} \; | |
sudo mkdir -p "/usr/local/zig" && sudo mv "$(find . -maxdepth 1 -type d | grep -v '^.$')"/* "/usr/local/zig" ; popd >/dev/null 2>&1 | |
continue-on-error: true | |
- name: Dos2Unix Everything | |
run: | | |
#Presets | |
set +x ; set +e | |
#--------------# | |
cd "$GITHUB_WORKSPACE/main" | |
find . -type f ! -path "./.git/*" -exec dos2unix {} \; 2>/dev/null | |
continue-on-error: true | |
- name: ActionLint | |
run: | | |
#Presets | |
set +x ; set +e | |
#--------------# | |
cd "$GITHUB_WORKSPACE/main" | |
find ".github/workflows" -type f -name "*ml" -exec actionlint {} \; | |
continue-on-error: true | |
- name: Shellcheck | |
run: | | |
#Presets | |
set +x ; set +e | |
#--------------# | |
cd "$GITHUB_WORKSPACE/main" | |
find ".github" -type f -name '*.sh' -exec shellcheck --exclude="SC2261" --severity=error "{}" \; 2>/dev/null | tee "${GITHUB_WORKSPACE}/main/.github/shellcheck.txt" | |
continue-on-error: true | |
- name: YAMLint | |
run: | | |
#Presets | |
set +x ; set +e | |
#--------------# | |
cd "$GITHUB_WORKSPACE/main/.github/scripts/aarch64_Linux/bins" | |
find . -name '*.yaml' -exec sh -c 'for file; do bins=$(yq .bins[] "$file"); if [ "$(echo "$bins" | wc -l)" -eq 1 ] && [ "$(echo "$bins" | wc -w)" -gt 1 ]; then echo "$file --> $file.invalid"; mv "$file" "$file.invalid" ; fi; done' sh {} + | |
cd "$GITHUB_WORKSPACE/main/.github/scripts/arm64_v8a_Android/bins" | |
find . -name '*.yaml' -exec sh -c 'for file; do bins=$(yq .bins[] "$file"); if [ "$(echo "$bins" | wc -l)" -eq 1 ] && [ "$(echo "$bins" | wc -w)" -gt 1 ]; then echo "$file --> $file.invalid"; mv "$file" "$file.invalid" ; fi; done' sh {} + | |
cd "$GITHUB_WORKSPACE/main/.github/scripts/x86_64_Linux/bins" | |
find . -name '*.yaml' -exec sh -c 'for file; do bins=$(yq .bins[] "$file"); if [ "$(echo "$bins" | wc -l)" -eq 1 ] && [ "$(echo "$bins" | wc -w)" -gt 1 ]; then echo "$file --> $file.invalid"; mv "$file" "$file.invalid" ; fi; done' sh {} + | |
cd "$GITHUB_WORKSPACE/main/.github/scripts/x64_Windows/bins" | |
find . -name '*.yaml' -exec sh -c 'for file; do bins=$(yq .bins[] "$file"); if [ "$(echo "$bins" | wc -l)" -eq 1 ] && [ "$(echo "$bins" | wc -w)" -gt 1 ]; then echo "$file --> $file.invalid"; mv "$file" "$file.invalid" ; fi; done' sh {} + | |
continue-on-error: true | |
- name: Generate GO_TARGETS | |
run: | | |
#Presets | |
set +x ; set +e | |
#--------------# | |
export PATH="$HOME/bin:$HOME/.cargo/bin:$HOME/.cargo/env:$HOME/.go/bin:$HOME/go/bin:/home/linuxbrew/.linuxbrew/bin:/home/linuxbrew/.linuxbrew/sbin:$HOME/.local/bin:$HOME/miniconda3/bin:$HOME/miniconda3/condabin:/usr/local/zig:/usr/local/zig/lib:/usr/local/zig/lib/include:/usr/local/musl/bin:/usr/local/musl/lib:/usr/local/musl/include:$PATH" | |
go tool dist list --json | jq -r '["| [GOOS](https://pkg.go.dev/internal/platform) | [GOARCH](https://github.com/golang/go/blob/master/src/go/build/syslist.go) | [CGO_SUPPORT](https://web.archive.org/web/20240614051126/https://medium.com/@pengcheng1222/exploring-cgo-enabled-in-go-23cf5cf2fe88) | [NATIVE](https://gist.github.com/asukakenji/f15ba7e588ac42795f421b48b8aede63) |", "| --- | --- | --- | --- |"] + (map("| \(.GOOS) | \(.GOARCH) | \(.CgoSupported | if . then "✅" else "❌" end) | \(.FirstClass | if . then "✅" else "❌" end) |")) | join("\n")' > "$GITHUB_WORKSPACE/main/Docs/GO_TARGETS.md" | |
continue-on-error: true | |
- name: Generate NIX_TARGETS | |
run: | | |
#Presets | |
set +x ; set +e | |
#--------------# | |
#https://github.com/NixOS/nixpkgs/blob/master/lib/systems/examples.nix | |
export PATH="$HOME/bin:$HOME/.cargo/bin:$HOME/.cargo/env:$HOME/.go/bin:$HOME/go/bin:/home/linuxbrew/.linuxbrew/bin:/home/linuxbrew/.linuxbrew/sbin:$HOME/.local/bin:$HOME/miniconda3/bin:$HOME/miniconda3/condabin:/usr/local/zig:/usr/local/zig/lib:/usr/local/zig/lib/include:/usr/local/musl/bin:/usr/local/musl/lib:/usr/local/musl/include:$PATH" | |
nix-instantiate --eval --strict --expr 'let pkgs = import <nixpkgs> {}; pkgsCross = pkgs.pkgsCross; in builtins.attrNames pkgsCross' | tr -d '[]"' | tr ' ' '\n' | sed '/^\s*$/d' | sed 's/^/pkgsCross./' > "$GITHUB_WORKSPACE/main/Docs/NIX_TARGETS.txt" | |
continue-on-error: true | |
- name: Generate RUST_TARGETS | |
run: | | |
#Presets | |
set +x ; set +e | |
#--------------# | |
export PATH="$HOME/bin:$HOME/.cargo/bin:$HOME/.cargo/env:$HOME/.go/bin:$HOME/go/bin:/home/linuxbrew/.linuxbrew/bin:/home/linuxbrew/.linuxbrew/sbin:$HOME/.local/bin:$HOME/miniconda3/bin:$HOME/miniconda3/condabin:/usr/local/zig:/usr/local/zig/lib:/usr/local/zig/lib/include:/usr/local/musl/bin:/usr/local/musl/lib:/usr/local/musl/include:$PATH" | |
rustup target list > "$GITHUB_WORKSPACE/main/Docs/RUST_TARGETS.txt" | |
continue-on-error: true | |
- name: Generate ZIG_TARGETS | |
run: | | |
#Presets | |
set +x ; set +e | |
#--------------# | |
export PATH="$HOME/bin:$HOME/.cargo/bin:$HOME/.cargo/env:$HOME/.go/bin:$HOME/go/bin:/home/linuxbrew/.linuxbrew/bin:/home/linuxbrew/.linuxbrew/sbin:$HOME/.local/bin:$HOME/miniconda3/bin:$HOME/miniconda3/condabin:/usr/local/zig:/usr/local/zig/lib:/usr/local/zig/lib/include:/usr/local/musl/bin:/usr/local/musl/lib:/usr/local/musl/include:$PATH" | |
zig targets | jq -r '.libc[]' > "$GITHUB_WORKSPACE/main/Docs/ZIG_TARGETS.txt" | |
continue-on-error: true | |
#------------------------------------------------------------------------------------# | |
- name: rClone BackUp Repo-I ("https://pub.ajam.dev/repos/$GITHUB_REPOSITORY") | |
run: | | |
# Presets | |
set +x ; set +e | |
#--------------# | |
#touch "$HOME/.rclone.conf" | |
echo "${{ secrets.RCLONE_CF_R2_PUB }}" > "$HOME/.rclone.conf" | |
#copy & chdir to Repo | |
cp -r "$GITHUB_WORKSPACE/main" "$SYSTMP/REPO" && cd "$SYSTMP/REPO" | |
#Git pull | |
git pull origin main --no-edit 2>/dev/null | |
#Del Bloat | |
rm -rf "$(pwd)/.git" | |
# rm -rf "$(pwd)/aarch64_arm64_Linux" | |
# rm -rf "$(pwd)/arm64_v8a_Android" | |
# rm -rf "$(pwd)/x86_64_Linux" | |
# rm -rf "$(pwd)/x64_Windows" | |
#Upload to Pub | |
wait ; echo | |
echo -e "[+] Syncing $GITHUB_REPOSITORY to pub.ajam.dev/repos/$GITHUB_REPOSITORY \n" | |
rclone sync "." "r2:/pub/repos/$GITHUB_REPOSITORY/" --user-agent="$USER_AGENT" --buffer-size="100M" --s3-upload-concurrency="500" --s3-chunk-size="100M" --multi-thread-streams="500" --checkers="2000" --transfers="1000" --check-first --checksum --copy-links --fast-list --progress | |
#Get JQ Module to convert size : https://users.aalto.fi/~tontti/posts/jq-and-human-readable-bytes/ | |
curl -qfsSL "https://pub.ajam.dev/utils/devscripts/jq/to_human_bytes.jq" -o "./to_human_bytes.jq" | |
##Scripts_aarch64_Linux | |
rclone lsjson --fast-list "r2:/pub/repos/Azathothas/Toolpacks/.github/scripts/aarch64_Linux/bins/" --exclude="*.yaml" | jq -r 'include "./to_human_bytes" ; .[] | select(.IsDir == false) | {name: (.Name), update_date: (.ModTime | split(".")[0]), source_url: "https://pub.ajam.dev/repos/Azathothas/Toolpacks/.github/scripts/aarch64_Linux/bins/\(.Path)"}' | jq . > "./metadata.json.tmp_aarch64_Linux" & | |
##Scripts_arm64_v8a_Android | |
rclone lsjson --fast-list "r2:/pub/repos/Azathothas/Toolpacks/.github/scripts/arm64_v8a_Android/bins/" --exclude="*.yaml" | jq -r 'include "./to_human_bytes" ; .[] | select(.IsDir == false) | {name: (.Name), update_date: (.ModTime | split(".")[0]), source_url: "https://pub.ajam.dev/repos/Azathothas/Toolpacks/.github/scripts/arm64_v8a_Android/bins/\(.Path)"}' | jq . > "./metadata.json.tmp_arm64_v8a_Android" & | |
##Scripts_x86_64_Linux | |
rclone lsjson --fast-list "r2:/pub/repos/Azathothas/Toolpacks/.github/scripts/x86_64_Linux/bins/" --exclude="*.yaml" | jq -r 'include "./to_human_bytes" ; .[] | select(.IsDir == false) | {name: (.Name), update_date: (.ModTime | split(".")[0]), source_url: "https://pub.ajam.dev/repos/Azathothas/Toolpacks/.github/scripts/x86_64_Linux/bins/\(.Path)"}' | jq . > "./metadata.json.tmp_x86_64_Linux" & | |
##Scripts_x64_Windows | |
rclone lsjson --fast-list "r2:/pub/repos/Azathothas/Toolpacks/.github/scripts/x64_Windows/bins/" --exclude="*.yaml" | jq -r 'include "./to_human_bytes" ; .[] | select(.IsDir == false) | {name: (.Name), update_date: (.ModTime | split(".")[0]), source_url: "https://pub.ajam.dev/repos/Azathothas/Toolpacks/.github/scripts/x64_Windows/bins/\(.Path)"}' | jq . > "./metadata.json.tmp_x64_Windows" & | |
##Clean Pretty Formatted | |
wait ; echo | |
echo "[" $(cat metadata.json.tmp_aarch64_Linux | tr '\n' ' ' | sed 's/}/},/g' | sed '$ s/,$//') "]" | sed '$s/,[[:space:]]*\]/\]/' | jq . | tee "./metadata.json_aarch64_Linux" & | |
echo "[" $(cat metadata.json.tmp_arm64_v8a_Android | tr '\n' ' ' | sed 's/}/},/g' | sed '$ s/,$//') "]" | sed '$s/,[[:space:]]*\]/\]/' | jq . | tee "./metadata.json_arm64_v8a_Android" & | |
echo "[" $(cat metadata.json.tmp_x86_64_Linux | tr '\n' ' ' | sed 's/}/},/g' | sed '$ s/,$//') "]" | sed '$s/,[[:space:]]*\]/\]/' | jq . | tee "./metadata.json_x86_64_Linux" & | |
echo "[" $(cat metadata.json.tmp_x64_Windows | tr '\n' ' ' | sed 's/}/},/g' | sed '$ s/,$//') "]" | sed '$s/,[[:space:]]*\]/\]/' | jq . | tee "./metadata.json_x64_Windows" & | |
##Update Metadata | |
wait ; echo | |
rclone copyto --checksum "./metadata.json_aarch64_Linux" "r2:/pub/repos/Azathothas/Toolpacks/.github/scripts/aarch64_Linux/bins/metadata.json" & | |
rclone copyto --checksum "./metadata.json_arm64_v8a_Android" "r2:/pub/repos/Azathothas/Toolpacks/.github/scripts/arm64_v8a_Android/bins/metadata.json" & | |
rclone copyto --checksum "./metadata.json_x86_64_Linux" "r2:/pub/repos/Azathothas/Toolpacks/.github/scripts/x86_64_Linux/bins/metadata.json" & | |
rclone copyto --checksum "./metadata.json_x64_Windows" "r2:/pub/repos/Azathothas/Toolpacks/.github/scripts/x64_Windows/bins/metadata.json" & | |
##Update Repo | |
wait ; echo | |
cat "./metadata.json_aarch64_Linux" > "$GITHUB_WORKSPACE/main/.github/scripts/aarch64_Linux/bins/metadata.json" | |
cat "./metadata.json_arm64_v8a_Android" > "$GITHUB_WORKSPACE/main/.github/scripts/arm64_v8a_Android/bins/metadata.json" | |
cat "./metadata.json_x86_64_Linux" > "$GITHUB_WORKSPACE/main/.github/scripts/x86_64_Linux/bins/metadata.json" | |
cat "./metadata.json_x64_Windows" > "$GITHUB_WORKSPACE/main/.github/scripts/x64_Windows/bins/metadata.json" | |
continue-on-error: true | |
#------------------------------------------------------------------------------------# | |
#-----------------------------------------------------# | |
#-----------------------------------------------------# | |
- name: Parse ("bin.ajam.dev/arm64_v8a_Android/") Bins | |
run: | | |
#Presets | |
set +x ; set +e | |
#--------------# | |
#Fetch from r2 | |
#Copy to $GITHUB_WORKSPACE/main/arm64_v8a_Android | |
rclone copyto --checksum "r2:/bin/arm64_v8a_Android/BLAKE3SUM.txt" "$GITHUB_WORKSPACE/main/arm64_v8a_Android/BLAKE3SUM.txt" 2>/dev/null | |
rclone copyto --checksum "r2:/bin/arm64_v8a_Android/FILE.txt" "$GITHUB_WORKSPACE/main/arm64_v8a_Android/FILE.txt" 2>/dev/null | |
rclone copyto --checksum "r2:/bin/arm64_v8a_Android/SIZE.txt" "$GITHUB_WORKSPACE/main/arm64_v8a_Android/SIZE.txt" 2>/dev/null | |
echo '```mathematica' > "$GITHUB_WORKSPACE/main/arm64_v8a_Android/SIZE.md" 2>/dev/null | |
cat "$GITHUB_WORKSPACE/main/arm64_v8a_Android/SIZE.txt" >> "$GITHUB_WORKSPACE/main/arm64_v8a_Android/SIZE.md" 2>/dev/null | |
echo '```' >> "$GITHUB_WORKSPACE/main/arm64_v8a_Android/SIZE.md" 2>/dev/null | |
rclone copyto --checksum "r2:/bin/arm64_v8a_Android/SHA256SUM.txt" "$GITHUB_WORKSPACE/main/arm64_v8a_Android/SHA256SUM.txt" 2>/dev/null | |
rclone copyto --checksum "r2:/bin/arm64_v8a_Android/METADATA.json" "$GITHUB_WORKSPACE/main/arm64_v8a_Android/METADATA.json" 2>/dev/null | |
continue-on-error: true | |
#-----------------------------------------------------# | |
#-----------------------------------------------------# | |
- name: Parse ("bin.ajam.dev/aarch64_arm64_Linux/") Bins | |
run: | | |
#Presets | |
set +x ; set +e | |
#--------------# | |
#Fetch from r2 | |
#Copy to $GITHUB_WORKSPACE/main/aarch64_arm64_Linux | |
rclone copyto --checksum "r2:/bin/aarch64_arm64_Linux/BLAKE3SUM.txt" "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/BLAKE3SUM.txt" 2>/dev/null | |
rclone copyto --checksum "r2:/bin/aarch64_arm64_Linux/FILE.txt" "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/FILE.txt" 2>/dev/null | |
rclone copyto --checksum "r2:/bin/aarch64_arm64_Linux/METADATA.AIO.json" "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/METADATA.AIO.json" 2>/dev/null | |
rclone copyto --checksum "r2:/bin/aarch64_arm64_Linux/METADATA.AIO.min.json" "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/METADATA.AIO.min.json" 2>/dev/null | |
rclone copyto --checksum "r2:/bin/aarch64_arm64_Linux/METADATA.AIO.toml" "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/METADATA.AIO.toml" 2>/dev/null | |
rclone copyto --checksum "r2:/bin/aarch64_arm64_Linux/METADATA.AIO.yaml" "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/METADATA.AIO.yaml" 2>/dev/null | |
rclone copyto --checksum "r2:/bin/aarch64_arm64_Linux/SIZE.txt" "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/SIZE.txt" 2>/dev/null | |
echo '```mathematica' > "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/SIZE.md" 2>/dev/null | |
cat "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/SIZE.txt" >> "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/SIZE.md" 2>/dev/null | |
echo '```' >> "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/SIZE.md" 2>/dev/null | |
rclone copyto --checksum "r2:/bin/aarch64_arm64_Linux/SHA256SUM.txt" "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/SHA256SUM.txt" 2>/dev/null | |
continue-on-error: true | |
#-----------------------------------------------------# | |
#-----------------------------------------------------# | |
- name: Parse ("bin.ajam.dev/x86_64_Linux/") Bins | |
run: | | |
#Presets | |
set +x ; set +e | |
#--------------# | |
#Fetch from r2 | |
#Copy to $GITHUB_WORKSPACE/main/x86_64_Linux | |
rclone copyto --checksum "r2:/bin/x86_64_Linux/BLAKE3SUM.txt" "$GITHUB_WORKSPACE/main/x86_64_Linux/BLAKE3SUM.txt" 2>/dev/null | |
rclone copyto --checksum "r2:/bin/x86_64_Linux/FILE.txt" "$GITHUB_WORKSPACE/main/x86_64_Linux/FILE.txt" 2>/dev/null | |
rclone copyto --checksum "r2:/bin/x86_64_Linux/METADATA.AIO.json" "$GITHUB_WORKSPACE/main/x86_64_Linux/METADATA.AIO.json" 2>/dev/null | |
rclone copyto --checksum "r2:/bin/x86_64_Linux/METADATA.AIO.min.json" "$GITHUB_WORKSPACE/main/x86_64_Linux/METADATA.AIO.min.json" 2>/dev/null | |
rclone copyto --checksum "r2:/bin/x86_64_Linux/METADATA.AIO.toml" "$GITHUB_WORKSPACE/main/x86_64_Linux/METADATA.AIO.toml" 2>/dev/null | |
rclone copyto --checksum "r2:/bin/x86_64_Linux/METADATA.AIO.yaml" "$GITHUB_WORKSPACE/main/x86_64_Linux/METADATA.AIO.yaml" 2>/dev/null | |
rclone copyto --checksum "r2:/bin/x86_64_Linux/SIZE.txt" "$GITHUB_WORKSPACE/main/x86_64_Linux/SIZE.txt" 2>/dev/null | |
echo '```mathematica' > "$GITHUB_WORKSPACE/main/x86_64_Linux/SIZE.md" 2>/dev/null | |
cat "$GITHUB_WORKSPACE/main/x86_64_Linux/SIZE.txt" >> "$GITHUB_WORKSPACE/main/x86_64_Linux/SIZE.md" 2>/dev/null | |
echo '```' >> "$GITHUB_WORKSPACE/main/x86_64_Linux/SIZE.md" 2>/dev/null | |
rclone copyto --checksum "r2:/bin/x86_64_Linux/SHA256SUM.txt" "$GITHUB_WORKSPACE/main/x86_64_Linux/SHA256SUM.txt" 2>/dev/null | |
continue-on-error: true | |
#-----------------------------------------------------# | |
#-----------------------------------------------------# | |
- name: Parse ("bin.ajam.dev/x64_Windows/") Bins | |
run: | | |
#Presets | |
set +x ; set +e | |
#--------------# | |
#Fetch from r2 | |
#Copy to $GITHUB_WORKSPACE/main/x64_Windows | |
rclone copyto --checksum "r2:/bin/x64_Windows/BLAKE3SUM.txt" "$GITHUB_WORKSPACE/main/x64_Windows/BLAKE3SUM.txt" 2>/dev/null | |
rclone copyto --checksum "r2:/bin/x64_Windows/FILE.txt" "$GITHUB_WORKSPACE/main/x64_Windows/FILE.txt" 2>/dev/null | |
rclone copyto --checksum "r2:/bin/x64_Windows/SIZE.txt" "$GITHUB_WORKSPACE/main/x64_Windows/SIZE.txt" 2>/dev/null | |
echo '```mathematica' > "$GITHUB_WORKSPACE/main/x64_Windows/SIZE.md" 2>/dev/null | |
cat "$GITHUB_WORKSPACE/main/x64_Windows/SIZE.txt" >> "$GITHUB_WORKSPACE/main/x64_Windows/SIZE.md" 2>/dev/null | |
echo '```' >> "$GITHUB_WORKSPACE/main/x64_Windows/SIZE.md" 2>/dev/null | |
rclone copyto --checksum "r2:/bin/x64_Windows/SHA256SUM.txt" "$GITHUB_WORKSPACE/main/x64_Windows/SHA256SUM.txt" 2>/dev/null | |
continue-on-error: true | |
#-----------------------------------------------------# | |
#------------------------------------------------------------------------------------# | |
- name: Generate Repo Metadata (git-sizer) | |
run: | | |
#Presets | |
set +x ; set +e | |
#--------------# | |
cd "$GITHUB_WORKSPACE/main" | |
#Dust sizes | |
echo '```mathematica' > "$GITHUB_WORKSPACE/main/.github/SIZE.md" | |
dust -b -c -i -r -n 99999999 "$GITHUB_WORKSPACE/main" | tee -a "$GITHUB_WORKSPACE/main/.github/SIZE.md" | |
dust -b -c -i -r -n 99999999 "$GITHUB_WORKSPACE/main" | tee "$GITHUB_WORKSPACE/main/.github/SIZE.txt" | |
echo '```' >> "$GITHUB_WORKSPACE/main/.github/SIZE.md" | |
continue-on-error: true | |
- name: Parse Build Logs (aarch64_arm64_Linux) | |
run: | | |
#Presets | |
set +x ; set +e ; ulimit -S -s unlimited ; ulimit -f unlimited 2>/dev/null | |
#--------------# | |
cd "$GITHUB_WORKSPACE/main" | |
#--------------------# | |
##aarch64_arm64_Linux## | |
curl -qfsSL "https://bin.ajam.dev/aarch64_arm64_Linux/BUILD.BIN.log.txt" -o "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/BUILD.BIN.log.txt" | |
#Secrets & Tokens | |
sed -i '/.*github_pat.*/Id' "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/BUILD.BIN.log.txt" | |
sed -i '/.*ghp_.*/Id' "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/BUILD.BIN.log.txt" | |
sed -i '/.*access_key_id.*/Id' "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/BUILD.BIN.log.txt" | |
sed -i '/.*secret_access_key.*/Id' "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/BUILD.BIN.log.txt" | |
sed -i '/.*cloudflarestorage.*/Id' "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/BUILD.BIN.log.txt" | |
echo -e '```mathematica' > "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/BUILD_ERROR.log.md" | |
#Broken GH Repos | |
echo -e "\n[-] Authentication failed --> # Invalid/Deleted GH Repos\n" >> "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/BUILD_ERROR.log.md" | |
grep -B5 -A3 -in ".* Authentication failed .*" "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/BUILD.BIN.log.txt" >> "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/BUILD_ERROR.log.md" | |
#Eget Fails | |
echo -e "\n\n[-] asset * not found --> # Eget failed to find that --asset\n" >> "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/BUILD_ERROR.log.md" | |
grep -B5 -in "asset .* not found" "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/BUILD.BIN.log.txt" >> "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/BUILD_ERROR.log.md" | |
#Failed URLs | |
echo -e "\n\n[-] Could not resolve host --> # Curl Failed to connect, usually Invalid URL\n" >> "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/BUILD_ERROR.log.md" | |
awk '/Could not resolve host/{print prev; print; found=1} {prev=$0} /^\[+\]/{exit} END{if (!found) print prev}' "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/BUILD.BIN.log.txt" >> "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/BUILD_ERROR.log.md" | |
#Failed Installs | |
echo -e "\n\n[-] command not found --> # Something failed to Install\n" >> "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/BUILD_ERROR.log.md" | |
grep -in "command not found" "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/BUILD.BIN.log.txt" >> "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/BUILD_ERROR.log.md" | |
#Dynamically Linked | |
echo -e "\n\n[-] Dynamically Linked --> # Not Statically Linked\n" >> "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/BUILD_ERROR.log.md" | |
grep -in "dynamically linked" "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/FILE.txt" >> "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/BUILD_ERROR.log.md" | |
#Not correct binary | |
echo -e "\n\n[-] Incorrect Arch --> # Not for aarch64\n" >> "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/BUILD_ERROR.log.md" | |
grep -ivn "aarch64\|text" "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/FILE.txt" >> "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/BUILD_ERROR.log.md" | |
#Build Failures | |
echo -e "\n\n[-] No such file or directory # Usually Because Build Failures\n" >> "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/BUILD_ERROR.log.md" | |
grep -in "No such file or directory" "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/BUILD.BIN.log.txt" >> "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/BUILD_ERROR.log.md" | |
#Empty Bins | |
echo -e "\n\n[-] Empty (0 Sized) Bins # Usually Because Symlinks\n" >> "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/BUILD_ERROR.log.md" | |
jq -r '.[] | select(.Size == "0 B") | "\(.Name) --> \(.Size)"' "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/METADATA.json" >> "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/BUILD_ERROR.log.md" | |
#END | |
echo -e '```' "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/BUILD_ERROR.log.md" | |
continue-on-error: true | |
- name: Parse Build Logs (x86_64_Linux) | |
run: | | |
#Presets | |
set +x ; set +e ; ulimit -S -s unlimited ; ulimit -f unlimited 2>/dev/null | |
#--------------# | |
cd "$GITHUB_WORKSPACE/main" | |
#--------------------# | |
##x86_64_Linux## | |
#Secrets & Tokens | |
curl -qfsSL "https://bin.ajam.dev/x86_64_Linux/BUILD.BIN.log.txt" -o "$GITHUB_WORKSPACE/main/x86_64_Linux/BUILD.BIN.log.txt" | |
sed -i '/.*github_pat.*/Id' "$GITHUB_WORKSPACE/main/x86_64_Linux/BUILD.BIN.log.txt" | |
sed -i '/.*ghp_.*/Id' "$GITHUB_WORKSPACE/main/x86_64_Linux/BUILD.BIN.log.txt" | |
sed -i '/.*access_key_id.*/Id' "$GITHUB_WORKSPACE/main/x86_64_Linux/BUILD.BIN.log.txt" | |
sed -i '/.*secret_access_key.*/Id' "$GITHUB_WORKSPACE/main/x86_64_Linux/BUILD.BIN.log.txt" | |
sed -i '/.*cloudflarestorage.*/Id' "$GITHUB_WORKSPACE/main/x86_64_Linux/BUILD.BIN.log.txt" | |
echo -e '```mathematica' > "$GITHUB_WORKSPACE/main/x86_64_Linux/BUILD_ERROR.log.md" | |
#Broken GH Repos | |
echo -e "\n[-] Authentication failed --> # Invalid/Deleted GH Repos\n" >> "$GITHUB_WORKSPACE/main/x86_64_Linux/BUILD_ERROR.log.md" | |
grep -B5 -A3 -in ".* Authentication failed .*" "$GITHUB_WORKSPACE/main/x86_64_Linux/BUILD.BIN.log.txt" >> "$GITHUB_WORKSPACE/main/x86_64_Linux/BUILD_ERROR.log.md" | |
#Eget Fails | |
echo -e "\n\n[-] asset * not found --> # Eget failed to find that --asset\n" >> "$GITHUB_WORKSPACE/main/x86_64_Linux/BUILD_ERROR.log.md" | |
grep -B5 -in "asset .* not found" "$GITHUB_WORKSPACE/main/x86_64_Linux/BUILD.BIN.log.txt" >> "$GITHUB_WORKSPACE/main/x86_64_Linux/BUILD_ERROR.log.md" | |
#Failed URLs | |
echo -e "\n\n[-] Could not resolve host --> # Curl Failed to connect, usually Invalid URL\n" >> "$GITHUB_WORKSPACE/main/x86_64_Linux/BUILD_ERROR.log.md" | |
awk '/Could not resolve host/{print prev; print; found=1} {prev=$0} /^\[+\]/{exit} END{if (!found) print prev}' "$GITHUB_WORKSPACE/main/x86_64_Linux/BUILD.BIN.log.txt" >> "$GITHUB_WORKSPACE/main/x86_64_Linux/BUILD_ERROR.log.md" | |
#Failed Installs | |
echo -e "\n\n[-] command not found --> # Something failed to Install\n" >> "$GITHUB_WORKSPACE/main/x86_64_Linux/BUILD_ERROR.log.md" | |
grep -in "command not found" "$GITHUB_WORKSPACE/main/x86_64_Linux/BUILD.BIN.log.txt" >> "$GITHUB_WORKSPACE/main/x86_64_Linux/BUILD_ERROR.log.md" | |
#Dynamically Linked | |
echo -e "\n\n[-] Dynamically Linked --> # Not Statically Linked\n" >> "$GITHUB_WORKSPACE/main/x86_64_Linux/BUILD_ERROR.log.md" | |
grep -in "dynamically linked" "$GITHUB_WORKSPACE/main/x86_64_Linux/FILE.txt" >> "$GITHUB_WORKSPACE/main/x86_64_Linux/BUILD_ERROR.log.md" | |
#Not correct binary | |
echo -e "\n\n[-] Incorrect Arch --> # Not for x86_64\n" >> "$GITHUB_WORKSPACE/main/x86_64_Linux/BUILD_ERROR.log.md" | |
grep -ivn "x86-64\|text" "$GITHUB_WORKSPACE/main/x86_64_Linux/FILE.txt" >> "$GITHUB_WORKSPACE/main/x86_64_Linux/BUILD_ERROR.log.md" | |
#Broken Builds | |
echo -e "\n\n[-] No such file or directory # Usually Because Build Failures\n" >> "$GITHUB_WORKSPACE/main/x86_64_Linux/BUILD_ERROR.log.md" | |
grep -in "No such file or directory" "$GITHUB_WORKSPACE/main/x86_64_Linux/BUILD.BIN.log.txt" >> "$GITHUB_WORKSPACE/main/x86_64_Linux/BUILD_ERROR.log.md" | |
#Empty Bins | |
echo -e "\n\n[-] Empty (0 Sized) Bins # Usually Because Symlinks\n" >> "$GITHUB_WORKSPACE/main/x86_64_Linux/BUILD_ERROR.log.md" | |
jq -r '.[] | select(.Size == "0 B") | "\(.Name) --> \(.Size)"' "$GITHUB_WORKSPACE/main/x86_64_Linux/METADATA.json" >> "$GITHUB_WORKSPACE/main/x86_64_Linux/BUILD_ERROR.log.md" | |
#END | |
echo -e '```' "$GITHUB_WORKSPACE/main/x86_64_Linux/BUILD_ERROR.log.md" | |
continue-on-error: true | |
- name: Generate Markdown (Main README.md) | |
run: | | |
#Presets | |
set +x ; set +e ; ulimit -S -s unlimited ; ulimit -f unlimited 2>/dev/null | |
#--------------# | |
cd "$GITHUB_WORKSPACE/main" | |
#Generate $GITHUB_WORKSPACE/main/README.md | |
cat "$GITHUB_WORKSPACE/main/INFO.md" > "$GITHUB_WORKSPACE/main/README.md" | |
##bin.ajam.dev/arm64_v8a_Android/ | |
unset MAIN_N BASE_N TOTAL_PACKAGES | |
MAIN_N="$(curl -qfsSL "https://bin.ajam.dev/METADATA.json" | jq '.[] | select(.host == "arm64-v8a-Android") | .main' | jq -r '.[].name' | sort -u | wc -l)" && export MAIN_N="${MAIN_N}" | |
BASE_N="$(curl -qfsSL "https://bin.ajam.dev/METADATA.json" | jq '.[] | select(.host == "arm64-v8a-Android") | .baseutils' | jq -r '.[].name' | sort -u | wc -l)" && export BASE_N="${BASE_N}" | |
export TOTAL_PACKAGES="$((MAIN_N + BASE_N))" && export TOTAL_PACKAGES="$TOTAL_PACKAGES" | |
sed "s/PLACEHOLDER_arm64_v8a_Android/$TOTAL_PACKAGES/g" -i "$GITHUB_WORKSPACE/main/README.md" | |
##bin.ajam.dev/aarch64_arm64_Linux | |
unset MAIN_N BASE_N TOTAL_PACKAGES | |
MAIN_N="$(curl -qfsSL "https://bin.ajam.dev/METADATA.json" | jq '.[] | select(.host == "aarch64-Linux") | .main' | jq -r '.[].name' | sort -u | wc -l)" && export MAIN_N="${MAIN_N}" | |
BASE_N="$(curl -qfsSL "https://bin.ajam.dev/METADATA.json" | jq '.[] | select(.host == "aarch64-Linux") | .baseutils' | jq -r '.[].name' | sort -u | wc -l)" && export BASE_N="${BASE_N}" | |
export TOTAL_PACKAGES="$((MAIN_N + BASE_N))" && export TOTAL_PACKAGES="$TOTAL_PACKAGES" | |
sed "s/PLACEHOLDER_aarch64_arm64_Linux/$TOTAL_PACKAGES/g" -i "$GITHUB_WORKSPACE/main/README.md" | |
##bin.ajam.dev/x86_64_Linux/ | |
unset MAIN_N BASE_N TOTAL_PACKAGES | |
MAIN_N="$(curl -qfsSL "https://bin.ajam.dev/METADATA.json" | jq '.[] | select(.host == "x86_64-Linux") | .main' | jq -r '.[].name' | sort -u | wc -l)" && export MAIN_N="${MAIN_N}" | |
BASE_N="$(curl -qfsSL "https://bin.ajam.dev/METADATA.json" | jq '.[] | select(.host == "x86_64-Linux") | .baseutils' | jq -r '.[].name' | sort -u | wc -l)" && export BASE_N="${BASE_N}" | |
export TOTAL_PACKAGES="$((MAIN_N + BASE_N))" && export TOTAL_PACKAGES="$TOTAL_PACKAGES" | |
sed "s/PLACEHOLDER_x86_64_Linux/$TOTAL_PACKAGES/g" -i "$GITHUB_WORKSPACE/main/README.md" | |
##bin.ajam.dev/x64_Windows/ | |
unset MAIN_N BASE_N TOTAL_PACKAGES | |
MAIN_N="$(curl -qfsSL "https://bin.ajam.dev/METADATA.json" | jq '.[] | select(.host == "AMD64-Windows_NT") | .main' | jq -r '.[].name' | sort -u | wc -l)" && export MAIN_N="${MAIN_N}" | |
BASE_N="$(curl -qfsSL "https://bin.ajam.dev/METADATA.json" | jq '.[] | select(.host == "AMD64-Windows_NT") | .baseutils' | jq -r '.[].name' | sort -u | wc -l)" && export BASE_N="${BASE_N}" | |
export TOTAL_PACKAGES="$((MAIN_N + BASE_N))" && export TOTAL_PACKAGES="$TOTAL_PACKAGES" | |
sed "s/PLACEHOLDER_x64_Windows/$TOTAL_PACKAGES/g" -i "$GITHUB_WORKSPACE/main/README.md" | |
continue-on-error: true | |
- name: Scrape Public Issues | |
run: | | |
#Presets | |
set +x ; set +e | |
#--------------# | |
#https://docs.github.com/en/rest/issues/issues | |
export DATE_CUTOFF="2024-00-00T00:00:00Z" | |
TEMP_ISSUES="$(mktemp -d)" && export TEMP_ISSUES="${TEMP_ISSUES}" | |
USER_AGENT="$(curl -qfsSL 'https://pub.ajam.dev/repos/Azathothas/Wordlists/Misc/User-Agents/ua_chrome_macos_latest.txt')" && export USER_AGENT="$USER_AGENT" | |
echo -e "\n[+] Using Temp Dir : ${TEMP_ISSUES}" | |
#Fetch: `static binary` | |
(for page in {1..10}; do | |
echo -e "\n[+] GH API: Static Binary ($page/10)\n" | |
curl -qfsSL -A "$USER_AGENT" "https://api.github.com/search/issues?q=static+binary+state:open+is:issue&filter=all&since=${DATE_CUTOFF}&sort=created&direction=desc&per_page=100&page=$page" -H "Authorization: Bearer ${GITHUB_TOKEN}" | jq -r '.items | sort_by(.created_at)[] | "\(.created_at // "NA") \(.title // "NA") --> \(.html_url // "NA") (\(.user.login // "NA"))"' | tee -a "${TEMP_ISSUES}/$(openssl rand -hex 3).txt" | |
sleep 2 | |
done) & | |
#Fetch: `static build` | |
(for page in {1..10}; do | |
echo -e "\n[+] GH API: Static Build ($page/10)\n" | |
curl -qfsSL -A "$USER_AGENT" "https://api.github.com/search/issues?q=static+build+state:open+is:issue&filter=all&since=${DATE_CUTOFF}&sort=created&direction=desc&per_page=100&page=$page" -H "Authorization: Bearer ${GITHUB_TOKEN}" | jq -r '.items | sort_by(.created_at)[] | "\(.created_at // "NA") \(.title // "NA") --> \(.html_url // "NA") (\(.user.login // "NA"))"' | tee -a "${TEMP_ISSUES}/$(openssl rand -hex 3).txt" | |
sleep 2 | |
done) & | |
wait ; echo | |
#Fetch: `static compile` | |
(for page in {1..10}; do | |
echo -e "\n[+] GH API: Static Compile ($page/10)\n" | |
curl -qfsSL -A "$USER_AGENT" "https://api.github.com/search/issues?q=static+compile+state:open+is:issue&filter=all&since=${DATE_CUTOFF}&sort=created&direction=desc&per_page=100&page=$page" -H "Authorization: Bearer ${GITHUB_TOKEN}" | jq -r '.items | sort_by(.created_at)[] | "\(.created_at // "NA") \(.title // "NA") --> \(.html_url // "NA") (\(.user.login // "NA"))"' | tee -a "${TEMP_ISSUES}/$(openssl rand -hex 3).txt" | |
sleep 2 | |
done) & | |
#Fetch: `musl binary` | |
(for page in {1..10}; do | |
echo -e "\n[+] GH API: MUSL Binary ($page/10)\n" | |
curl -qfsSL -A "$USER_AGENT" "https://api.github.com/search/issues?q=musl+binary+state:open+is:issue&filter=all&since=${DATE_CUTOFF}&sort=created&direction=desc&per_page=100&page=$page" -H "Authorization: Bearer ${GITHUB_TOKEN}" | jq -r '.items | sort_by(.created_at)[] | "\(.created_at // "NA") \(.title // "NA") --> \(.html_url // "NA") (\(.user.login // "NA"))"' | tee -a "${TEMP_ISSUES}/$(openssl rand -hex 3).txt" | |
sleep 2 | |
done) & | |
wait ; echo | |
#Fetch: `musl build` | |
(for page in {1..10}; do | |
echo -e "\n[+] GH API: MUSL Build ($page/10)\n" | |
curl -qfsSL -A "$USER_AGENT" "https://api.github.com/search/issues?q=musl+build+state:open+is:issue&filter=all&since=${DATE_CUTOFF}&sort=created&direction=desc&per_page=100&page=$page" -H "Authorization: Bearer ${GITHUB_TOKEN}" | jq -r '.items | sort_by(.created_at)[] | "\(.created_at // "NA") \(.title // "NA") --> \(.html_url // "NA") (\(.user.login // "NA"))"' | tee -a "${TEMP_ISSUES}/$(openssl rand -hex 3).txt" | |
sleep 2 | |
done) & | |
#Fetch: `musl compile` | |
(for page in {1..10}; do | |
echo -e "\n[+] GH API: MUSL Compile ($page/10)\n" | |
curl -qfsSL -A "$USER_AGENT" "https://api.github.com/search/issues?q=musl+compile+state:open+is:issue&filter=all&since=${DATE_CUTOFF}&sort=created&direction=desc&per_page=100&page=$page" -H "Authorization: Bearer ${GITHUB_TOKEN}" | jq -r '.items | sort_by(.created_at)[] | "\(.created_at // "NA") \(.title // "NA") --> \(.html_url // "NA") (\(.user.login // "NA"))"' | tee -a "${TEMP_ISSUES}/$(openssl rand -hex 3).txt" | |
sleep 2 | |
done) & | |
wait ; echo | |
#Fetch: `release binary` | |
(for page in {1..10}; do | |
echo -e "\n[+] GH API: Release Binary ($page/10)\n" | |
curl -qfsSL -A "$USER_AGENT" "https://api.github.com/search/issues?q=release+binary+state:open+is:issue&filter=all&since=${DATE_CUTOFF}&sort=created&direction=desc&per_page=100&page=$page" -H "Authorization: Bearer ${GITHUB_TOKEN}" | jq -r '.items | sort_by(.created_at)[] | "\(.created_at // "NA") \(.title // "NA") --> \(.html_url // "NA") (\(.user.login // "NA"))"' | tee -a "${TEMP_ISSUES}/$(openssl rand -hex 3).txt" | |
sleep 2 | |
done) & | |
#Fetch: `release static` | |
(for page in {1..10}; do | |
echo -e "\n[+] GH API: Release Binary ($page/10)\n" | |
curl -qfsSL -A "$USER_AGENT" "https://api.github.com/search/issues?q=release+static+state:open+is:issue&filter=all&since=${DATE_CUTOFF}&sort=created&direction=desc&per_page=100&page=$page" -H "Authorization: Bearer ${GITHUB_TOKEN}" | jq -r '.items | sort_by(.created_at)[] | "\(.created_at // "NA") \(.title // "NA") --> \(.html_url // "NA") (\(.user.login // "NA"))"' | tee -a "${TEMP_ISSUES}/$(openssl rand -hex 3).txt" | |
sleep 2 | |
done) & | |
wait ; echo | |
#Sort | |
find "${TEMP_ISSUES}" -name "*.txt" -exec cat {} + | grep '^[0-9]' | grep -v '\[bot\])' | sort --unique --reverse -o "$GITHUB_WORKSPACE/main/.github/pub_issues.txt" | |
#find "${TEMP_ISSUES}" -name "*.txt" -exec cat {} + | grep '^[0-9]' | grep -v '\[bot\])' | sort --numeric-sort --reverse --unique -o "$GITHUB_WORKSPACE/main/.github/pub_issues.txt" | |
#find "${TEMP_ISSUES}" -name "*.txt" -exec cat {} + | grep '^[0-9]' | grep -v '\[bot\])' | sort --version-sort --reverse --unique -o "$GITHUB_WORKSPACE/main/.github/pub_issues.txt" | |
continue-on-error: true | |
#- name: Ensure Healthy Commit Nums (RESET ON >= 1000) | |
# run: | | |
# #Presets | |
# set +x ; set +e | |
# #--------------# | |
# pushd "$(mktemp -d)" > /dev/null 2>&1 && git clone --filter="blob:none" "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY" | |
# cd "$(find . -maxdepth 1 -type d -exec basename {} \; | grep -Ev '^\.$' | xargs -I {} realpath {})" | |
# CLONED_DIR="$(realpath .)" && export CLONED_DIR="$CLONED_DIR" | |
# COMMIT_NUMS="$(git rev-list --count HEAD)" && export COMMIT_NUMS="$COMMIT_NUMS" | |
# popd > /dev/null 2>&1 ; rm -rf "$CLONED_DIR" 2>/dev/null | |
# cd "$GITHUB_WORKSPACE/main" | |
# if [ "$COMMIT_NUMS" -gt 1000 ]; then | |
# echo -e "\n[+] Total number of commits exceeds 1000. ($COMMIT_NUMS)\n" | |
# cd "$GITHUB_WORKSPACE/main" | |
# git config "user.name" "Azathothas" | |
# git config "user.email" "[email protected]" | |
# git checkout --orphan temp | |
# git add --all --verbose && git commit -m "Purge (Re:Init)" | |
# git branch -D "main" | |
# git branch -m "main" | |
# git push --set-upstream origin main --force | |
# echo "RESET_HISTORY=YES" >> "$GITHUB_ENV" | |
# else | |
# echo -e "\n[+] Total number of commits looks healthy. ($COMMIT_NUMS)\n" | |
# echo "RESET_HISTORY=NO" >> "$GITHUB_ENV" | |
# fi | |
# pushd "$GITHUB_WORKSPACE/main" >/dev/null 2>&1 && git pull origin main && popd >/dev/null 2>&1 | |
# continue-on-error: true | |
- name: Get DateTime | |
run: | | |
# Date Time | |
NEPALI_TIME=$(TZ='Asia/Kathmandu' date +'%Y-%m-%d (%I:%M:%S %p)') | |
echo "NEPALI_TIME=$NEPALI_TIME" >> $GITHUB_ENV | |
continue-on-error: true | |
# - name: Git Pull | |
# run: | | |
# cd "$GITHUB_WORKSPACE/main" && git pull origin main | |
# continue-on-error: true | |
- uses: stefanzweifel/git-auto-commit-action@v5 | |
with: | |
repository: ./main | |
commit_user_name: Azathothas # defaults to "github-actions[bot]" | |
commit_user_email: [email protected] # defaults to "41898282+github-actions[bot]@users.noreply.github.com" | |
commit_message: "✅ Updated (Metadata) 🛍️ <-- ${{ env.NEPALI_TIME }} ⌚" | |
#push_options: '--force' | |
continue-on-error: true | |
#-----------------------------------------------------# | |
- name: rClone Generate Metadata (MAIN) | |
run: | | |
# Presets | |
set +x ; set +e | |
#--------------# | |
#touch "$HOME/.rclone.conf" | |
echo "${{ secrets.RCLONE_CF_R2_PUB }}" > "$HOME/.rclone.conf" | |
#Get JQ Module to convert size : https://users.aalto.fi/~tontti/posts/jq-and-human-readable-bytes/ | |
curl -qfsSL "https://pub.ajam.dev/utils/devscripts/jq/to_human_bytes.jq" -o "./to_human_bytes.jq" | |
#--------------------# | |
##arm64_v8a_Android## | |
rclone lsjson --fast-list "r2:/bin/arm64_v8a_Android/" | jq -r 'include "./to_human_bytes" ; .[] | select(.Size != 0 and .Size != -1 and (.Name | test("\\.(7z|ascii|bz2|gz|jq|json|log|md|png|rar|svg|tar|tgz|tmp|txt|upx|zip)$") | not)) | {name: (.Name), build_date: (.ModTime | split(".")[0]), build_log, build_script, b3sum, description, download_url: "https://bin.ajam.dev/arm64_v8a_Android/\(.Path)", extra_bins, note, sha256, size: (.Size | tonumber | bytes), repo_url, web_url}' | jq -s 'sort_by(.name)' | jq '.[]' > "./metadata.json.tmp_arm64_v8a_Android" & | |
##aarch64_arm64_Linux## | |
rclone lsjson --fast-list "r2:/bin/aarch64_arm64_Linux/" | jq -r 'include "./to_human_bytes" ; .[] | select(.Size != 0 and .Size != -1 and (.Name | test("\\.(7z|ascii|bz2|gz|jq|json|log|md|png|rar|svg|tar|tgz|tmp|txt|upx|zip)$") | not)) | {name: (.Name), build_date: (.ModTime | split(".")[0]), build_log, build_script, b3sum, description, download_url: "https://bin.ajam.dev/aarch64_arm64_Linux/\(.Path)", extra_bins, note, sha256, size: (.Size | tonumber | bytes), repo_url, web_url}' | jq -s 'sort_by(.name)' | jq '.[]' > "./metadata.json.tmp_aarch64_arm64_Linux" & | |
##x86_64_Linux## | |
rclone lsjson --fast-list "r2:/bin/x86_64_Linux/" | jq -r 'include "./to_human_bytes" ; .[] | select(.Size != 0 and .Size != -1 and (.Name | test("\\.(7z|ascii|bz2|gz|jq|json|log|md|png|rar|svg|tar|tgz|tmp|txt|upx|zip)$") | not)) | {name: (.Name), build_date: (.ModTime | split(".")[0]), build_log, build_script, b3sum, description, download_url: "https://bin.ajam.dev/x86_64_Linux/\(.Path)", extra_bins, note, sha256, size: (.Size | tonumber | bytes), repo_url, web_url}' | jq -s 'sort_by(.name)' | jq '.[]' > "./metadata.json.tmp_x86_64_Linux" & | |
##x64_Windows## | |
rclone lsjson --fast-list "r2:/bin/x64_Windows/" | jq -r 'include "./to_human_bytes" ; .[] | select(.Size != 0 and .Size != -1 and (.Name | test("\\.(7z|ascii|bz2|gz|jq|json|log|md|png|rar|svg|tar|tgz|tmp|txt|upx|zip)$") | not)) | {name: (.Name), build_date: (.ModTime | split(".")[0]), build_log, build_script, b3sum, description, download_url: "https://bin.ajam.dev/x64_Windows/\(.Path)", extra_bins, note, sha256, size: (.Size | tonumber | bytes), repo_url, web_url}' | jq -s 'sort_by(.name)' | jq '.[]' > "./metadata.json.tmp_x64_Windows" & | |
##Clean Pretty Formatted | |
wait ; echo | |
echo "[" $(cat metadata.json.tmp_arm64_v8a_Android | tr '\n' ' ' | sed 's/}/},/g' | sed '$ s/,$//') "]" | sed '$s/,[[:space:]]*\]/\]/' | jq . | tee "./metadata.json_arm64_v8a_Android" & | |
echo "[" $(cat metadata.json.tmp_aarch64_arm64_Linux | tr '\n' ' ' | sed 's/}/},/g' | sed '$ s/,$//') "]" | sed '$s/,[[:space:]]*\]/\]/' | jq . | tee "./metadata.json_aarch64_arm64_Linux" & | |
echo "[" $(cat metadata.json.tmp_x86_64_Linux | tr '\n' ' ' | sed 's/}/},/g' | sed '$ s/,$//') "]" | sed '$s/,[[:space:]]*\]/\]/' | jq . | tee "./metadata.json_x86_64_Linux" & | |
echo "[" $(cat metadata.json.tmp_x64_Windows | tr '\n' ' ' | sed 's/}/},/g' | sed '$ s/,$//') "]" | sed '$s/,[[:space:]]*\]/\]/' | jq . | tee "./metadata.json_x64_Windows" & | |
##Update Metadata | |
rclone copyto --checksum "./metadata.json_arm64_v8a_Android" "r2:/bin/arm64_v8a_Android/METADATA.json.tmp" --check-first --checkers 2000 --transfers 1000 --progress & | |
rclone copyto --checksum "./metadata.json_aarch64_arm64_Linux" "r2:/bin/aarch64_arm64_Linux/METADATA.json.tmp" --check-first --checkers 2000 --transfers 1000 --progress & | |
rclone copyto --checksum "./metadata.json_x86_64_Linux" "r2:/bin/x86_64_Linux/METADATA.json.tmp" --check-first --checkers 2000 --transfers 1000 --progress & | |
rclone copyto --checksum "./metadata.json_x64_Windows" "r2:/bin/x64_Windows/METADATA.json.tmp" --check-first --checkers 2000 --transfers 1000 --progress & | |
wait ; echo | |
continue-on-error: true | |
- name: rClone Generate Metadata (BASEUTILS) | |
run: | | |
# Presets | |
set +x ; set +e | |
#--------------# | |
#touch "$HOME/.rclone.conf" | |
echo "${{ secrets.RCLONE_CF_R2_PUB }}" > "$HOME/.rclone.conf" | |
#Get JQ Module to convert size : https://users.aalto.fi/~tontti/posts/jq-and-human-readable-bytes/ | |
curl -qfsSL "https://pub.ajam.dev/utils/devscripts/jq/to_human_bytes.jq" -o "./to_human_bytes.jq" | |
#--------------------# | |
##arm64_v8a_Android## | |
rclone copy "r2:/bin/arm64_v8a_Android/Baseutils/" "./metadata_Baseutils_arm64_v8a_Android_TEMP_JSON/" --include="INFO.json" --user-agent="$USER_AGENT" --buffer-size="10M" --s3-upload-concurrency="50" --s3-chunk-size="10M" --multi-thread-streams="50" --checkers="2000" --transfers="100" --retries="10" --check-first --checksum --copy-links --fast-list --progress & | |
##aarch64_arm64_Linux## | |
rclone copy "r2:/bin/aarch64_arm64_Linux/Baseutils/" "./metadata_Baseutils_aarch64_arm64_Linux_TEMP_JSON/" --include="INFO.json" --user-agent="$USER_AGENT" --buffer-size="10M" --s3-upload-concurrency="50" --s3-chunk-size="10M" --multi-thread-streams="50" --checkers="2000" --transfers="100" --retries="10" --check-first --checksum --copy-links --fast-list --progress & | |
##x86_64_Linux## | |
rclone copy "r2:/bin/x86_64_Linux/Baseutils/" "./metadata_Baseutils_x86_64_Linux_TEMP_JSON/" --include="INFO.json" --user-agent="$USER_AGENT" --buffer-size="10M" --s3-upload-concurrency="50" --s3-chunk-size="10M" --multi-thread-streams="50" --checkers="2000" --transfers="100" --retries="10" --check-first --checksum --copy-links --fast-list --progress & | |
##x64_Windows## | |
rclone copy "r2:/bin/x64_Windows/Baseutils/" "./metadata_Baseutils_x64_Windows_TEMP_JSON/" --include="INFO.json" --user-agent="$USER_AGENT" --buffer-size="10M" --s3-upload-concurrency="50" --s3-chunk-size="10M" --multi-thread-streams="50" --checkers="2000" --transfers="100" --retries="10" --check-first --checksum --copy-links --fast-list --progress & | |
##Merge (INFO) | |
wait ; echo | |
find "./metadata_Baseutils_arm64_v8a_Android_TEMP_JSON/" -iname "INFO.json" -exec jq -c '.[]' {} \; > "./metadata_Baseutils_arm64_v8a_Android_INFO_ALL.json" & | |
##aarch64_arm64_Linux## | |
find "./metadata_Baseutils_aarch64_arm64_Linux_TEMP_JSON/" -iname "INFO.json" -exec jq -c '.[]' {} \; > "./metadata_Baseutils_aarch64_arm64_Linux_INFO_ALL.json" & | |
find "./metadata_Baseutils_x86_64_Linux_TEMP_JSON/" -iname "INFO.json" -exec jq -c '.[]' {} \; > "./metadata_Baseutils_x86_64_Linux_INFO_ALL.json" & | |
find "./metadata_Baseutils_x64_Windows_TEMP_JSON/" -iname "INFO.json" -exec jq -c '.[]' {} \; > "./metadata_Baseutils_x64_Windows_INFO_ALL.json" & | |
##Enrich | |
wait ; echo | |
rclone lsjson --fast-list "r2:/bin/arm64_v8a_Android/Baseutils/" --files-only | jq -r 'include "./to_human_bytes" ; .[] | select(.Size != 0 and .Size != -1 and (.Name | test("\\.(7z|ascii|bz2|gz|jq|json|log|md|png|rar|svg|tar|tgz|tmp|txt|upx|zip)$") | not)) | {name: (.Name), build_date: (.ModTime | split(".")[0]), build_log, build_script, b3sum, description, download_url: "https://bin.ajam.dev/arm64_v8a_Android/Baseutils/\(.Path)", extra_bins, note, sha256, size: (.Size | tonumber | bytes), repo_url, web_url}' | jq -s 'sort_by(.name)' | jq '.[]' > "./metadata_Baseutils.json.tmp_arm64_v8a_Android" & | |
##aarch64_arm64_Linux## | |
rclone lsjson --fast-list "r2:/bin/aarch64_arm64_Linux/Baseutils/" --files-only | jq -r 'include "./to_human_bytes" ; .[] | select(.Size != 0 and .Size != -1 and (.Name | test("\\.(7z|ascii|bz2|gz|jq|json|log|md|png|rar|svg|tar|tgz|tmp|txt|upx|zip)$") | not)) | {name: (.Name), build_date: (.ModTime | split(".")[0]), build_log, build_script, b3sum, description, download_url: "https://bin.ajam.dev/aarch64_arm64_Linux/Baseutils/\(.Path)", extra_bins, note, sha256, size: (.Size | tonumber | bytes), repo_url, web_url}' | jq -s 'sort_by(.name)' | jq '.[]' > "./metadata_Baseutils.json.tmp_aarch64_arm64_Linux" & | |
##x86_64_Linux## | |
rclone lsjson --fast-list "r2:/bin/x86_64_Linux/Baseutils/" --files-only | jq -r 'include "./to_human_bytes" ; .[] | select(.Size != 0 and .Size != -1 and (.Name | test("\\.(7z|ascii|bz2|gz|jq|json|log|md|png|rar|svg|tar|tgz|tmp|txt|upx|zip)$") | not)) | {name: (.Name), build_date: (.ModTime | split(".")[0]), build_log, build_script, b3sum, description, download_url: "https://bin.ajam.dev/x86_64_Linux/Baseutils/\(.Path)", extra_bins, note, sha256, size: (.Size | tonumber | bytes), repo_url, web_url}' | jq -s 'sort_by(.name)' | jq '.[]' > "./metadata_Baseutils.json.tmp_x86_64_Linux" & | |
##x64_Windows## | |
rclone lsjson --fast-list "r2:/bin/x64_Windows/Baseutils/" --files-only | jq -r 'include "./to_human_bytes" ; .[] | select(.Size != 0 and .Size != -1 and (.Name | test("\\.(7z|ascii|bz2|gz|jq|json|log|md|png|rar|svg|tar|tgz|tmp|txt|upx|zip)$") | not)) | {name: (.Name), build_date: (.ModTime | split(".")[0]), build_log, build_script, b3sum, description, download_url: "https://bin.ajam.dev/x64_Windows/Baseutils/\(.Path)", extra_bins, note, sha256, size: (.Size | tonumber | bytes), repo_url, web_url}' | jq -s 'sort_by(.name)' | jq '.[]' > "./metadata_Baseutils.json.tmp_x64_Windows" & | |
##Clean | |
wait ; echo | |
sed -i 's/: null/: ""/g' "./metadata_Baseutils.json.tmp_arm64_v8a_Android" & | |
sed -i 's/: null/: ""/g' "./metadata_Baseutils.json.tmp_aarch64_arm64_Linux" & | |
sed -i 's/: null/: ""/g' "./metadata_Baseutils.json.tmp_x86_64_Linux" & | |
sed -i 's/: null/: ""/g' "./metadata_Baseutils.json.tmp_x64_Windows" & | |
##Merge (ALL) | |
wait ; echo | |
jq -s 'sort_by(.download_url)' "./metadata_Baseutils.json.tmp_arm64_v8a_Android" "./metadata_Baseutils_arm64_v8a_Android_INFO_ALL.json" > "./metadata_Baseutils.json_arm64_v8a_Android" & | |
jq -s 'sort_by(.download_url)' "./metadata_Baseutils.json.tmp_aarch64_arm64_Linux" "./metadata_Baseutils_aarch64_arm64_Linux_INFO_ALL.json" > "./metadata_Baseutils.json_aarch64_arm64_Linux" & | |
jq -s 'sort_by(.download_url)' "./metadata_Baseutils.json.tmp_x86_64_Linux" "./metadata_Baseutils_x86_64_Linux_INFO_ALL.json" > "./metadata_Baseutils.json_x86_64_Linux" & | |
jq -s 'sort_by(.download_url)' "./metadata_Baseutils.json.tmp_x64_Windows" "./metadata_Baseutils_x64_Windows_INFO_ALL.json" > "./metadata_Baseutils.json_x64_Windows" & | |
##Update | |
rclone copyto --checksum "./metadata_Baseutils.json_arm64_v8a_Android" "r2:/bin/arm64_v8a_Android/Baseutils/METADATA.json" & | |
rclone copyto --checksum "./metadata_Baseutils.json_aarch64_arm64_Linux" "r2:/bin/aarch64_arm64_Linux/Baseutils/METADATA.json" & | |
rclone copyto --checksum "./metadata_Baseutils.json_x86_64_Linux" "r2:/bin/x86_64_Linux/Baseutils/METADATA.json" & | |
rclone copyto --checksum "./metadata_Baseutils.json_x64_Windows" "r2:/bin/x64_Windows/Baseutils/METADATA.json" & | |
wait ; echo | |
continue-on-error: true | |
#------------------------------------------------------------------------------------# | |
- name: rClone BackUp Repo-II ("https://pub.ajam.dev/repos/$GITHUB_REPOSITORY") | |
run: | | |
# Presets | |
set +x ; set +e | |
#--------------# | |
#touch "$HOME/.rclone.conf" | |
echo "${{ secrets.RCLONE_CF_R2_PUB }}" > "$HOME/.rclone.conf" | |
#chdir to Repo | |
cd "$GITHUB_WORKSPACE/main" | |
#Git pull | |
git pull origin main --no-edit 2>/dev/null | |
#Del Bloat | |
rm -rf "$(pwd)/.git" | |
#Upload to Pub | |
echo -e "[+] Syncing $GITHUB_REPOSITORY to pub.ajam.dev/repos/$GITHUB_REPOSITORY \n" | |
rclone sync "." "r2:/pub/repos/$GITHUB_REPOSITORY/" --user-agent="$USER_AGENT" --buffer-size="100M" --s3-upload-concurrency="500" --s3-chunk-size="100M" --multi-thread-streams="500" --checkers="2000" --transfers="1000" --check-first --checksum --copy-links --fast-list --progress | |
continue-on-error: true | |
#------------------------------------------------------------------------------------# | |
#------------------------------------------------------------------------------------# | |
#------------------------------------------------------------------------------------# | |
#------------------------------------------------------------------------------------# | |
enrich-arm64v8a-Android: | |
runs-on: ubuntu-latest | |
needs: [check-post] | |
steps: | |
- name: Checkout repository | |
uses: actions/checkout@v4 | |
with: | |
path: main | |
filter: "blob:none" #https://github.blog/2020-12-21-get-up-to-speed-with-partial-clone-and-shallow-clone/ | |
- name: Setup Env | |
run: | | |
#presets | |
set +x ; set +e | |
#tmp | |
SYSTMP="$(dirname $(mktemp -u))" && export SYSTMP="$SYSTMP" | |
#GH ENV | |
echo "SYSTMP=$SYSTMP" >> "$GITHUB_ENV" | |
#GH Dirs | |
mkdir -p "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux" | |
mkdir -p "$GITHUB_WORKSPACE/main/arm64_v8a_Android" | |
mkdir -p "$GITHUB_WORKSPACE/main/x64_Windows" | |
mkdir -p "$GITHUB_WORKSPACE/main/x86_64_Linux" | |
#-------------# | |
mkdir -p "$HOME/bin" | |
##Setup rClone | |
mkdir -p "$HOME/.config/rclone" | |
echo "${{ secrets.RCLONE_CF_R2_PUB }}" > "$HOME/.config/rclone/rclone.conf" | |
export RCLONE_STATS="120s" ; echo "RCLONE_STATS=$RCLONE_STATS" >> "$GITHUB_ENV" | |
##User-Agent | |
USER_AGENT="$(curl -qfsSL 'https://pub.ajam.dev/repos/Azathothas/Wordlists/Misc/User-Agents/ua_chrome_macos_latest.txt')" && export USER_AGENT="$USER_AGENT" | |
echo "USER_AGENT=$USER_AGENT" >> "$GITHUB_ENV" | |
continue-on-error: true | |
- name: Install Addons | |
run: | | |
#presets | |
set +x ; set +e | |
#-------------# | |
bash <(curl -qfsSL "https://pub.ajam.dev/repos/Azathothas/Arsenal/misc/Linux/install_dev_tools.sh") | |
sudo apt-get update -y -qq && sudo apt-get install kmod -y -qq | |
sudo apt-get update -y -qq && sudo apt-get install fuse3 -y -qq | |
sudo apt-get update -y -qq && sudo apt-get install libfuse3-dev -y -qq | |
sudo sed 's/#user_allow_other/user_allow_other/g' -i "/etc/fuse.conf" | |
continue-on-error: true | |
- name: rClone Check Metadata (bin.ajam.dev/arm64_v8a_Android) | |
run: | | |
#presets | |
set +x ; set +e | |
#-------------# | |
BINDIR="${SYSTMP}/toolpack_arm64_v8a_Android" && export BINDIR="${BINDIR}" | |
mkdir -p "${BINDIR}" | |
rclone mount "r2:/bin/arm64_v8a_Android/" "${BINDIR}" \ | |
--daemon \ | |
--allow-other \ | |
--no-modtime \ | |
--read-only \ | |
--attr-timeout 15s \ | |
--checkers 100 \ | |
--buffer-size 512M \ | |
--dir-cache-time 5m \ | |
--low-level-retries 10 \ | |
--multi-thread-streams 16 \ | |
--poll-interval 15s \ | |
--transfers 16 \ | |
--vfs-read-wait 0 \ | |
--vfs-cache-mode full \ | |
--vfs-cache-max-size 20G \ | |
--vfs-cache-poll-interval 15s \ | |
--vfs-read-chunk-size 128M \ | |
--vfs-read-chunk-size-limit off \ | |
--user-agent="${USER_AGENT}" ; df -h | |
#File | |
cd "${BINDIR}" && find "./" -maxdepth 1 -type f | grep -v -E '\.7z$|\.gz$|\.jq$|\.json$|\.log$|\.md$|\.tar$|\.tgz$|\.tmp$|\.txt$|\.upx$|\.zip$|Baseutils/' | sort | xargs file | sed 's/^.\///' > "${SYSTMP}/arm64_v8a_Android_FILE" | |
rclone copyto "${SYSTMP}/arm64_v8a_Android_FILE" "r2:/bin/arm64_v8a_Android/FILE.txt" --user-agent="${USER_AGENT}" --s3-upload-concurrency="500" --s3-chunk-size="100M" --multi-thread-streams="500" --checkers="2000" --transfers="1000" --retries="10" --check-first --checksum --copy-links --fast-list --progress | |
#Size (Dust) | |
dust --depth 1 -v '\.7z$|\.gz$|\.jq$|\.json$|\.log$|\.md$|\.tar$|\.tgz$|\.tmp$|\.txt$|\.upx$|\.zip$|Baseutils/' --only-file --no-percent-bars --no-colors --ignore_hidden --reverse --number-of-lines 99999999 "${BINDIR}" | tee "${SYSTMP}/arm64_v8a_Android_SIZE.txt" | |
rclone copyto "${SYSTMP}/arm64_v8a_Android_SIZE.txt" "r2:/bin/arm64_v8a_Android/SIZE.txt" --user-agent="${USER_AGENT}" --s3-upload-concurrency="500" --s3-chunk-size="100M" --multi-thread-streams="500" --checkers="2000" --transfers="1000" --retries="10" --check-first --checksum --copy-links --fast-list --progress | |
#BLAKE3SUM | |
cd "${BINDIR}" && find "./" -maxdepth 1 -type f | grep -v -E '\.7z$|\.gz$|\.jq$|\.json$|\.log$|\.md$|\.tar$|\.tgz$|\.tmp$|\.txt$|\.upx$|\.zip$|Baseutils/' | sort | xargs b3sum | sed 's/\.\///g' > "${SYSTMP}/arm64_v8a_Android_BLAKE3SUM" | |
rclone copyto "${SYSTMP}/arm64_v8a_Android_BLAKE3SUM" "r2:/bin/arm64_v8a_Android/BLAKE3SUM.txt" --user-agent="${USER_AGENT}" --s3-upload-concurrency="500" --s3-chunk-size="100M" --multi-thread-streams="500" --checkers="2000" --transfers="1000" --retries="10" --check-first --checksum --copy-links --fast-list --progress | |
#SHA256SUM | |
cd "${BINDIR}" && find "./" -maxdepth 1 -type f | grep -v -E '\.7z$|\.gz$|\.jq$|\.json$|\.log$|\.md$|\.tar$|\.tgz$|\.tmp$|\.txt$|\.upx$|\.zip$|Baseutils/' | sort | xargs sha256sum | sed 's/\.\///g' > "${SYSTMP}/arm64_v8a_Android_SHA256SUM" | |
rclone copyto "${SYSTMP}/arm64_v8a_Android_SHA256SUM" "r2:/bin/arm64_v8a_Android/SHA256SUM.txt" --user-agent="${USER_AGENT}" --s3-upload-concurrency="500" --s3-chunk-size="100M" --multi-thread-streams="500" --checkers="2000" --transfers="1000" --retries="10" --check-first --checksum --copy-links --fast-list --progress | |
##Purge | |
#rm -rf "${BINDIR}" | |
sudo fusermount3 -uz "${BINDIR}" ; df -h | |
continue-on-error: true | |
- name: rClone Enrich Metadata (bin.ajam.dev/arm64_v8a_Android) | |
run: | | |
# Presets | |
set +x ; set +e | |
#--------------# | |
##Run | |
chmod +x "$GITHUB_WORKSPACE/main/.github/scripts/arm64_v8a_Android/gen_meta.sh" | |
dos2unix --quiet "$GITHUB_WORKSPACE/main/.github/scripts/arm64_v8a_Android/gen_meta.sh" | |
bash "$GITHUB_WORKSPACE/main/.github/scripts/arm64_v8a_Android/gen_meta.sh" | |
cd "$GITHUB_WORKSPACE/main/.github/scripts/arm64_v8a_Android/bins/" && yq ea '[.]' *.yaml > "$GITHUB_WORKSPACE/main/arm64_v8a_Android/BINS.yaml" | |
continue-on-error: true | |
#------------------------------------------------------------------------------------# | |
#------------------------------------------------------------------------------------# | |
#------------------------------------------------------------------------------------# | |
#------------------------------------------------------------------------------------# | |
enrich-arm64-Linux: | |
runs-on: ubuntu-latest | |
needs: [check-post] | |
steps: | |
- name: Checkout repository | |
uses: actions/checkout@v4 | |
with: | |
path: main | |
filter: "blob:none" #https://github.blog/2020-12-21-get-up-to-speed-with-partial-clone-and-shallow-clone/ | |
- name: Setup Env | |
run: | | |
#presets | |
set +x ; set +e | |
#tmp | |
SYSTMP="$(dirname $(mktemp -u))" && export SYSTMP="$SYSTMP" | |
#GH ENV | |
echo "SYSTMP=$SYSTMP" >> "$GITHUB_ENV" | |
#GH Dirs | |
mkdir -p "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux" | |
mkdir -p "$GITHUB_WORKSPACE/main/arm64_v8a_Android" | |
mkdir -p "$GITHUB_WORKSPACE/main/x64_Windows" | |
mkdir -p "$GITHUB_WORKSPACE/main/x86_64_Linux" | |
#-------------# | |
mkdir -p "$HOME/bin" | |
##Setup rClone | |
mkdir -p "$HOME/.config/rclone" | |
echo "${{ secrets.RCLONE_CF_R2_PUB }}" > "$HOME/.config/rclone/rclone.conf" | |
export RCLONE_STATS="120s" ; echo "RCLONE_STATS=$RCLONE_STATS" >> "$GITHUB_ENV" | |
##User-Agent | |
USER_AGENT="$(curl -qfsSL 'https://pub.ajam.dev/repos/Azathothas/Wordlists/Misc/User-Agents/ua_chrome_macos_latest.txt')" && export USER_AGENT="$USER_AGENT" | |
echo "USER_AGENT=$USER_AGENT" >> "$GITHUB_ENV" | |
continue-on-error: true | |
- name: Install Addons | |
run: | | |
#presets | |
set +x ; set +e | |
#-------------# | |
bash <(curl -qfsSL "https://pub.ajam.dev/repos/Azathothas/Arsenal/misc/Linux/install_dev_tools.sh") | |
sudo apt-get update -y -qq && sudo apt-get install kmod -y -qq | |
sudo apt-get update -y -qq && sudo apt-get install fuse3 -y -qq | |
sudo apt-get update -y -qq && sudo apt-get install libfuse3-dev -y -qq | |
sudo sed 's/#user_allow_other/user_allow_other/g' -i "/etc/fuse.conf" | |
continue-on-error: true | |
- name: rClone Check Metadata (bin.ajam.dev/aarch64_arm64_Linux) | |
run: | | |
#presets | |
set +x ; set +e | |
#-------------# | |
BINDIR="${SYSTMP}/toolpack_aarch64" && export BINDIR="${BINDIR}" | |
mkdir -p "${BINDIR}" | |
rclone mount "r2:/bin/aarch64_arm64_Linux/" "${BINDIR}" \ | |
--daemon \ | |
--allow-other \ | |
--no-modtime \ | |
--read-only \ | |
--attr-timeout 15s \ | |
--checkers 100 \ | |
--buffer-size 512M \ | |
--dir-cache-time 5m \ | |
--low-level-retries 10 \ | |
--multi-thread-streams 16 \ | |
--poll-interval 15s \ | |
--transfers 16 \ | |
--vfs-read-wait 0 \ | |
--vfs-cache-mode full \ | |
--vfs-cache-max-size 20G \ | |
--vfs-cache-poll-interval 15s \ | |
--vfs-read-chunk-size 128M \ | |
--vfs-read-chunk-size-limit off \ | |
--user-agent="${USER_AGENT}" ; df -h | |
#File | |
cd "${BINDIR}" && find "./" -maxdepth 1 -type f | grep -v -E '\.7z$|\.gz$|\.jq$|\.json$|\.log$|\.md$|\.tar$|\.tgz$|\.tmp$|\.txt$|\.upx$|\.zip$|Baseutils/' | sort | xargs file | sed 's/^.\///' > "${SYSTMP}/aarch64_Linux_FILE" | |
rclone copyto "${SYSTMP}/aarch64_Linux_FILE" "r2:/bin/aarch64_arm64_Linux/FILE.txt" --user-agent="${USER_AGENT}" --s3-upload-concurrency="500" --s3-chunk-size="100M" --multi-thread-streams="500" --checkers="2000" --transfers="1000" --retries="10" --check-first --checksum --copy-links --fast-list --progress | |
#Size (Dust) | |
dust --depth 1 -v '\.7z$|\.gz$|\.jq$|\.json$|\.log$|\.md$|\.tar$|\.tgz$|\.tmp$|\.txt$|\.upx$|\.zip$|Baseutils/' --only-file --no-percent-bars --no-colors --ignore_hidden --reverse --number-of-lines 99999999 "${BINDIR}" | tee "${SYSTMP}/aarch64_Linux_SIZE.txt" | |
rclone copyto "${SYSTMP}/aarch64_Linux_SIZE.txt" "r2:/bin/aarch64_arm64_Linux/SIZE.txt" --user-agent="${USER_AGENT}" --s3-upload-concurrency="500" --s3-chunk-size="100M" --multi-thread-streams="500" --checkers="2000" --transfers="1000" --retries="10" --check-first --checksum --copy-links --fast-list --progress | |
#BLAKE3SUM | |
cd "${BINDIR}" && find "./" -maxdepth 1 -type f | grep -v -E '\.7z$|\.gz$|\.jq$|\.json$|\.log$|\.md$|\.tar$|\.tgz$|\.tmp$|\.txt$|\.upx$|\.zip$|Baseutils/' | sort | xargs b3sum | sed 's/\.\///g' > "${SYSTMP}/aarch64_Linux_BLAKE3SUM" | |
rclone copyto "${SYSTMP}/aarch64_Linux_BLAKE3SUM" "r2:/bin/aarch64_arm64_Linux/BLAKE3SUM.txt" --user-agent="${USER_AGENT}" --s3-upload-concurrency="500" --s3-chunk-size="100M" --multi-thread-streams="500" --checkers="2000" --transfers="1000" --retries="10" --check-first --checksum --copy-links --fast-list --progress | |
#SHA256SUM | |
cd "${BINDIR}" && find "./" -maxdepth 1 -type f | grep -v -E '\.7z$|\.gz$|\.jq$|\.json$|\.log$|\.md$|\.tar$|\.tgz$|\.tmp$|\.txt$|\.upx$|\.zip$|Baseutils/' | sort | xargs sha256sum | sed 's/\.\///g' > "${SYSTMP}/aarch64_Linux_SHA256SUM" | |
rclone copyto "${SYSTMP}/aarch64_Linux_SHA256SUM" "r2:/bin/aarch64_arm64_Linux/SHA256SUM.txt" --user-agent="${USER_AGENT}" --s3-upload-concurrency="500" --s3-chunk-size="100M" --multi-thread-streams="500" --checkers="2000" --transfers="1000" --retries="10" --check-first --checksum --copy-links --fast-list --progress | |
##Purge | |
#rm -rf "${BINDIR}" | |
sudo fusermount3 -uz "${BINDIR}" ; df -h | |
continue-on-error: true | |
- name: rClone Enrich Metadata (bin.ajam.dev/aarch64_arm64_Linux) | |
run: | | |
# Presets | |
set +x ; set +e | |
#--------------# | |
##Run | |
chmod +x "$GITHUB_WORKSPACE/main/.github/scripts/aarch64_Linux/gen_meta.sh" | |
dos2unix --quiet "$GITHUB_WORKSPACE/main/.github/scripts/aarch64_Linux/gen_meta.sh" | |
bash "$GITHUB_WORKSPACE/main/.github/scripts/aarch64_Linux/gen_meta.sh" | |
cd "$GITHUB_WORKSPACE/main/.github/scripts/aarch64_Linux/bins/" && yq ea '[.]' *.yaml > "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/BINS.yaml" | |
continue-on-error: true | |
#------------------------------------------------------------------------------------# | |
#------------------------------------------------------------------------------------# | |
#------------------------------------------------------------------------------------# | |
#------------------------------------------------------------------------------------# | |
enrich-x86-64-Linux: | |
runs-on: ubuntu-latest | |
needs: [check-post] | |
steps: | |
- name: Checkout repository | |
uses: actions/checkout@v4 | |
with: | |
path: main | |
filter: "blob:none" #https://github.blog/2020-12-21-get-up-to-speed-with-partial-clone-and-shallow-clone/ | |
- name: Setup Env | |
run: | | |
#presets | |
set +x ; set +e | |
#tmp | |
SYSTMP="$(dirname $(mktemp -u))" && export SYSTMP="$SYSTMP" | |
#GH ENV | |
echo "SYSTMP=$SYSTMP" >> "$GITHUB_ENV" | |
#GH Dirs | |
mkdir -p "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux" | |
mkdir -p "$GITHUB_WORKSPACE/main/arm64_v8a_Android" | |
mkdir -p "$GITHUB_WORKSPACE/main/x64_Windows" | |
mkdir -p "$GITHUB_WORKSPACE/main/x86_64_Linux" | |
#-------------# | |
mkdir -p "$HOME/bin" | |
##Setup rClone | |
mkdir -p "$HOME/.config/rclone" | |
echo "${{ secrets.RCLONE_CF_R2_PUB }}" > "$HOME/.config/rclone/rclone.conf" | |
export RCLONE_STATS="120s" ; echo "RCLONE_STATS=$RCLONE_STATS" >> "$GITHUB_ENV" | |
##User-Agent | |
USER_AGENT="$(curl -qfsSL 'https://pub.ajam.dev/repos/Azathothas/Wordlists/Misc/User-Agents/ua_chrome_macos_latest.txt')" && export USER_AGENT="$USER_AGENT" | |
echo "USER_AGENT=$USER_AGENT" >> "$GITHUB_ENV" | |
continue-on-error: true | |
- name: Install Addons | |
run: | | |
#presets | |
set +x ; set +e | |
#-------------# | |
bash <(curl -qfsSL "https://pub.ajam.dev/repos/Azathothas/Arsenal/misc/Linux/install_dev_tools.sh") | |
sudo apt-get update -y -qq && sudo apt-get install kmod -y -qq | |
sudo apt-get update -y -qq && sudo apt-get install fuse3 -y -qq | |
sudo apt-get update -y -qq && sudo apt-get install libfuse3-dev -y -qq | |
sudo sed 's/#user_allow_other/user_allow_other/g' -i "/etc/fuse.conf" | |
continue-on-error: true | |
- name: rClone Check Metadata (bin.ajam.dev/x86_64_Linux) | |
run: | | |
#presets | |
set +x ; set +e | |
#-------------# | |
BINDIR="${SYSTMP}/toolpack_x86_64" && export BINDIR="${BINDIR}" | |
mkdir -p "${BINDIR}" | |
rclone mount "r2:/bin/x86_64_Linux/" "${BINDIR}" \ | |
--daemon \ | |
--allow-other \ | |
--no-modtime \ | |
--read-only \ | |
--attr-timeout 15s \ | |
--checkers 100 \ | |
--buffer-size 512M \ | |
--dir-cache-time 5m \ | |
--low-level-retries 10 \ | |
--multi-thread-streams 16 \ | |
--poll-interval 15s \ | |
--transfers 16 \ | |
--vfs-read-wait 0 \ | |
--vfs-cache-mode full \ | |
--vfs-cache-max-size 20G \ | |
--vfs-cache-poll-interval 15s \ | |
--vfs-read-chunk-size 128M \ | |
--vfs-read-chunk-size-limit off \ | |
--user-agent="${USER_AGENT}" ; df -h | |
#File | |
cd "${BINDIR}" && find "./" -maxdepth 1 -type f | grep -v -E '\.7z$|\.gz$|\.jq$|\.json$|\.log$|\.md$|\.tar$|\.tgz$|\.tmp$|\.txt$|\.upx$|\.zip$|Baseutils/' | sort | xargs file | sed 's/^.\///' > "${SYSTMP}/x86_64_Linux_FILE" | |
rclone copyto "${SYSTMP}/x86_64_Linux_FILE" "r2:/bin/x86_64_Linux/FILE.txt" --user-agent="$USER_AGENT" --s3-upload-concurrency="500" --s3-chunk-size="100M" --multi-thread-streams="500" --checkers="2000" --transfers="1000" --retries="10" --check-first --checksum --copy-links --fast-list --progress | |
#Size (Dust) | |
dust --depth 1 -v '\.7z$|\.gz$|\.jq$|\.json$|\.log$|\.md$|\.tar$|\.tgz$|\.tmp$|\.txt$|\.upx$|\.zip$|Baseutils/' --only-file --no-percent-bars --no-colors --ignore_hidden --reverse --number-of-lines 99999999 "${BINDIR}" | tee "${SYSTMP}/x86_64_Linux_SIZE.txt" | |
rclone copyto "${SYSTMP}/x86_64_Linux_SIZE.txt" "r2:/bin/x86_64_Linux/SIZE.txt" --user-agent="$USER_AGENT" --s3-upload-concurrency="500" --s3-chunk-size="100M" --multi-thread-streams="500" --checkers="2000" --transfers="1000" --retries="10" --check-first --checksum --copy-links --fast-list --progress | |
#BLAKE3SUM | |
cd "${BINDIR}" && find "./" -maxdepth 1 -type f | grep -v -E '\.7z$|\.gz$|\.jq$|\.json$|\.log$|\.md$|\.tar$|\.tgz$|\.tmp$|\.txt$|\.upx$|\.zip$|Baseutils/' | sort | xargs b3sum | sed 's/\.\///g' > "${SYSTMP}/x86_64_Linux_BLAKE3SUM" | |
rclone copyto "${SYSTMP}/x86_64_Linux_BLAKE3SUM" "r2:/bin/x86_64_Linux/BLAKE3SUM.txt" --user-agent="$USER_AGENT" --s3-upload-concurrency="500" --s3-chunk-size="100M" --multi-thread-streams="500" --checkers="2000" --transfers="1000" --retries="10" --check-first --checksum --copy-links --fast-list --progress | |
#SHA256SUM | |
cd "${BINDIR}" && find "./" -maxdepth 1 -type f | grep -v -E '\.7z$|\.gz$|\.jq$|\.json$|\.log$|\.md$|\.tar$|\.tgz$|\.tmp$|\.txt$|\.upx$|\.zip$|Baseutils/' | sort | xargs sha256sum | sed 's/\.\///g' > "${SYSTMP}/x86_64_Linux_SHA256SUM" | |
rclone copyto "${SYSTMP}/x86_64_Linux_SHA256SUM" "r2:/bin/x86_64_Linux/SHA256SUM.txt" --user-agent="$USER_AGENT" --s3-upload-concurrency="500" --s3-chunk-size="100M" --multi-thread-streams="500" --checkers="2000" --transfers="1000" --retries="10" --check-first --checksum --copy-links --fast-list --progress | |
##Purge | |
#rm -rf "${BINDIR}" | |
sudo fusermount3 -uz "${BINDIR}" ; df -h | |
continue-on-error: true | |
- name: rClone Enrich Metadata (bin.ajam.dev/x86_64_Linux) | |
run: | | |
# Presets | |
set +x ; set +e | |
#--------------# | |
##Run | |
chmod +x "$GITHUB_WORKSPACE/main/.github/scripts/x86_64_Linux/gen_meta.sh" | |
dos2unix --quiet "$GITHUB_WORKSPACE/main/.github/scripts/x86_64_Linux/gen_meta.sh" | |
bash "$GITHUB_WORKSPACE/main/.github/scripts/x86_64_Linux/gen_meta.sh" | |
cd "$GITHUB_WORKSPACE/main/.github/scripts/x86_64_Linux/bins/" && yq ea '[.]' *.yaml > "$GITHUB_WORKSPACE/main/x86_64_Linux/BINS.yaml" | |
continue-on-error: true | |
#------------------------------------------------------------------------------------# | |
#------------------------------------------------------------------------------------# | |
#------------------------------------------------------------------------------------# | |
#------------------------------------------------------------------------------------# | |
enrich-x64-Windows: | |
runs-on: ubuntu-latest | |
needs: [check-post] | |
steps: | |
- name: Checkout repository | |
uses: actions/checkout@v4 | |
with: | |
path: main | |
filter: "blob:none" #https://github.blog/2020-12-21-get-up-to-speed-with-partial-clone-and-shallow-clone/ | |
- name: Setup Env | |
run: | | |
#presets | |
set +x ; set +e | |
#tmp | |
SYSTMP="$(dirname $(mktemp -u))" && export SYSTMP="$SYSTMP" | |
#GH ENV | |
echo "SYSTMP=$SYSTMP" >> "$GITHUB_ENV" | |
#GH Dirs | |
mkdir -p "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux" | |
mkdir -p "$GITHUB_WORKSPACE/main/arm64_v8a_Android" | |
mkdir -p "$GITHUB_WORKSPACE/main/x64_Windows" | |
mkdir -p "$GITHUB_WORKSPACE/main/x86_64_Linux" | |
#-------------# | |
mkdir -p "$HOME/bin" | |
##Setup rClone | |
mkdir -p "$HOME/.config/rclone" | |
echo "${{ secrets.RCLONE_CF_R2_PUB }}" > "$HOME/.config/rclone/rclone.conf" | |
export RCLONE_STATS="120s" ; echo "RCLONE_STATS=$RCLONE_STATS" >> "$GITHUB_ENV" | |
##User-Agent | |
USER_AGENT="$(curl -qfsSL 'https://pub.ajam.dev/repos/Azathothas/Wordlists/Misc/User-Agents/ua_chrome_macos_latest.txt')" && export USER_AGENT="$USER_AGENT" | |
echo "USER_AGENT=$USER_AGENT" >> "$GITHUB_ENV" | |
continue-on-error: true | |
- name: Install Addons | |
run: | | |
#presets | |
set +x ; set +e | |
#-------------# | |
bash <(curl -qfsSL "https://pub.ajam.dev/repos/Azathothas/Arsenal/misc/Linux/install_dev_tools.sh") | |
sudo apt-get update -y -qq && sudo apt-get install kmod -y -qq | |
sudo apt-get update -y -qq && sudo apt-get install fuse3 -y -qq | |
sudo apt-get update -y -qq && sudo apt-get install libfuse3-dev -y -qq | |
sudo sed 's/#user_allow_other/user_allow_other/g' -i "/etc/fuse.conf" | |
continue-on-error: true | |
- name: rClone Check Metadata (bin.ajam.dev/x64_Windows) | |
run: | | |
#presets | |
set +x ; set +e | |
#-------------# | |
BINDIR="${SYSTMP}/toolpack_x86_64" && export BINDIR="${BINDIR}" | |
mkdir -p "${BINDIR}" | |
rclone mount "r2:/bin/x64_Windows/" "${BINDIR}" \ | |
--daemon \ | |
--allow-other \ | |
--no-modtime \ | |
--read-only \ | |
--attr-timeout 15s \ | |
--checkers 100 \ | |
--buffer-size 512M \ | |
--dir-cache-time 5m \ | |
--low-level-retries 10 \ | |
--multi-thread-streams 16 \ | |
--poll-interval 15s \ | |
--transfers 16 \ | |
--vfs-read-wait 0 \ | |
--vfs-cache-mode full \ | |
--vfs-cache-max-size 20G \ | |
--vfs-cache-poll-interval 15s \ | |
--vfs-read-chunk-size 128M \ | |
--vfs-read-chunk-size-limit off \ | |
--user-agent="${USER_AGENT}" ; df -h | |
#File | |
cd "${BINDIR}" && find "./" -maxdepth 1 -type f | grep -v -E '\.7z$|\.gz$|\.jq$|\.json$|\.log$|\.md$|\.tar$|\.tgz$|\.tmp$|\.txt$|\.upx$|\.zip$|Baseutils/' | sort | xargs file | sed 's/^.\///' > "${SYSTMP}/x64_Windows_FILE" | |
rclone copyto "${SYSTMP}/x64_Windows_FILE" "r2:/bin/x64_Windows/FILE.txt" --user-agent="$USER_AGENT" --s3-upload-concurrency="500" --s3-chunk-size="100M" --multi-thread-streams="500" --checkers="2000" --transfers="1000" --retries="10" --check-first --checksum --copy-links --fast-list --progress | |
#Size (Dust) | |
dust --depth 1 -v '\.7z$|\.gz$|\.jq$|\.json$|\.log$|\.md$|\.tar$|\.tgz$|\.tmp$|\.txt$|\.upx$|\.zip$|Baseutils/' --only-file --no-percent-bars --no-colors --ignore_hidden --reverse --number-of-lines 99999999 "${BINDIR}" | tee "${SYSTMP}/x64_Windows_SIZE.txt" | |
rclone copyto "${SYSTMP}/x64_Windows_SIZE.txt" "r2:/bin/x64_Windows/SIZE.txt" --user-agent="$USER_AGENT" --s3-upload-concurrency="500" --s3-chunk-size="100M" --multi-thread-streams="500" --checkers="2000" --transfers="1000" --retries="10" --check-first --checksum --copy-links --fast-list --progress | |
#BLAKE3SUM | |
cd "${BINDIR}" && find "./" -maxdepth 1 -type f | grep -v -E '\.7z$|\.gz$|\.jq$|\.json$|\.log$|\.md$|\.tar$|\.tgz$|\.tmp$|\.txt$|\.upx$|\.zip$|Baseutils/' | sort | xargs b3sum | sed 's/\.\///g' > "${SYSTMP}/x64_Windows_BLAKE3SUM" | |
rclone copyto "${SYSTMP}/x64_Windows_BLAKE3SUM" "r2:/bin/x64_Windows/BLAKE3SUM.txt" --user-agent="$USER_AGENT" --s3-upload-concurrency="500" --s3-chunk-size="100M" --multi-thread-streams="500" --checkers="2000" --transfers="1000" --retries="10" --check-first --checksum --copy-links --fast-list --progress | |
#SHA256SUM | |
cd "${BINDIR}" && find "./" -maxdepth 1 -type f | grep -v -E '\.7z$|\.gz$|\.jq$|\.json$|\.log$|\.md$|\.tar$|\.tgz$|\.tmp$|\.txt$|\.upx$|\.zip$|Baseutils/' | sort | xargs sha256sum | sed 's/\.\///g' > "${SYSTMP}/x64_Windows_SHA256SUM" | |
rclone copyto "${SYSTMP}/x64_Windows_SHA256SUM" "r2:/bin/x64_Windows/SHA256SUM.txt" --user-agent="$USER_AGENT" --s3-upload-concurrency="500" --s3-chunk-size="100M" --multi-thread-streams="500" --checkers="2000" --transfers="1000" --retries="10" --check-first --checksum --copy-links --fast-list --progress | |
##Purge | |
#rm -rf "${BINDIR}" | |
sudo fusermount3 -uz "${BINDIR}" ; df -h | |
continue-on-error: true | |
- name: rClone Enrich Metadata (bin.ajam.dev/x64_Windows) | |
run: | | |
# Presets | |
set +x ; set +e | |
#--------------# | |
##Run | |
chmod +x "$GITHUB_WORKSPACE/main/.github/scripts/x64_Windows/gen_meta.sh" | |
dos2unix --quiet "$GITHUB_WORKSPACE/main/.github/scripts/x64_Windows/gen_meta.sh" | |
bash "$GITHUB_WORKSPACE/main/.github/scripts/x64_Windows/gen_meta.sh" | |
cd "$GITHUB_WORKSPACE/main/.github/scripts/x64_Windows/bins/" && yq ea '[.]' *.yaml > "$GITHUB_WORKSPACE/main/x64_Windows/BINS.yaml" | |
continue-on-error: true | |
#------------------------------------------------------------------------------------# | |
#------------------------------------------------------------------------------------# | |
#------------------------------------------------------------------------------------# | |
#------------------------------------------------------------------------------------# | |
merge-all: | |
runs-on: ubuntu-latest | |
needs: [enrich-arm64v8a-Android, enrich-arm64-Linux, enrich-x86-64-Linux, enrich-x64-Windows] | |
permissions: | |
contents: write | |
steps: | |
- name: Checkout repository | |
uses: actions/checkout@v4 | |
with: | |
path: main | |
filter: "blob:none" #https://github.blog/2020-12-21-get-up-to-speed-with-partial-clone-and-shallow-clone/ | |
- name: Setup Env | |
run: | | |
#presets | |
set +x ; set +e | |
#tmp | |
SYSTMP="$(dirname $(mktemp -u))" && export SYSTMP="$SYSTMP" | |
#GH ENV | |
echo "SYSTMP=$SYSTMP" >> "$GITHUB_ENV" | |
#GH Dirs | |
mkdir -p "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux" | |
mkdir -p "$GITHUB_WORKSPACE/main/arm64_v8a_Android" | |
mkdir -p "$GITHUB_WORKSPACE/main/x64_Windows" | |
mkdir -p "$GITHUB_WORKSPACE/main/x86_64_Linux" | |
#-------------# | |
mkdir -p "$HOME/bin" | |
##Setup rClone | |
mkdir -p "$HOME/.config/rclone" | |
echo "${{ secrets.RCLONE_CF_R2_PUB }}" > "$HOME/.config/rclone/rclone.conf" | |
export RCLONE_STATS="120s" ; echo "RCLONE_STATS=$RCLONE_STATS" >> "$GITHUB_ENV" | |
##User-Agent | |
USER_AGENT="$(curl -qfsSL 'https://pub.ajam.dev/repos/Azathothas/Wordlists/Misc/User-Agents/ua_chrome_macos_latest.txt')" && export USER_AGENT="$USER_AGENT" | |
echo "USER_AGENT=$USER_AGENT" >> "$GITHUB_ENV" | |
continue-on-error: true | |
- name: Install Addons | |
run: | | |
#presets | |
set +x ; set +e | |
#-------------# | |
bash <(curl -qfsSL "https://pub.ajam.dev/repos/Azathothas/Arsenal/misc/Linux/install_dev_tools.sh") | |
continue-on-error: true | |
- name: Merge & Update ("bin.ajam.dev/METADATA.json") | |
run: | | |
#presets | |
set +x ; set +e | |
#-------------# | |
##Fetch | |
#arm64_v8a_Android | |
curl -qfsSL "https://bin.ajam.dev/arm64_v8a_Android/METADATA.json" -o "$SYSTMP/METADATA.json" || echo '{}' > "$SYSTMP/METADATA.json" | |
curl -qfsSL "https://bin.ajam.dev/arm64_v8a_Android/Baseutils/METADATA.json" -o "$SYSTMP/Baseutils.json" || echo '{}' > "$SYSTMP/Baseutils.json" | |
jq -n --slurpfile baseutils "$SYSTMP/Baseutils.json" '{ main: input, baseutils: $baseutils[0] }' "$SYSTMP/METADATA.json" | jq -s . > "$SYSTMP/arm64_v8a_Android.json" | |
#aarch64_arm64_Linux | |
curl -qfsSL "https://bin.ajam.dev/aarch64_arm64_Linux/METADATA.json" -o "$SYSTMP/METADATA.json" || echo '{}' > "$SYSTMP/METADATA.json" | |
curl -qfsSL "https://bin.ajam.dev/aarch64_arm64_Linux/Baseutils/METADATA.json" -o "$SYSTMP/Baseutils.json" || echo '{}' > "$SYSTMP/Baseutils.json" | |
#cat "$SYSTMP/Baseutils.json" | jq -n --argjson baseutils "$(jq -n 'input')" '{ main: input, baseutils: $baseutils }' "$SYSTMP/METADATA.json" | jq -s . > "$SYSTMP/aarch64_arm64_Linux.json" | |
jq -n --slurpfile baseutils "$SYSTMP/Baseutils.json" '{ main: input, baseutils: $baseutils[0] }' "$SYSTMP/METADATA.json" | jq -s . > "$SYSTMP/aarch64_arm64_Linux.json" | |
#x86_64_Linux | |
curl -qfsSL "https://bin.ajam.dev/x86_64_Linux/METADATA.json" -o "$SYSTMP/METADATA.json" || echo '{}' > "$SYSTMP/METADATA.json" | |
curl -qfsSL "https://bin.ajam.dev/x86_64_Linux/Baseutils/METADATA.json" -o "$SYSTMP/Baseutils.json" || echo '{}' > "$SYSTMP/Baseutils.json" | |
#cat "$SYSTMP/Baseutils.json" | jq -n --argjson baseutils "$(jq -n 'input')" '{ main: input, baseutils: $baseutils }' "$SYSTMP/METADATA.json" | jq -s . > "$SYSTMP/x86_64_Linux.json" | |
jq -n --slurpfile baseutils "$SYSTMP/Baseutils.json" '{ main: input, baseutils: $baseutils[0] }' "$SYSTMP/METADATA.json" | jq -s . > "$SYSTMP/x86_64_Linux.json" | |
#x64_Windows | |
curl -qfsSL "https://bin.ajam.dev/x64_Windows/METADATA.json" -o "$SYSTMP/METADATA.json" || echo '{}' > "$SYSTMP/METADATA.json" | |
curl -qfsSL "https://bin.ajam.dev/x64_Windows/Baseutils/METADATA.json" -o "$SYSTMP/Baseutils.json" || echo '{}' > "$SYSTMP/Baseutils.json" | |
jq -n --slurpfile baseutils "$SYSTMP/Baseutils.json" '{ main: input, baseutils: $baseutils[0] }' "$SYSTMP/METADATA.json" | jq -s . > "$SYSTMP/x64_Windows.json" | |
##Merge | |
if jq empty "$SYSTMP/aarch64_arm64_Linux.json" "$SYSTMP/x86_64_Linux.json" "$SYSTMP/arm64_v8a_Android.json" "$SYSTMP/x64_Windows.json" 2>/dev/null; then | |
#jq -s '[{ "host": "aarch64-Linux" } + .[0][]], [{ "host": "x86_64-Linux" } + .[1][]], [{ "host": "arm64-v8a-Android" } + .[2][]], [{ "host": "AMD64-Windows_NT" } + .[3][]]' "$SYSTMP/aarch64_arm64_Linux.json" "$SYSTMP/x86_64_Linux.json" "$SYSTMP/arm64_v8a_Android.json" "$SYSTMP/x64_Windows.json" | jq 'sort_by(.host)' > "$SYSTMP/METADATA.json" | |
jq -s '[{ "host": "aarch64-Linux" } + .[0][], { "host": "x86_64-Linux" } + .[1][], { "host": "arm64-v8a-Android" } + .[2][], { "host": "AMD64-Windows_NT" } + .[3][]]' "$SYSTMP/aarch64_arm64_Linux.json" "$SYSTMP/x86_64_Linux.json" "$SYSTMP/arm64_v8a_Android.json" "$SYSTMP/x64_Windows.json" | jq 'sort_by(.host)' > "$SYSTMP/METADATA.json" | |
if jq empty "$SYSTMP/METADATA.json" 2>/dev/null; then | |
##rClone Update | |
echo -e "\n Updating Metadata.json\n" | |
rclone copyto --checksum "$SYSTMP/METADATA.json" "r2:/bin/METADATA.json" --check-first --checkers 2000 --transfers 1000 | |
##Copy to REPO | |
rsync -av --checksum "$SYSTMP/METADATA.json" "$GITHUB_WORKSPACE/main/METADATA.json" | |
#YAML | |
cat "$GITHUB_WORKSPACE/main/METADATA.json" | yj -jy > "$GITHUB_WORKSPACE/main/METADATA.yaml" | |
fi | |
fi | |
continue-on-error: true | |
- name: Git Pull | |
run: | | |
cd "$GITHUB_WORKSPACE/main" && git pull origin main | |
continue-on-error: true | |
- name: Update Repo | |
run: | | |
#presets | |
set +x ; set +e | |
#-------------# | |
#arm64_v8a_Android | |
rclone copyto --checksum "r2:/bin/arm64_v8a_Android/METADATA.json" "$GITHUB_WORKSPACE/main/arm64_v8a_Android/METADATA.json" --user-agent="$USER_AGENT" --buffer-size="100M" --s3-upload-concurrency="500" --s3-chunk-size="100M" --multi-thread-streams="500" --checkers="2000" --transfers="1000" --check-first --checksum --copy-links --fast-list --progress | |
#curl -qfsSL "https://bin.ajam.dev/arm64_v8a_Android/METADATA.json" -o "$GITHUB_WORKSPACE/main/arm64_v8a_Android/METADATA.json" | |
#aarch64_arm64_Linux | |
rclone copyto --checksum "r2:/bin/aarch64_arm64_Linux/METADATA.json" "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/METADATA.json" --user-agent="$USER_AGENT" --buffer-size="100M" --s3-upload-concurrency="500" --s3-chunk-size="100M" --multi-thread-streams="500" --checkers="2000" --transfers="1000" --check-first --checksum --copy-links --fast-list --progress | |
#curl -qfsSL "https://bin.ajam.dev/aarch64_arm64_Linux/METADATA.json" -o "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/METADATA.json" | |
#x86_64_Linux | |
rclone copyto --checksum "r2:/bin/x86_64_Linux/METADATA.json" "$GITHUB_WORKSPACE/main/x86_64_Linux/METADATA.json" --user-agent="$USER_AGENT" --buffer-size="100M" --s3-upload-concurrency="500" --s3-chunk-size="100M" --multi-thread-streams="500" --checkers="2000" --transfers="1000" --check-first --checksum --copy-links --fast-list --progress | |
#curl -qfsSL "https://bin.ajam.dev/x86_64_Linux/METADATA.json" -o "$GITHUB_WORKSPACE/main/x86_64_Linux/METADATA.json" | |
#x64_Windows | |
rclone copyto --checksum "r2:/bin/x64_Windows/METADATA.json" "$GITHUB_WORKSPACE/main/x64_Windows/METADATA.json" --user-agent="$USER_AGENT" --buffer-size="100M" --s3-upload-concurrency="500" --s3-chunk-size="100M" --multi-thread-streams="500" --checkers="2000" --transfers="1000" --check-first --checksum --copy-links --fast-list --progress | |
continue-on-error: true | |
- name: Generate README (aarch64) | |
run: | | |
#presets | |
set +x ; set +e | |
#-------------# | |
if jq empty "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/METADATA.json" 2>/dev/null; then | |
#Convert to YAML | |
cat "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/METADATA.json" | yj -jy > cat "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/METADATA.yaml" | |
#Gen README | |
cat "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/METADATA.json" | jq -r '.[] | "### \(.name // "" | if . == null or . == "" then "" else . end)\n- **Log:** \(.build_log // "" | if . == null or . == "" then "" else . end)\n- **Script:** \(.build_script // "" | if . == null or . == "" then "" else . end)\n- **Description:** \(.description // "" | if . == null or . == "" then "" else . end)\n- **Note:** \(.note // "" | if . == null or . == "" then "" else . end)\n- **Author:** \(.repo_author // "" | if . == null or . == "" then "" else . end)\n- **WebURL:** [\(.web_url // "" | if . == null or . == "" then "" else . end)](\(.web_url // "" | if . == null or . == "" then "" else . end))\n- **Repository:** [\(.repo_url // "" | if . == null or . == "" then "" else . end)](\(.repo_url // "" | if . == null or . == "" then "" else . end))\n- **Stars:** \(.repo_stars // "" | if . == null or . == "" then "" else . end)\n- **Version:** \(.repo_version // "" | if . == null or . == "" then "" else . end)\n- **Updated:** \(.repo_updated // "" | if . == null or . == "" then "" else . end)\n- **Size:** \(.size // "" | if . == null or . == "" then "" else . end)\n- **SHA256SUM:** \(.sha256 // "" | if . == null or . == "" then "" else . end)\n- **B3SUM:** \(.b3sum // "" | if . == null or . == "" then "" else . end)\n- **Source:** \(.download_url // "" | if . == null or . == "" then "" else . end)\n- **Language:** \(.repo_language // "" | if . == null or . == "" then "" else . end)\n- **License:** \(.repo_license // "" | if . == null or . == "" then "" else . end)\n\n---\n"' > "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/DETAILED.md" | |
{ echo "| Name | Description | Author | WebURL | Repository | Stars | Version | Updated | Size | SHA256SUM | B3SUM | Source | Language | License |" ; echo "| ---- | ----------- | ------ | ------ | ---------- | ----- | ------- | ------- | ---- | --- | ------|------ | -------- | ------- |" ; cat "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/METADATA.json" | jq -r '.[] | "| \(.name // "" | if . == null or . == "" then "" else . end) | \(.description // "" | if . == null or . == "" then "" else . end) | \(.repo_author // "" | if . == null or . == "" then "" else . end) | [\(.web_url // "" | if . == null or . == "" then "" else . end)](\(.web_url // "" | if . == null or . == "" then "" else . end)) | [\(.repo_url // "" | if . == null or . == "" then "" else . end)](\(.repo_url // "" | if . == null or . == "" then "" else . end)) | \(.repo_stars // "" | if . == null or . == "" then "" else . end) | \(.repo_version // "" | if . == null or . == "" then "" else . end) | \(.repo_updated // "" | if . == null or . == "" then "" else . end) | \(.repo_size // "" | if . == null or . == "" then "" else . end) | \(.sha256 // "" | if . == null or . == "" then "" else . end) | \(.b3sum // "" | if . == null or . == "" then "" else . end) | \(.download_url // "" | if . == null or . == "" then "" else . end) | \(.repo_language // "" | if . == null or . == "" then "" else . end) | \(.repo_license // "" | if . == null or . == "" then "" else . end) |"' ; } > "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/README.md" | |
sed 's/||//g' -i "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/README.md" | |
fi | |
continue-on-error: true | |
- name: Generate README (arm64-v8a) | |
run: | | |
#presets | |
set +x ; set +e | |
#-------------# | |
if jq empty "$GITHUB_WORKSPACE/main/arm64_v8a_Android/METADATA.json" 2>/dev/null; then | |
#Convert to YAML | |
cat "$GITHUB_WORKSPACE/main/arm64_v8a_Android/METADATA.json" | yj -jy > cat "$GITHUB_WORKSPACE/main/arm64_v8a_Android/METADATA.yaml" | |
#Gen README | |
cat "$GITHUB_WORKSPACE/main/arm64_v8a_Android/METADATA.json" | jq -r '.[] | "### \(.name // "" | if . == null or . == "" then "" else . end)\n- **Log:** \(.build_log // "" | if . == null or . == "" then "" else . end)\n- **Script:** \(.build_script // "" | if . == null or . == "" then "" else . end)\n- **Description:** \(.description // "" | if . == null or . == "" then "" else . end)\n- **Note:** \(.note // "" | if . == null or . == "" then "" else . end)\n- **Author:** \(.repo_author // "" | if . == null or . == "" then "" else . end)\n- **WebURL:** [\(.web_url // "" | if . == null or . == "" then "" else . end)](\(.web_url // "" | if . == null or . == "" then "" else . end))\n- **Repository:** [\(.repo_url // "" | if . == null or . == "" then "" else . end)](\(.repo_url // "" | if . == null or . == "" then "" else . end))\n- **Stars:** \(.repo_stars // "" | if . == null or . == "" then "" else . end)\n- **Version:** \(.repo_version // "" | if . == null or . == "" then "" else . end)\n- **Updated:** \(.repo_updated // "" | if . == null or . == "" then "" else . end)\n- **Size:** \(.size // "" | if . == null or . == "" then "" else . end)\n- **SHA256SUM:** \(.sha256 // "" | if . == null or . == "" then "" else . end)\n- **B3SUM:** \(.b3sum // "" | if . == null or . == "" then "" else . end)\n- **Source:** \(.download_url // "" | if . == null or . == "" then "" else . end)\n- **Language:** \(.repo_language // "" | if . == null or . == "" then "" else . end)\n- **License:** \(.repo_license // "" | if . == null or . == "" then "" else . end)\n\n---\n"' > "$GITHUB_WORKSPACE/main/arm64_v8a_Android/DETAILED.md" | |
{ echo "| Name | Description | Author | WebURL | Repository | Stars | Version | Updated | Size | SHA256SUM | B3SUM | Source | Language | License |" ; echo "| ---- | ----------- | ------ | ------ | ---------- | ----- | ------- | ------- | ---- | --- | ------|------ | -------- | ------- |" ; cat "$GITHUB_WORKSPACE/main/arm64_v8a_Android/METADATA.json" | jq -r '.[] | "| \(.name // "" | if . == null or . == "" then "" else . end) | \(.description // "" | if . == null or . == "" then "" else . end) | \(.repo_author // "" | if . == null or . == "" then "" else . end) | [\(.web_url // "" | if . == null or . == "" then "" else . end)](\(.web_url // "" | if . == null or . == "" then "" else . end)) | [\(.repo_url // "" | if . == null or . == "" then "" else . end)](\(.repo_url // "" | if . == null or . == "" then "" else . end)) | \(.repo_stars // "" | if . == null or . == "" then "" else . end) | \(.repo_version // "" | if . == null or . == "" then "" else . end) | \(.repo_updated // "" | if . == null or . == "" then "" else . end) | \(.repo_size // "" | if . == null or . == "" then "" else . end) | \(.sha256 // "" | if . == null or . == "" then "" else . end) | \(.b3sum // "" | if . == null or . == "" then "" else . end) | \(.download_url // "" | if . == null or . == "" then "" else . end) | \(.repo_language // "" | if . == null or . == "" then "" else . end) | \(.repo_license // "" | if . == null or . == "" then "" else . end) |"' ; } > "$GITHUB_WORKSPACE/main/arm64_v8a_Android/README.md" | |
sed 's/||//g' -i "$GITHUB_WORKSPACE/main/arm64_v8a_Android/README.md" | |
fi | |
continue-on-error: true | |
- name: Generate README (x86_64) | |
run: | | |
#presets | |
set +x ; set +e | |
#-------------# | |
if jq empty "$GITHUB_WORKSPACE/main/x86_64_Linux/METADATA.json" 2>/dev/null; then | |
#Convert to YAML | |
cat "$GITHUB_WORKSPACE/main/x86_64_Linux/METADATA.json" | yj -jy > cat "$GITHUB_WORKSPACE/main/x86_64_Linux/METADATA.yaml" | |
#Gen README | |
cat "$GITHUB_WORKSPACE/main/x86_64_Linux/METADATA.json" | jq -r '.[] | "### \(.name // "" | if . == null or . == "" then "" else . end)\n- **Log:** \(.build_log // "" | if . == null or . == "" then "" else . end)\n- **Script:** \(.build_script // "" | if . == null or . == "" then "" else . end)\n- **Description:** \(.description // "" | if . == null or . == "" then "" else . end)\n- **Note:** \(.note // "" | if . == null or . == "" then "" else . end)\n- **Author:** \(.repo_author // "" | if . == null or . == "" then "" else . end)\n- **WebURL:** [\(.web_url // "" | if . == null or . == "" then "" else . end)](\(.web_url // "" | if . == null or . == "" then "" else . end))\n- **Repository:** [\(.repo_url // "" | if . == null or . == "" then "" else . end)](\(.repo_url // "" | if . == null or . == "" then "" else . end))\n- **Stars:** \(.repo_stars // "" | if . == null or . == "" then "" else . end)\n- **Version:** \(.repo_version // "" | if . == null or . == "" then "" else . end)\n- **Updated:** \(.repo_updated // "" | if . == null or . == "" then "" else . end)\n- **Size:** \(.size // "" | if . == null or . == "" then "" else . end)\n- **SHA256SUM:** \(.sha256 // "" | if . == null or . == "" then "" else . end)\n- **B3SUM:** \(.b3sum // "" | if . == null or . == "" then "" else . end)\n- **Source:** \(.download_url // "" | if . == null or . == "" then "" else . end)\n- **Language:** \(.repo_language // "" | if . == null or . == "" then "" else . end)\n- **License:** \(.repo_license // "" | if . == null or . == "" then "" else . end)\n\n---\n"' > "$GITHUB_WORKSPACE/main/x86_64_Linux/DETAILED.md" | |
{ echo "| Name | Description | Author | WebURL | Repository | Stars | Version | Updated | Size | SHA256SUM | B3SUM | Source | Language | License |" ; echo "| ---- | ----------- | ------ | ------ | ---------- | ----- | ------- | ------- | ---- | --- | ------|------ | -------- | ------- |" ; cat "$GITHUB_WORKSPACE/main/x86_64_Linux/METADATA.json" | jq -r '.[] | "| \(.name // "" | if . == null or . == "" then "" else . end) | \(.description // "" | if . == null or . == "" then "" else . end) | \(.repo_author // "" | if . == null or . == "" then "" else . end) | [\(.web_url // "" | if . == null or . == "" then "" else . end)](\(.web_url // "" | if . == null or . == "" then "" else . end)) | [\(.repo_url // "" | if . == null or . == "" then "" else . end)](\(.repo_url // "" | if . == null or . == "" then "" else . end)) | \(.repo_stars // "" | if . == null or . == "" then "" else . end) | \(.repo_version // "" | if . == null or . == "" then "" else . end) | \(.repo_updated // "" | if . == null or . == "" then "" else . end) | \(.repo_size // "" | if . == null or . == "" then "" else . end) | \(.sha256 // "" | if . == null or . == "" then "" else . end) | \(.b3sum // "" | if . == null or . == "" then "" else . end) | \(.download_url // "" | if . == null or . == "" then "" else . end) | \(.repo_language // "" | if . == null or . == "" then "" else . end) | \(.repo_license // "" | if . == null or . == "" then "" else . end) |"' ; } > "$GITHUB_WORKSPACE/main/x86_64_Linux/README.md" | |
sed 's/||//g' -i "$GITHUB_WORKSPACE/main/x86_64_Linux/README.md" | |
fi | |
continue-on-error: true | |
- name: Generate README (X64-Windows) | |
run: | | |
#presets | |
set +x ; set +e | |
#-------------# | |
if jq empty "$GITHUB_WORKSPACE/main/x64_Windows/METADATA.json" 2>/dev/null; then | |
#Convert to YAML | |
cat "$GITHUB_WORKSPACE/main/x64_Windows/METADATA.json" | yj -jy > cat "$GITHUB_WORKSPACE/main/x64_Windows/METADATA.yaml" | |
#Gen README | |
cat "$GITHUB_WORKSPACE/main/x64_Windows/METADATA.json" | jq -r '.[] | "### \(.name // "" | if . == null or . == "" then "" else . end)\n- **Log:** \(.build_log // "" | if . == null or . == "" then "" else . end)\n- **Script:** \(.build_script // "" | if . == null or . == "" then "" else . end)\n- **Description:** \(.description // "" | if . == null or . == "" then "" else . end)\n- **Note:** \(.note // "" | if . == null or . == "" then "" else . end)\n- **Author:** \(.repo_author // "" | if . == null or . == "" then "" else . end)\n- **WebURL:** [\(.web_url // "" | if . == null or . == "" then "" else . end)](\(.web_url // "" | if . == null or . == "" then "" else . end))\n- **Repository:** [\(.repo_url // "" | if . == null or . == "" then "" else . end)](\(.repo_url // "" | if . == null or . == "" then "" else . end))\n- **Stars:** \(.repo_stars // "" | if . == null or . == "" then "" else . end)\n- **Version:** \(.repo_version // "" | if . == null or . == "" then "" else . end)\n- **Updated:** \(.repo_updated // "" | if . == null or . == "" then "" else . end)\n- **Size:** \(.size // "" | if . == null or . == "" then "" else . end)\n- **SHA256SUM:** \(.sha256 // "" | if . == null or . == "" then "" else . end)\n- **B3SUM:** \(.b3sum // "" | if . == null or . == "" then "" else . end)\n- **Source:** \(.download_url // "" | if . == null or . == "" then "" else . end)\n- **Language:** \(.repo_language // "" | if . == null or . == "" then "" else . end)\n- **License:** \(.repo_license // "" | if . == null or . == "" then "" else . end)\n\n---\n"' > "$GITHUB_WORKSPACE/main/x64_Windows/DETAILED.md" | |
{ echo "| Name | Description | Author | WebURL | Repository | Stars | Version | Updated | Size | SHA256SUM | B3SUM | Source | Language | License |" ; echo "| ---- | ----------- | ------ | ------ | ---------- | ----- | ------- | ------- | ---- | --- | ------|------ | -------- | ------- |" ; cat "$GITHUB_WORKSPACE/main/x64_Windows/METADATA.json" | jq -r '.[] | "| \(.name // "" | if . == null or . == "" then "" else . end) | \(.description // "" | if . == null or . == "" then "" else . end) | \(.repo_author // "" | if . == null or . == "" then "" else . end) | [\(.web_url // "" | if . == null or . == "" then "" else . end)](\(.web_url // "" | if . == null or . == "" then "" else . end)) | [\(.repo_url // "" | if . == null or . == "" then "" else . end)](\(.repo_url // "" | if . == null or . == "" then "" else . end)) | \(.repo_stars // "" | if . == null or . == "" then "" else . end) | \(.repo_version // "" | if . == null or . == "" then "" else . end) | \(.repo_updated // "" | if . == null or . == "" then "" else . end) | \(.repo_size // "" | if . == null or . == "" then "" else . end) | \(.sha256 // "" | if . == null or . == "" then "" else . end) | \(.b3sum // "" | if . == null or . == "" then "" else . end) | \(.download_url // "" | if . == null or . == "" then "" else . end) | \(.repo_language // "" | if . == null or . == "" then "" else . end) | \(.repo_license // "" | if . == null or . == "" then "" else . end) |"' ; } > "$GITHUB_WORKSPACE/main/x64_Windows/README.md" | |
sed 's/||//g' -i "$GITHUB_WORKSPACE/main/x64_Windows/README.md" | |
fi | |
continue-on-error: true | |
- name: Check Health (arm64_v8a_Android) | |
run: | | |
# Presets | |
set +x ; set +e | |
#--------------# | |
rclone copyto "r2:/bin/arm64_v8a_Android/METADATA.json" "$SYSTMP/METADATA.json" --user-agent="$USER_AGENT" --s3-upload-concurrency="500" --s3-chunk-size="100M" --multi-thread-streams="500" --checkers="2000" --transfers="1000" --retries="10" --check-first --checksum --copy-links --fast-list --progress | |
#Duplicates | |
jq -r 'group_by(.b3sum, .sha256) | map(select(length > 1)) | map( map("Name: \(.name)\nDESCR: \(.description)\nDownload URL: \(.download_url)\nREPO: \(.repo_url)\nWeb URL: \(.web_url)\n"),"---------------------") | flatten[]' "$SYSTMP/METADATA.json" > "$GITHUB_WORKSPACE/main/arm64_v8a_Android/DUPLICATES.txt" | |
#Duplicates (overwrites) | |
yq e '.[] | .bins[]' "$GITHUB_WORKSPACE/main/arm64_v8a_Android/BINS.yaml" | sort | uniq -c | awk '$1 > 1 {print $2}' > "$SYSTMP/duplicate_bins.txt" | |
readarray -t dupes < "$SYSTMP/duplicate_bins.txt" | |
for dupe in "${dupes[@]}"; do | |
names_and_paths=$(yq e ".[] | select(.bins[] == \"$dupe\") | .name + \" (\(.path))\"" "$GITHUB_WORKSPACE/main/arm64_v8a_Android/BINS.yaml" | paste -sd ' ') | |
echo "$dupe --> $names_and_paths" | |
done | tee "$GITHUB_WORKSPACE/main/arm64_v8a_Android/DUPLICATES_OVERWRITES.txt" | |
#Dates | |
jq -r '.[] | "\(.build_date) --> [\(.name) \(.repo_info) (\(.repo_updated))]"' "$SYSTMP/METADATA.json" | sort --numeric-sort > "$GITHUB_WORKSPACE/main/arm64_v8a_Android/BUILD_DATES.txt" | |
continue-on-error: true | |
- name: Check Health (aarch64_arm64_Linux) | |
run: | | |
# Presets | |
set +x ; set +e | |
#--------------# | |
rclone copyto "r2:/bin/aarch64_arm64_Linux/METADATA.json" "$SYSTMP/METADATA.json" --user-agent="$USER_AGENT" --s3-upload-concurrency="500" --s3-chunk-size="100M" --multi-thread-streams="500" --checkers="2000" --transfers="1000" --retries="10" --check-first --checksum --copy-links --fast-list --progress | |
#Duplicates | |
jq -r 'group_by(.b3sum, .sha256) | map(select(length > 1)) | map( map("Name: \(.name)\nDESCR: \(.description)\nDownload URL: \(.download_url)\nREPO: \(.repo_url)\nWeb URL: \(.web_url)\n"),"---------------------") | flatten[]' "$SYSTMP/METADATA.json" > "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/DUPLICATES.txt" | |
#Duplicates (overwrites) | |
yq e '.[] | .bins[]' "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/BINS.yaml" | sort | uniq -c | awk '$1 > 1 {print $2}' > "$SYSTMP/duplicate_bins.txt" | |
readarray -t dupes < "$SYSTMP/duplicate_bins.txt" | |
for dupe in "${dupes[@]}"; do | |
names_and_paths=$(yq e ".[] | select(.bins[] == \"$dupe\") | .name + \" (\(.path))\"" "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/BINS.yaml" | paste -sd ' ') | |
echo "$dupe --> $names_and_paths" | |
done | tee "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/DUPLICATES_OVERWRITES.txt" | |
#Dates | |
jq -r '.[] | "\(.build_date) --> [\(.name) \(.repo_info) (\(.repo_updated))]"' "$SYSTMP/METADATA.json" | sort --numeric-sort > "$GITHUB_WORKSPACE/main/aarch64_arm64_Linux/BUILD_DATES.txt" | |
continue-on-error: true | |
- name: Check Health (x86_64_Linux) | |
run: | | |
# Presets | |
set +x ; set +e | |
#--------------# | |
rclone copyto "r2:/bin/x86_64_Linux/METADATA.json" "$SYSTMP/METADATA.json" --user-agent="$USER_AGENT" --s3-upload-concurrency="500" --s3-chunk-size="100M" --multi-thread-streams="500" --checkers="2000" --transfers="1000" --retries="10" --check-first --checksum --copy-links --fast-list --progress | |
#Duplicates | |
jq -r 'group_by(.b3sum, .sha256) | map(select(length > 1)) | map( map("Name: \(.name)\nDESCR: \(.description)\nDownload URL: \(.download_url)\nREPO: \(.repo_url)\nWeb URL: \(.web_url)\n"),"---------------------") | flatten[]' "$SYSTMP/METADATA.json" > "$GITHUB_WORKSPACE/main/x86_64_Linux/DUPLICATES.txt" | |
#Duplicates (overwrites) | |
yq e '.[] | .bins[]' "$GITHUB_WORKSPACE/main/x86_64_Linux/BINS.yaml" | sort | uniq -c | awk '$1 > 1 {print $2}' > "$SYSTMP/duplicate_bins.txt" | |
readarray -t dupes < "$SYSTMP/duplicate_bins.txt" | |
for dupe in "${dupes[@]}"; do | |
names_and_paths=$(yq e ".[] | select(.bins[] == \"$dupe\") | .name + \" (\(.path))\"" "$GITHUB_WORKSPACE/main/x86_64_Linux/BINS.yaml" | paste -sd ' ') | |
echo "$dupe --> $names_and_paths" | |
done | tee "$GITHUB_WORKSPACE/main/x86_64_Linux/DUPLICATES_OVERWRITES.txt" | |
#Dates | |
jq -r '.[] | "\(.build_date) --> [\(.name) \(.repo_info) (\(.repo_updated))]"' "$SYSTMP/METADATA.json" | sort --numeric-sort > "$GITHUB_WORKSPACE/main/x86_64_Linux/BUILD_DATES.txt" | |
continue-on-error: true | |
- name: Check Health (x64_Windows) | |
run: | | |
# Presets | |
set +x ; set +e | |
#--------------# | |
rclone copyto "r2:/bin/x64_Windows/METADATA.json" "$SYSTMP/METADATA.json" --user-agent="$USER_AGENT" --s3-upload-concurrency="500" --s3-chunk-size="100M" --multi-thread-streams="500" --checkers="2000" --transfers="1000" --retries="10" --check-first --checksum --copy-links --fast-list --progress | |
#Duplicates | |
jq -r 'group_by(.b3sum, .sha256) | map(select(length > 1)) | map( map("Name: \(.name)\nDESCR: \(.description)\nDownload URL: \(.download_url)\nREPO: \(.repo_url)\nWeb URL: \(.web_url)\n"),"---------------------") | flatten[]' "$SYSTMP/METADATA.json" > "$GITHUB_WORKSPACE/main/x64_Windows/DUPLICATES.txt" | |
#Duplicates (overwrites) | |
yq e '.[] | .bins[]' "$GITHUB_WORKSPACE/main/x64_Windows/BINS.yaml" | sort | uniq -c | awk '$1 > 1 {print $2}' > "$SYSTMP/duplicate_bins.txt" | |
readarray -t dupes < "$SYSTMP/duplicate_bins.txt" | |
for dupe in "${dupes[@]}"; do | |
names_and_paths=$(yq e ".[] | select(.bins[] == \"$dupe\") | .name + \" (\(.path))\"" "$GITHUB_WORKSPACE/main/x64_Windows/BINS.yaml" | paste -sd ' ') | |
echo "$dupe --> $names_and_paths" | |
done | tee "$GITHUB_WORKSPACE/main/x64_Windows/DUPLICATES_OVERWRITES.txt" | |
#Dates | |
jq -r '.[] | "\(.build_date) --> [\(.name) \(.repo_info) (\(.repo_updated))]"' "$SYSTMP/METADATA.json" | sort --numeric-sort > "$GITHUB_WORKSPACE/main/x64_Windows/BUILD_DATES.txt" | |
continue-on-error: true | |
- name: Get DateTime | |
run: | | |
# Date Time | |
NEPALI_TIME=$(TZ='Asia/Kathmandu' date +'%Y-%m-%d (%I:%M:%S %p)') | |
echo "NEPALI_TIME=$NEPALI_TIME" >> $GITHUB_ENV | |
continue-on-error: true | |
- uses: stefanzweifel/git-auto-commit-action@v5 | |
with: | |
repository: ./main | |
commit_user_name: Azathothas # defaults to "github-actions[bot]" | |
commit_user_email: [email protected] # defaults to "41898282+github-actions[bot]@users.noreply.github.com" | |
commit_message: "✅ Updated (Metadata) 🛍️ <-- ${{ env.NEPALI_TIME }} ⌚" | |
#push_options: '--force' | |
continue-on-error: true | |
#------------------------------------------------------------------------------------# |