From ca91c5b774f3b52a970c7f165bd65533e8864c16 Mon Sep 17 00:00:00 2001 From: Christian Rauch Date: Thu, 15 Feb 2024 22:37:51 +0100 Subject: [PATCH 1/8] update actions/setup-python --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5e31e6da..ccbb2519 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -15,7 +15,7 @@ jobs: steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: "3.x" From 57222fba71912b91cd62eda9772541fe05ad15f9 Mon Sep 17 00:00:00 2001 From: Christian Rauch Date: Thu, 15 Feb 2024 22:47:31 +0100 Subject: [PATCH 2/8] update conda-incubator/setup-miniconda --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ccbb2519..1c3ac820 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -19,7 +19,7 @@ jobs: with: python-version: "3.x" - - uses: conda-incubator/setup-miniconda@v2 + - uses: conda-incubator/setup-miniconda@v3 if: runner.os == 'Windows' - name: install Microsoft Visual C++ (Windows) From 0a439d7d43329479e5c1c2375533127bd6f60f77 Mon Sep 17 00:00:00 2001 From: Christian Rauch Date: Sun, 31 Dec 2023 22:27:47 +0100 Subject: [PATCH 3/8] update upload-artifact and download-artifact --- .github/workflows/main.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 1c3ac820..944a09df 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -92,9 +92,9 @@ jobs: file ./build/ROSProjectManager-*-*-*.zip - name: upload artifact - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: - name: plugin_archive_artifact + name: plugin_archive_artifact_${{ matrix.config.name }} if-no-files-found: error path: | ./build/ROSProjectManager-*-*-*.zip @@ -106,9 +106,10 @@ jobs: needs: build steps: - name: download artifact - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: - name: plugin_archive_artifact + pattern: plugin_archive_artifact_* + merge-multiple: true path: ./ - name: create release From 325eddaa56b0eec711dfa58004cefb9d96b12e42 Mon Sep 17 00:00:00 2001 From: Christian Rauch Date: Fri, 16 Feb 2024 20:02:51 +0100 Subject: [PATCH 4/8] remove conda-incubator/setup-miniconda --- .github/workflows/main.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 944a09df..be83f23f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -19,9 +19,6 @@ jobs: with: python-version: "3.x" - - uses: conda-incubator/setup-miniconda@v3 - if: runner.os == 'Windows' - - name: install Microsoft Visual C++ (Windows) uses: ilammy/msvc-dev-cmd@v1 if: runner.os == 'Windows' From eece9bff22f5df31e667c95551b404584177ac77 Mon Sep 17 00:00:00 2001 From: Christian Rauch Date: Fri, 16 Feb 2024 23:50:03 +0100 Subject: [PATCH 5/8] set the GET response timeout of 30 minutes --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index cb0ac94c..c034679d 100755 --- a/setup.py +++ b/setup.py @@ -34,7 +34,7 @@ "x64": "64"} def download_check_fail(url, expected_type): - response = requests.get(url, allow_redirects=True) + response = requests.get(url, allow_redirects=True, timeout=1800) if not response.ok: raise RuntimeError("error retrieving "+response.url) if response.headers.get('content-type') != expected_type: From b4ae4f0ab867d4b487c0873c023761202e6d0c06 Mon Sep 17 00:00:00 2001 From: Christian Rauch Date: Sat, 17 Feb 2024 00:35:54 +0100 Subject: [PATCH 6/8] flush download output --- setup.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/setup.py b/setup.py index c034679d..de2612a3 100755 --- a/setup.py +++ b/setup.py @@ -34,12 +34,13 @@ "x64": "64"} def download_check_fail(url, expected_type): + print("download URL:", url, flush=True) response = requests.get(url, allow_redirects=True, timeout=1800) if not response.ok: raise RuntimeError("error retrieving "+response.url) if response.headers.get('content-type') != expected_type: print("Warning: invalid content type, expected '{}', got '{}'".format( - expected_type, response.headers.get('content-type'))) + expected_type, response.headers.get('content-type')), flush=True) return response def read_downloadable_archives(package): @@ -86,8 +87,6 @@ def qtc_download_check_extract(cfg, dir_install): for archive_name in archive_names: url_archive = base_url+"/"+archive_name - print("download", url_archive) - content = download_check_fail(url_archive, "application/x-7z-compressed").content if md5sums[archive_name] != hashlib.md5(content).hexdigest(): @@ -162,8 +161,6 @@ def qt_download_check_extract(cfg, dir_install): for package_name, package_version, archive_name in archives_match: url_archive = base_url+'/'+package_name+'/'+package_version+archive_name - print("download", url_archive) - content = download_check_fail(url_archive, "application/x-7z-compressed").content sha1sum = download_check_fail(url_archive+".sha1", "application/x-7z-compressed").text From 322d3a21c647f64166b7b18fd4c3d70c898f910c Mon Sep 17 00:00:00 2001 From: Christian Rauch Date: Sat, 17 Feb 2024 01:06:36 +0100 Subject: [PATCH 7/8] install tqdm --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index be83f23f..aa0f7cb6 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -55,7 +55,7 @@ jobs: - name: install Qt and Qt Creator shell: bash run: | - pip install pyyaml requests py7zr + pip install pyyaml requests py7zr tqdm_loggable python setup.py --export_variables cat env >> $GITHUB_ENV From c6620592f181ead767382450583b9d0485f37a5b Mon Sep 17 00:00:00 2001 From: Christian Rauch Date: Sat, 17 Feb 2024 01:06:01 +0100 Subject: [PATCH 8/8] show extraction progress --- setup.py | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index de2612a3..c16a8285 100755 --- a/setup.py +++ b/setup.py @@ -11,6 +11,8 @@ import py7zr import requests import yaml +from tqdm_loggable.auto import tqdm + url_repo_qtc_fmt = "https://download.qt.io/{release_type}_releases/qtcreator/{qtcv_maj}/{qtcv_full}/installer_source/{os}_{arch}/" @@ -47,6 +49,27 @@ def read_downloadable_archives(package): archive_names = io.StringIO(package.find("DownloadableArchives").text) return list(csv.reader(archive_names, delimiter=',', skipinitialspace=True))[0] +def extract_progress(archive_bytes, archive_name, destination_path): + class ExtractProgressBar(py7zr.callbacks.ExtractCallback, tqdm): + def __init__(self, *args, total_bytes, **kwargs): + super().__init__(self, *args, total=total_bytes, **kwargs) + def report_start_preparation(self): + pass + def report_start(self, processing_file_path, processing_bytes): + pass + def report_end(self, processing_file_path, wrote_bytes): + self.update(int(wrote_bytes)) + def report_postprocess(self): + self.update(int(self.total)) + def report_warning(self, message): + pass + + with py7zr.SevenZipFile(io.BytesIO(archive_bytes)) as zf: + with ExtractProgressBar(unit='B', unit_scale=True, miniters=1, + total_bytes=sum([f.uncompressed for f in zf.files]), + desc=archive_name) as cb_progress: + zf.extractall(path=destination_path, callback=cb_progress) + def qtc_download_check_extract(cfg, dir_install): # if the Qt Creator version contains '-beta' or '-rc' we have to # use the "development" repo, otherwise use the "official" repo @@ -92,7 +115,7 @@ def qtc_download_check_extract(cfg, dir_install): if md5sums[archive_name] != hashlib.md5(content).hexdigest(): raise RuntimeError(archive_name+" MD5 hash sum does not match") - py7zr.SevenZipFile(io.BytesIO(content)).extractall(dir_install_qt) + extract_progress(content, archive_name, dir_install_qt) if cfg['os'] == "Darwin": dir_install_qt = os.path.join(dir_install_qt, "Qt Creator.app", "Contents", "Resources") @@ -168,7 +191,7 @@ def qt_download_check_extract(cfg, dir_install): if sha1sum != hashlib.sha1(content).hexdigest(): raise RuntimeError(archive_name+" SHA1 hash sum does not match") - py7zr.SevenZipFile(io.BytesIO(content)).extractall(dir_install) + extract_progress(content, archive_name, dir_install) qt_path = os.path.join(dir_install, "{}.{}.0".format(ver_maj, ver_min)) qt_archs = os.listdir(qt_path)