diff --git a/build-driver/build.py b/build-driver/build.py index 3baa2983..6003787f 100755 --- a/build-driver/build.py +++ b/build-driver/build.py @@ -30,7 +30,8 @@ class JobProperties: version: str release_name: str grml_name: str - isoname: str + iso_name: str # name of the resulting .ISO files + sources_name: str # name of the resulting sources tarball def usage(program_name): @@ -95,7 +96,7 @@ def run_grml_live( version: str, release_name: str, grml_name: str, - isoname: str, + iso_name: str, old_iso_path: Path | None, ): env = dict(os.environ) @@ -134,7 +135,7 @@ def run_grml_live( "-g", grml_name, "-i", - isoname, + iso_name, "-o", output_dir, ] @@ -223,7 +224,7 @@ def download_file(url: str, local_path: Path): run_x(["curl", "-#fSL", "--output", local_path, url]) -def should_skip_sources(build_config: dict, env: dict) -> bool: +def skip_sources_requested(build_config: dict, env: dict) -> bool: if env.get("SKIP_SOURCES", "") == "1": return True if build_config.get("skip_sources", False) is True: @@ -265,7 +266,7 @@ def build( job_properties.version, job_properties.release_name, job_properties.grml_name, - job_properties.isoname, + job_properties.iso_name, old_iso_path, ) @@ -349,10 +350,7 @@ def download_old_dpkg_list_last_release(tmp_dir: Path, last_release_version: str return None -def download_old_iso(tmp_dir: Path, old_iso_url: str | None) -> Path | None: - if old_iso_url is None: - return None - +def download_old_iso(tmp_dir: Path, old_iso_url: str) -> Path | None: path = tmp_dir / "old.iso" with ci_section(f"Downloading old ISO {old_iso_url} to {path!s}"): @@ -361,6 +359,22 @@ def download_old_iso(tmp_dir: Path, old_iso_url: str | None) -> Path | None: return path +def download_old_sources(tmp_dir: Path, old_iso_url: str) -> Path | None: + path = tmp_dir / "old-sources.tar" + + # https://.../2024-12-18_10_03_44/grml_isos/grml...iso + # => https://.../2024-12-18_10_03_44/ , _, grml...iso + old_base_url, _, old_iso_name = old_iso_url.rsplit("/", 2) + # grml-something.iso => grml-something-sources.tar + old_sources_name = old_iso_name.rsplit(".", 1)[0] + "-sources.tar" + old_sources_url = f"{old_base_url}/{old_sources_name}" + + with ci_section(f"Downloading old Sources {old_sources_url} to {path!s}"): + download_file(old_sources_url, path) + + return path + + def main(program_name: str, argv: list[str]) -> int: print(f"I: {program_name} started with {argv=}") try: @@ -398,8 +412,12 @@ def main(program_name: str, argv: list[str]) -> int: build_config = load_config(build_config_file) - skip_sources = should_skip_sources(build_config, dict(os.environ)) - classes = get_grml_live_classes(arch, flavor, classes_for_mode, skip_sources) + skip_sources = skip_sources_requested(build_config, dict(os.environ)) + # skip SOURCES in release mode as grml-live would re-download all sources, + # possibly mismatching the versions. Also we do not prepare a working DNS, + # so it would just fail. In the future, grml-live should support reusing + # the sources tarball and fetching just the necessary differences. + classes = get_grml_live_classes(arch, flavor, classes_for_mode, skip_sources or build_mode == "release") build_grml_name = f"grml-{flavor}-{arch}" last_release_version = build_config["last_release"] @@ -414,6 +432,7 @@ def main(program_name: str, argv: list[str]) -> int: if build_mode == "release": old_iso_url = build_config["base_iso"][flavor][arch] build_version = build_config["release_version"] + artifact_basename = f"grml-{flavor}-{build_version}-{arch}" job_properties = JobProperties( job_timestamp=datetime.datetime.now(), @@ -427,7 +446,8 @@ def main(program_name: str, argv: list[str]) -> int: # f.e. "Glumpad Grumbirn" release_name=build_config["release_name"], grml_name=build_grml_name, - isoname=f"grml-{flavor}-{build_version}-{arch}.iso", + iso_name=f"{artifact_basename}.iso", + sources_name=f"{artifact_basename}-sources.tar", ) elif build_mode == "daily": @@ -436,6 +456,7 @@ def main(program_name: str, argv: list[str]) -> int: CI_PIPELINE_IID = os.getenv("CI_PIPELINE_IID", "0") build_version = f"d{date_stamp}b{CI_PIPELINE_IID}" build_release_name = f"daily{date_stamp}build{CI_PIPELINE_IID}{debian_suite}" + artifact_basename = f"grml-{flavor}-{build_release_name}-{arch}" job_properties = JobProperties( job_timestamp=datetime.datetime.now(), @@ -446,7 +467,8 @@ def main(program_name: str, argv: list[str]) -> int: version=build_version, release_name=build_release_name, grml_name=build_grml_name, - isoname=f"grml-{flavor}-{build_release_name}-{arch}.iso", + iso_name=f"{artifact_basename}.iso", + sources_name=f"{artifact_basename}-sources.tar", ) else: @@ -474,7 +496,14 @@ def main(program_name: str, argv: list[str]) -> int: old_dpkg_list_previous_build = cache_dir / "dpkg.list" old_dpkg_list_last_release = download_old_dpkg_list_last_release(tmp_dir, last_release_version, flavor) - old_iso_path = download_old_iso(tmp_dir, old_iso_url) + if old_iso_url is None: + old_iso_path = None + else: + old_iso_path = download_old_iso(tmp_dir, old_iso_url) + if skip_sources or old_iso_url is None: + old_sources_path = None + else: + old_sources_path = download_old_sources(tmp_dir, old_iso_url) with results_mover(build_dir, output_dir): build( @@ -492,6 +521,9 @@ def main(program_name: str, argv: list[str]) -> int: old_dpkg_list_previous_build.parent.mkdir(exist_ok=True) shutil.copyfile(new_dpkg_list, old_dpkg_list_previous_build) + if old_sources_path: + old_sources_path.rename(build_dir / job_properties.sources_name) + print("I: Success.") return 0