From d54238884edc54c379148aca21415ed05a96f3f1 Mon Sep 17 00:00:00 2001 From: vmagro Date: Wed, 21 Feb 2024 10:24:53 -0800 Subject: [PATCH] [antlir] delete more unused code Summary: Seriously, never ending Test Plan: waitforsandcastle Differential Revision: D54001074 --- antlir/bzl/container_opts.bzl | 23 ---- antlir/bzl/container_opts.shape.bzl | 15 -- antlir/bzl/image_layer_alias.bzl | 43 ------ antlir/bzl/tests/ImageCppUnittestTest.cpp | 28 ---- antlir/bzl/tests/ImageHostnameTest.cpp | 16 --- antlir/bzl/tests/check_rpm_names.py | 32 ----- antlir/bzl/tests/coverage_test_helper.py | 11 -- antlir/bzl/tests/image_rust_unittest.rs | 14 -- antlir/bzl/tests/rpms_with_reason.py | 128 ------------------ antlir/bzl/tests/test_hoist.py | 74 ---------- antlir/bzl/tests/test_image_hostname.py | 14 -- .../bzl/tests/test_image_python_unittest.py | 97 ------------- .../tests/test_image_unittest_repo_server.py | 36 ----- antlir/bzl/tests/test_unsanitized_env.py | 16 --- 14 files changed, 547 deletions(-) delete mode 100644 antlir/bzl/container_opts.bzl delete mode 100644 antlir/bzl/container_opts.shape.bzl delete mode 100644 antlir/bzl/image_layer_alias.bzl delete mode 100644 antlir/bzl/tests/ImageCppUnittestTest.cpp delete mode 100644 antlir/bzl/tests/ImageHostnameTest.cpp delete mode 100644 antlir/bzl/tests/check_rpm_names.py delete mode 100644 antlir/bzl/tests/coverage_test_helper.py delete mode 100644 antlir/bzl/tests/image_rust_unittest.rs delete mode 100644 antlir/bzl/tests/rpms_with_reason.py delete mode 100644 antlir/bzl/tests/test_hoist.py delete mode 100644 antlir/bzl/tests/test_image_hostname.py delete mode 100644 antlir/bzl/tests/test_image_python_unittest.py delete mode 100644 antlir/bzl/tests/test_image_unittest_repo_server.py delete mode 100644 antlir/bzl/tests/test_unsanitized_env.py diff --git a/antlir/bzl/container_opts.bzl b/antlir/bzl/container_opts.bzl deleted file mode 100644 index 2a207b90f21..00000000000 --- a/antlir/bzl/container_opts.bzl +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -load("@bazel_skylib//lib:types.bzl", "types") -load("//antlir/bzl:shape.bzl", "shape") -load(":container_opts.shape.bzl", "container_opts_t") -load(":structs.bzl", "structs") - -def _new_container_opts_t(**kwargs): - return container_opts_t( - **kwargs - ) - -def normalize_container_opts(container_opts): - if not container_opts: - container_opts = {} - if types.is_dict(container_opts): - return _new_container_opts_t(**container_opts) - if shape.is_instance(container_opts, container_opts_t): - return _new_container_opts_t(**shape.as_serializable_dict(container_opts)) - return _new_container_opts_t(**structs.to_dict(container_opts)) diff --git a/antlir/bzl/container_opts.shape.bzl b/antlir/bzl/container_opts.shape.bzl deleted file mode 100644 index a1a3b686f1a..00000000000 --- a/antlir/bzl/container_opts.shape.bzl +++ /dev/null @@ -1,15 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -load("//antlir/bzl:shape.bzl", "shape") - -# -# Prefer to keep this default-initializable to avoid having to update a -# bunch of tests and other Python callsites. -container_opts_t = shape.shape( - # Future: move `boot` in here, too. - boot_await_dbus = shape.field(bool, default = True), - boot_await_system_running = shape.field(bool, default = False), -) diff --git a/antlir/bzl/image_layer_alias.bzl b/antlir/bzl/image_layer_alias.bzl deleted file mode 100644 index 81d1c9646c4..00000000000 --- a/antlir/bzl/image_layer_alias.bzl +++ /dev/null @@ -1,43 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -load("//antlir/bzl:build_defs.bzl", "alias") -load(":antlir2_shim.bzl", "antlir2_shim") - -""" -USE WITH CARE -- this was added to aid in implementing `released_layer`. -Most people should avoid aliases, and should instead use an absolute target -path to refer to the original layer. If you think you need this, talk to -the `antlir` team. - -The output of this target is meant to be indistinguishable from the source -layer. Both Buck outputs share the same `buck-image-out` subvolume, -minimizing space usage. This target increments the refcount of the -subvolume in `buck-image-out`, ensuring that it will live on as long any -single reference exists. - -Take care: the `mountconfig.json` field `build_source` will point at the -ORIGINAL layer, which can be unexpected for the consumer. At present, I see -no reason to rewrite this configuration, but this can be revised in the -future. - -Of course, there are some differences between the targets from the point of -view of Buck: - - They have different paths -- that's the point! - - They may have different visibility settings. - - Their "type" attribute will differ. -""" - -def image_layer_alias(name, layer, runtime = None, visibility = None, antlir2 = None): - visibility = visibility or [] - - if antlir2_shim.upgrade_or_shadow_layer( - antlir2 = antlir2, - name = name, - fn = alias, - actual = layer, - visibility = visibility, - ) != "upgrade": - fail("antlir1 is dead") diff --git a/antlir/bzl/tests/ImageCppUnittestTest.cpp b/antlir/bzl/tests/ImageCppUnittestTest.cpp deleted file mode 100644 index fb6b1127a0a..00000000000 --- a/antlir/bzl/tests/ImageCppUnittestTest.cpp +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) Meta Platforms, Inc. and affiliates. - * - * This source code is licensed under the MIT license found in the - * LICENSE file in the root directory of this source tree. - */ - -#include -#include -#include -#include -#include - -TEST(ImageCppUnittest, TestContainer) { - ASSERT_STREQ("nobody", ::getenv("USER")); - // Future: add more assertions about the container setup as needed -} - -TEST(ImageCppUnittest, TestEnv) { - // Ensure that per-test `env` settings do reach the container. - ASSERT_STREQ("meow", std::getenv("kitteh")); - // Ensure that the container's environment is sanitized. - // - // Unlike the Python test, we don't check the environment against an - // allowlist, but only because it's considerably more hassle to figure out - // how to do this in GTest. - ASSERT_EQ(nullptr, ::getenv("BUCK_BUILD_ID")); -} diff --git a/antlir/bzl/tests/ImageHostnameTest.cpp b/antlir/bzl/tests/ImageHostnameTest.cpp deleted file mode 100644 index 7b7edb942bf..00000000000 --- a/antlir/bzl/tests/ImageHostnameTest.cpp +++ /dev/null @@ -1,16 +0,0 @@ -/* - * Copyright (c) Meta Platforms, Inc. and affiliates. - * - * This source code is licensed under the MIT license found in the - * LICENSE file in the root directory of this source tree. - */ - -#include -#include - -TEST(ImageHostname, TestContainer) { - // Ensure the hostname configuration was propagated inside the container - std::array hostname; - gethostname(hostname.data(), sizeof(hostname)); - ASSERT_STREQ("test-hostname.com", hostname.data()); -} diff --git a/antlir/bzl/tests/check_rpm_names.py b/antlir/bzl/tests/check_rpm_names.py deleted file mode 100644 index 4e9c5170935..00000000000 --- a/antlir/bzl/tests/check_rpm_names.py +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import importlib.resources -import subprocess -from typing import Set - - -def get_rpms() -> Set[str]: - return { - rpm - for rpm in subprocess.check_output( - ["rpm", "-qa", "--queryformat", "%{NAME}\n"], - text=True, - ).split("\n") - if rpm - } - - -def check_rpm_names(test_case, package, resource: str) -> None: - expected = { - # `rpms-with-reason` adds a TAB-separated reason to the RPM name - s.split("\t")[0].strip() - for s in importlib.resources.read_text(package, resource).splitlines() - } - test_case.assertEqual( - expected, - get_rpms(), - ) diff --git a/antlir/bzl/tests/coverage_test_helper.py b/antlir/bzl/tests/coverage_test_helper.py deleted file mode 100644 index 1987d2d7650..00000000000 --- a/antlir/bzl/tests/coverage_test_helper.py +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - - -def coverage_test_helper() -> int: - one = 10 + 3 - 12 - print("This function should be called {one} time to achieve 100% coverage") - return one diff --git a/antlir/bzl/tests/image_rust_unittest.rs b/antlir/bzl/tests/image_rust_unittest.rs deleted file mode 100644 index 52dba80f4d9..00000000000 --- a/antlir/bzl/tests/image_rust_unittest.rs +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright (c) Meta Platforms, Inc. and affiliates. - * - * This source code is licensed under the MIT license found in the - * LICENSE file in the root directory of this source tree. - */ - -use std::env; - -#[test] -fn test_env() { - assert_eq!(env::var("kitteh").unwrap(), "meow"); - assert_eq!(env::var("dogsgo").unwrap(), "woof"); -} diff --git a/antlir/bzl/tests/rpms_with_reason.py b/antlir/bzl/tests/rpms_with_reason.py deleted file mode 100644 index 24974b3ad52..00000000000 --- a/antlir/bzl/tests/rpms_with_reason.py +++ /dev/null @@ -1,128 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -""" -This tool helps control package bloat in your image layers. - -Usage: - - image.layer( - name = "your-layer-rpms-with-reason", - parent_layer = ":your-layer", - features = [feature.install_buck_runnable( - "//antlir/bzl/tests:rpms-with-reason", - "/rpms-with-reason", - )], - ) - command_alias( - name = "print-your-layer-rpms", - exe = "//antlir/nspawn_in_subvol:run", - args = [ - "--layer=$(location :base.c8.rc-rpms-with-reason)", - "--user=root", - "--", - "/rpms-with-reason", - # Pro-tip: put these in a list variable and share it with the - # `feature.rpms_install` feature making your layer. - "WANTED-RPM1", - "WANTED-RPM2", - ], - ) - - buck run :print-your-layer-rpms > my-rpm-list - -This outputs a `image.test_rpm_names`-compatible list of RPMs installed in -`:your-layer`, one per line, with a TAB-separated annotation showing whether -this RPM is in one of these categories: - - a dependency of a protected RPM (e.g. `systemd`) - - a dependency of a wanted RPM you specified on the command-line - - a wanted RPM - - none of the above -- marked "NOT REQUIRED" - -The annotation for removable packages also estimates the container size -savings from removing that package, its dependents, and its unused -dependencies. Caveat: this assumes the removal of the dependencies you -marked as WANTED. - -Future: It would be pretty easy to let this derive wanted RPMs from -`feature` targets. -""" - -import re -import subprocess -from concurrent.futures import ThreadPoolExecutor -from functools import partial - - -def fmt_required_by(rpm, wanted, required_by, cost=None) -> str: - notes = [] - if rpm in wanted: - notes.append("wanted") - required_by = required_by - {rpm} - if required_by: - notes.append("required by: " + " ".join(sorted(required_by))) - if not notes: - notes.append("NOT REQUIRED") - elif cost: - notes.append("remove to free: " + cost) - return f"{rpm}\t{'; '.join(notes)}" - - -def fmt_single_rpm(rpm: str, wanted) -> str: - if rpm == "gpg-pubkey": - return f"{rpm}\tfor RPM signature checking" - - p = subprocess.run( - ["dnf", "remove", "--assumeno", rpm], - stderr=subprocess.PIPE, - stdout=subprocess.PIPE, - text=True, - check=False, - ) - m = re.search(" removing the following protected packages: (.*)\n", p.stderr) - if m: - return fmt_required_by(rpm, wanted, {p for p in m.group(1).split() if p}) - - if re.search("\nRemove +[0-9]+ Packages?\n", p.stdout): - removed = [l.split()[0] for l in p.stdout.split("\n") if l.startswith(" ")] - assert "Package" == removed[0], removed[0] - removed = removed[1:] - return fmt_required_by( - rpm, - wanted, - wanted.intersection(removed), - # pyre-fixme[16]: Optional type has no attribute `group`. - cost=re.search("\nFreed space: (.*)\n", p.stdout).group(1), - ) - - raise AssertionError(p) - - -def print_rpms_with_reason(wanted_rpms) -> None: - wanted = set(wanted_rpms) - # Bug alert: I was too lazy to make this handle packages that exist in - # both i686 & x86_64 architectures, but it should be OK for our usage. - rpms = ( - subprocess.check_output( - ["rpm", "-qa", "--queryformat", "%{NAME}\n"], - text=True, - ) - .strip() - .split("\n") - ) - with ThreadPoolExecutor() as executor: - lines = executor.map(partial(fmt_single_rpm, wanted=wanted), sorted(set(rpms))) - for line in lines: - print(line) - - -def main() -> None: - import sys - - print_rpms_with_reason(sys.argv[1:]) - - -if __name__ == "__main__": - main() diff --git a/antlir/bzl/tests/test_hoist.py b/antlir/bzl/tests/test_hoist.py deleted file mode 100644 index bde0c3f6972..00000000000 --- a/antlir/bzl/tests/test_hoist.py +++ /dev/null @@ -1,74 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import importlib.resources -import os -import unittest - - -def fs_tree(path): - root = {} - for (dirpath, _, filenames) in os.walk(path): - parts = os.path.relpath(dirpath, path).split(os.sep) - - node = root - for p in parts: - node = node.setdefault(p, {}) - - for fn in filenames: - node[fn] = None - return root - - -class HoistTest(unittest.TestCase): - def test_simple_file(self): - with importlib.resources.path(__package__, "test_simple_file") as path: - self.assertTrue(os.path.isfile(path), "hoist file missing") - - def test_out_file(self): - with importlib.resources.path(__package__, "test_out_file") as rpath: - self.assertTrue(os.path.isdir(rpath), "hoist root is not a folder") - - ref = { - ".": { - "file1": None, - }, - } - self.assertEqual(fs_tree(rpath), ref, "wrong hoist output") - - def test_simple_folder(self): - with importlib.resources.path(__package__, "test_simple_folder") as rpath: - ref = { - ".": { - "file1.rpm": None, - "file2": None, - }, - } - self.assertEqual(fs_tree(rpath), ref, "wrong hoist output") - - def test_simple_selector(self): - with importlib.resources.path(__package__, "test_simple_selector") as rpath: - ref = { - ".": { - "file1": None, - "file2.rpm": None, - }, - "folder1": { - "file1.rpm": None, - "file2": None, - }, - } - self.assertEqual(fs_tree(rpath), ref, "wrong hoist output") - - def test_selector_flat(self): - with importlib.resources.path(__package__, "test_selector_flat") as rpath: - ref = { - ".": { - "file1.rpm": None, - "file2.rpm": None, - }, - } - self.assertEqual(fs_tree(rpath), ref, "wrong hoist output") diff --git a/antlir/bzl/tests/test_image_hostname.py b/antlir/bzl/tests/test_image_hostname.py deleted file mode 100644 index e71fb2c1f1d..00000000000 --- a/antlir/bzl/tests/test_image_hostname.py +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import socket -import unittest - - -class ImagePythonUnittestTest(unittest.TestCase): - def test_container(self) -> None: - # Ensure the hostname configuration was propagated inside the container - self.assertEqual("test-hostname.com", socket.gethostname()) diff --git a/antlir/bzl/tests/test_image_python_unittest.py b/antlir/bzl/tests/test_image_python_unittest.py deleted file mode 100644 index 11d4fa94944..00000000000 --- a/antlir/bzl/tests/test_image_python_unittest.py +++ /dev/null @@ -1,97 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import getpass -import os -import subprocess -import unittest - -from antlir.bzl.tests.coverage_test_helper import coverage_test_helper - - -class ImagePythonUnittestTest(unittest.TestCase): - def test_container(self) -> None: - # This should cause our 100% coverage assertion to pass. - coverage_test_helper() - self.assertEqual("nobody", getpass.getuser()) - # Test `internal_only_logs_tmpfs`: container /logs should be writable - with open("/logs/garfield", "w") as catlog: - catlog.write("Feed me.") - # Future: add more assertions here as it becomes necessary what - # aspects of test containers we actually care about. - - def test_env(self) -> None: - # Ensure that per-test `env` settings do reach the container. - self.assertEqual("meow", os.environ.pop("kitteh")) - # Ensure that the container's environment is sanitized. - env_allowlist = { - # Antlir internals - "ANTLIR_CONTAINER_IS_NOT_PART_OF_A_BUILD_STEP", - "ANTLIR_BUCK", - # Session basics - "HOME", - "LOGNAME", - "NOTIFY_SOCKET", - "PATH", - "TERM", - "USER", - # Provided by the shell running the test - "PWD", - "SHLVL", - # `nspawn --as-pid2` sets these 2, although they're quite silly. - "container", - "container_uuid", # our nspawn runtime actually sets this to '' - "container_host_id", # systemd 246+ - "container_host_version_id", # systemd 246+ - # These 2 are another `systemd` artifact, appearing when we pass - # FDs into the container. - "LISTEN_FDS", - "LISTEN_PID", - # PAR noise that doesn't start with `FB_PAR_` (filtered below) - "PAR_LAUNCH_TIMESTAMP", - "SCRIBE_LOG_USAGE", - "LC_ALL", - "LC_CTYPE", - # FB test runner - "TEST_PILOT", - # FB "tcc coverage" mode seems to set these, mostly via - # testinfra/testpilot/integration/python/adapters/coverage.py - "COVERAGE_RCFILE", - "PLATFORM", - "PYTHON_SUBPROCESS_COVERAGE", - "PY_IMPL", - "PY_MAJOR", - "PY_MINOR", - } - for var in os.environ: - if var.startswith("FB_PAR_"): # Set for non-in-place build modes - continue - self.assertIn(var, env_allowlist) - # If the allowlist proves unmaintainable, Buck guarantees that this - # variable is set, and it is NOT explicitly passed into containers, - # so it ought to be absent. See also `test-unsanitized-env`. - self.assertNotIn("BUCK_BUILD_ID", os.environ) - - def test_layer_mount(self) -> None: - # Verify that `/meownt` exists and is a mount point - self.assertTrue(os.path.exists("/meownt")) - subprocess.check_output(["/usr/bin/mountpoint", "-q", "/meownt"]) - - # Verify that `/layer_mount` exists and is a mount point - self.assertTrue(os.path.exists("/layer_mount")) - subprocess.check_output(["/usr/bin/mountpoint", "-q", "/layer_mount"]) - - def test_wrapped_runnable_args(self) -> None: - self.assertTrue(os.path.exists("/foo/bar/installed/print-arg")) - - # Check arg with spaces to make sure the executable wrapping - # works as expected - self.assertEqual( - subprocess.check_output( - ["/foo/bar/installed/print-arg", "space cadet"], text=True - ), - "space cadet\n", - ) diff --git a/antlir/bzl/tests/test_image_unittest_repo_server.py b/antlir/bzl/tests/test_image_unittest_repo_server.py deleted file mode 100644 index b555ad3a608..00000000000 --- a/antlir/bzl/tests/test_image_unittest_repo_server.py +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import os -import subprocess -import unittest - -from antlir.config import antlir_dep -from antlir.fs_utils import temp_dir -from antlir.rpm.find_snapshot import snapshot_install_dir -from antlir.tests.flavor_helpers import get_rpm_installers_supported - - -class ImageUnittestTestRepoServer(unittest.TestCase): - def test_install_rpm(self) -> None: - snapshot_dir = snapshot_install_dir(antlir_dep("rpm:repo-snapshot-for-tests")) - for prog in get_rpm_installers_supported(): - with temp_dir() as td: - os.mkdir(td / ".meta") - subprocess.check_call( - [ - snapshot_dir / prog / "bin" / prog, - f"--installroot={td}", - "install", - "--assumeyes", - "rpm-test-carrot", - ] - ) - # We don't need a full rendered subvol test, since the - # contents of the filesystem is checked by other tests. - # (e.g. `test-yum-dnf-from-snapshot`, `test-image-layer`) - with open(td / "rpm_test/carrot.txt") as infile: - self.assertEqual("carrot 2 rc0\n", infile.read()) diff --git a/antlir/bzl/tests/test_unsanitized_env.py b/antlir/bzl/tests/test_unsanitized_env.py deleted file mode 100644 index 6142766d08e..00000000000 --- a/antlir/bzl/tests/test_unsanitized_env.py +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import os -import unittest - - -class UnsanitizedEnvTest(unittest.TestCase): - def test_env(self) -> None: - if "IS_BUCK2" not in os.environ: - self.assertTrue("BUCK_BUILD_ID" in os.environ) - # Comes from the test's `env` - self.assertEqual("meow", os.environ["kitteh"])