Consolidate all python targets in the tools folder (#80408)

Summary:
All buck targets that points to caffe2/tools folder are now moved to tools/BUCK.
This also eliminates all python library/binary import in pt_defs.bzl, which caused T124308913.

Test Plan: CI

Differential Revision: D37468313

Pull Request resolved: pytorch/pytorch#80408
Approved by: https://github.com/seemethere, https://github.com/malfet

Author: linbinyu

.github/workflows/_buck-build-test.yml

@@ -62,7 +62,15 @@ jobs:
           command: |
             sh scripts/buck_setup.sh
 
-      - name: Build C10
+      - name: Build tools
+        run: |
+          buck build tools: --keep-going
+
+      - name: Run tools tests
+        run: |
+          buck test tools:selective_build_test tools:gen_oplist_test tools:gen_operators_yaml_test
+
+      - name: Build c10
         run: |
           buck build c10:c10
 

.lintrunner.toml

@@ -16,6 +16,7 @@ exclude_patterns = [
     'torch/lib/**',
     'venv/**',
     '**/*.pyi',
+    'tools/test/test_selective_build.py',
 ]
 command = [
     'python3',
@@ -145,6 +146,10 @@ include_patterns = [
 exclude_patterns = [
     # (linbinyu) copied from internal repo
     'tools/code_analyzer/gen_operators_yaml.py',
+    'tools/gen_vulkan_spv.py',
+    'tools/test/gen_operators_yaml_test.py',
+    'tools/test/gen_oplist_test.py',
+    'tools/test/test_selective_build.py',
 ]
 command = [
     'python3',
@@ -334,6 +339,7 @@ exclude_patterns = [
 command = [
     'python3',
     'tools/linter/adapters/grep_linter.py',
+    # @lint-ignore TXT2
     '--pattern=	',
     '--linter-name=TABS',
     '--error-name=saw some tabs',
@@ -565,6 +571,9 @@ include_patterns = [
     'torch/_decomp/**/*.py',
     'test/onnx/**/*.py',
 ]
+exclude_patterns = [
+    'tools/gen_vulkan_spv.py',
+]
 command = [
     'python3',
     'tools/linter/adapters/black_linter.py',

buckbuild.bzl

@@ -3,8 +3,6 @@
 
 load("@bazel_skylib//lib:paths.bzl", "paths")
 load("//tools/build_defs:fb_native_wrapper.bzl", "fb_native")
-load("//tools/build_defs:fb_python_binary.bzl", "fb_python_binary")
-load("//tools/build_defs:fb_python_library.bzl", "fb_python_library")
 load("//tools/build_defs:fb_xplat_cxx_library.bzl", "fb_xplat_cxx_library")
 load("//tools/build_defs:fb_xplat_genrule.bzl", "fb_xplat_genrule")
 load("//tools/build_defs:fbsource_utils.bzl", "is_arvr_mode")
@@ -416,7 +414,7 @@ def gen_aten_files(
         name = name,
         default_outs = ["."],
         outs = get_aten_generated_files(backends),
-        cmd = "$(exe {}:gen_aten_bin) ".format(ROOT) + " ".join([
+        cmd = "$(exe {}torchgen:gen) ".format(ROOT_PATH) + " ".join([
             "--source-path $(location {}:aten_src_path)/aten/src/ATen".format(ROOT),
             "--install_dir $OUT",
         ] + extra_params),
@@ -442,7 +440,7 @@ def gen_aten_unboxing_files(
         name = genrule_name,
         default_outs = ["."],
         outs = get_unboxing_generated_files(),
-        cmd = "$(exe {}:gen_unboxing_bin) ".format(ROOT) + " ".join([
+        cmd = "$(exe {}tools:gen_unboxing_bin) ".format(ROOT_PATH) + " ".join([
             "--source-path $(location {}:aten_src_path)/aten/src/ATen".format(ROOT),
             "--install_dir $OUT",
         ] + extra_params),
@@ -515,7 +513,7 @@ def pt_operator_query_codegen(
     # @lint-ignore BUCKLINT
     fb_native.genrule(
         name = oplist_dir_name,
-        cmd = ("$(exe {}:gen_oplist) ".format(ROOT) +
+        cmd = ("$(exe {}tools:gen_oplist) ".format(ROOT_PATH) +
                "--model_file_list_path $(@query_outputs 'attrfilter(labels, pt_operator_library, deps(set({deps})))') " +
                ("" if enforce_traced_op_list else "--allow_include_all_overloads ") +
                "--output_dir $OUT ").format(deps = " ".join(["\"{}\"".format(d) for d in deps])),
@@ -620,7 +618,7 @@ def gen_aten_libtorch_files(name, extra_params = [], compatible_with = [], apple
         outs = get_generate_code_bin_outs(),
         default_outs = ["."],
         bash = "mkdir -p tools && " +
-               "$(exe {}tools/setup_helpers:generate_code_bin) ".format(ROOT_PATH) + " ".join(
+               "$(exe {}tools:generate_code_bin) ".format(ROOT_PATH) + " ".join(
             # Mobile build only needs libtorch - skip python bindings for now, except
             # for ovrsource, which needs Python bindings.
             (["--subset libtorch"] if not is_arvr_mode() else []) + [
@@ -630,7 +628,7 @@ def gen_aten_libtorch_files(name, extra_params = [], compatible_with = [], apple
             ] + extra_params,
         ),
         cmd_exe = "@powershell -Command New-Item -Path tools -ItemType Directory -Force; " +
-                  "$(exe {}tools/setup_helpers:generate_code_bin) ".format(ROOT_PATH) + " ".join(
+                  "$(exe {}tools:generate_code_bin) ".format(ROOT_PATH) + " ".join(
             # Mobile build only needs libtorch - skip python bindings for now, except
             # for ovrsource, which needs Python bindings.
             (["--subset libtorch"] if not is_arvr_mode() else []) + [
@@ -950,7 +948,7 @@ def define_buck_targets(
             "torch/csrc/api/include/torch/version.h.in",
             "version.txt",
         ],
-        cmd = "$(exe {}tools/setup_helpers:gen-version-header) ".format(ROOT_PATH) + " ".join([
+        cmd = "$(exe {}tools:gen-version-header) ".format(ROOT_PATH) + " ".join([
             "--template-path",
             "torch/csrc/api/include/torch/version.h.in",
             "--version-path",
@@ -995,28 +993,13 @@ def define_buck_targets(
         ],
     )
 
-    fb_python_library(
-        name = "substitutelib",
-        srcs = ["tools/substitute.py"],
-        base_module = "",
-    )
-
-    fb_python_binary(
-        name = "substitute",
-        main_module = "tools.substitute",
-        visibility = ["PUBLIC"],
-        deps = [
-            ":substitutelib",
-        ],
-    )
-
     # @lint-ignore BUCKLINT
     fb_native.genrule(
         name = "generate_aten_config",
         srcs = [
             "aten/src/ATen/Config.h.in",
         ],
-        cmd = "$(exe :substitute) " + " ".join([
+        cmd = "$(exe {}tools:substitute) ".format(ROOT_PATH) + " ".join([
             "--install_dir",
             "$OUT",
             "--input-file",
@@ -1072,79 +1055,6 @@ def define_buck_targets(
         default_outs = ["."],
     )
 
-    fb_python_binary(
-        name = "gen_aten_bin",
-        main_module = "torchgen.gen",
-        visibility = [
-            "PUBLIC",
-        ],
-        deps = [
-            ROOT_PATH + "torchgen:torchgen",
-        ],
-    )
-
-    fb_python_binary(
-        name = "gen_unboxing_bin",
-        main_module = "tools.jit.gen_unboxing",
-        visibility = [
-            "PUBLIC",
-        ],
-        deps = [
-            ROOT_PATH + "tools/jit:jit",
-        ],
-    )
-
-    fb_python_library(
-        name = "gen_oplist_lib",
-        srcs = subdir_glob([
-            ("tools/code_analyzer", "gen_oplist.py"),
-            ("tools/code_analyzer", "gen_op_registration_allowlist.py"),
-        ]),
-        base_module = "",
-        tests = [
-            ":gen_oplist_test",
-        ],
-        deps = [
-            third_party("pyyaml"),
-            ROOT_PATH + "tools/lite_interpreter:gen_selected_mobile_ops_header",
-            ROOT_PATH + "torchgen:torchgen",
-        ],
-    )
-
-    fb_python_library(
-        name = "gen_operators_yaml_lib",
-        srcs = subdir_glob([
-            ("tools/code_analyzer", "gen_operators_yaml.py"),
-            ("tools/code_analyzer", "gen_op_registration_allowlist.py"),
-        ]),
-        base_module = "",
-        tests = [
-            ":gen_operators_yaml_test",
-        ],
-        deps = [
-            third_party("pyyaml"),
-            ROOT_PATH + "torchgen:torchgen",
-        ],
-    )
-
-    fb_python_binary(
-        name = "gen_oplist",
-        main_module = "gen_oplist",
-        visibility = ["PUBLIC"],
-        deps = [
-            ":gen_oplist_lib",
-        ],
-    )
-
-    fb_python_binary(
-        name = "gen_operators_yaml",
-        main_module = "gen_operators_yaml",
-        visibility = ["PUBLIC"],
-        deps = [
-            ":gen_operators_yaml_lib",
-        ],
-    )
-
     gen_aten_files(
         name = "gen_aten",
         extra_flags = get_aten_codegen_extra_params(USED_PT_BACKENDS),

cmake/VulkanCodegen.cmake

@@ -62,7 +62,7 @@ if(NOT USE_VULKAN_SHADERC_RUNTIME)
   execute_process(
     COMMAND
     "${PYTHON_EXECUTABLE}"
-    ${CMAKE_CURRENT_LIST_DIR}/../aten/src/ATen/gen_vulkan_spv.py
+    ${CMAKE_CURRENT_LIST_DIR}/../tools/gen_vulkan_spv.py
     --glsl-path ${CMAKE_CURRENT_LIST_DIR}/../aten/src/ATen/native/vulkan/glsl
     --output-path ${VULKAN_GEN_OUTPUT_PATH}
     --glslc-path=${GLSLC_PATH}

pt_ops.bzl

@@ -39,7 +39,7 @@ def pt_operator_library(
         name = name,
         out = "model_operators.yaml",
         cmd = (
-            "$(exe {root}:gen_operators_yaml) " +
+            "$(exe {exe}) " +
             "{optionally_root_ops} " +
             "{optionally_training_root_ops} " +
             "--rule_name {rule_name} " +
@@ -52,7 +52,7 @@ def pt_operator_library(
             "{optionally_model_traced_backends} " +
             "{optionally_include_all_operators}"
         ).format(
-            root = "//" if IS_OSS else "//xplat/caffe2",
+            exe = "//tools:gen_operators_yaml" if IS_OSS else "//xplat/caffe2/tools:gen_operators_yaml",
             rule_name = name,
             model_name = model_name,
             dep_graph_yaml = "none" if IS_OSS else "$(location //xplat/caffe2:pytorch_op_deps)/fb/pytorch_op_deps.yaml ",