Add iree-c-embed-data and iree-flatcc-cli to release packages. (i…

…ree-org#18001)

Progress on iree-org#16203.

First, this renames `generate_embed_data` to `iree-c-embed-data`. The
name here dates back to having a single implementation shared across
multiple projects using Bazel (and Google's internal Blaze). There isn't
much code in the tool, but we've been using it for long enough to give
it a name matching our other tools IMO.

Next, this includes `iree-c-embed-data` and `iree-flatcc-cli` in the
`iree-runtime` Python package (https://pypi.org/project/iree-runtime/).
Both of these host tools are required for building large parts of the
runtime (embed data for builtins and bitcode, flatcc for schemas), so
including them in packages will allow users (and pkgci workflows) to get
them without needing to build from source.

build_tools/bazel/iree_bytecode_module.bzl

@@ -6,7 +6,7 @@
 
 """Rules for compiling IREE executables, modules, and archives."""
 
-load("//build_tools/embed_data:build_defs.bzl", "c_embed_data")
+load("//build_tools/embed_data:build_defs.bzl", "iree_c_embed_data")
 
 # TODO(benvanik): port to a full starlark rule, document, etc.
 
@@ -81,7 +81,7 @@ def iree_bytecode_module(
 
     # Embed the module for use in C.
     if c_identifier:
-        c_embed_data(
+        iree_c_embed_data(
             name = "%s_c" % (name),
             identifier = c_identifier,
             srcs = [module_name],

build_tools/bazel_to_cmake/bazel_to_cmake_converter.py

@@ -531,7 +531,7 @@ def cc_binary(
             f")\n\n"
         )
 
-    def c_embed_data(
+    def iree_c_embed_data(
         self,
         name,
         srcs,
@@ -555,6 +555,7 @@ def c_embed_data(
             "H_FILE_OUTPUT", h_file_output
         )
         testonly_block = self._convert_option_block("TESTONLY", testonly)
+        strip_prefix_block = self._convert_option_block("STRIP_PREFIX", strip_prefix)
         identifier_block = self._convert_string_arg_block("IDENTIFIER", identifier)
         flatten_block = self._convert_option_block("FLATTEN", flatten)
         deps_block = self._convert_target_list_block("DEPS", deps)
@@ -568,6 +569,7 @@ def c_embed_data(
             f"{h_file_output_block}"
             f"{identifier_block}"
             f"{testonly_block}"
+            f"{strip_prefix_block}"
             f"{flatten_block}"
             f"  PUBLIC\n)\n\n"
         )

build_tools/cmake/iree_c_embed_data.cmake

@@ -8,7 +8,7 @@ include(CMakeParseArguments)
 
 # iree_c_embed_data()
 #
-# CMake function to imitate Bazel's c_embed_data rule.
+# CMake function matching the iree_c_embed_data rule.
 #
 # Parameters:
 # PACKAGE: Name of the package (overrides actual path)
@@ -96,8 +96,8 @@ function(iree_c_embed_data)
 
   add_custom_command(
     OUTPUT "${_RULE_H_FILE_OUTPUT}" "${_RULE_C_FILE_OUTPUT}"
-    COMMAND generate_embed_data ${_ARGS} ${_RESOLVED_SRCS}
-    DEPENDS generate_embed_data ${_RESOLVED_SRCS}
+    COMMAND iree-c-embed-data ${_ARGS} ${_RESOLVED_SRCS}
+    DEPENDS iree-c-embed-data ${_RESOLVED_SRCS}
   )
 
   if(_RULE_TESTONLY)

build_tools/embed_data/BUILD.bazel

@@ -6,7 +6,7 @@
 
 # Generates source files with embedded file contents.
 
-load(":build_defs.bzl", "c_embed_data")
+load(":build_defs.bzl", "iree_c_embed_data")
 
 package(
     default_visibility = ["//visibility:public"],
@@ -15,11 +15,11 @@ package(
 )
 
 cc_binary(
-    name = "generate_embed_data",
-    srcs = ["generate_embed_data_main.cc"],
+    name = "iree-c-embed-data",
+    srcs = ["iree-c-embed-data-main.cc"],
 )
 
-c_embed_data(
+iree_c_embed_data(
     name = "testembed1",
     # do not sort
     srcs = [
@@ -31,7 +31,7 @@ c_embed_data(
     h_file_output = "testembed1.h",
 )
 
-c_embed_data(
+iree_c_embed_data(
     name = "testembed2",
     srcs = [
         "data/file3.bin",
@@ -42,8 +42,8 @@ c_embed_data(
 )
 
 cc_test(
-    name = "c_embed_data_test",
-    srcs = ["c_embed_data_test.cc"],
+    name = "iree_c_embed_data_test",
+    srcs = ["iree_c_embed_data_test.cc"],
     deps = [
         ":testembed1",
         ":testembed2",

build_tools/embed_data/CMakeLists.txt

@@ -6,23 +6,23 @@
 
 # Just import if IREE_HOST_BIN_DIR is set (e.g. when cross-compiling).
 if(IREE_HOST_BIN_DIR)
-  iree_import_binary(NAME generate_embed_data)
-  install(IMPORTED_RUNTIME_ARTIFACTS generate_embed_data
-          COMPONENT generate_embed_data
+  iree_import_binary(NAME iree-c-embed-data)
+  install(IMPORTED_RUNTIME_ARTIFACTS iree-c-embed-data
+          COMPONENT iree-c-embed-data
           RUNTIME DESTINATION bin
           BUNDLE DESTINATION bin)
   return()
 endif()
 
-add_executable(generate_embed_data)
-target_sources(generate_embed_data PRIVATE generate_embed_data_main.cc)
-set_target_properties(generate_embed_data PROPERTIES
-  OUTPUT_NAME generate_embed_data
+add_executable(iree-c-embed-data)
+target_sources(iree-c-embed-data PRIVATE iree-c-embed-data-main.cc)
+set_target_properties(iree-c-embed-data PROPERTIES
+  OUTPUT_NAME iree-c-embed-data
   RUNTIME_OUTPUT_DIRECTORY "${IREE_BINARY_DIR}/tools"
 )
 
-install(TARGETS generate_embed_data
-        COMPONENT IREETools-CompilerExtra
+install(TARGETS iree-c-embed-data
+        COMPONENT IREETools-Runtime
         RUNTIME DESTINATION bin
         BUNDLE DESTINATION bin)
 
@@ -55,9 +55,9 @@ iree_c_embed_data(
 
 iree_cc_test(
   NAME
-    "c_embed_data_test"
+    "iree_c_embed_data_test"
   SRCS
-    "c_embed_data_test.cc"
+    "iree_c_embed_data_test.cc"
   DEPS
     ::testembed1
     ::testembed2

build_tools/embed_data/build_defs.bzl

@@ -13,7 +13,7 @@ def clean_dep(dep):
     """
     return str(Label(dep))
 
-def c_embed_data(
+def iree_c_embed_data(
         name,
         srcs,
         c_file_output,
@@ -22,7 +22,7 @@ def c_embed_data(
         strip_prefix = None,
         flatten = False,
         identifier = None,
-        generator = clean_dep("//build_tools/embed_data:generate_embed_data"),
+        generator = clean_dep("//build_tools/embed_data:iree-c-embed-data"),
         **kwargs):
     """Embeds 'srcs' into a C module.
 

build_tools/third_party/cuda/BUILD.template

@@ -7,7 +7,7 @@
 
 load("@bazel_skylib//rules:common_settings.bzl", "bool_flag")
 # Variables and string substitutions don't work here because of course they don't
-load("%IREE_REPO_ALIAS%//build_tools/embed_data:build_defs.bzl", "c_embed_data")
+load("%IREE_REPO_ALIAS%//build_tools/embed_data:build_defs.bzl", "iree_c_embed_data")
 
 package(default_visibility = ["//visibility:public"])
 
@@ -42,14 +42,14 @@ cc_library(
     includes = ["include"],
 )
 
-c_embed_data(
+iree_c_embed_data(
     name = "libdevice_embedded",
     srcs = [
         LIBDEVICE_REL_PATH,
     ],
     c_file_output = "iree_cuda/libdevice_embedded.c",
     flatten = True,
-    generator = "%IREE_REPO_ALIAS%//build_tools/embed_data:generate_embed_data",
+    generator = "%IREE_REPO_ALIAS%//build_tools/embed_data:iree-c-embed-data",
     h_file_output = "iree_cuda/libdevice_embedded.h",
     includes = [
         "iree_cuda",

compiler/src/iree/compiler/Dialect/HAL/BUILD.bazel

@@ -4,15 +4,15 @@
 # See https://llvm.org/LICENSE.txt for license information.
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
-load("//build_tools/embed_data:build_defs.bzl", "c_embed_data")
+load("//build_tools/embed_data:build_defs.bzl", "iree_c_embed_data")
 
 package(
     default_visibility = ["//visibility:public"],
     features = ["layering_check"],
     licenses = ["notice"],  # Apache 2.0
 )
 
-c_embed_data(
+iree_c_embed_data(
     name = "hal_imports",
     srcs = ["hal.imports.mlir"],
     c_file_output = "hal.imports.c",

compiler/src/iree/compiler/Dialect/Stream/Builtins/BUILD.bazel

@@ -4,15 +4,15 @@
 # See https://llvm.org/LICENSE.txt for license information.
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
-load("//build_tools/embed_data:build_defs.bzl", "c_embed_data")
+load("//build_tools/embed_data:build_defs.bzl", "iree_c_embed_data")
 
 package(
     default_visibility = ["//visibility:public"],
     features = ["layering_check"],
     licenses = ["notice"],  # Apache 2.0
 )
 
-c_embed_data(
+iree_c_embed_data(
     name = "Builtins",
     srcs = [
         "fill_i16.mlir",

compiler/src/iree/compiler/Dialect/VMVX/BUILD.bazel

@@ -4,15 +4,15 @@
 # See https://llvm.org/LICENSE.txt for license information.
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
-load("//build_tools/embed_data:build_defs.bzl", "c_embed_data")
+load("//build_tools/embed_data:build_defs.bzl", "iree_c_embed_data")
 
 package(
     default_visibility = ["//visibility:public"],
     features = ["layering_check"],
     licenses = ["notice"],  # Apache 2.0
 )
 
-c_embed_data(
+iree_c_embed_data(
     name = "vmvx_imports",
     srcs = ["vmvx.imports.mlir"],
     c_file_output = "vmvx.imports.c",

compiler/src/iree/compiler/Modules/Check/BUILD.bazel

@@ -4,15 +4,15 @@
 # See https://llvm.org/LICENSE.txt for license information.
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
-load("//build_tools/embed_data:build_defs.bzl", "c_embed_data")
+load("//build_tools/embed_data:build_defs.bzl", "iree_c_embed_data")
 
 package(
     default_visibility = ["//visibility:public"],
     features = ["layering_check"],
     licenses = ["notice"],  # Apache 2.0
 )
 
-c_embed_data(
+iree_c_embed_data(
     name = "check_imports",
     srcs = ["check.imports.mlir"],
     c_file_output = "check.imports.c",

compiler/src/iree/compiler/Modules/HAL/Inline/BUILD.bazel

@@ -4,15 +4,15 @@
 # See https://llvm.org/LICENSE.txt for license information.
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
-load("//build_tools/embed_data:build_defs.bzl", "c_embed_data")
+load("//build_tools/embed_data:build_defs.bzl", "iree_c_embed_data")
 
 package(
     default_visibility = ["//visibility:public"],
     features = ["layering_check"],
     licenses = ["notice"],  # Apache 2.0
 )
 
-c_embed_data(
+iree_c_embed_data(
     name = "hal_inline_imports",
     srcs = ["hal_inline.imports.mlir"],
     c_file_output = "hal_inline.imports.c",

compiler/src/iree/compiler/Modules/HAL/Loader/BUILD.bazel

@@ -4,15 +4,15 @@
 # See https://llvm.org/LICENSE.txt for license information.
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
-load("//build_tools/embed_data:build_defs.bzl", "c_embed_data")
+load("//build_tools/embed_data:build_defs.bzl", "iree_c_embed_data")
 
 package(
     default_visibility = ["//visibility:public"],
     features = ["layering_check"],
     licenses = ["notice"],  # Apache 2.0
 )
 
-c_embed_data(
+iree_c_embed_data(
     name = "hal_loader_imports",
     srcs = ["hal_loader.imports.mlir"],
     c_file_output = "hal_loader.imports.c",

compiler/src/iree/compiler/Modules/IO/Parameters/BUILD.bazel

@@ -4,15 +4,15 @@
 # See https://llvm.org/LICENSE.txt for license information.
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
-load("//build_tools/embed_data:build_defs.bzl", "c_embed_data")
+load("//build_tools/embed_data:build_defs.bzl", "iree_c_embed_data")
 
 package(
     default_visibility = ["//visibility:public"],
     features = ["layering_check"],
     licenses = ["notice"],  # Apache 2.0
 )
 
-c_embed_data(
+iree_c_embed_data(
     name = "io_parameters_imports",
     srcs = ["io_parameters.imports.mlir"],
     c_file_output = "io_parameters.imports.c",

experimental/web/sample_static/build_sample.sh

@@ -57,7 +57,7 @@ mkdir -p "${BINARY_DIR}"
 ###############################################################################
 
 COMPILE_TOOL="${INSTALL_ROOT}/bin/iree-compile"
-EMBED_DATA_TOOL="${INSTALL_ROOT}/bin/generate_embed_data"
+EMBED_DATA_TOOL="${INSTALL_ROOT}/bin/iree-c-embed-data"
 INPUT_NAME="mnist"
 INPUT_PATH="${ROOT_DIR}/samples/models/mnist.mlir"
 

experimental/webgpu/shaders/BUILD.bazel

@@ -4,15 +4,15 @@
 # See https://llvm.org/LICENSE.txt for license information.
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
-load("//build_tools/embed_data:build_defs.bzl", "c_embed_data")
+load("//build_tools/embed_data:build_defs.bzl", "iree_c_embed_data")
 
 package(
     default_visibility = ["//visibility:public"],
     features = ["layering_check"],
     licenses = ["notice"],  # Apache 2.0
 )
 
-c_embed_data(
+iree_c_embed_data(
     name = "shaders",
     srcs = [
         "fill_buffer.wgsl",