Support loading shaders from a FILE and as SPIRV-BIN (#1059)

* Support loading shaders from a FILE and as SPIRV-BIN

This is convenient in a number of cases. For example, one can more
conveniently refer to a standalone shader source file. And one can
pass-through a SPIR-V file that may be using a shader-only extension
that Amber is unaware of.

* check-format complaint: use a pointer

---------

Co-authored-by: Nicolai Hähnle <[email protected]>

docs/amber_script.md

@@ -111,7 +111,7 @@ file system, before falling back to the standard file system.
 
 Shader programs are declared using the `SHADER` command. \
 Shaders can be declared as `PASSTHROUGH`, with inlined source or using source
-from a `VIRTUAL_FILE`.
+from a `VIRTUAL_FILE` or from a `FILE` in the file system.
 
 Pass-through shader:
 
@@ -132,12 +132,12 @@ SHADER {shader_type} {shader_name} {shader_format} [ TARGET_ENV {target_env} ]
 END
 ```
 
-Shader using source from `VIRTUAL_FILE`:
+Shader using source from `VIRTUAL_FILE` or `FILE`:
 
 ```groovy
 # Creates a shader of |shader_type| with the given |shader_name|. The shader
 # will be of |shader_format|. The shader will use the virtual file with |path|.
-SHADER {shader_type} {shader_name} {shader_format} [ TARGET_ENV {target_env} ] VIRTUAL_FILE {path}
+SHADER {shader_type} {shader_name} {shader_format} [ TARGET_ENV {target_env} ] ( VIRTUAL_FILE | FILE ) {path}
 ```
 
 `{shader_name}` is used to identify the shader to attach to `PIPELINE`s,
@@ -164,19 +164,21 @@ can not contain compute shaders, and must contain a vertex shader and a fragment
 shader. Ray tracing pipeline can contain only shaders of ray tracing types:
 ray generation, any hit, closest hit, miss, intersection, and callable shaders.
 
-The provided `multi` shader can only be used with `SPIRV-ASM` and `SPIRV-HEX`
-and allows for providing multiple shaders in a single module (so the `vertex`
-and `fragment` shaders can be provided together.)
+The provided `multi` shader can only be used with `SPIRV-ASM`, `SPIRV-HEX`, and
+`SPIRV-BIN` and allows for providing multiple shaders in a single module (so the
+`vertex` and `fragment` shaders can be provided together.)
 
-Note, `SPIRV-ASM` and `SPIRV-HEX` can also be used with each of the other shader
-types, but in that case must only provide a single shader type in the module.
+Note, `SPIRV-ASM`, `SPIRV-HEX`, and `SPIRV-BIN` can also be used with each of
+the other shader types, but in that case must only provide a single shader type
+in the module.
 
 #### Shader Format
  * `GLSL`  (with glslang)
  * `HLSL`  (with dxc or glslang if dxc disabled)
  * `SPIRV-ASM` (with spirv-as; specifying `TARGET_ENV` is _highly recommended_
     in this case, as explained below)
  * `SPIRV-HEX` (decoded straight to SPIR-V)
+ * `SPIRV-BIN` (read as binary SPIR-V, only with `FILE`)
  * `OPENCL-C` (with clspv)
 
 ### Target environment
@@ -190,8 +192,8 @@ SPIR-V environment. For example:
  * `vulkan1.2`
 
 Check the help text of the corresponding tool (e.g. spirv-as, glslangValidator)
-for the full list. The `SPIRV-HEX` shader format is not affected by the target
-environment.
+for the full list. The `SPIRV-HEX` and `SPIRV-BIN` shader formats are not
+affected by the target environment.
 
 The specified target environment for the shader overrides the default (`spv1.0`)
 or the one specified on the command line.

include/amber/amber.h

@@ -101,6 +101,9 @@ class Delegate {
   virtual amber::Result LoadBufferData(const std::string file_name,
                                        BufferDataFileType file_type,
                                        amber::BufferInfo* buffer) const = 0;
+  /// Load a raw file
+  virtual amber::Result LoadFile(const std::string file_name,
+                                 std::vector<char>* buffer) const = 0;
 
   /// Mechanism for gathering timing from 'TIME_EXECUTION'
   virtual void ReportExecutionTiming(double){}

include/amber/shader_info.h

@@ -29,6 +29,7 @@ enum ShaderFormat {
   kShaderFormatHlsl,
   kShaderFormatSpirvAsm,
   kShaderFormatSpirvHex,
+  kShaderFormatSpirvBin,
   kShaderFormatOpenCLC,
 };
 

samples/amber.cc

@@ -410,6 +410,14 @@ class SampleDelegate : public amber::Delegate {
     return {};
   }
 
+  amber::Result LoadFile(const std::string file_name,
+                         std::vector<char>* buffer) const override {
+    *buffer = ReadFile(path_ + file_name);
+    if (buffer->empty())
+      return amber::Result("Failed to load file " + file_name);
+    return {};
+  }
+
  private:
   bool log_graphics_calls_ = false;
   bool log_graphics_calls_time_ = false;

src/amberscript/parser.cc

@@ -440,6 +440,8 @@ Result Parser::ToShaderFormat(const std::string& str, ShaderFormat* fmt) {
     *fmt = kShaderFormatSpirvAsm;
   else if (str == "SPIRV-HEX")
     *fmt = kShaderFormatSpirvHex;
+  else if (str == "SPIRV-BIN")
+    *fmt = kShaderFormatSpirvBin;
   else if (str == "OPENCL-C")
     *fmt = kShaderFormatOpenCLC;
   else
@@ -526,19 +528,33 @@ Result Parser::ParseShaderBlock() {
   }
 
   token = tokenizer_->PeekNextToken();
-  if (token->IsIdentifier() && token->AsString() == "VIRTUAL_FILE") {
-    tokenizer_->NextToken();  // Skip VIRTUAL_FILE
+  if (token->IsIdentifier() &&
+      (token->AsString() == "VIRTUAL_FILE" || token->AsString() == "FILE")) {
+    bool isVirtual = token->AsString() == "VIRTUAL_FILE";
+    tokenizer_->NextToken();  // Skip VIRTUAL_FILE or FILE
 
     token = tokenizer_->NextToken();
     if (!token->IsIdentifier() && !token->IsString())
-      return Result("expected virtual file path after VIRTUAL_FILE");
+      return Result("expected file path after VIRTUAL_FILE or FILE");
 
     auto path = token->AsString();
 
     std::string data;
-    r = script_->GetVirtualFile(path, &data);
-    if (!r.IsSuccess())
-      return r;
+    if (isVirtual) {
+      r = script_->GetVirtualFile(path, &data);
+      if (!r.IsSuccess())
+        return r;
+    } else {
+      if (!delegate_)
+        return Result("missing delegate for loading shader file");
+
+      std::vector<char> buffer;
+      r = delegate_->LoadFile(path, &buffer);
+      if (!r.IsSuccess())
+        return r;
+
+      data.insert(data.begin(), buffer.begin(), buffer.end());
+    }
 
     shader->SetData(data);
     shader->SetFilePath(path);

src/amberscript/parser_buffer_test.cc

@@ -48,6 +48,10 @@ class DummyDelegate : public amber::Delegate {
 
     return {};
   }
+
+  amber::Result LoadFile(const std::string, std::vector<char>*) const override {
+    return Result("DummyDelegate::LoadFile not implemented");
+  }
 };
 
 TEST_F(AmberScriptParserTest, BufferData) {

src/shader_compiler.cc

@@ -16,6 +16,7 @@
 
 #include <algorithm>
 #include <cstdlib>
+#include <cstring>
 #include <iterator>
 #include <string>
 #include <utility>
@@ -123,6 +124,9 @@ std::pair<Result, std::vector<uint32_t>> ShaderCompiler::Compile(
     Result r = ParseHex(shader->GetData(), &results);
     if (!r.IsSuccess())
       return {Result("Unable to parse shader hex."), {}};
+  } else if (shader->GetFormat() == kShaderFormatSpirvBin) {
+    results.resize(shader->GetData().size() / 4);
+    memcpy(results.data(), shader->GetData().data(), shader->GetData().size());
 
 #if AMBER_ENABLE_SHADERC
   } else if (shader->GetFormat() == kShaderFormatGlsl) {

tests/cases/shader_file.amber

@@ -0,0 +1,25 @@
+#!amber
+# Copyright 2024 Advanced Micro Devices, Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+SHADER compute shader GLSL FILE shader_file.comp
+BUFFER buffer DATA_TYPE uint32 DATA 0 END
+
+PIPELINE compute the_pipeline
+  ATTACH shader
+  BIND BUFFER buffer AS storage DESCRIPTOR_SET 0 BINDING 0
+END
+
+RUN the_pipeline 1 1 1
+EXPECT buffer IDX 0 EQ 1

tests/cases/shader_file.comp

@@ -0,0 +1,22 @@
+// Copyright 2024 Advanced Micro Devices, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#version 430
+layout(set = 0, binding = 0) buffer buf {
+  uint x;
+};
+
+void main() {
+  x = 1;
+}

tests/cases/shader_spirv_bin.amber

@@ -0,0 +1,25 @@
+#!amber
+# Copyright 2024 Advanced Micro Devices, Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+SHADER compute shader SPIRV-BIN FILE shader_spirv_bin.spv
+BUFFER buffer DATA_TYPE uint32 DATA 0 END
+
+PIPELINE compute the_pipeline
+  ATTACH shader
+  BIND BUFFER buffer AS storage DESCRIPTOR_SET 0 BINDING 0
+END
+
+RUN the_pipeline 1 1 1
+EXPECT buffer IDX 0 EQ 1

tools/amber-mode.el

@@ -46,7 +46,7 @@
 ;;               "tessellation_control" "multi" "framebuffer" "graphics" "uniform"
 ;;               "storage" "push_constant" "color" "depth_stencil" "EQ" "NE" "LT"
 ;;               "LE" "GT" "GE" "EQ_RGB" "EQ_RGBA" "EQ_BUFFER" "GLSL" "HLSL"
-;;               "SPIRV-ASM" "SPIRV-HEX" "OPENCL-C"
+;;               "SPIRV-ASM" "SPIRV-HEX" "SPIRV-BIN" "OPENCL-C"
 ;;               ) t)
 
 ;; (regexp-opt '(

tools/amber-syntax.vim

@@ -63,7 +63,7 @@ syn keyword amberComparator EQ_HISTOGRAM_EMD_BUFFER
 syn keyword amberKeyword compute vertex geometry fragment graphics
 syn keyword amberKeyword tessellation_evaulation tessellation_control multi
 
-syn keyword amberFormat GLSL HLSL SPIRV-ASM SPIRV-HEX OPENCL-C
+syn keyword amberFormat GLSL HLSL SPIRV-ASM SPIRV-HEX SPIRV-BIN OPENCL-C
 
 syn keyword amberTopology point_list line_list line_list_with_adjacency
 syn keyword amberTopology line_strip line_strip_with_adjacency triangle_list

tools/amber.sublime-syntax

@@ -42,7 +42,7 @@ contexts:
       scope: constant.character.escape.amber
     - match: '\b(EQ|NE|LT|LE|GT|GE|EQ_RGB|EQ_RGBA|EQ_BUFFER|RMSE_BUFFER)\b'
       scope: constant.character.esape.amber
-    - match: '\b(GLSL|HLSL|SPIRV-ASM|SPIRV-HEX|OPENCL-C)\b'
+    - match: '\b(GLSL|HLSL|SPIRV-ASM|SPIRV-HEX|SPIRV-BIN|OPENCL-C)\b'
       scope: constant.character.escape.amber
 
     - match: '\b(point_list|line_list|line_list_with_adjacency|line_strip)\b'