shader_bundle.cc

Go to the documentation of this file.

// Copyright 2013 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
 
#include "impeller/compiler/shader_bundle.h"
 
#include <filesystem>
#include <sstream>
 
#include "flutter/fml/file.h"
#include "flutter/fml/mapping.h"
#include "impeller/compiler/compiler.h"
#include "impeller/compiler/reflector.h"
#include "impeller/compiler/source_options.h"
#include "impeller/compiler/types.h"
 
#include "impeller/compiler/utilities.h"
#include "impeller/runtime_stage/runtime_stage.h"
#include "impeller/shader_bundle/shader_bundle_flatbuffers.h"
#include "third_party/json/include/nlohmann/json.hpp"
 
namespace impeller {
namespace compiler {
 
std::optional<ShaderBundleConfig> ParseShaderBundleConfig(
    const std::string& bundle_config_json,
    std::ostream& error_stream) {
  auto json = nlohmann::json::parse(bundle_config_json, nullptr, false);
  if (json.is_discarded() || !json.is_object()) {
    error_stream << "The shader bundle is not a valid JSON object."
                 << std::endl;
    return std::nullopt;
  }
 
  ShaderBundleConfig bundle;
  for (auto& [shader_name, shader_value] : json.items()) {
    if (bundle.find(shader_name) != bundle.end()) {
      error_stream << "Duplicate shader \"" << shader_name << "\"."
                   << std::endl;
      return std::nullopt;
    }
    if (!shader_value.is_object()) {
      error_stream << "Invalid shader entry \"" << shader_name
                   << "\": Entry is not a JSON object." << std::endl;
      return std::nullopt;
    }
 
    ShaderConfig shader;
 
    if (!shader_value.contains("file")) {
      error_stream << "Invalid shader entry \"" << shader_name
                   << "\": Missing required \"file\" field." << std::endl;
      return std::nullopt;
    }
    shader.source_file_name = shader_value["file"];
 
    if (!shader_value.contains("type")) {
      error_stream << "Invalid shader entry \"" << shader_name
                   << "\": Missing required \"type\" field." << std::endl;
      return std::nullopt;
    }
    shader.type = SourceTypeFromString(shader_value["type"]);
    if (shader.type == SourceType::kUnknown) {
      error_stream << "Invalid shader entry \"" << shader_name
                   << "\": Shader type " << shader_value["type"]
                   << " is unknown." << std::endl;
      return std::nullopt;
    }
 
    shader.language = shader_value.contains("language")
                          ? ToSourceLanguage(shader_value["language"])
                          : SourceLanguage::kGLSL;
    if (shader.language == SourceLanguage::kUnknown) {
      error_stream << "Invalid shader entry \"" << shader_name
                   << "\": Unknown language type " << shader_value["language"]
                   << "." << std::endl;
      return std::nullopt;
    }
 
    shader.entry_point = shader_value.contains("entry_point")
                             ? shader_value["entry_point"]
                             : "main";
 
    bundle[shader_name] = shader;
  }
 
  return bundle;
}
 
std::vector<std::string_view> GetShaderBundleTargetPlatformDefines(
    TargetPlatform platform) {
  switch (platform) {
    case TargetPlatform::kMetalIOS:
      return {"IMPELLER_TARGET_METAL", "IMPELLER_TARGET_METAL_IOS"};
    case TargetPlatform::kMetalDesktop:
      return {"IMPELLER_TARGET_METAL", "IMPELLER_TARGET_METAL_DESKTOP"};
    case TargetPlatform::kOpenGLES:
      return {"IMPELLER_TARGET_OPENGLES"};
    case TargetPlatform::kOpenGLDesktop:
      return {"IMPELLER_TARGET_OPENGL"};
    case TargetPlatform::kVulkan:
      return {"IMPELLER_TARGET_VULKAN"};
    case TargetPlatform::kSkSL:
    case TargetPlatform::kRuntimeStageMetal:
    case TargetPlatform::kRuntimeStageGLES:
    case TargetPlatform::kRuntimeStageGLES3:
    case TargetPlatform::kRuntimeStageVulkan:
    case TargetPlatform::kUnknown:
      return {};
  }
  return {};
}
 
static std::unique_ptr<fb::shaderbundle::BackendShaderT>
GenerateShaderBackendFB(TargetPlatform target_platform,
                        SourceOptions& options,
                        const std::string& shader_name,
                        const ShaderConfig& shader_config,
                        std::set<std::string>* out_dependencies) {
  auto result = std::make_unique<fb::shaderbundle::BackendShaderT>();
 
  std::shared_ptr<fml::FileMapping> source_file_mapping =
      fml::FileMapping::CreateReadOnly(shader_config.source_file_name);
  if (!source_file_mapping) {
    std::cerr << "Could not open file for bundled shader \"" << shader_name
              << "\"." << std::endl;
    return nullptr;
  }
 
  /// Override options.
  options.target_platform = target_platform;
  options.file_name = shader_name;  // This is just used for error messages.
  options.type = shader_config.type;
  options.source_language = shader_config.language;
  options.entry_point_name = EntryPointFunctionNameFromSourceName(
      shader_config.source_file_name, options.type, options.source_language,
      shader_config.entry_point);
 
  // Inject the platform-discriminating defines (e.g. IMPELLER_TARGET_METAL) so
  // bundled shaders can specialize per backend. These are added to a local copy
  // because `options` is shared across every backend compiled here; pushing
  // onto it directly would accumulate defines from previously compiled
  // backends.
  SourceOptions backend_options = options;
  backend_options.target_platform = target_platform;
  for (const auto& define :
       GetShaderBundleTargetPlatformDefines(target_platform)) {
    backend_options.defines.emplace_back(define);
  }
 
  Reflector::Options reflector_options;
  reflector_options.target_platform = target_platform;
  reflector_options.entry_point_name = options.entry_point_name;
  reflector_options.shader_name = shader_name;
 
  Compiler compiler(source_file_mapping, backend_options, reflector_options);
  if (!compiler.IsValid()) {
    std::cerr << "Compilation failed for bundled shader \"" << shader_name
              << "\"." << std::endl;
    std::cerr << compiler.GetErrorMessages() << std::endl;
    return nullptr;
  }
 
  // Record dependencies so the caller can emit a depfile. The shader's
  // source file plus every transitive `#include` that contributed to
  // the compilation. The same source is compiled across multiple
  // target platforms; the std::set dedupes naturally.
  if (out_dependencies) {
    out_dependencies->insert(shader_config.source_file_name);
    for (const auto& included : compiler.GetIncludedFileNames()) {
      out_dependencies->insert(included);
    }
  }
 
  auto reflector = compiler.GetReflector();
  if (reflector == nullptr) {
    std::cerr << "Could not create reflector for bundled shader \""
              << shader_name << "\"." << std::endl;
    return nullptr;
  }
 
  auto bundle_data = reflector->GetShaderBundleData();
  if (!bundle_data) {
    std::cerr << "Bundled shader information was nil for \"" << shader_name
              << "\"." << std::endl;
    return nullptr;
  }
 
  result = bundle_data->CreateFlatbuffer();
  if (!result) {
    std::cerr << "Failed to create flatbuffer for bundled shader \""
              << shader_name << "\"." << std::endl;
    return nullptr;
  }
 
  return result;
}
 
static std::unique_ptr<fb::shaderbundle::ShaderT> GenerateShaderFB(
    SourceOptions options,
    const std::string& shader_name,
    const ShaderConfig& shader_config,
    std::set<std::string>* out_dependencies) {
  auto result = std::make_unique<fb::shaderbundle::ShaderT>();
  result->name = shader_name;
  result->metal_ios =
      GenerateShaderBackendFB(TargetPlatform::kMetalIOS, options, shader_name,
                              shader_config, out_dependencies);
  if (!result->metal_ios) {
    return nullptr;
  }
  result->metal_desktop =
      GenerateShaderBackendFB(TargetPlatform::kMetalDesktop, options,
                              shader_name, shader_config, out_dependencies);
  if (!result->metal_desktop) {
    return nullptr;
  }
  result->opengl_es =
      GenerateShaderBackendFB(TargetPlatform::kOpenGLES, options, shader_name,
                              shader_config, out_dependencies);
  if (!result->opengl_es) {
    return nullptr;
  }
  result->opengl_desktop =
      GenerateShaderBackendFB(TargetPlatform::kOpenGLDesktop, options,
                              shader_name, shader_config, out_dependencies);
  if (!result->opengl_desktop) {
    return nullptr;
  }
  result->vulkan =
      GenerateShaderBackendFB(TargetPlatform::kVulkan, options, shader_name,
                              shader_config, out_dependencies);
  if (!result->vulkan) {
    return nullptr;
  }
  return result;
}
 
std::optional<fb::shaderbundle::ShaderBundleT> GenerateShaderBundleFlatbuffer(
    const std::string& bundle_config_json,
    const SourceOptions& options,
    std::set<std::string>* out_dependencies) {
  // --------------------------------------------------------------------------
  /// 1. Parse the bundle configuration.
  ///
 
  std::optional<ShaderBundleConfig> bundle_config =
      ParseShaderBundleConfig(bundle_config_json, std::cerr);
  if (!bundle_config) {
    return std::nullopt;
  }
 
  // --------------------------------------------------------------------------
  /// 2. Build the deserialized shader bundle.
  ///
 
  fb::shaderbundle::ShaderBundleT shader_bundle;
  shader_bundle.format_version = static_cast<uint32_t>(
      fb::shaderbundle::ShaderBundleFormatVersion::kVersion);
 
  for (const auto& [shader_name, shader_config] : bundle_config.value()) {
    std::unique_ptr<fb::shaderbundle::ShaderT> shader =
        GenerateShaderFB(options, shader_name, shader_config, out_dependencies);
    if (!shader) {
      return std::nullopt;
    }
    shader_bundle.shaders.push_back(std::move(shader));
  }
 
  return shader_bundle;
}
 
/// Write a Ninja-style depfile listing every source file (including
/// `#include`d headers) that contributed to the shader bundle at
/// `target`.
///
/// Format mirrors `Compiler::CreateDepfileContents` for single-shader
/// compiles: `<target>: <dep1> <dep2> ... <depN>\n`.
/// See
/// https://github.com/ninja-build/ninja/blob/master/src/depfile_parser.cc#L28
static bool OutputBundleDepfile(const Switches& switches,
                                const std::string& target,
                                const std::set<std::string>& dependencies) {
  std::stringstream stream;
  stream << target << ":";
  for (const auto& dep : dependencies) {
    stream << " " << dep;
  }
  stream << "\n";
  const auto contents = std::make_shared<std::string>(stream.str());
  const fml::NonOwnedMapping mapping(
      reinterpret_cast<const uint8_t*>(contents->data()), contents->size(),
      [contents](auto, auto) {});
 
  // Pass the relative path straight through; fml::WriteAtomically
  // resolves it against switches.working_directory (a directory fd
  // representing the build system's intended working dir, which may
  // differ from std::filesystem::current_path()).
  if (!fml::WriteAtomically(*switches.working_directory,
                            Utf8FromPath(switches.depfile_path).c_str(),
                            mapping)) {
    std::cerr << "Could not write depfile to " << switches.depfile_path
              << std::endl;
    return false;
  }
  return true;
}
 
bool GenerateShaderBundle(Switches& switches) {
  // --------------------------------------------------------------------------
  /// 1. Parse the shader bundle and generate the flatbuffer result.
  ///
  /// Collect dependencies along the way so a depfile can be emitted
  /// after the bundle is written. The same source file is compiled
  /// across multiple target platforms; the std::set dedupes naturally.
  ///
 
  std::set<std::string> dependencies;
  const bool want_depfile = !switches.depfile_path.empty();
  auto shader_bundle = GenerateShaderBundleFlatbuffer(
      switches.shader_bundle, switches.CreateSourceOptions(),
      want_depfile ? &dependencies : nullptr);
  if (!shader_bundle.has_value()) {
    // Specific error messages are already handled by
    // GenerateShaderBundleFlatbuffer.
    return false;
  }
 
  // --------------------------------------------------------------------------
  /// 2. Serialize the shader bundle and write to disk.
  ///
 
  auto builder = std::make_shared<flatbuffers::FlatBufferBuilder>();
  builder->Finish(fb::shaderbundle::ShaderBundle::Pack(*builder.get(),
                                                       &shader_bundle.value()),
                  fb::shaderbundle::ShaderBundleIdentifier());
  auto mapping = std::make_shared<fml::NonOwnedMapping>(
      builder->GetBufferPointer(), builder->GetSize(),
      [builder](auto, auto) {});
 
  auto sl_file_name = std::filesystem::absolute(
      std::filesystem::current_path() / switches.sl_file_name);
 
  if (!fml::WriteAtomically(*switches.working_directory,         //
                            Utf8FromPath(sl_file_name).c_str(),  //
                            *mapping                             //
                            )) {
    std::cerr << "Could not write file to " << switches.sl_file_name
              << std::endl;
    return false;
  }
  // Tools that consume the runtime stage data expect the access mode to
  // be 0644.
  if (!SetPermissiveAccess(sl_file_name)) {
    return false;
  }
 
  // --------------------------------------------------------------------------
  /// 3. Output a depfile if one was requested.
  ///
  /// Lets build systems (notably Dart's `hooks` framework, which
  /// `flutter_gpu_shaders`' `buildShaderBundleJson` consumer goes
  /// through) rerun the bundle build when any contributing source file
  /// or `#include`d header changes.
 
  if (want_depfile) {
    if (!OutputBundleDepfile(switches, Utf8FromPath(sl_file_name),
                             dependencies)) {
      return false;
    }
  }
 
  return true;
}
 
}  // namespace compiler
}  // namespace impeller