external/vulkancts/vkscpc/vkscpc.cpp

   1 /*-------------------------------------------------------------------------
   2  * Vulkan CTS Framework
   3  * --------------------
   4  *
   5  * Copyright (c) 2021 The Khronos Group Inc.
   6  *
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *      http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  *
  19  *-------------------------------------------------------------------------*/
  20
  21 #include <iostream>
  22 #include <fstream>
  23 #include <sstream>
  24 #include <json/json.h>
  25 #include "deCommandLine.hpp"
  26 #include "deDirectoryIterator.hpp"
  27 #include "tcuCommandLine.hpp"
  28 #include "tcuPlatform.hpp"
  29 #include "tcuTestContext.hpp"
  30 #include "tcuResource.hpp"
  31 #include "tcuTestLog.hpp"
  32 #include "vkPlatform.hpp"
  33 #include "vktTestCase.hpp"
  34 #include "vksStructsVKSC.hpp"
  35 #include "vksCacheBuilder.hpp"
  36
  37 namespace opt
  38 {
  39
  40 DE_DECLARE_COMMAND_LINE_OPT(CompilerDataPath,   std::string);
  41 DE_DECLARE_COMMAND_LINE_OPT(CompilerOutputFile, std::string);
  42 DE_DECLARE_COMMAND_LINE_OPT(LogFile,                    std::string);
  43 DE_DECLARE_COMMAND_LINE_OPT(FilePrefix,                 std::string);
  44
  45 void registerOptions (de::cmdline::Parser& parser)
  46 {
  47         using de::cmdline::Option;
  48         using de::cmdline::NamedValue;
  49
  50         parser << Option<CompilerDataPath>              ("p", "path",           "Offline pipeline data directory",              "");
  51         parser << Option<CompilerOutputFile>    ("o", "out",            "Output file with pipeline cache",              "");
  52         parser << Option<LogFile>                               ("l", "log",            "Log file",                                                             "dummy.log");
  53         parser << Option<FilePrefix>                    ("x", "prefix",         "Prefix for input files",                               "");
  54 }
  55
  56 }
  57
  58 enum PipelineType
  59 {
  60         PT_UNDEFINED_PIPELINE = 0,
  61         PT_GRAPHICS_PIPELINE,
  62         PT_COMPUTE_PIPELINE,
  63 };
  64
  65 void importFilesForExternalCompiler (vksc_server::VulkanPipelineCacheInput&     input,
  66                                                                          const std::string&                                             path,
  67                                                                          const std::string&                                             filePrefix)
  68 {
  69         vksc_server::json::Context context;
  70
  71         for (de::DirectoryIterator iter(path); iter.hasItem(); iter.next())
  72         {
  73                 const de::FilePath                                                      filePath                                        = iter.getItem();
  74                 if (filePath.getType() != de::FilePath::TYPE_FILE)
  75                         continue;
  76                 if (filePath.getFileExtension() != "json")
  77                         continue;
  78                 if (!filePrefix.empty() && filePath.getBaseName().find(filePrefix) != 0)
  79                         continue;
  80
  81                 std::string                                                                     fileContents;
  82                 {
  83                         std::ifstream file(filePath.getPath());
  84                         std::stringstream buffer;
  85                         buffer << file.rdbuf();
  86                         fileContents = buffer.str();
  87                 }
  88
  89                 Json::Value                                                                     jsonRoot;
  90                 std::string                                                                     errors;
  91                 bool                                                                            parsingSuccessful                       = context.reader->parse(fileContents.c_str(), fileContents.c_str() + fileContents.size(), &jsonRoot, &errors);
  92                 if (!parsingSuccessful)
  93                         TCU_THROW(InternalError, (std::string("JSON parsing error. File ") + filePath.getPath() + " Error : " + errors).c_str());
  94
  95                 // decide what pipeline type will be created later
  96                 PipelineType pipelineType = PT_UNDEFINED_PIPELINE;
  97                 if (jsonRoot.isMember("GraphicsPipelineState"))
  98                         pipelineType = PT_GRAPHICS_PIPELINE;
  99                 else if (jsonRoot.isMember("ComputePipelineState"))
 100                         pipelineType = PT_COMPUTE_PIPELINE;
 101                 if(pipelineType == PT_UNDEFINED_PIPELINE)
 102                         TCU_THROW(InternalError, (std::string("JSON - unknown pipeline. File ") + filePath.getPath()).c_str());
 103
 104                 const Json::Value&                                                      jsonGraphicsPipelineState       = jsonRoot["GraphicsPipelineState"];
 105                 const Json::Value&                                                      jsonComputePipelineState        = jsonRoot["ComputePipelineState"];
 106                 const Json::Value&                                                      jsonPipelineState                       = (pipelineType == PT_GRAPHICS_PIPELINE ) ? jsonGraphicsPipelineState : jsonComputePipelineState;
 107                 vksc_server::VulkanJsonPipelineDescription      pipelineDescription;
 108
 109                 {
 110                         const Json::Value&      jsonSamplerYcbcrConversions             = jsonPipelineState["YcbcrSamplers"];
 111                         if (!jsonSamplerYcbcrConversions.isNull())
 112                         {
 113                                 for (Json::ArrayIndex i = 0; i < jsonSamplerYcbcrConversions.size(); ++i)
 114                                 {
 115                                         const Json::Value::Members      membersNames    = jsonSamplerYcbcrConversions[i].getMemberNames();
 116                                         const Json::Value&                      value                   = jsonSamplerYcbcrConversions[i][membersNames[0]];
 117                                         deUint64                                        index;
 118                                         std::istringstream(membersNames[0]) >> index;
 119                                         input.samplerYcbcrConversions[vk::VkSamplerYcbcrConversion(index)] =  std::string(fileContents.begin() + value.getOffsetStart(), fileContents.begin() + value.getOffsetLimit());
 120                                 }
 121                         }
 122
 123                         const Json::Value&      jsonSamplers                                    = jsonPipelineState["ImmutableSamplers"];
 124                         if (!jsonSamplers.isNull())
 125                         {
 126                                 for (Json::ArrayIndex i = 0; i < jsonSamplers.size(); ++i)
 127                                 {
 128                                         const Json::Value::Members      membersNames    = jsonSamplers[i].getMemberNames();
 129                                         const Json::Value&                      value                   = jsonSamplers[i][membersNames[0]];
 130                                         deUint64                                        index;
 131                                         std::istringstream(membersNames[0]) >> index;
 132                                         input.samplers[vk::VkSampler(index)] = std::string(fileContents.begin() + value.getOffsetStart(), fileContents.begin() + value.getOffsetLimit());
 133                                 }
 134                         }
 135
 136                         const Json::Value&      jsonDescriptorSetLayouts                = jsonPipelineState["DescriptorSetLayouts"];
 137                         if (!jsonDescriptorSetLayouts.isNull())
 138                         {
 139                                 for (Json::ArrayIndex i = 0; i < jsonDescriptorSetLayouts.size(); ++i)
 140                                 {
 141                                         const Json::Value::Members      membersNames    = jsonDescriptorSetLayouts[i].getMemberNames();
 142                                         const Json::Value&                      value                   = jsonDescriptorSetLayouts[i][membersNames[0]];
 143                                         deUint64                                        index;
 144                                         std::istringstream(membersNames[0]) >> index;
 145                                         input.descriptorSetLayouts[vk::VkDescriptorSetLayout(index)] = std::string(fileContents.begin() + value.getOffsetStart(), fileContents.begin() + value.getOffsetLimit());
 146                                 }
 147                         }
 148
 149                         deUint64                                                pipelineLayoutHandle    = 0u;
 150                         deUint64                                                renderPassHandle                = 0u;
 151                         std::map<std::string, deUint64> stages;
 152
 153                         const Json::Value&      jsonComputePipeline                             = jsonPipelineState["ComputePipeline"];
 154                         if (!jsonComputePipeline.isNull())
 155                         {
 156                                 pipelineDescription.pipelineContents                    = std::string(fileContents.begin() + jsonComputePipeline.getOffsetStart(), fileContents.begin() + jsonComputePipeline.getOffsetLimit());
 157                                 pipelineLayoutHandle                                                    = jsonComputePipeline["layout"].asUInt64();
 158
 159                                 const Json::Value&      jsonStage                                       = jsonComputePipeline["stage"];
 160                                 stages[jsonStage["stage"].asString()]                   = jsonStage["module"].asUInt64();
 161                         }
 162
 163                         const Json::Value&      jsonGraphicsPipeline                    = jsonPipelineState["GraphicsPipeline"];
 164                         if (!jsonGraphicsPipeline.isNull())
 165                         {
 166                                 pipelineDescription.pipelineContents                    = std::string(fileContents.begin() + jsonGraphicsPipeline.getOffsetStart(), fileContents.begin() + jsonGraphicsPipeline.getOffsetLimit());
 167                                 pipelineLayoutHandle                                                    = jsonGraphicsPipeline["layout"].asUInt64();
 168                                 renderPassHandle                                                                = jsonGraphicsPipeline["renderPass"].asUInt64();
 169
 170                                 const Json::Value&      jsonStages = jsonGraphicsPipeline["pStages"];
 171                                 for (Json::ArrayIndex i = 0; i < jsonStages.size(); ++i)
 172                                         stages[jsonStages[i]["stage"].asString()] = jsonStages[i]["module"].asUInt64();
 173                         }
 174
 175                         const Json::Value&      jsonPipelineLayout                              = jsonPipelineState["PipelineLayout"];
 176                         if (!jsonPipelineLayout.isNull() && pipelineLayoutHandle != 0u)
 177                         {
 178                                 input.pipelineLayouts[vk::VkPipelineLayout(pipelineLayoutHandle)] = std::string(fileContents.begin() + jsonPipelineLayout.getOffsetStart(), fileContents.begin() + jsonPipelineLayout.getOffsetLimit());
 179                         }
 180
 181                         const Json::Value&      jsonRenderPass                                  = jsonPipelineState["Renderpass"];
 182                         if (!jsonRenderPass.isNull() && renderPassHandle != 0u)
 183                         {
 184                                 input.renderPasses[vk::VkRenderPass(renderPassHandle)] = std::string(fileContents.begin() + jsonRenderPass.getOffsetStart(), fileContents.begin() + jsonRenderPass.getOffsetLimit());
 185                         }
 186
 187                         const Json::Value&      jsonRenderPass2                                 = jsonPipelineState["Renderpass2"];
 188                         if (!jsonRenderPass2.isNull() && renderPassHandle != 0u)
 189                         {
 190                                 input.renderPasses[vk::VkRenderPass(renderPassHandle)] = std::string(fileContents.begin() + jsonRenderPass.getOffsetStart(), fileContents.begin() + jsonRenderPass.getOffsetLimit());
 191                         }
 192
 193                         const Json::Value&      jsonShaderFileNames                             = jsonPipelineState["ShaderFileNames"];
 194                         if (!jsonShaderFileNames.isNull())
 195                         {
 196                                 for (Json::ArrayIndex i = 0; i < jsonShaderFileNames.size(); ++i)
 197                                 {
 198                                         std::string                             stageName       = jsonShaderFileNames[i]["stage"].asString();
 199                                         std::string                             fileName        = jsonShaderFileNames[i]["filename"].asString();
 200                                         auto it = stages.find(stageName);
 201                                         if(it == end(stages))
 202                                                 TCU_THROW(InternalError, (std::string("JSON - missing shader stage. File ") + filePath.getPath()).c_str());
 203
 204                                         de::FilePath                    shaderPath      (path);
 205                                         shaderPath.join(de::FilePath(fileName));
 206                                         std::ifstream                   iFile           (shaderPath.getPath(), std::ios::in | std::ios::binary);
 207                                         if(!iFile)
 208                                                 TCU_THROW(InternalError, (std::string("JSON - missing shader file ") + fileName + ". File " + filePath.getPath()).c_str());
 209
 210                                         auto                                    fileBegin       = iFile.tellg();
 211                                         iFile.seekg(0, std::ios::end);
 212                                         auto                                    fileEnd         = iFile.tellg();
 213                                         iFile.seekg(0, std::ios::beg);
 214                                         std::size_t                             fileSize        = static_cast<std::size_t>(fileEnd - fileBegin);
 215                                         std::vector<deUint8>    shaderData      (fileSize);
 216
 217                                         iFile.read(reinterpret_cast<char*>(shaderData.data()), fileSize);
 218                                         if (iFile.fail())
 219                                                 TCU_THROW(InternalError, (std::string("JSON - error reading shader file ") + fileName + ". File " + filePath.getPath()).c_str());
 220
 221                                         vk::VkShaderModuleCreateInfo smCI
 222                                         {
 223                                                 VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO,            // VkStructureType                              sType;
 224                                                 DE_NULL,                                                                                        // const void*                                  pNext;
 225                                                 vk::VkShaderModuleCreateFlags(0u),                                      // VkShaderModuleCreateFlags    flags;
 226                                                 fileSize,                                                                                       // deUintptr                                    codeSize;
 227                                                 reinterpret_cast<deUint32*>(shaderData.data())          // const deUint32*                              pCode;
 228                                         };
 229
 230                                         input.shaderModules[vk::VkShaderModule(it->second)] = vksc_server::json::writeJSON_VkShaderModuleCreateInfo(smCI);
 231                                 }
 232                         }
 233
 234                         const Json::Value&      jsonPhysicalDeviceFeatures              = jsonPipelineState["PhysicalDeviceFeatures"];
 235                         if (!jsonPhysicalDeviceFeatures.isNull())
 236                         {
 237                                 pipelineDescription.deviceFeatures                              = std::string(fileContents.begin() + jsonPhysicalDeviceFeatures.getOffsetStart(), fileContents.begin() + jsonPhysicalDeviceFeatures.getOffsetLimit());
 238                         }
 239                 }
 240
 241                 const Json::Value&      jsonEnabledExtensions                           = jsonRoot["EnabledExtensions"];
 242                 if (!jsonEnabledExtensions.isNull())
 243                 {
 244                         for (Json::ArrayIndex i = 0; i < jsonEnabledExtensions.size(); ++i)
 245                                 pipelineDescription.deviceExtensions.push_back(jsonEnabledExtensions[i].asString());
 246                 }
 247
 248                 const Json::Value&      jsonPipelineUUID                                        = jsonRoot["PipelineUUID"];
 249                 if (!jsonPipelineUUID.isNull())
 250                 {
 251                         pipelineDescription.id.sType                    = VK_STRUCTURE_TYPE_PIPELINE_OFFLINE_CREATE_INFO;
 252                         pipelineDescription.id.pNext                    = DE_NULL;
 253                         for (Json::ArrayIndex i = 0; i < jsonPipelineUUID.size(); ++i)
 254                                 pipelineDescription.id.pipelineIdentifier[i] = deUint8(jsonPipelineUUID[i].asUInt());
 255                         pipelineDescription.id.matchControl             = VK_PIPELINE_MATCH_CONTROL_APPLICATION_UUID_EXACT_MATCH;
 256                         pipelineDescription.id.poolEntrySize    = 0u;
 257                 }
 258                 input.pipelines.push_back(pipelineDescription);
 259         }
 260 }
 261
 262 tcu::Platform* createPlatform(void);
 263
 264 int main (int argc, char** argv)
 265 {
 266         de::cmdline::CommandLine        cmdLine;
 267
 268         // Parse command line.
 269         {
 270                 de::cmdline::Parser     parser;
 271                 opt::registerOptions(parser);
 272
 273                 if (!parser.parse(argc, argv, &cmdLine, std::cerr))
 274                 {
 275                         parser.help(std::cout);
 276                         return EXIT_FAILURE;
 277                 }
 278         }
 279
 280         try
 281         {
 282                 // load JSON files into VulkanPipelineCacheInput
 283                 vksc_server::VulkanPipelineCacheInput                   input;
 284                 importFilesForExternalCompiler(input, cmdLine.getOption<opt::CompilerDataPath>(), cmdLine.getOption<opt::FilePrefix>());
 285
 286                 // create Vulkan instance
 287                 tcu::CommandLine                                cmdLineDummy    {"--deqp-vk-device-id=0"};
 288                 tcu::DirArchive                                 archive                 {""};
 289                 tcu::TestLog                                    log                             { cmdLine.getOption<opt::LogFile>().c_str() }; log.supressLogging(true);
 290                 de::SharedPtr<tcu::Platform>    platform                {createPlatform()};
 291                 de::SharedPtr<vk::Library>              library                 {platform->getVulkanPlatform().createLibrary(vk::Platform::LIBRARY_TYPE_VULKAN, DE_NULL)};
 292                 tcu::TestContext                                tcx                             {*platform, archive, log, cmdLineDummy, nullptr};
 293                 vk::BinaryCollection                    collection              {};
 294                 vkt::Context                                    context                 (tcx, library->getPlatformInterface(), collection, de::SharedPtr<vk::ResourceInterface>{new vk::ResourceInterfaceStandard{ tcx }});
 295
 296                 // create pipeline cache
 297                 std::vector<deUint8>                    binary                  = vksc_server::buildPipelineCache(
 298                                                                                                                         input,
 299                                                                                                                         library->getPlatformInterface(),
 300                                                                                                                         context.getInstance(),
 301                                                                                                                         context.getInstanceInterface(),
 302                                                                                                                         context.getPhysicalDevice(),
 303                                                                                                                         context.getUniversalQueueFamilyIndex());
 304
 305                 // write pipeline cache to output file
 306                 std::ofstream                                   oFile                   (cmdLine.getOption<opt::CompilerOutputFile>().c_str(), std::ios::out | std::ios::binary);
 307                 if (!oFile)
 308                         TCU_THROW(InternalError, (std::string("Cannot create file : ") + cmdLine.getOption<opt::CompilerOutputFile>().c_str()));
 309                 oFile.write(reinterpret_cast<char*>(binary.data()), binary.size());
 310         }
 311         catch (const std::exception& e)
 312         {
 313                 std::cout << e.what() << std::endl;
 314         }
 315
 316         return EXIT_SUCCESS;
 317 }