From 2401fc0a726ce696f555108320fe602b35a5de82 Mon Sep 17 00:00:00 2001 From: Andrey Tuganov Date: Tue, 3 Oct 2017 17:36:37 -0400 Subject: [PATCH] Refactored MARK-V API - switched from C to C++ - moved MARK-V model creation from backend to frontend - The same MARK-V model object can be used to encode/decode multiple files - Added MARK-V model factory (currently only one option) - Added --validate option to spirv-markv (run validation while encoding/decoding) --- include/spirv-tools/markv.h | 91 ---- source/comp/CMakeLists.txt | 2 +- source/comp/markv.h | 64 +++ source/comp/markv_autogen.cpp | 58 --- source/comp/markv_autogen.h | 60 --- source/comp/markv_codec.cpp | 422 +++++------------- source/comp/markv_model.h | 176 ++++++++ test/comp/CMakeLists.txt | 6 +- test/comp/markv_codec_test.cpp | 70 +-- tools/CMakeLists.txt | 5 +- tools/comp/markv.cpp | 71 +-- tools/comp/markv_model_factory.cpp | 34 ++ tools/comp/markv_model_factory.h | 32 ++ tools/comp/markv_model_shader_default.cpp | 112 +++++ tools/comp/markv_model_shader_default.h | 30 ++ .../markv_model_shader_default_autogen.inc | 0 tools/stats/stats_analyzer.cpp | 5 +- 17 files changed, 615 insertions(+), 623 deletions(-) delete mode 100644 include/spirv-tools/markv.h create mode 100644 source/comp/markv.h delete mode 100644 source/comp/markv_autogen.cpp delete mode 100644 source/comp/markv_autogen.h create mode 100644 source/comp/markv_model.h create mode 100644 tools/comp/markv_model_factory.cpp create mode 100644 tools/comp/markv_model_factory.h create mode 100644 tools/comp/markv_model_shader_default.cpp create mode 100644 tools/comp/markv_model_shader_default.h rename source/comp/markv_autogen.inc => tools/comp/markv_model_shader_default_autogen.inc (100%) diff --git a/include/spirv-tools/markv.h b/include/spirv-tools/markv.h deleted file mode 100644 index 9941d435..00000000 --- a/include/spirv-tools/markv.h +++ /dev/null @@ -1,91 +0,0 @@ -// Copyright (c) 2017 Google Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// MARK-V is a compression format for SPIR-V binaries. It strips away -// non-essential information (such as result ids which can be regenerated) and -// uses various bit reduction techiniques to reduce the size of the binary. -// -// WIP: MARK-V codec is in early stages of development. At the moment it only -// can encode and decode some SPIR-V files and only if exacly the same build of -// software is used (is doesn't write or handle version numbers yet). - -#ifndef SPIRV_TOOLS_MARKV_H_ -#define SPIRV_TOOLS_MARKV_H_ - -#include "libspirv.h" - -#ifdef __cplusplus -extern "C" { -#endif - -typedef struct spv_markv_binary_t { - uint8_t* data; - size_t length; -} spv_markv_binary_t; - -typedef spv_markv_binary_t* spv_markv_binary; -typedef const spv_markv_binary_t* const_spv_markv_binary; - -typedef struct spv_markv_encoder_options_t spv_markv_encoder_options_t; -typedef spv_markv_encoder_options_t* spv_markv_encoder_options; -typedef const spv_markv_encoder_options_t* spv_const_markv_encoder_options; - -typedef struct spv_markv_decoder_options_t spv_markv_decoder_options_t; -typedef spv_markv_decoder_options_t* spv_markv_decoder_options; -typedef const spv_markv_decoder_options_t* spv_const_markv_decoder_options; - -// Creates spv_markv_encoder_options with default options. Returns a valid -// options object. The object remains valid until it is passed into -// spvMarkvEncoderOptionsDestroy. -spv_markv_encoder_options spvMarkvEncoderOptionsCreate(); - -// Destroys the given spv_markv_encoder_options object. -void spvMarkvEncoderOptionsDestroy(spv_markv_encoder_options options); - -// Creates spv_markv_decoder_options with default options. Returns a valid -// options object. The object remains valid until it is passed into -// spvMarkvDecoderOptionsDestroy. -spv_markv_decoder_options spvMarkvDecoderOptionsCreate(); - -// Destroys the given spv_markv_decoder_options object. -void spvMarkvDecoderOptionsDestroy(spv_markv_decoder_options options); - -// Encodes the given SPIR-V binary to MARK-V binary. -// If |comments| is not nullptr, it would contain a textual description of -// how encoding was done (with snippets of disassembly and bit sequences). -spv_result_t spvSpirvToMarkv(spv_const_context context, - const uint32_t* spirv_words, - size_t spirv_num_words, - spv_const_markv_encoder_options options, - spv_markv_binary* markv_binary, - spv_text* comments, spv_diagnostic* diagnostic); - -// Decodes a SPIR-V binary from the given MARK-V binary. -// If |comments| is not nullptr, it would contain a textual description of -// how decoding was done (with snippets of disassembly and bit sequences). -spv_result_t spvMarkvToSpirv(spv_const_context context, - const uint8_t* markv_data, - size_t markv_size_bytes, - spv_const_markv_decoder_options options, - spv_binary* spirv_binary, - spv_text* comments, spv_diagnostic* diagnostic); - -// Destroys MARK-V binary created by spvSpirvToMarkv(). -void spvMarkvBinaryDestroy(spv_markv_binary binary); - -#ifdef __cplusplus -} -#endif - -#endif // SPIRV_TOOLS_MARKV_H_ diff --git a/source/comp/CMakeLists.txt b/source/comp/CMakeLists.txt index cb6be7ae..b1d3e5c8 100644 --- a/source/comp/CMakeLists.txt +++ b/source/comp/CMakeLists.txt @@ -13,7 +13,7 @@ # limitations under the License. if(SPIRV_BUILD_COMPRESSION) - add_library(SPIRV-Tools-comp markv_codec.cpp markv_autogen.cpp) + add_library(SPIRV-Tools-comp markv_codec.cpp) spvtools_default_compile_options(SPIRV-Tools-comp) target_include_directories(SPIRV-Tools-comp diff --git a/source/comp/markv.h b/source/comp/markv.h new file mode 100644 index 00000000..063c210e --- /dev/null +++ b/source/comp/markv.h @@ -0,0 +1,64 @@ +// Copyright (c) 2017 Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// MARK-V is a compression format for SPIR-V binaries. It strips away +// non-essential information (such as result ids which can be regenerated) and +// uses various bit reduction techiniques to reduce the size of the binary and +// make it more similar to other compressed SPIR-V files to further improve +// compression of the dataset. + +#ifndef SPIRV_TOOLS_MARKV_HPP_ +#define SPIRV_TOOLS_MARKV_HPP_ + +#include +#include + +#include "markv_model.h" +#include "spirv-tools/libspirv.hpp" + +namespace spvtools { + +struct MarkvEncoderOptions { + bool validate_spirv_binary = false; +}; + +struct MarkvDecoderOptions { + bool validate_spirv_binary = false; +}; + +// Encodes the given SPIR-V binary to MARK-V binary. +// If |comments| is not nullptr, it would contain a textual description of +// how encoding was done (with snippets of disassembly and bit sequences). +spv_result_t SpirvToMarkv(spv_const_context context, + const std::vector& spirv, + const MarkvEncoderOptions& options, + const MarkvModel& markv_model, + MessageConsumer message_consumer, + std::vector* markv, + std::string* comments); + +// Decodes a SPIR-V binary from the given MARK-V binary. +// If |comments| is not nullptr, it would contain a textual description of +// how decoding was done (with snippets of disassembly and bit sequences). +spv_result_t MarkvToSpirv(spv_const_context context, + const std::vector& markv, + const MarkvDecoderOptions& options, + const MarkvModel& markv_model, + MessageConsumer message_consumer, + std::vector* spirv, + std::string* comments); + +} // namespace spvtools + +#endif // SPIRV_TOOLS_MARKV_HPP_ diff --git a/source/comp/markv_autogen.cpp b/source/comp/markv_autogen.cpp deleted file mode 100644 index 3e89e14f..00000000 --- a/source/comp/markv_autogen.cpp +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright (c) 2017 Google Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "markv_autogen.h" - -#include -#include -#include -#include -#include -#include - -#include "spirv/1.2/spirv.h" - -using spvutils::HuffmanCodec; - -namespace { - -// Signals that the value is not in the coding scheme and a fallback method -// needs to be used. -const uint64_t kMarkvNoneOfTheAbove = GetMarkvNonOfTheAbove(); - -inline uint32_t CombineOpcodeAndNumOperands(uint32_t opcode, - uint32_t num_operands) { - return opcode | (num_operands << 16); -} - -} // namespace - -// The following file contains autogenerated statistical coding rules. -// Generated by running spirv-stats on representative corpus of shaders with -// flags: -// --codegen_opcode_and_num_operands_hist -// --codegen_opcode_and_num_operands_markov_huffman_codecs -// --codegen_literal_string_huffman_codecs -// --codegen_non_id_word_huffman_codecs -// --codegen_id_descriptor_huffman_codecs -// -// Example: -// find -type f -print0 | xargs -0 -s 2000000 -// ~/SPIRV-Tools/build/tools/spirv-stats -v -// --codegen_opcode_and_num_operands_hist -// --codegen_opcode_and_num_operands_markov_huffman_codecs -// --codegen_literal_string_huffman_codecs --codegen_non_id_word_huffman_codecs -// --codegen_id_descriptor_huffman_codecs -o -// ~/SPIRV-Tools/source/comp/markv_autogen.inc -#include "markv_autogen.inc" diff --git a/source/comp/markv_autogen.h b/source/comp/markv_autogen.h deleted file mode 100644 index dd92c34f..00000000 --- a/source/comp/markv_autogen.h +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright (c) 2017 Google Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef LIBSPIRV_COMP_MARKV_AUTOGEN_H_ -#define LIBSPIRV_COMP_MARKV_AUTOGEN_H_ - -#include -#include -#include -#include - -#include "util/huffman_codec.h" - -inline uint64_t GetMarkvNonOfTheAbove() { - // Magic number. - return 1111111111111111111; -} - -// Returns of histogram of CombineOpcodeAndNumOperands(opcode, num_operands). -std::map GetOpcodeAndNumOperandsHist(); - -// Returns Huffman codecs based on a Markov chain of histograms of -// CombineOpcodeAndNumOperands(opcode, num_operands). -// Map prev_opcode -> codec. -std::map>> - GetOpcodeAndNumOperandsMarkovHuffmanCodecs(); - -// Returns Huffman codecs for literal strings. -// Map opcode -> codec. -std::map>> - GetLiteralStringHuffmanCodecs(); - -// Returns Huffman codecs for single-word non-id operand slots. -// Map -> codec. -std::map, - std::unique_ptr>> - GetNonIdWordHuffmanCodecs(); - -// Returns Huffman codecs for id descriptors used by common operand slots. -// Map -> codec. -std::map, - std::unique_ptr>> - GetIdDescriptorHuffmanCodecs(); - -// Returns a set of all descriptors which are encodable by at least one codec -// returned by GetIdDescriptorHuffmanCodecs(). -std::unordered_set GetDescriptorsWithCodingScheme(); - -#endif // LIBSPIRV_COMP_MARKV_AUTOGEN_H_ diff --git a/source/comp/markv_codec.cpp b/source/comp/markv_codec.cpp index 082e1166..9ab4ef86 100644 --- a/source/comp/markv_codec.cpp +++ b/source/comp/markv_codec.cpp @@ -19,9 +19,6 @@ // MARK-V is a compression format for SPIR-V binaries. It strips away // non-essential information (such as result ids which can be regenerated) and // uses various bit reduction techiniques to reduce the size of the binary. -// -// MarkvModel is a flatbuffers object containing a set of rules defining how -// compression/decompression is done (coding schemes, dictionaries). #include #include @@ -48,11 +45,11 @@ #include "ext_inst.h" #include "id_descriptor.h" #include "instruction.h" -#include "markv_autogen.h" +#include "markv.h" +#include "markv_model.h" #include "opcode.h" #include "operand.h" #include "spirv-tools/libspirv.h" -#include "spirv-tools/markv.h" #include "spirv_endian.h" #include "spirv_validator_options.h" #include "util/bit_stream.h" @@ -74,13 +71,7 @@ using spvutils::HuffmanCodec; using MoveToFront = spvutils::MoveToFront; using MultiMoveToFront = spvutils::MultiMoveToFront; -struct spv_markv_encoder_options_t { - bool validate_spirv_binary = false; -}; - -struct spv_markv_decoder_options_t { - bool validate_spirv_binary = false; -}; +namespace spvtools { namespace { @@ -155,7 +146,7 @@ const uint32_t kMarkvMaxPresumedAccessIndex = 31; // Signals that the value is not in the coding scheme and a fallback method // needs to be used. -const uint64_t kMarkvNoneOfTheAbove = GetMarkvNonOfTheAbove(); +const uint64_t kMarkvNoneOfTheAbove = MarkvModel::GetMarkvNoneOfTheAbove(); // Mtf ranks smaller than this are encoded with Huffman coding. const uint32_t kMtfSmallestRankEncodedByValue = 10; @@ -208,208 +199,6 @@ GetMtfHuffmanCodecs() { return codecs; } -// Encoding/decoding model containing various constants and codecs. -class MarkvModel { - public: - MarkvModel() - : mtf_huffman_codecs_(GetMtfHuffmanCodecs()), - opcode_and_num_operands_huffman_codec_(GetOpcodeAndNumOperandsHist()), - opcode_and_num_operands_markov_huffman_codecs_( - GetOpcodeAndNumOperandsMarkovHuffmanCodecs()), - non_id_word_huffman_codecs_(GetNonIdWordHuffmanCodecs()), - id_descriptor_huffman_codecs_(GetIdDescriptorHuffmanCodecs()), - descriptors_with_coding_scheme_(GetDescriptorsWithCodingScheme()), - literal_string_huffman_codecs_(GetLiteralStringHuffmanCodecs()) {} - - size_t opcode_chunk_length() const { return 7; } - size_t num_operands_chunk_length() const { return 3; } - size_t mtf_rank_chunk_length() const { return 5; } - - size_t u16_chunk_length() const { return 4; } - size_t s16_chunk_length() const { return 4; } - size_t s16_block_exponent() const { return 6; } - - size_t u32_chunk_length() const { return 8; } - size_t s32_chunk_length() const { return 8; } - size_t s32_block_exponent() const { return 10; } - - size_t u64_chunk_length() const { return 8; } - size_t s64_chunk_length() const { return 8; } - size_t s64_block_exponent() const { return 10; } - - // Returns Huffman codec for ranks of the mtf with given |handle|. - // Different mtfs can use different rank distributions. - // May return nullptr if the codec doesn't exist. - const HuffmanCodec* GetMtfHuffmanCodec(uint64_t handle) const { - const auto it = mtf_huffman_codecs_.find(handle); - if (it == mtf_huffman_codecs_.end()) - return nullptr; - return it->second.get(); - } - - // Returns a codec for common opcode_and_num_operands words for the given - // previous opcode. May return nullptr if the codec doesn't exist. - const HuffmanCodec* GetOpcodeAndNumOperandsMarkovHuffmanCodec( - uint32_t prev_opcode) const { - if (prev_opcode == SpvOpNop) - return &opcode_and_num_operands_huffman_codec_; - - const auto it = - opcode_and_num_operands_markov_huffman_codecs_.find(prev_opcode); - if (it == opcode_and_num_operands_markov_huffman_codecs_.end()) - return nullptr; - return it->second.get(); - } - - // Returns a codec for common non-id words used for given operand slot. - // Operand slot is defined by the opcode and the operand index. - // May return nullptr if the codec doesn't exist. - const HuffmanCodec* GetNonIdWordHuffmanCodec( - uint32_t opcode, uint32_t operand_index) const { - const auto it = non_id_word_huffman_codecs_.find( - std::pair(opcode, operand_index)); - if (it == non_id_word_huffman_codecs_.end()) - return nullptr; - return it->second.get(); - } - - // Returns a codec for common id descriptos used for given operand slot. - // Operand slot is defined by the opcode and the operand index. - // May return nullptr if the codec doesn't exist. - const HuffmanCodec* GetIdDescriptorHuffmanCodec( - uint32_t opcode, uint32_t operand_index) const { - const auto it = id_descriptor_huffman_codecs_.find( - std::pair(opcode, operand_index)); - if (it == id_descriptor_huffman_codecs_.end()) - return nullptr; - return it->second.get(); - } - - // Returns a codec for common strings used by the given opcode. - // Operand slot is defined by the opcode and the operand index. - // May return nullptr if the codec doesn't exist. - const HuffmanCodec* GetLiteralStringHuffmanCodec( - uint32_t opcode) const { - const auto it = literal_string_huffman_codecs_.find(opcode); - if (it == literal_string_huffman_codecs_.end()) - return nullptr; - return it->second.get(); - } - - bool DescriptorHasCodingScheme(uint32_t descriptor) const { - return descriptors_with_coding_scheme_.count(descriptor); - } - - private: - // Huffman codecs for move-to-front ranks. The map key is mtf handle. Doesn't - // need to contain a different codec for every handle as most use one and the - // same. - std::map>> - mtf_huffman_codecs_; - - // Huffman codec for base-rate of opcode_and_num_operands. - HuffmanCodec opcode_and_num_operands_huffman_codec_; - - // Huffman codecs for opcode_and_num_operands. The map key is previous opcode. - std::map>> - opcode_and_num_operands_markov_huffman_codecs_; - - // Huffman codecs for non-id single-word operand values. - // The map key is pair . - std::map, - std::unique_ptr>> - non_id_word_huffman_codecs_; - - // Huffman codecs for id descriptors. The map key is pair - // . - std::map, - std::unique_ptr>> - id_descriptor_huffman_codecs_; - - std::unordered_set descriptors_with_coding_scheme_; - - // Huffman codecs for literal strings. The map key is the opcode of the - // current instruction. This assumes, that there is no more than one literal - // string operand per instruction, but would still work even if this is not - // the case. Names and debug information strings are not collected. - std::map>> - literal_string_huffman_codecs_; -}; - -const MarkvModel* GetDefaultModel() { - static MarkvModel model; - return &model; -} - -// Returns chunk length used for variable length encoding of spirv operand -// words. Returns zero if operand type corresponds to potentially multiple -// words or a word which is not expected to profit from variable width encoding. -// Chunk length is selected based on the size of expected value. -// Most of these values will later be encoded with probability-based coding, -// but variable width integer coding is a good quick solution. -// TODO(atgoo@github.com): Put this in MarkvModel flatbuffer. -size_t GetOperandVariableWidthChunkLength(spv_operand_type_t type) { - switch (type) { - case SPV_OPERAND_TYPE_TYPE_ID: - return 4; - case SPV_OPERAND_TYPE_RESULT_ID: - case SPV_OPERAND_TYPE_ID: - case SPV_OPERAND_TYPE_SCOPE_ID: - case SPV_OPERAND_TYPE_MEMORY_SEMANTICS_ID: - return 8; - case SPV_OPERAND_TYPE_LITERAL_INTEGER: - case SPV_OPERAND_TYPE_OPTIONAL_LITERAL_INTEGER: - return 6; - case SPV_OPERAND_TYPE_CAPABILITY: - return 6; - case SPV_OPERAND_TYPE_SOURCE_LANGUAGE: - case SPV_OPERAND_TYPE_EXECUTION_MODEL: - return 3; - case SPV_OPERAND_TYPE_ADDRESSING_MODEL: - case SPV_OPERAND_TYPE_MEMORY_MODEL: - return 2; - case SPV_OPERAND_TYPE_EXECUTION_MODE: - return 6; - case SPV_OPERAND_TYPE_STORAGE_CLASS: - return 4; - case SPV_OPERAND_TYPE_DIMENSIONALITY: - case SPV_OPERAND_TYPE_SAMPLER_ADDRESSING_MODE: - return 3; - case SPV_OPERAND_TYPE_SAMPLER_FILTER_MODE: - return 2; - case SPV_OPERAND_TYPE_SAMPLER_IMAGE_FORMAT: - return 6; - case SPV_OPERAND_TYPE_FP_ROUNDING_MODE: - case SPV_OPERAND_TYPE_LINKAGE_TYPE: - case SPV_OPERAND_TYPE_ACCESS_QUALIFIER: - case SPV_OPERAND_TYPE_OPTIONAL_ACCESS_QUALIFIER: - return 2; - case SPV_OPERAND_TYPE_FUNCTION_PARAMETER_ATTRIBUTE: - return 3; - case SPV_OPERAND_TYPE_DECORATION: - case SPV_OPERAND_TYPE_BUILT_IN: - return 6; - case SPV_OPERAND_TYPE_GROUP_OPERATION: - case SPV_OPERAND_TYPE_KERNEL_ENQ_FLAGS: - case SPV_OPERAND_TYPE_KERNEL_PROFILING_INFO: - return 2; - case SPV_OPERAND_TYPE_FP_FAST_MATH_MODE: - case SPV_OPERAND_TYPE_FUNCTION_CONTROL: - case SPV_OPERAND_TYPE_LOOP_CONTROL: - case SPV_OPERAND_TYPE_IMAGE: - case SPV_OPERAND_TYPE_OPTIONAL_IMAGE: - case SPV_OPERAND_TYPE_OPTIONAL_MEMORY_ACCESS: - case SPV_OPERAND_TYPE_SELECTION_CONTROL: - return 4; - case SPV_OPERAND_TYPE_EXTENSION_INSTRUCTION_NUMBER: - case SPV_OPERAND_TYPE_TYPED_LITERAL_NUMBER: - return 6; - default: - return 0; - } - return 0; -} - // Returns true if the opcode has a fixed number of operands. May return a // false negative. bool OpcodeHasFixedNumberOfOperands(SpvOp opcode) { @@ -534,21 +323,6 @@ class CommentLogger { bool use_delimiter_ = false; }; -// Creates spv_text object containing text from |str|. -// The returned value is owned by the caller and needs to be destroyed with -// spvTextDestroy. -spv_text CreateSpvText(const std::string& str) { - spv_text out = new spv_text_t(); - assert(out); - char* cstr = new char[str.length() + 1]; - assert(cstr); - std::strncpy(cstr, str.c_str(), str.length()); - cstr[str.length()] = '\0'; - out->str = cstr; - out->length = str.length(); - return out; -} - // Base class for MARK-V encoder and decoder. Contains common functionality // such as: // - Validator connection and validation state. @@ -561,10 +335,6 @@ class MarkvCodecBase { MarkvCodecBase() = delete; - void SetModel(const MarkvModel* model) { - model_ = model; - } - protected: struct MarkvHeader { MarkvHeader() { @@ -585,10 +355,14 @@ class MarkvCodecBase { uint32_t spirv_generator; }; + // |model| is owned by the caller, must be not null and valid during the + // lifetime of the codec. explicit MarkvCodecBase(spv_const_context context, - spv_validator_options validator_options) + spv_validator_options validator_options, + const MarkvModel* model) : validator_options_(validator_options), grammar_(context), - model_(GetDefaultModel()), context_(context), + model_(model), mtf_huffman_codecs_(GetMtfHuffmanCodecs()), + context_(context), vstate_(validator_options ? new ValidationState_t(context, validator_options_) : nullptr) {} @@ -713,9 +487,21 @@ class MarkvCodecBase { vstate_->setIdBound(id_bound); } + // Returns Huffman codec for ranks of the mtf with given |handle|. + // Different mtfs can use different rank distributions. + // May return nullptr if the codec doesn't exist. + const spvutils::HuffmanCodec* GetMtfHuffmanCodec(uint64_t handle) const { + const auto it = mtf_huffman_codecs_.find(handle); + if (it == mtf_huffman_codecs_.end()) + return nullptr; + return it->second.get(); + } + spv_validator_options validator_options_ = nullptr; const libspirv::AssemblyGrammar grammar_; MarkvHeader header_; + + // MARK-V model, not owned. const MarkvModel* model_ = nullptr; // Current instruction, current operand and current operand index. @@ -755,6 +541,12 @@ class MarkvCodecBase { // Container/computer for id descriptors. IdDescriptorCollection id_descriptors_; + // Huffman codecs for move-to-front ranks. The map key is mtf handle. Doesn't + // need to contain a different codec for every handle as most use one and the + // same. + std::map>> + mtf_huffman_codecs_; + private: spv_const_context context_ = nullptr; @@ -777,9 +569,12 @@ class MarkvCodecBase { // on how encoding was done, which can later be accessed with GetComments(). class MarkvEncoder : public MarkvCodecBase { public: + // |model| is owned by the caller, must be not null and valid during the + // lifetime of MarkvEncoder. MarkvEncoder(spv_const_context context, - spv_const_markv_encoder_options options) - : MarkvCodecBase(context, GetValidatorOptions(options)), + const MarkvEncoderOptions& options, + const MarkvModel* model) + : MarkvCodecBase(context, GetValidatorOptions(options), model), options_(options) { (void) options_; } @@ -804,19 +599,20 @@ class MarkvEncoder : public MarkvCodecBase { // into a single buffer and returns it as spv_markv_binary. The returned // value is owned by the caller and needs to be destroyed with // spvMarkvBinaryDestroy(). - spv_markv_binary GetMarkvBinary() { + std::vector GetMarkvBinary() { header_.markv_length_in_bits = static_cast(sizeof(header_) * 8 + writer_.GetNumBits()); + header_.markv_model = + (model_->model_type() << 16) | model_->model_version(); + const size_t num_bytes = sizeof(header_) + writer_.GetDataSizeBytes(); + std::vector markv(num_bytes); - spv_markv_binary markv_binary = new spv_markv_binary_t(); - markv_binary->data = new uint8_t[num_bytes]; - markv_binary->length = num_bytes; assert(writer_.GetData()); - std::memcpy(markv_binary->data, &header_, sizeof(header_)); - std::memcpy(markv_binary->data + sizeof(header_), + std::memcpy(markv.data(), &header_, sizeof(header_)); + std::memcpy(markv.data() + sizeof(header_), writer_.GetData(), writer_.GetDataSizeBytes()); - return markv_binary; + return markv; } // Creates an internal logger which writes comments on the encoding process. @@ -854,8 +650,8 @@ class MarkvEncoder : public MarkvCodecBase { private: // Creates and returns validator options. Returned value owned by the caller. static spv_validator_options GetValidatorOptions( - spv_const_markv_encoder_options options) { - return options->validate_spirv_binary ? + const MarkvEncoderOptions& options) { + return options.validate_spirv_binary ? spvValidatorOptionsCreate() : nullptr; } @@ -896,7 +692,7 @@ class MarkvEncoder : public MarkvCodecBase { // Encodes a literal number operand and writes it to the bit stream. spv_result_t EncodeLiteralNumber(const spv_parsed_operand_t& operand); - spv_const_markv_encoder_options options_; + MarkvEncoderOptions options_; // Bit stream where encoded instructions are written. BitWriterWord64 writer_; @@ -912,12 +708,14 @@ class MarkvEncoder : public MarkvCodecBase { // Decodes MARK-V buffers written by MarkvEncoder. class MarkvDecoder : public MarkvCodecBase { public: + // |model| is owned by the caller, must be not null and valid during the + // lifetime of MarkvEncoder. MarkvDecoder(spv_const_context context, - const uint8_t* markv_data, - size_t markv_size_bytes, - spv_const_markv_decoder_options options) - : MarkvCodecBase(context, GetValidatorOptions(options)), - options_(options), reader_(markv_data, markv_size_bytes) { + const std::vector& markv, + const MarkvDecoderOptions& options, + const MarkvModel* model) + : MarkvCodecBase(context, GetValidatorOptions(options), model), + options_(options), reader_(markv) { (void) options_; SetIdBound(1); parsed_operands_.reserve(25); @@ -938,8 +736,8 @@ class MarkvDecoder : public MarkvCodecBase { // Creates and returns validator options. Returned value owned by the caller. static spv_validator_options GetValidatorOptions( - spv_const_markv_decoder_options options) { - return options->validate_spirv_binary ? + const MarkvDecoderOptions& options) { + return options.validate_spirv_binary ? spvValidatorOptionsCreate() : nullptr; } @@ -1024,7 +822,7 @@ class MarkvDecoder : public MarkvCodecBase { // kind SPV_NUMBER_NONE. void RecordNumberType(); - spv_const_markv_decoder_options options_; + MarkvDecoderOptions options_; // Temporary sink where decoded SPIR-V words are written. Once it contains the // entire module, the container is moved and returned. @@ -1690,7 +1488,8 @@ spv_result_t MarkvEncoder::EncodeNonIdWord(uint32_t word) { } // Fallback encoding. - const size_t chunk_length = GetOperandVariableWidthChunkLength(operand_.type); + const size_t chunk_length = + model_->GetOperandVariableWidthChunkLength(operand_.type); if (chunk_length) { writer_.WriteVariableWidthU32(word, chunk_length); } else { @@ -1718,7 +1517,8 @@ spv_result_t MarkvDecoder::DecodeNonIdWord(uint32_t* word) { // Received kMarkvNoneOfTheAbove signal, use fallback decoding. } - const size_t chunk_length = GetOperandVariableWidthChunkLength(operand_.type); + const size_t chunk_length = + model_->GetOperandVariableWidthChunkLength(operand_.type); if (chunk_length) { if (!reader_.ReadVariableWidthU32(word, chunk_length)) return Diag(SPV_ERROR_INVALID_BINARY) @@ -1817,10 +1617,10 @@ spv_result_t MarkvDecoder::DecodeOpcodeAndNumberOfOperands( spv_result_t MarkvEncoder::EncodeMtfRankHuffman(uint32_t rank, uint64_t mtf, uint64_t fallback_method) { - const auto* codec = model_->GetMtfHuffmanCodec(mtf); + const auto* codec = GetMtfHuffmanCodec(mtf); if (!codec) { assert(fallback_method != kMtfNone); - codec = model_->GetMtfHuffmanCodec(fallback_method); + codec = GetMtfHuffmanCodec(fallback_method); } if (!codec) @@ -1850,10 +1650,10 @@ spv_result_t MarkvEncoder::EncodeMtfRankHuffman(uint32_t rank, uint64_t mtf, spv_result_t MarkvDecoder::DecodeMtfRankHuffman( uint64_t mtf, uint32_t fallback_method, uint32_t* rank) { - const auto* codec = model_->GetMtfHuffmanCodec(mtf); + const auto* codec = GetMtfHuffmanCodec(mtf); if (!codec) { assert(fallback_method != kMtfNone); - codec = model_->GetMtfHuffmanCodec(fallback_method); + codec = GetMtfHuffmanCodec(fallback_method); } if (!codec) @@ -2493,6 +2293,17 @@ spv_result_t MarkvDecoder::DecodeModule(std::vector* spirv_binary) { return Diag(SPV_ERROR_INVALID_BINARY) << "MARK-V binary and the codec have different versions"; + const uint32_t model_type = header_.markv_model >> 16; + const uint32_t model_version = header_.markv_model & 0xFFFF; + if (model_type != model_->model_type()) + return Diag(SPV_ERROR_INVALID_BINARY) + << "MARK-V binary and the codec use different MARK-V models"; + + if (model_version != model_->model_version()) + return Diag(SPV_ERROR_INVALID_BINARY) + << "MARK-V binary and the codec use different versions if the same " + << "MARK-V model"; + spirv_.reserve(header_.markv_length_in_bits / 2); // Heuristic. spirv_.resize(5, 0); spirv_[0] = kSpirvMagicNumber; @@ -3026,19 +2837,17 @@ spv_result_t EncodeInstruction( } // namespace -spv_result_t spvSpirvToMarkv(spv_const_context context, - const uint32_t* spirv_words, - const size_t spirv_num_words, - spv_const_markv_encoder_options options, - spv_markv_binary* markv_binary, - spv_text* comments, spv_diagnostic* diagnostic) { +spv_result_t SpirvToMarkv(spv_const_context context, + const std::vector& spirv, + const MarkvEncoderOptions& options, + const MarkvModel& markv_model, + MessageConsumer message_consumer, + std::vector* markv, + std::string* comments) { spv_context_t hijack_context = *context; - if (diagnostic) { - *diagnostic = nullptr; - libspirv::UseDiagnosticAsMessageConsumer(&hijack_context, diagnostic); - } + SetContextMessageConsumer(&hijack_context, message_consumer); - spv_const_binary_t spirv_binary = {spirv_words, spirv_num_words}; + spv_const_binary_t spirv_binary = {spirv.data(), spirv.size()}; spv_endianness_t endian; spv_position_t position = {}; @@ -3055,13 +2864,13 @@ spv_result_t spvSpirvToMarkv(spv_const_context context, << "Invalid SPIR-V header."; } - MarkvEncoder encoder(&hijack_context, options); + MarkvEncoder encoder(&hijack_context, options, &markv_model); if (comments) { encoder.CreateCommentsLogger(); spv_text text = nullptr; - if (spvBinaryToText(&hijack_context, spirv_words, spirv_num_words, + if (spvBinaryToText(&hijack_context, spirv.data(), spirv.size(), SPV_BINARY_TO_TEXT_OPTION_NO_HEADER, &text, nullptr) != SPV_SUCCESS) { return DiagnosticStream(position, hijack_context.consumer, @@ -3074,72 +2883,41 @@ spv_result_t spvSpirvToMarkv(spv_const_context context, } if (spvBinaryParse( - &hijack_context, &encoder, spirv_words, spirv_num_words, EncodeHeader, - EncodeInstruction, diagnostic) != SPV_SUCCESS) { + &hijack_context, &encoder, spirv.data(), spirv.size(), EncodeHeader, + EncodeInstruction, nullptr) != SPV_SUCCESS) { return DiagnosticStream(position, hijack_context.consumer, SPV_ERROR_INVALID_BINARY) << "Unable to encode to MARK-V."; } if (comments) - *comments = CreateSpvText(encoder.GetComments()); + *comments = encoder.GetComments(); - *markv_binary = encoder.GetMarkvBinary(); + *markv = encoder.GetMarkvBinary(); return SPV_SUCCESS; } -spv_result_t spvMarkvToSpirv(spv_const_context context, - const uint8_t* markv_data, - size_t markv_size_bytes, - spv_const_markv_decoder_options options, - spv_binary* spirv_binary, - spv_text* /* comments */, - spv_diagnostic* diagnostic) { +spv_result_t MarkvToSpirv(spv_const_context context, + const std::vector& markv, + const MarkvDecoderOptions& options, + const MarkvModel& markv_model, + MessageConsumer message_consumer, + std::vector* spirv, + std::string* /* comments */) { spv_position_t position = {}; spv_context_t hijack_context = *context; - if (diagnostic) { - *diagnostic = nullptr; - libspirv::UseDiagnosticAsMessageConsumer(&hijack_context, diagnostic); - } - - MarkvDecoder decoder(&hijack_context, markv_data, markv_size_bytes, options); + SetContextMessageConsumer(&hijack_context, message_consumer); - std::vector words; + MarkvDecoder decoder(&hijack_context, markv, options, &markv_model); - if (decoder.DecodeModule(&words) != SPV_SUCCESS) { + if (decoder.DecodeModule(spirv) != SPV_SUCCESS) { return DiagnosticStream(position, hijack_context.consumer, SPV_ERROR_INVALID_BINARY) << "Unable to decode MARK-V."; } - assert(!words.empty()); - - *spirv_binary = new spv_binary_t(); - (*spirv_binary)->code = new uint32_t[words.size()]; - (*spirv_binary)->wordCount = words.size(); - std::memcpy((*spirv_binary)->code, words.data(), 4 * words.size()); - + assert(!spirv->empty()); return SPV_SUCCESS; } -void spvMarkvBinaryDestroy(spv_markv_binary binary) { - if (!binary) return; - delete[] binary->data; - delete binary; -} - -spv_markv_encoder_options spvMarkvEncoderOptionsCreate() { - return new spv_markv_encoder_options_t; -} - -void spvMarkvEncoderOptionsDestroy(spv_markv_encoder_options options) { - delete options; -} - -spv_markv_decoder_options spvMarkvDecoderOptionsCreate() { - return new spv_markv_decoder_options_t; -} - -void spvMarkvDecoderOptionsDestroy(spv_markv_decoder_options options) { - delete options; -} +} // namespave spvtools diff --git a/source/comp/markv_model.h b/source/comp/markv_model.h new file mode 100644 index 00000000..f656df45 --- /dev/null +++ b/source/comp/markv_model.h @@ -0,0 +1,176 @@ +// Copyright (c) 2017 Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef LIBSPIRV_COMP_MARKV_MODEL_H_ +#define LIBSPIRV_COMP_MARKV_MODEL_H_ + +#include +#include +#include + +#include "spirv/1.2/spirv.h" +#include "spirv-tools/libspirv.h" +#include "util/huffman_codec.h" + +namespace spvtools { + +// Base class for MARK-V models. +// The class contains encoding/decoding model with various constants and +// codecs used by the compression algorithm. +class MarkvModel { + public: + MarkvModel() : operand_chunk_lengths_( + static_cast(SPV_OPERAND_TYPE_NUM_OPERAND_TYPES), 0) {} + + uint32_t model_type() const { return model_type_; } + uint32_t model_version() const { return model_version_; } + + uint32_t opcode_chunk_length() const { return opcode_chunk_length_; } + uint32_t num_operands_chunk_length() const { return num_operands_chunk_length_; } + uint32_t mtf_rank_chunk_length() const { return mtf_rank_chunk_length_; } + + uint32_t u64_chunk_length() const { return u64_chunk_length_; } + uint32_t s64_chunk_length() const { return s64_chunk_length_; } + uint32_t s64_block_exponent() const { return s64_block_exponent_; } + + // Returns a codec for common opcode_and_num_operands words for the given + // previous opcode. May return nullptr if the codec doesn't exist. + const spvutils::HuffmanCodec* GetOpcodeAndNumOperandsMarkovHuffmanCodec( + uint32_t prev_opcode) const { + if (prev_opcode == SpvOpNop) + return opcode_and_num_operands_huffman_codec_.get(); + + const auto it = + opcode_and_num_operands_markov_huffman_codecs_.find(prev_opcode); + if (it == opcode_and_num_operands_markov_huffman_codecs_.end()) + return nullptr; + return it->second.get(); + } + + // Returns a codec for common non-id words used for given operand slot. + // Operand slot is defined by the opcode and the operand index. + // May return nullptr if the codec doesn't exist. + const spvutils::HuffmanCodec* GetNonIdWordHuffmanCodec( + uint32_t opcode, uint32_t operand_index) const { + const auto it = non_id_word_huffman_codecs_.find( + std::pair(opcode, operand_index)); + if (it == non_id_word_huffman_codecs_.end()) + return nullptr; + return it->second.get(); + } + + // Returns a codec for common id descriptos used for given operand slot. + // Operand slot is defined by the opcode and the operand index. + // May return nullptr if the codec doesn't exist. + const spvutils::HuffmanCodec* GetIdDescriptorHuffmanCodec( + uint32_t opcode, uint32_t operand_index) const { + const auto it = id_descriptor_huffman_codecs_.find( + std::pair(opcode, operand_index)); + if (it == id_descriptor_huffman_codecs_.end()) + return nullptr; + return it->second.get(); + } + + // Returns a codec for common strings used by the given opcode. + // Operand slot is defined by the opcode and the operand index. + // May return nullptr if the codec doesn't exist. + const spvutils::HuffmanCodec* GetLiteralStringHuffmanCodec( + uint32_t opcode) const { + const auto it = literal_string_huffman_codecs_.find(opcode); + if (it == literal_string_huffman_codecs_.end()) + return nullptr; + return it->second.get(); + } + + // Checks if |descriptor| has a coding scheme in any of + // id_descriptor_huffman_codecs_. + bool DescriptorHasCodingScheme(uint32_t descriptor) const { + return descriptors_with_coding_scheme_.count(descriptor); + } + + // Returns chunk length used for variable length encoding of spirv operand + // words. + uint32_t GetOperandVariableWidthChunkLength(spv_operand_type_t type) const { + return operand_chunk_lengths_.at(static_cast(type)); + } + + // Sets model type. + void SetModelType(uint32_t in_model_type) { + model_type_ = in_model_type; + } + + // Sets model version. + void SetModelVersion(uint32_t in_model_version) { + model_version_ = in_model_version; + } + + // Returns value used by Huffman codecs as a signal that a value is not in the + // coding table. + static uint64_t GetMarkvNoneOfTheAbove() { + // Magic number. + return 1111111111111111111; + } + + MarkvModel(const MarkvModel&) = delete; + const MarkvModel& operator=(const MarkvModel&) = delete; + + protected: + // Huffman codec for base-rate of opcode_and_num_operands. + std::unique_ptr> + opcode_and_num_operands_huffman_codec_; + + // Huffman codecs for opcode_and_num_operands. The map key is previous opcode. + std::map>> + opcode_and_num_operands_markov_huffman_codecs_; + + // Huffman codecs for non-id single-word operand values. + // The map key is pair . + std::map, + std::unique_ptr>> non_id_word_huffman_codecs_; + + // Huffman codecs for id descriptors. The map key is pair + // . + std::map, + std::unique_ptr>> id_descriptor_huffman_codecs_; + + // Set of all descriptors which have a coding scheme in any of + // id_descriptor_huffman_codecs_. + std::unordered_set descriptors_with_coding_scheme_; + + // Huffman codecs for literal strings. The map key is the opcode of the + // current instruction. This assumes, that there is no more than one literal + // string operand per instruction, but would still work even if this is not + // the case. Names and debug information strings are not collected. + std::map>> + literal_string_huffman_codecs_; + + // Chunk lengths used for variable width encoding of operands (index is + // spv_operand_type of the operand). + std::vector operand_chunk_lengths_; + + uint32_t opcode_chunk_length_ = 7; + uint32_t num_operands_chunk_length_ = 3; + uint32_t mtf_rank_chunk_length_ = 5; + + uint32_t u64_chunk_length_ = 8; + uint32_t s64_chunk_length_ = 8; + uint32_t s64_block_exponent_ = 10; + + uint32_t model_type_ = 0; + uint32_t model_version_ = 0; +}; + +} // namespace spvtools + +#endif // LIBSPIRV_COMP_MARKV_MODEL_H_ diff --git a/test/comp/CMakeLists.txt b/test/comp/CMakeLists.txt index 80339fb1..9243fa26 100644 --- a/test/comp/CMakeLists.txt +++ b/test/comp/CMakeLists.txt @@ -19,7 +19,11 @@ set(VAL_TEST_COMMON_SRCS if(SPIRV_BUILD_COMPRESSION) add_spvtools_unittest(TARGET markv_codec - SRCS markv_codec_test.cpp ${VAL_TEST_COMMON_SRCS} + SRCS + markv_codec_test.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../../tools/comp/markv_model_factory.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../../tools/comp/markv_model_shader_default.cpp + ${VAL_TEST_COMMON_SRCS} LIBS SPIRV-Tools-comp ${SPIRV_TOOLS} ) endif(SPIRV_BUILD_COMPRESSION) diff --git a/test/comp/markv_codec_test.cpp b/test/comp/markv_codec_test.cpp index 711069ca..09d39a8c 100644 --- a/test/comp/markv_codec_test.cpp +++ b/test/comp/markv_codec_test.cpp @@ -19,8 +19,9 @@ #include #include "gmock/gmock.h" -#include "spirv-tools/markv.h" +#include "source/comp/markv.h" #include "test_fixture.h" +#include "tools/comp/markv_model_factory.h" #include "unit_spirv.h" namespace { @@ -82,52 +83,15 @@ void Disassemble(const std::vector& words, spvTextDestroy(text); } -// Encodes SPIR-V |words| to |markv_binary|. |comments| context snippets of -// disassembly and bit sequences for debugging. -void Encode(const std::vector& words, - spv_markv_binary* markv_binary, - std::string* comments, - spv_target_env env = SPV_ENV_UNIVERSAL_1_2) { - ScopedContext ctx(env); - SetContextMessageConsumer(ctx.context, DiagnosticsMessageHandler); - - std::unique_ptr> options( - spvMarkvEncoderOptionsCreate(), &spvMarkvEncoderOptionsDestroy); - spv_text spv_text_comments; - ASSERT_EQ(SPV_SUCCESS, spvSpirvToMarkv(ctx.context, words.data(), - words.size(), options.get(), - markv_binary, &spv_text_comments, - nullptr)); - - *comments = std::string(spv_text_comments->str, spv_text_comments->length); - spvTextDestroy(spv_text_comments); -} - -// Decodes |markv_binary| to SPIR-V |words|. -void Decode(const spv_markv_binary markv_binary, - std::vector* words, - spv_target_env env = SPV_ENV_UNIVERSAL_1_2) { - ScopedContext ctx(env); - SetContextMessageConsumer(ctx.context, DiagnosticsMessageHandler); - - spv_binary spirv_binary = nullptr; - std::unique_ptr> options( - spvMarkvDecoderOptionsCreate(), &spvMarkvDecoderOptionsDestroy); - ASSERT_EQ(SPV_SUCCESS, spvMarkvToSpirv(ctx.context, markv_binary->data, - markv_binary->length, options.get(), - &spirv_binary, nullptr, nullptr)); - - *words = std::vector( - spirv_binary->code, spirv_binary->code + spirv_binary->wordCount); - - spvBinaryDestroy(spirv_binary); -} - // Encodes/decodes |original|, assembles/dissasembles |original|, then compares // the results of the two operations. void TestEncodeDecode(const std::string& original_text) { + ScopedContext ctx(SPV_ENV_UNIVERSAL_1_2); + std::unique_ptr model = + spvtools::CreateMarkvModel(spvtools::kMarkvModelShaderDefault); + spvtools::MarkvEncoderOptions encoder_options; + spvtools::MarkvDecoderOptions decoder_options; + std::vector expected_binary; Compile(original_text, &expected_binary); ASSERT_FALSE(expected_binary.empty()); @@ -141,17 +105,17 @@ void TestEncodeDecode(const std::string& original_text) { SPV_TEXT_TO_BINARY_OPTION_PRESERVE_NUMERIC_IDS); ASSERT_FALSE(binary_to_encode.empty()); - spv_markv_binary markv_binary = nullptr; + std::vector markv; std::string encoder_comments; - Encode(binary_to_encode, &markv_binary, &encoder_comments); - ASSERT_NE(nullptr, markv_binary); - - // std::cerr << encoder_comments << std::endl; - // std::cerr << "SPIR-V size: " << expected_binary.size() * 4 << std::endl; - // std::cerr << "MARK-V size: " << markv_binary->length << std::endl; + ASSERT_EQ(SPV_SUCCESS, spvtools::SpirvToMarkv( + ctx.context, binary_to_encode, encoder_options, *model, + DiagnosticsMessageHandler, &markv, &encoder_comments)); + ASSERT_FALSE(markv.empty()); std::vector decoded_binary; - Decode(markv_binary, &decoded_binary); + ASSERT_EQ(SPV_SUCCESS, spvtools::MarkvToSpirv( + ctx.context, markv, decoder_options, *model, + DiagnosticsMessageHandler, &decoded_binary, nullptr)); ASSERT_FALSE(decoded_binary.empty()); EXPECT_EQ(expected_binary, decoded_binary) << encoder_comments; @@ -161,8 +125,6 @@ void TestEncodeDecode(const std::string& original_text) { ASSERT_FALSE(decoded_text.empty()); EXPECT_EQ(expected_text, decoded_text) << encoder_comments; - - spvMarkvBinaryDestroy(markv_binary); } void TestEncodeDecodeShaderMainBody(const std::string& body) { diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index fa4ade7e..1219720f 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -61,7 +61,10 @@ if (NOT ${SPIRV_SKIP_EXECUTABLES}) spirv-cfg spirv-link) if(SPIRV_BUILD_COMPRESSION) - add_spvtools_tool(TARGET spirv-markv SRCS comp/markv.cpp + add_spvtools_tool(TARGET spirv-markv + SRCS comp/markv.cpp + comp/markv_model_factory.cpp + comp/markv_model_shader_default.cpp LIBS SPIRV-Tools-comp ${SPIRV_TOOLS}) target_include_directories(spirv-markv PRIVATE ${spirv-tools_SOURCE_DIR} ${SPIRV_HEADER_INCLUDE_DIR}) diff --git a/tools/comp/markv.cpp b/tools/comp/markv.cpp index f9df9ca6..b107cdd2 100644 --- a/tools/comp/markv.cpp +++ b/tools/comp/markv.cpp @@ -20,9 +20,10 @@ #include #include +#include "markv_model_factory.h" +#include "source/comp/markv.h" #include "source/spirv_target_env.h" #include "source/table.h" -#include "spirv-tools/markv.h" #include "tools/io.h" namespace { @@ -63,6 +64,7 @@ Options: -h, --help Print this help. --comments Write codec comments to stdout. --version Display MARK-V codec version. + --validate Validate SPIR-V while encoding or decoding. -o Set the output filename. Output goes to standard output if this option is @@ -119,6 +121,7 @@ int main(int argc, char** argv) { } bool want_comments = false; + bool validate_spirv_binary = false; for (int argi = 2; argi < argc; ++argi) { if ('-' == argv[argi][0]) { @@ -143,6 +146,8 @@ int main(int argc, char** argv) { } else if (0 == strcmp(argv[argi], "--version")) { fprintf(stderr, "error: Not implemented\n"); return 1; + } else if (0 == strcmp(argv[argi], "--validate")) { + validate_spirv_binary = true; } else { print_usage(argv[0]); return 1; @@ -179,69 +184,69 @@ int main(int argc, char** argv) { const bool write_to_stdout = output_filename == nullptr || 0 == strcmp(output_filename, "-"); - spv_text comments = nullptr; - spv_text* comments_ptr = want_comments ? &comments : nullptr; + std::string comments; + std::string* comments_ptr = want_comments ? &comments : nullptr; ScopedContext ctx(SPV_ENV_UNIVERSAL_1_2); - SetContextMessageConsumer(ctx.context, DiagnosticsMessageHandler); + + std::unique_ptr model = + spvtools::CreateMarkvModel(spvtools::kMarkvModelShaderDefault); if (task == kEncode) { - std::vector contents; - if (!ReadFile(input_filename, "rb", &contents)) return 1; + std::vector spirv; + if (!ReadFile(input_filename, "rb", &spirv)) return 1; + + spvtools::MarkvEncoderOptions options; + options.validate_spirv_binary = validate_spirv_binary; - std::unique_ptr> options( - spvMarkvEncoderOptionsCreate(), &spvMarkvEncoderOptionsDestroy); - spv_markv_binary markv_binary = nullptr; + std::vector markv; - if (SPV_SUCCESS != - spvSpirvToMarkv(ctx.context, contents.data(), contents.size(), - options.get(), &markv_binary, comments_ptr, nullptr)) { + if (SPV_SUCCESS != spvtools::SpirvToMarkv( + ctx.context, spirv, options, *model, DiagnosticsMessageHandler, + &markv, comments_ptr)) { std::cerr << "error: Failed to encode " << input_filename << " to MARK-V " << std::endl; return 1; } if (want_comments) { - if (!WriteFile(nullptr, "w", comments->str, - comments->length)) return 1; + if (!WriteFile(nullptr, "w", comments.c_str(), + comments.length())) return 1; } if (!want_comments || !write_to_stdout) { - if (!WriteFile(output_filename, "wb", markv_binary->data, - markv_binary->length)) return 1; + if (!WriteFile(output_filename, "wb", markv.data(), + markv.size())) return 1; } } else if (task == kDecode) { - std::vector contents; - if (!ReadFile(input_filename, "rb", &contents)) return 1; + std::vector markv; + if (!ReadFile(input_filename, "rb", &markv)) return 1; - std::unique_ptr> options( - spvMarkvDecoderOptionsCreate(), &spvMarkvDecoderOptionsDestroy); - spv_binary spirv_binary = nullptr; + spvtools::MarkvDecoderOptions options; + options.validate_spirv_binary = validate_spirv_binary; - if (SPV_SUCCESS != - spvMarkvToSpirv(ctx.context, contents.data(), contents.size(), - options.get(), &spirv_binary, comments_ptr, nullptr)) { - std::cerr << "error: Failed to encode " << input_filename << " to MARK-V " + std::vector spirv; + + if (SPV_SUCCESS != spvtools::MarkvToSpirv( + ctx.context, markv, options, *model, DiagnosticsMessageHandler, + &spirv, comments_ptr)) { + std::cerr << "error: Failed to decode " << input_filename << " to SPIR-V " << std::endl; return 1; } if (want_comments) { - if (!WriteFile(nullptr, "w", comments->str, - comments->length)) return 1; + if (!WriteFile(nullptr, "w", comments.c_str(), + comments.length())) return 1; } if (!want_comments || !write_to_stdout) { - if (!WriteFile(output_filename, "wb", spirv_binary->code, - spirv_binary->wordCount)) return 1; + if (!WriteFile(output_filename, "wb", spirv.data(), + spirv.size())) return 1; } } else { assert(false && "Unknown task"); } - spvTextDestroy(comments); - return 0; } diff --git a/tools/comp/markv_model_factory.cpp b/tools/comp/markv_model_factory.cpp new file mode 100644 index 00000000..c0a833bb --- /dev/null +++ b/tools/comp/markv_model_factory.cpp @@ -0,0 +1,34 @@ +// Copyright (c) 2017 Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "markv_model_factory.h" +#include "markv_model_shader_default.h" + +namespace spvtools { + +std::unique_ptr CreateMarkvModel(MarkvModelType type) { + std::unique_ptr model; + switch (type) { + case kMarkvModelShaderDefault: { + model.reset(new MarkvModelShaderDefault()); + break; + } + } + + model->SetModelType(static_cast(type)); + + return model; +} + +} // namespace spvtools diff --git a/tools/comp/markv_model_factory.h b/tools/comp/markv_model_factory.h new file mode 100644 index 00000000..235efe0e --- /dev/null +++ b/tools/comp/markv_model_factory.h @@ -0,0 +1,32 @@ +// Copyright (c) 2017 Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef SPIRV_TOOLS_COMP_MARKV_MODEL_FACTORY_H_ +#define SPIRV_TOOLS_COMP_MARKV_MODEL_FACTORY_H_ + +#include + +#include "source/comp/markv_model.h" + +namespace spvtools { + +enum MarkvModelType { + kMarkvModelShaderDefault = 1, +}; + +std::unique_ptr CreateMarkvModel(MarkvModelType type); + +} // namespace spvtools + +#endif // SPIRV_TOOLS_COMP_MARKV_MODEL_FACTORY_H_ diff --git a/tools/comp/markv_model_shader_default.cpp b/tools/comp/markv_model_shader_default.cpp new file mode 100644 index 00000000..a58c103f --- /dev/null +++ b/tools/comp/markv_model_shader_default.cpp @@ -0,0 +1,112 @@ +// Copyright (c) 2017 Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "markv_model_shader_default.h" + +#include +#include +#include +#include +#include +#include + +using spvutils::HuffmanCodec; + +namespace spvtools { + +namespace { + +// Signals that the value is not in the coding scheme and a fallback method +// needs to be used. +const uint64_t kMarkvNoneOfTheAbove = MarkvModel::GetMarkvNoneOfTheAbove(); + +inline uint32_t CombineOpcodeAndNumOperands(uint32_t opcode, + uint32_t num_operands) { + return opcode | (num_operands << 16); +} + +// The following file contains autogenerated statistical coding rules. +// Generated by running spirv-stats on representative corpus of shaders with +// flags: +// --codegen_opcode_and_num_operands_hist +// --codegen_opcode_and_num_operands_markov_huffman_codecs +// --codegen_literal_string_huffman_codecs +// --codegen_non_id_word_huffman_codecs +// --codegen_id_descriptor_huffman_codecs +// +// Example: +// find -type f -print0 | xargs -0 -s 2000000 +// ~/SPIRV-Tools/build/tools/spirv-stats -v +// --codegen_opcode_and_num_operands_hist +// --codegen_opcode_and_num_operands_markov_huffman_codecs +// --codegen_literal_string_huffman_codecs --codegen_non_id_word_huffman_codecs +// --codegen_id_descriptor_huffman_codecs -o +// ~/SPIRV-Tools/source/comp/markv_autogen.inc +#include "markv_model_shader_default_autogen.inc" + +} // namespace + +MarkvModelShaderDefault::MarkvModelShaderDefault() { + const uint16_t kVersionNumber = 0; + SetModelVersion(kVersionNumber); + + opcode_and_num_operands_huffman_codec_.reset( + new HuffmanCodec(GetOpcodeAndNumOperandsHist())); + opcode_and_num_operands_markov_huffman_codecs_ = + GetOpcodeAndNumOperandsMarkovHuffmanCodecs(); + non_id_word_huffman_codecs_ = GetNonIdWordHuffmanCodecs(); + id_descriptor_huffman_codecs_ = GetIdDescriptorHuffmanCodecs(); + descriptors_with_coding_scheme_ = GetDescriptorsWithCodingScheme(); + literal_string_huffman_codecs_ = GetLiteralStringHuffmanCodecs(); + + operand_chunk_lengths_[SPV_OPERAND_TYPE_TYPE_ID] = 4; + operand_chunk_lengths_[SPV_OPERAND_TYPE_RESULT_ID] = 8; + operand_chunk_lengths_[SPV_OPERAND_TYPE_ID] = 8; + operand_chunk_lengths_[SPV_OPERAND_TYPE_SCOPE_ID] = 8; + operand_chunk_lengths_[SPV_OPERAND_TYPE_MEMORY_SEMANTICS_ID] = 8; + operand_chunk_lengths_[SPV_OPERAND_TYPE_LITERAL_INTEGER] = 6; + operand_chunk_lengths_[SPV_OPERAND_TYPE_OPTIONAL_LITERAL_INTEGER] = 6; + operand_chunk_lengths_[SPV_OPERAND_TYPE_CAPABILITY] = 6; + operand_chunk_lengths_[SPV_OPERAND_TYPE_SOURCE_LANGUAGE] = 3; + operand_chunk_lengths_[SPV_OPERAND_TYPE_EXECUTION_MODEL] = 3; + operand_chunk_lengths_[SPV_OPERAND_TYPE_ADDRESSING_MODEL] = 2; + operand_chunk_lengths_[SPV_OPERAND_TYPE_MEMORY_MODEL] = 2; + operand_chunk_lengths_[SPV_OPERAND_TYPE_EXECUTION_MODE] = 6; + operand_chunk_lengths_[SPV_OPERAND_TYPE_STORAGE_CLASS] = 4; + operand_chunk_lengths_[SPV_OPERAND_TYPE_DIMENSIONALITY] = 3; + operand_chunk_lengths_[SPV_OPERAND_TYPE_SAMPLER_ADDRESSING_MODE] = 3; + operand_chunk_lengths_[SPV_OPERAND_TYPE_SAMPLER_FILTER_MODE] = 2; + operand_chunk_lengths_[SPV_OPERAND_TYPE_SAMPLER_IMAGE_FORMAT] = 6; + operand_chunk_lengths_[SPV_OPERAND_TYPE_FP_ROUNDING_MODE] = 2; + operand_chunk_lengths_[SPV_OPERAND_TYPE_LINKAGE_TYPE] = 2; + operand_chunk_lengths_[SPV_OPERAND_TYPE_ACCESS_QUALIFIER] = 2; + operand_chunk_lengths_[SPV_OPERAND_TYPE_OPTIONAL_ACCESS_QUALIFIER] = 2; + operand_chunk_lengths_[SPV_OPERAND_TYPE_FUNCTION_PARAMETER_ATTRIBUTE] = 3; + operand_chunk_lengths_[SPV_OPERAND_TYPE_DECORATION] = 6; + operand_chunk_lengths_[SPV_OPERAND_TYPE_BUILT_IN] = 6; + operand_chunk_lengths_[SPV_OPERAND_TYPE_GROUP_OPERATION] = 2; + operand_chunk_lengths_[SPV_OPERAND_TYPE_KERNEL_ENQ_FLAGS] = 2; + operand_chunk_lengths_[SPV_OPERAND_TYPE_KERNEL_PROFILING_INFO] = 2; + operand_chunk_lengths_[SPV_OPERAND_TYPE_FP_FAST_MATH_MODE] = 4; + operand_chunk_lengths_[SPV_OPERAND_TYPE_FUNCTION_CONTROL] = 4; + operand_chunk_lengths_[SPV_OPERAND_TYPE_LOOP_CONTROL] = 4; + operand_chunk_lengths_[SPV_OPERAND_TYPE_IMAGE] = 4; + operand_chunk_lengths_[SPV_OPERAND_TYPE_OPTIONAL_IMAGE] = 4; + operand_chunk_lengths_[SPV_OPERAND_TYPE_OPTIONAL_MEMORY_ACCESS] = 4; + operand_chunk_lengths_[SPV_OPERAND_TYPE_SELECTION_CONTROL] = 4; + operand_chunk_lengths_[SPV_OPERAND_TYPE_EXTENSION_INSTRUCTION_NUMBER] = 6; + operand_chunk_lengths_[SPV_OPERAND_TYPE_TYPED_LITERAL_NUMBER] = 6; +} + +} // namespace spvtools diff --git a/tools/comp/markv_model_shader_default.h b/tools/comp/markv_model_shader_default.h new file mode 100644 index 00000000..c338b78a --- /dev/null +++ b/tools/comp/markv_model_shader_default.h @@ -0,0 +1,30 @@ +// Copyright (c) 2017 Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef SPIRV_TOOLS_COMP_DEFAULT_SHADER_MARKV_MODEL_H_ +#define SPIRV_TOOLS_COMP_DEFAULT_SHADER_MARKV_MODEL_H_ + +#include "source/comp/markv_model.h" + +namespace spvtools { + +// MARK-V model designed to be a default model for shader compression. +class MarkvModelShaderDefault : public MarkvModel { + public: + MarkvModelShaderDefault(); +}; + +} // namespace spvtools + +#endif // SPIRV_TOOLS_COMP_DEFAULT_SHADER_MARKV_MODEL_H_ diff --git a/source/comp/markv_autogen.inc b/tools/comp/markv_model_shader_default_autogen.inc similarity index 100% rename from source/comp/markv_autogen.inc rename to tools/comp/markv_model_shader_default_autogen.inc diff --git a/tools/stats/stats_analyzer.cpp b/tools/stats/stats_analyzer.cpp index cb2085db..b9d2c66c 100644 --- a/tools/stats/stats_analyzer.cpp +++ b/tools/stats/stats_analyzer.cpp @@ -25,7 +25,7 @@ #include "spirv/1.2/spirv.h" #include "source/enum_string_mapping.h" -#include "source/comp/markv_autogen.h" +#include "source/comp/markv_model.h" #include "source/opcode.h" #include "source/operand.h" #include "source/spirv_constant.h" @@ -38,7 +38,8 @@ namespace { // Signals that the value is not in the coding scheme and a fallback method // needs to be used. -const uint64_t kMarkvNoneOfTheAbove = GetMarkvNonOfTheAbove(); +const uint64_t kMarkvNoneOfTheAbove = + spvtools::MarkvModel::GetMarkvNoneOfTheAbove(); inline uint32_t CombineOpcodeAndNumOperands(uint32_t opcode, uint32_t num_operands) { -- 2.34.1