+++ /dev/null
-// Copyright (c) 2017 Google Inc.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// MARK-V is a compression format for SPIR-V binaries. It strips away
-// non-essential information (such as result ids which can be regenerated) and
-// uses various bit reduction techiniques to reduce the size of the binary.
-//
-// WIP: MARK-V codec is in early stages of development. At the moment it only
-// can encode and decode some SPIR-V files and only if exacly the same build of
-// software is used (is doesn't write or handle version numbers yet).
-
-#ifndef SPIRV_TOOLS_MARKV_H_
-#define SPIRV_TOOLS_MARKV_H_
-
-#include "libspirv.h"
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-typedef struct spv_markv_binary_t {
- uint8_t* data;
- size_t length;
-} spv_markv_binary_t;
-
-typedef spv_markv_binary_t* spv_markv_binary;
-typedef const spv_markv_binary_t* const_spv_markv_binary;
-
-typedef struct spv_markv_encoder_options_t spv_markv_encoder_options_t;
-typedef spv_markv_encoder_options_t* spv_markv_encoder_options;
-typedef const spv_markv_encoder_options_t* spv_const_markv_encoder_options;
-
-typedef struct spv_markv_decoder_options_t spv_markv_decoder_options_t;
-typedef spv_markv_decoder_options_t* spv_markv_decoder_options;
-typedef const spv_markv_decoder_options_t* spv_const_markv_decoder_options;
-
-// Creates spv_markv_encoder_options with default options. Returns a valid
-// options object. The object remains valid until it is passed into
-// spvMarkvEncoderOptionsDestroy.
-spv_markv_encoder_options spvMarkvEncoderOptionsCreate();
-
-// Destroys the given spv_markv_encoder_options object.
-void spvMarkvEncoderOptionsDestroy(spv_markv_encoder_options options);
-
-// Creates spv_markv_decoder_options with default options. Returns a valid
-// options object. The object remains valid until it is passed into
-// spvMarkvDecoderOptionsDestroy.
-spv_markv_decoder_options spvMarkvDecoderOptionsCreate();
-
-// Destroys the given spv_markv_decoder_options object.
-void spvMarkvDecoderOptionsDestroy(spv_markv_decoder_options options);
-
-// Encodes the given SPIR-V binary to MARK-V binary.
-// If |comments| is not nullptr, it would contain a textual description of
-// how encoding was done (with snippets of disassembly and bit sequences).
-spv_result_t spvSpirvToMarkv(spv_const_context context,
- const uint32_t* spirv_words,
- size_t spirv_num_words,
- spv_const_markv_encoder_options options,
- spv_markv_binary* markv_binary,
- spv_text* comments, spv_diagnostic* diagnostic);
-
-// Decodes a SPIR-V binary from the given MARK-V binary.
-// If |comments| is not nullptr, it would contain a textual description of
-// how decoding was done (with snippets of disassembly and bit sequences).
-spv_result_t spvMarkvToSpirv(spv_const_context context,
- const uint8_t* markv_data,
- size_t markv_size_bytes,
- spv_const_markv_decoder_options options,
- spv_binary* spirv_binary,
- spv_text* comments, spv_diagnostic* diagnostic);
-
-// Destroys MARK-V binary created by spvSpirvToMarkv().
-void spvMarkvBinaryDestroy(spv_markv_binary binary);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif // SPIRV_TOOLS_MARKV_H_
# limitations under the License.
if(SPIRV_BUILD_COMPRESSION)
- add_library(SPIRV-Tools-comp markv_codec.cpp markv_autogen.cpp)
+ add_library(SPIRV-Tools-comp markv_codec.cpp)
spvtools_default_compile_options(SPIRV-Tools-comp)
target_include_directories(SPIRV-Tools-comp
--- /dev/null
+// Copyright (c) 2017 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// MARK-V is a compression format for SPIR-V binaries. It strips away
+// non-essential information (such as result ids which can be regenerated) and
+// uses various bit reduction techiniques to reduce the size of the binary and
+// make it more similar to other compressed SPIR-V files to further improve
+// compression of the dataset.
+
+#ifndef SPIRV_TOOLS_MARKV_HPP_
+#define SPIRV_TOOLS_MARKV_HPP_
+
+#include <string>
+#include <vector>
+
+#include "markv_model.h"
+#include "spirv-tools/libspirv.hpp"
+
+namespace spvtools {
+
+struct MarkvEncoderOptions {
+ bool validate_spirv_binary = false;
+};
+
+struct MarkvDecoderOptions {
+ bool validate_spirv_binary = false;
+};
+
+// Encodes the given SPIR-V binary to MARK-V binary.
+// If |comments| is not nullptr, it would contain a textual description of
+// how encoding was done (with snippets of disassembly and bit sequences).
+spv_result_t SpirvToMarkv(spv_const_context context,
+ const std::vector<uint32_t>& spirv,
+ const MarkvEncoderOptions& options,
+ const MarkvModel& markv_model,
+ MessageConsumer message_consumer,
+ std::vector<uint8_t>* markv,
+ std::string* comments);
+
+// Decodes a SPIR-V binary from the given MARK-V binary.
+// If |comments| is not nullptr, it would contain a textual description of
+// how decoding was done (with snippets of disassembly and bit sequences).
+spv_result_t MarkvToSpirv(spv_const_context context,
+ const std::vector<uint8_t>& markv,
+ const MarkvDecoderOptions& options,
+ const MarkvModel& markv_model,
+ MessageConsumer message_consumer,
+ std::vector<uint32_t>* spirv,
+ std::string* comments);
+
+} // namespace spvtools
+
+#endif // SPIRV_TOOLS_MARKV_HPP_
+++ /dev/null
-// Copyright (c) 2017 Google Inc.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include "markv_autogen.h"
-
-#include <algorithm>
-#include <map>
-#include <memory>
-#include <vector>
-#include <unordered_map>
-#include <unordered_set>
-
-#include "spirv/1.2/spirv.h"
-
-using spvutils::HuffmanCodec;
-
-namespace {
-
-// Signals that the value is not in the coding scheme and a fallback method
-// needs to be used.
-const uint64_t kMarkvNoneOfTheAbove = GetMarkvNonOfTheAbove();
-
-inline uint32_t CombineOpcodeAndNumOperands(uint32_t opcode,
- uint32_t num_operands) {
- return opcode | (num_operands << 16);
-}
-
-} // namespace
-
-// The following file contains autogenerated statistical coding rules.
-// Generated by running spirv-stats on representative corpus of shaders with
-// flags:
-// --codegen_opcode_and_num_operands_hist
-// --codegen_opcode_and_num_operands_markov_huffman_codecs
-// --codegen_literal_string_huffman_codecs
-// --codegen_non_id_word_huffman_codecs
-// --codegen_id_descriptor_huffman_codecs
-//
-// Example:
-// find <SHADER_CORPUS_DIR> -type f -print0 | xargs -0 -s 2000000
-// ~/SPIRV-Tools/build/tools/spirv-stats -v
-// --codegen_opcode_and_num_operands_hist
-// --codegen_opcode_and_num_operands_markov_huffman_codecs
-// --codegen_literal_string_huffman_codecs --codegen_non_id_word_huffman_codecs
-// --codegen_id_descriptor_huffman_codecs -o
-// ~/SPIRV-Tools/source/comp/markv_autogen.inc
-#include "markv_autogen.inc"
+++ /dev/null
-// Copyright (c) 2017 Google Inc.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#ifndef LIBSPIRV_COMP_MARKV_AUTOGEN_H_
-#define LIBSPIRV_COMP_MARKV_AUTOGEN_H_
-
-#include <map>
-#include <memory>
-#include <numeric>
-#include <unordered_set>
-
-#include "util/huffman_codec.h"
-
-inline uint64_t GetMarkvNonOfTheAbove() {
- // Magic number.
- return 1111111111111111111;
-}
-
-// Returns of histogram of CombineOpcodeAndNumOperands(opcode, num_operands).
-std::map<uint64_t, uint32_t> GetOpcodeAndNumOperandsHist();
-
-// Returns Huffman codecs based on a Markov chain of histograms of
-// CombineOpcodeAndNumOperands(opcode, num_operands).
-// Map prev_opcode -> codec.
-std::map<uint32_t, std::unique_ptr<spvutils::HuffmanCodec<uint64_t>>>
- GetOpcodeAndNumOperandsMarkovHuffmanCodecs();
-
-// Returns Huffman codecs for literal strings.
-// Map opcode -> codec.
-std::map<uint32_t, std::unique_ptr<spvutils::HuffmanCodec<std::string>>>
- GetLiteralStringHuffmanCodecs();
-
-// Returns Huffman codecs for single-word non-id operand slots.
-// Map <opcode, operand_index> -> codec.
-std::map<std::pair<uint32_t, uint32_t>,
- std::unique_ptr<spvutils::HuffmanCodec<uint64_t>>>
- GetNonIdWordHuffmanCodecs();
-
-// Returns Huffman codecs for id descriptors used by common operand slots.
-// Map <opcode, operand_index> -> codec.
-std::map<std::pair<uint32_t, uint32_t>,
- std::unique_ptr<spvutils::HuffmanCodec<uint64_t>>>
- GetIdDescriptorHuffmanCodecs();
-
-// Returns a set of all descriptors which are encodable by at least one codec
-// returned by GetIdDescriptorHuffmanCodecs().
-std::unordered_set<uint32_t> GetDescriptorsWithCodingScheme();
-
-#endif // LIBSPIRV_COMP_MARKV_AUTOGEN_H_
// MARK-V is a compression format for SPIR-V binaries. It strips away
// non-essential information (such as result ids which can be regenerated) and
// uses various bit reduction techiniques to reduce the size of the binary.
-//
-// MarkvModel is a flatbuffers object containing a set of rules defining how
-// compression/decompression is done (coding schemes, dictionaries).
#include <algorithm>
#include <cassert>
#include "ext_inst.h"
#include "id_descriptor.h"
#include "instruction.h"
-#include "markv_autogen.h"
+#include "markv.h"
+#include "markv_model.h"
#include "opcode.h"
#include "operand.h"
#include "spirv-tools/libspirv.h"
-#include "spirv-tools/markv.h"
#include "spirv_endian.h"
#include "spirv_validator_options.h"
#include "util/bit_stream.h"
using MoveToFront = spvutils::MoveToFront<uint32_t>;
using MultiMoveToFront = spvutils::MultiMoveToFront<uint32_t>;
-struct spv_markv_encoder_options_t {
- bool validate_spirv_binary = false;
-};
-
-struct spv_markv_decoder_options_t {
- bool validate_spirv_binary = false;
-};
+namespace spvtools {
namespace {
// Signals that the value is not in the coding scheme and a fallback method
// needs to be used.
-const uint64_t kMarkvNoneOfTheAbove = GetMarkvNonOfTheAbove();
+const uint64_t kMarkvNoneOfTheAbove = MarkvModel::GetMarkvNoneOfTheAbove();
// Mtf ranks smaller than this are encoded with Huffman coding.
const uint32_t kMtfSmallestRankEncodedByValue = 10;
return codecs;
}
-// Encoding/decoding model containing various constants and codecs.
-class MarkvModel {
- public:
- MarkvModel()
- : mtf_huffman_codecs_(GetMtfHuffmanCodecs()),
- opcode_and_num_operands_huffman_codec_(GetOpcodeAndNumOperandsHist()),
- opcode_and_num_operands_markov_huffman_codecs_(
- GetOpcodeAndNumOperandsMarkovHuffmanCodecs()),
- non_id_word_huffman_codecs_(GetNonIdWordHuffmanCodecs()),
- id_descriptor_huffman_codecs_(GetIdDescriptorHuffmanCodecs()),
- descriptors_with_coding_scheme_(GetDescriptorsWithCodingScheme()),
- literal_string_huffman_codecs_(GetLiteralStringHuffmanCodecs()) {}
-
- size_t opcode_chunk_length() const { return 7; }
- size_t num_operands_chunk_length() const { return 3; }
- size_t mtf_rank_chunk_length() const { return 5; }
-
- size_t u16_chunk_length() const { return 4; }
- size_t s16_chunk_length() const { return 4; }
- size_t s16_block_exponent() const { return 6; }
-
- size_t u32_chunk_length() const { return 8; }
- size_t s32_chunk_length() const { return 8; }
- size_t s32_block_exponent() const { return 10; }
-
- size_t u64_chunk_length() const { return 8; }
- size_t s64_chunk_length() const { return 8; }
- size_t s64_block_exponent() const { return 10; }
-
- // Returns Huffman codec for ranks of the mtf with given |handle|.
- // Different mtfs can use different rank distributions.
- // May return nullptr if the codec doesn't exist.
- const HuffmanCodec<uint32_t>* GetMtfHuffmanCodec(uint64_t handle) const {
- const auto it = mtf_huffman_codecs_.find(handle);
- if (it == mtf_huffman_codecs_.end())
- return nullptr;
- return it->second.get();
- }
-
- // Returns a codec for common opcode_and_num_operands words for the given
- // previous opcode. May return nullptr if the codec doesn't exist.
- const HuffmanCodec<uint64_t>* GetOpcodeAndNumOperandsMarkovHuffmanCodec(
- uint32_t prev_opcode) const {
- if (prev_opcode == SpvOpNop)
- return &opcode_and_num_operands_huffman_codec_;
-
- const auto it =
- opcode_and_num_operands_markov_huffman_codecs_.find(prev_opcode);
- if (it == opcode_and_num_operands_markov_huffman_codecs_.end())
- return nullptr;
- return it->second.get();
- }
-
- // Returns a codec for common non-id words used for given operand slot.
- // Operand slot is defined by the opcode and the operand index.
- // May return nullptr if the codec doesn't exist.
- const HuffmanCodec<uint64_t>* GetNonIdWordHuffmanCodec(
- uint32_t opcode, uint32_t operand_index) const {
- const auto it = non_id_word_huffman_codecs_.find(
- std::pair<uint32_t, uint32_t>(opcode, operand_index));
- if (it == non_id_word_huffman_codecs_.end())
- return nullptr;
- return it->second.get();
- }
-
- // Returns a codec for common id descriptos used for given operand slot.
- // Operand slot is defined by the opcode and the operand index.
- // May return nullptr if the codec doesn't exist.
- const HuffmanCodec<uint64_t>* GetIdDescriptorHuffmanCodec(
- uint32_t opcode, uint32_t operand_index) const {
- const auto it = id_descriptor_huffman_codecs_.find(
- std::pair<uint32_t, uint32_t>(opcode, operand_index));
- if (it == id_descriptor_huffman_codecs_.end())
- return nullptr;
- return it->second.get();
- }
-
- // Returns a codec for common strings used by the given opcode.
- // Operand slot is defined by the opcode and the operand index.
- // May return nullptr if the codec doesn't exist.
- const HuffmanCodec<std::string>* GetLiteralStringHuffmanCodec(
- uint32_t opcode) const {
- const auto it = literal_string_huffman_codecs_.find(opcode);
- if (it == literal_string_huffman_codecs_.end())
- return nullptr;
- return it->second.get();
- }
-
- bool DescriptorHasCodingScheme(uint32_t descriptor) const {
- return descriptors_with_coding_scheme_.count(descriptor);
- }
-
- private:
- // Huffman codecs for move-to-front ranks. The map key is mtf handle. Doesn't
- // need to contain a different codec for every handle as most use one and the
- // same.
- std::map<uint64_t, std::unique_ptr<HuffmanCodec<uint32_t>>>
- mtf_huffman_codecs_;
-
- // Huffman codec for base-rate of opcode_and_num_operands.
- HuffmanCodec<uint64_t> opcode_and_num_operands_huffman_codec_;
-
- // Huffman codecs for opcode_and_num_operands. The map key is previous opcode.
- std::map<uint32_t, std::unique_ptr<HuffmanCodec<uint64_t>>>
- opcode_and_num_operands_markov_huffman_codecs_;
-
- // Huffman codecs for non-id single-word operand values.
- // The map key is pair <opcode, operand_index>.
- std::map<std::pair<uint32_t, uint32_t>,
- std::unique_ptr<HuffmanCodec<uint64_t>>>
- non_id_word_huffman_codecs_;
-
- // Huffman codecs for id descriptors. The map key is pair
- // <opcode, operand_index>.
- std::map<std::pair<uint32_t, uint32_t>,
- std::unique_ptr<HuffmanCodec<uint64_t>>>
- id_descriptor_huffman_codecs_;
-
- std::unordered_set<uint32_t> descriptors_with_coding_scheme_;
-
- // Huffman codecs for literal strings. The map key is the opcode of the
- // current instruction. This assumes, that there is no more than one literal
- // string operand per instruction, but would still work even if this is not
- // the case. Names and debug information strings are not collected.
- std::map<uint32_t, std::unique_ptr<HuffmanCodec<std::string>>>
- literal_string_huffman_codecs_;
-};
-
-const MarkvModel* GetDefaultModel() {
- static MarkvModel model;
- return &model;
-}
-
-// Returns chunk length used for variable length encoding of spirv operand
-// words. Returns zero if operand type corresponds to potentially multiple
-// words or a word which is not expected to profit from variable width encoding.
-// Chunk length is selected based on the size of expected value.
-// Most of these values will later be encoded with probability-based coding,
-// but variable width integer coding is a good quick solution.
-// TODO(atgoo@github.com): Put this in MarkvModel flatbuffer.
-size_t GetOperandVariableWidthChunkLength(spv_operand_type_t type) {
- switch (type) {
- case SPV_OPERAND_TYPE_TYPE_ID:
- return 4;
- case SPV_OPERAND_TYPE_RESULT_ID:
- case SPV_OPERAND_TYPE_ID:
- case SPV_OPERAND_TYPE_SCOPE_ID:
- case SPV_OPERAND_TYPE_MEMORY_SEMANTICS_ID:
- return 8;
- case SPV_OPERAND_TYPE_LITERAL_INTEGER:
- case SPV_OPERAND_TYPE_OPTIONAL_LITERAL_INTEGER:
- return 6;
- case SPV_OPERAND_TYPE_CAPABILITY:
- return 6;
- case SPV_OPERAND_TYPE_SOURCE_LANGUAGE:
- case SPV_OPERAND_TYPE_EXECUTION_MODEL:
- return 3;
- case SPV_OPERAND_TYPE_ADDRESSING_MODEL:
- case SPV_OPERAND_TYPE_MEMORY_MODEL:
- return 2;
- case SPV_OPERAND_TYPE_EXECUTION_MODE:
- return 6;
- case SPV_OPERAND_TYPE_STORAGE_CLASS:
- return 4;
- case SPV_OPERAND_TYPE_DIMENSIONALITY:
- case SPV_OPERAND_TYPE_SAMPLER_ADDRESSING_MODE:
- return 3;
- case SPV_OPERAND_TYPE_SAMPLER_FILTER_MODE:
- return 2;
- case SPV_OPERAND_TYPE_SAMPLER_IMAGE_FORMAT:
- return 6;
- case SPV_OPERAND_TYPE_FP_ROUNDING_MODE:
- case SPV_OPERAND_TYPE_LINKAGE_TYPE:
- case SPV_OPERAND_TYPE_ACCESS_QUALIFIER:
- case SPV_OPERAND_TYPE_OPTIONAL_ACCESS_QUALIFIER:
- return 2;
- case SPV_OPERAND_TYPE_FUNCTION_PARAMETER_ATTRIBUTE:
- return 3;
- case SPV_OPERAND_TYPE_DECORATION:
- case SPV_OPERAND_TYPE_BUILT_IN:
- return 6;
- case SPV_OPERAND_TYPE_GROUP_OPERATION:
- case SPV_OPERAND_TYPE_KERNEL_ENQ_FLAGS:
- case SPV_OPERAND_TYPE_KERNEL_PROFILING_INFO:
- return 2;
- case SPV_OPERAND_TYPE_FP_FAST_MATH_MODE:
- case SPV_OPERAND_TYPE_FUNCTION_CONTROL:
- case SPV_OPERAND_TYPE_LOOP_CONTROL:
- case SPV_OPERAND_TYPE_IMAGE:
- case SPV_OPERAND_TYPE_OPTIONAL_IMAGE:
- case SPV_OPERAND_TYPE_OPTIONAL_MEMORY_ACCESS:
- case SPV_OPERAND_TYPE_SELECTION_CONTROL:
- return 4;
- case SPV_OPERAND_TYPE_EXTENSION_INSTRUCTION_NUMBER:
- case SPV_OPERAND_TYPE_TYPED_LITERAL_NUMBER:
- return 6;
- default:
- return 0;
- }
- return 0;
-}
-
// Returns true if the opcode has a fixed number of operands. May return a
// false negative.
bool OpcodeHasFixedNumberOfOperands(SpvOp opcode) {
bool use_delimiter_ = false;
};
-// Creates spv_text object containing text from |str|.
-// The returned value is owned by the caller and needs to be destroyed with
-// spvTextDestroy.
-spv_text CreateSpvText(const std::string& str) {
- spv_text out = new spv_text_t();
- assert(out);
- char* cstr = new char[str.length() + 1];
- assert(cstr);
- std::strncpy(cstr, str.c_str(), str.length());
- cstr[str.length()] = '\0';
- out->str = cstr;
- out->length = str.length();
- return out;
-}
-
// Base class for MARK-V encoder and decoder. Contains common functionality
// such as:
// - Validator connection and validation state.
MarkvCodecBase() = delete;
- void SetModel(const MarkvModel* model) {
- model_ = model;
- }
-
protected:
struct MarkvHeader {
MarkvHeader() {
uint32_t spirv_generator;
};
+ // |model| is owned by the caller, must be not null and valid during the
+ // lifetime of the codec.
explicit MarkvCodecBase(spv_const_context context,
- spv_validator_options validator_options)
+ spv_validator_options validator_options,
+ const MarkvModel* model)
: validator_options_(validator_options), grammar_(context),
- model_(GetDefaultModel()), context_(context),
+ model_(model), mtf_huffman_codecs_(GetMtfHuffmanCodecs()),
+ context_(context),
vstate_(validator_options ?
new ValidationState_t(context, validator_options_) : nullptr) {}
vstate_->setIdBound(id_bound);
}
+ // Returns Huffman codec for ranks of the mtf with given |handle|.
+ // Different mtfs can use different rank distributions.
+ // May return nullptr if the codec doesn't exist.
+ const spvutils::HuffmanCodec<uint32_t>* GetMtfHuffmanCodec(uint64_t handle) const {
+ const auto it = mtf_huffman_codecs_.find(handle);
+ if (it == mtf_huffman_codecs_.end())
+ return nullptr;
+ return it->second.get();
+ }
+
spv_validator_options validator_options_ = nullptr;
const libspirv::AssemblyGrammar grammar_;
MarkvHeader header_;
+
+ // MARK-V model, not owned.
const MarkvModel* model_ = nullptr;
// Current instruction, current operand and current operand index.
// Container/computer for id descriptors.
IdDescriptorCollection id_descriptors_;
+ // Huffman codecs for move-to-front ranks. The map key is mtf handle. Doesn't
+ // need to contain a different codec for every handle as most use one and the
+ // same.
+ std::map<uint64_t, std::unique_ptr<HuffmanCodec<uint32_t>>>
+ mtf_huffman_codecs_;
+
private:
spv_const_context context_ = nullptr;
// on how encoding was done, which can later be accessed with GetComments().
class MarkvEncoder : public MarkvCodecBase {
public:
+ // |model| is owned by the caller, must be not null and valid during the
+ // lifetime of MarkvEncoder.
MarkvEncoder(spv_const_context context,
- spv_const_markv_encoder_options options)
- : MarkvCodecBase(context, GetValidatorOptions(options)),
+ const MarkvEncoderOptions& options,
+ const MarkvModel* model)
+ : MarkvCodecBase(context, GetValidatorOptions(options), model),
options_(options) {
(void) options_;
}
// into a single buffer and returns it as spv_markv_binary. The returned
// value is owned by the caller and needs to be destroyed with
// spvMarkvBinaryDestroy().
- spv_markv_binary GetMarkvBinary() {
+ std::vector<uint8_t> GetMarkvBinary() {
header_.markv_length_in_bits =
static_cast<uint32_t>(sizeof(header_) * 8 + writer_.GetNumBits());
+ header_.markv_model =
+ (model_->model_type() << 16) | model_->model_version();
+
const size_t num_bytes = sizeof(header_) + writer_.GetDataSizeBytes();
+ std::vector<uint8_t> markv(num_bytes);
- spv_markv_binary markv_binary = new spv_markv_binary_t();
- markv_binary->data = new uint8_t[num_bytes];
- markv_binary->length = num_bytes;
assert(writer_.GetData());
- std::memcpy(markv_binary->data, &header_, sizeof(header_));
- std::memcpy(markv_binary->data + sizeof(header_),
+ std::memcpy(markv.data(), &header_, sizeof(header_));
+ std::memcpy(markv.data() + sizeof(header_),
writer_.GetData(), writer_.GetDataSizeBytes());
- return markv_binary;
+ return markv;
}
// Creates an internal logger which writes comments on the encoding process.
private:
// Creates and returns validator options. Returned value owned by the caller.
static spv_validator_options GetValidatorOptions(
- spv_const_markv_encoder_options options) {
- return options->validate_spirv_binary ?
+ const MarkvEncoderOptions& options) {
+ return options.validate_spirv_binary ?
spvValidatorOptionsCreate() : nullptr;
}
// Encodes a literal number operand and writes it to the bit stream.
spv_result_t EncodeLiteralNumber(const spv_parsed_operand_t& operand);
- spv_const_markv_encoder_options options_;
+ MarkvEncoderOptions options_;
// Bit stream where encoded instructions are written.
BitWriterWord64 writer_;
// Decodes MARK-V buffers written by MarkvEncoder.
class MarkvDecoder : public MarkvCodecBase {
public:
+ // |model| is owned by the caller, must be not null and valid during the
+ // lifetime of MarkvEncoder.
MarkvDecoder(spv_const_context context,
- const uint8_t* markv_data,
- size_t markv_size_bytes,
- spv_const_markv_decoder_options options)
- : MarkvCodecBase(context, GetValidatorOptions(options)),
- options_(options), reader_(markv_data, markv_size_bytes) {
+ const std::vector<uint8_t>& markv,
+ const MarkvDecoderOptions& options,
+ const MarkvModel* model)
+ : MarkvCodecBase(context, GetValidatorOptions(options), model),
+ options_(options), reader_(markv) {
(void) options_;
SetIdBound(1);
parsed_operands_.reserve(25);
// Creates and returns validator options. Returned value owned by the caller.
static spv_validator_options GetValidatorOptions(
- spv_const_markv_decoder_options options) {
- return options->validate_spirv_binary ?
+ const MarkvDecoderOptions& options) {
+ return options.validate_spirv_binary ?
spvValidatorOptionsCreate() : nullptr;
}
// kind SPV_NUMBER_NONE.
void RecordNumberType();
- spv_const_markv_decoder_options options_;
+ MarkvDecoderOptions options_;
// Temporary sink where decoded SPIR-V words are written. Once it contains the
// entire module, the container is moved and returned.
}
// Fallback encoding.
- const size_t chunk_length = GetOperandVariableWidthChunkLength(operand_.type);
+ const size_t chunk_length =
+ model_->GetOperandVariableWidthChunkLength(operand_.type);
if (chunk_length) {
writer_.WriteVariableWidthU32(word, chunk_length);
} else {
// Received kMarkvNoneOfTheAbove signal, use fallback decoding.
}
- const size_t chunk_length = GetOperandVariableWidthChunkLength(operand_.type);
+ const size_t chunk_length =
+ model_->GetOperandVariableWidthChunkLength(operand_.type);
if (chunk_length) {
if (!reader_.ReadVariableWidthU32(word, chunk_length))
return Diag(SPV_ERROR_INVALID_BINARY)
spv_result_t MarkvEncoder::EncodeMtfRankHuffman(uint32_t rank, uint64_t mtf,
uint64_t fallback_method) {
- const auto* codec = model_->GetMtfHuffmanCodec(mtf);
+ const auto* codec = GetMtfHuffmanCodec(mtf);
if (!codec) {
assert(fallback_method != kMtfNone);
- codec = model_->GetMtfHuffmanCodec(fallback_method);
+ codec = GetMtfHuffmanCodec(fallback_method);
}
if (!codec)
spv_result_t MarkvDecoder::DecodeMtfRankHuffman(
uint64_t mtf, uint32_t fallback_method, uint32_t* rank) {
- const auto* codec = model_->GetMtfHuffmanCodec(mtf);
+ const auto* codec = GetMtfHuffmanCodec(mtf);
if (!codec) {
assert(fallback_method != kMtfNone);
- codec = model_->GetMtfHuffmanCodec(fallback_method);
+ codec = GetMtfHuffmanCodec(fallback_method);
}
if (!codec)
return Diag(SPV_ERROR_INVALID_BINARY)
<< "MARK-V binary and the codec have different versions";
+ const uint32_t model_type = header_.markv_model >> 16;
+ const uint32_t model_version = header_.markv_model & 0xFFFF;
+ if (model_type != model_->model_type())
+ return Diag(SPV_ERROR_INVALID_BINARY)
+ << "MARK-V binary and the codec use different MARK-V models";
+
+ if (model_version != model_->model_version())
+ return Diag(SPV_ERROR_INVALID_BINARY)
+ << "MARK-V binary and the codec use different versions if the same "
+ << "MARK-V model";
+
spirv_.reserve(header_.markv_length_in_bits / 2); // Heuristic.
spirv_.resize(5, 0);
spirv_[0] = kSpirvMagicNumber;
} // namespace
-spv_result_t spvSpirvToMarkv(spv_const_context context,
- const uint32_t* spirv_words,
- const size_t spirv_num_words,
- spv_const_markv_encoder_options options,
- spv_markv_binary* markv_binary,
- spv_text* comments, spv_diagnostic* diagnostic) {
+spv_result_t SpirvToMarkv(spv_const_context context,
+ const std::vector<uint32_t>& spirv,
+ const MarkvEncoderOptions& options,
+ const MarkvModel& markv_model,
+ MessageConsumer message_consumer,
+ std::vector<uint8_t>* markv,
+ std::string* comments) {
spv_context_t hijack_context = *context;
- if (diagnostic) {
- *diagnostic = nullptr;
- libspirv::UseDiagnosticAsMessageConsumer(&hijack_context, diagnostic);
- }
+ SetContextMessageConsumer(&hijack_context, message_consumer);
- spv_const_binary_t spirv_binary = {spirv_words, spirv_num_words};
+ spv_const_binary_t spirv_binary = {spirv.data(), spirv.size()};
spv_endianness_t endian;
spv_position_t position = {};
<< "Invalid SPIR-V header.";
}
- MarkvEncoder encoder(&hijack_context, options);
+ MarkvEncoder encoder(&hijack_context, options, &markv_model);
if (comments) {
encoder.CreateCommentsLogger();
spv_text text = nullptr;
- if (spvBinaryToText(&hijack_context, spirv_words, spirv_num_words,
+ if (spvBinaryToText(&hijack_context, spirv.data(), spirv.size(),
SPV_BINARY_TO_TEXT_OPTION_NO_HEADER, &text, nullptr)
!= SPV_SUCCESS) {
return DiagnosticStream(position, hijack_context.consumer,
}
if (spvBinaryParse(
- &hijack_context, &encoder, spirv_words, spirv_num_words, EncodeHeader,
- EncodeInstruction, diagnostic) != SPV_SUCCESS) {
+ &hijack_context, &encoder, spirv.data(), spirv.size(), EncodeHeader,
+ EncodeInstruction, nullptr) != SPV_SUCCESS) {
return DiagnosticStream(position, hijack_context.consumer,
SPV_ERROR_INVALID_BINARY)
<< "Unable to encode to MARK-V.";
}
if (comments)
- *comments = CreateSpvText(encoder.GetComments());
+ *comments = encoder.GetComments();
- *markv_binary = encoder.GetMarkvBinary();
+ *markv = encoder.GetMarkvBinary();
return SPV_SUCCESS;
}
-spv_result_t spvMarkvToSpirv(spv_const_context context,
- const uint8_t* markv_data,
- size_t markv_size_bytes,
- spv_const_markv_decoder_options options,
- spv_binary* spirv_binary,
- spv_text* /* comments */,
- spv_diagnostic* diagnostic) {
+spv_result_t MarkvToSpirv(spv_const_context context,
+ const std::vector<uint8_t>& markv,
+ const MarkvDecoderOptions& options,
+ const MarkvModel& markv_model,
+ MessageConsumer message_consumer,
+ std::vector<uint32_t>* spirv,
+ std::string* /* comments */) {
spv_position_t position = {};
spv_context_t hijack_context = *context;
- if (diagnostic) {
- *diagnostic = nullptr;
- libspirv::UseDiagnosticAsMessageConsumer(&hijack_context, diagnostic);
- }
-
- MarkvDecoder decoder(&hijack_context, markv_data, markv_size_bytes, options);
+ SetContextMessageConsumer(&hijack_context, message_consumer);
- std::vector<uint32_t> words;
+ MarkvDecoder decoder(&hijack_context, markv, options, &markv_model);
- if (decoder.DecodeModule(&words) != SPV_SUCCESS) {
+ if (decoder.DecodeModule(spirv) != SPV_SUCCESS) {
return DiagnosticStream(position, hijack_context.consumer,
SPV_ERROR_INVALID_BINARY)
<< "Unable to decode MARK-V.";
}
- assert(!words.empty());
-
- *spirv_binary = new spv_binary_t();
- (*spirv_binary)->code = new uint32_t[words.size()];
- (*spirv_binary)->wordCount = words.size();
- std::memcpy((*spirv_binary)->code, words.data(), 4 * words.size());
-
+ assert(!spirv->empty());
return SPV_SUCCESS;
}
-void spvMarkvBinaryDestroy(spv_markv_binary binary) {
- if (!binary) return;
- delete[] binary->data;
- delete binary;
-}
-
-spv_markv_encoder_options spvMarkvEncoderOptionsCreate() {
- return new spv_markv_encoder_options_t;
-}
-
-void spvMarkvEncoderOptionsDestroy(spv_markv_encoder_options options) {
- delete options;
-}
-
-spv_markv_decoder_options spvMarkvDecoderOptionsCreate() {
- return new spv_markv_decoder_options_t;
-}
-
-void spvMarkvDecoderOptionsDestroy(spv_markv_decoder_options options) {
- delete options;
-}
+} // namespave spvtools
--- /dev/null
+// Copyright (c) 2017 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef LIBSPIRV_COMP_MARKV_MODEL_H_
+#define LIBSPIRV_COMP_MARKV_MODEL_H_
+
+#include <map>
+#include <unordered_set>
+#include <vector>
+
+#include "spirv/1.2/spirv.h"
+#include "spirv-tools/libspirv.h"
+#include "util/huffman_codec.h"
+
+namespace spvtools {
+
+// Base class for MARK-V models.
+// The class contains encoding/decoding model with various constants and
+// codecs used by the compression algorithm.
+class MarkvModel {
+ public:
+ MarkvModel() : operand_chunk_lengths_(
+ static_cast<size_t>(SPV_OPERAND_TYPE_NUM_OPERAND_TYPES), 0) {}
+
+ uint32_t model_type() const { return model_type_; }
+ uint32_t model_version() const { return model_version_; }
+
+ uint32_t opcode_chunk_length() const { return opcode_chunk_length_; }
+ uint32_t num_operands_chunk_length() const { return num_operands_chunk_length_; }
+ uint32_t mtf_rank_chunk_length() const { return mtf_rank_chunk_length_; }
+
+ uint32_t u64_chunk_length() const { return u64_chunk_length_; }
+ uint32_t s64_chunk_length() const { return s64_chunk_length_; }
+ uint32_t s64_block_exponent() const { return s64_block_exponent_; }
+
+ // Returns a codec for common opcode_and_num_operands words for the given
+ // previous opcode. May return nullptr if the codec doesn't exist.
+ const spvutils::HuffmanCodec<uint64_t>* GetOpcodeAndNumOperandsMarkovHuffmanCodec(
+ uint32_t prev_opcode) const {
+ if (prev_opcode == SpvOpNop)
+ return opcode_and_num_operands_huffman_codec_.get();
+
+ const auto it =
+ opcode_and_num_operands_markov_huffman_codecs_.find(prev_opcode);
+ if (it == opcode_and_num_operands_markov_huffman_codecs_.end())
+ return nullptr;
+ return it->second.get();
+ }
+
+ // Returns a codec for common non-id words used for given operand slot.
+ // Operand slot is defined by the opcode and the operand index.
+ // May return nullptr if the codec doesn't exist.
+ const spvutils::HuffmanCodec<uint64_t>* GetNonIdWordHuffmanCodec(
+ uint32_t opcode, uint32_t operand_index) const {
+ const auto it = non_id_word_huffman_codecs_.find(
+ std::pair<uint32_t, uint32_t>(opcode, operand_index));
+ if (it == non_id_word_huffman_codecs_.end())
+ return nullptr;
+ return it->second.get();
+ }
+
+ // Returns a codec for common id descriptos used for given operand slot.
+ // Operand slot is defined by the opcode and the operand index.
+ // May return nullptr if the codec doesn't exist.
+ const spvutils::HuffmanCodec<uint64_t>* GetIdDescriptorHuffmanCodec(
+ uint32_t opcode, uint32_t operand_index) const {
+ const auto it = id_descriptor_huffman_codecs_.find(
+ std::pair<uint32_t, uint32_t>(opcode, operand_index));
+ if (it == id_descriptor_huffman_codecs_.end())
+ return nullptr;
+ return it->second.get();
+ }
+
+ // Returns a codec for common strings used by the given opcode.
+ // Operand slot is defined by the opcode and the operand index.
+ // May return nullptr if the codec doesn't exist.
+ const spvutils::HuffmanCodec<std::string>* GetLiteralStringHuffmanCodec(
+ uint32_t opcode) const {
+ const auto it = literal_string_huffman_codecs_.find(opcode);
+ if (it == literal_string_huffman_codecs_.end())
+ return nullptr;
+ return it->second.get();
+ }
+
+ // Checks if |descriptor| has a coding scheme in any of
+ // id_descriptor_huffman_codecs_.
+ bool DescriptorHasCodingScheme(uint32_t descriptor) const {
+ return descriptors_with_coding_scheme_.count(descriptor);
+ }
+
+ // Returns chunk length used for variable length encoding of spirv operand
+ // words.
+ uint32_t GetOperandVariableWidthChunkLength(spv_operand_type_t type) const {
+ return operand_chunk_lengths_.at(static_cast<size_t>(type));
+ }
+
+ // Sets model type.
+ void SetModelType(uint32_t in_model_type) {
+ model_type_ = in_model_type;
+ }
+
+ // Sets model version.
+ void SetModelVersion(uint32_t in_model_version) {
+ model_version_ = in_model_version;
+ }
+
+ // Returns value used by Huffman codecs as a signal that a value is not in the
+ // coding table.
+ static uint64_t GetMarkvNoneOfTheAbove() {
+ // Magic number.
+ return 1111111111111111111;
+ }
+
+ MarkvModel(const MarkvModel&) = delete;
+ const MarkvModel& operator=(const MarkvModel&) = delete;
+
+ protected:
+ // Huffman codec for base-rate of opcode_and_num_operands.
+ std::unique_ptr<spvutils::HuffmanCodec<uint64_t>>
+ opcode_and_num_operands_huffman_codec_;
+
+ // Huffman codecs for opcode_and_num_operands. The map key is previous opcode.
+ std::map<uint32_t, std::unique_ptr<spvutils::HuffmanCodec<uint64_t>>>
+ opcode_and_num_operands_markov_huffman_codecs_;
+
+ // Huffman codecs for non-id single-word operand values.
+ // The map key is pair <opcode, operand_index>.
+ std::map<std::pair<uint32_t, uint32_t>,
+ std::unique_ptr<spvutils::HuffmanCodec<uint64_t>>> non_id_word_huffman_codecs_;
+
+ // Huffman codecs for id descriptors. The map key is pair
+ // <opcode, operand_index>.
+ std::map<std::pair<uint32_t, uint32_t>,
+ std::unique_ptr<spvutils::HuffmanCodec<uint64_t>>> id_descriptor_huffman_codecs_;
+
+ // Set of all descriptors which have a coding scheme in any of
+ // id_descriptor_huffman_codecs_.
+ std::unordered_set<uint32_t> descriptors_with_coding_scheme_;
+
+ // Huffman codecs for literal strings. The map key is the opcode of the
+ // current instruction. This assumes, that there is no more than one literal
+ // string operand per instruction, but would still work even if this is not
+ // the case. Names and debug information strings are not collected.
+ std::map<uint32_t, std::unique_ptr<spvutils::HuffmanCodec<std::string>>>
+ literal_string_huffman_codecs_;
+
+ // Chunk lengths used for variable width encoding of operands (index is
+ // spv_operand_type of the operand).
+ std::vector<uint32_t> operand_chunk_lengths_;
+
+ uint32_t opcode_chunk_length_ = 7;
+ uint32_t num_operands_chunk_length_ = 3;
+ uint32_t mtf_rank_chunk_length_ = 5;
+
+ uint32_t u64_chunk_length_ = 8;
+ uint32_t s64_chunk_length_ = 8;
+ uint32_t s64_block_exponent_ = 10;
+
+ uint32_t model_type_ = 0;
+ uint32_t model_version_ = 0;
+};
+
+} // namespace spvtools
+
+#endif // LIBSPIRV_COMP_MARKV_MODEL_H_
if(SPIRV_BUILD_COMPRESSION)
add_spvtools_unittest(TARGET markv_codec
- SRCS markv_codec_test.cpp ${VAL_TEST_COMMON_SRCS}
+ SRCS
+ markv_codec_test.cpp
+ ${CMAKE_CURRENT_SOURCE_DIR}/../../tools/comp/markv_model_factory.cpp
+ ${CMAKE_CURRENT_SOURCE_DIR}/../../tools/comp/markv_model_shader_default.cpp
+ ${VAL_TEST_COMMON_SRCS}
LIBS SPIRV-Tools-comp ${SPIRV_TOOLS}
)
endif(SPIRV_BUILD_COMPRESSION)
#include <string>
#include "gmock/gmock.h"
-#include "spirv-tools/markv.h"
+#include "source/comp/markv.h"
#include "test_fixture.h"
+#include "tools/comp/markv_model_factory.h"
#include "unit_spirv.h"
namespace {
spvTextDestroy(text);
}
-// Encodes SPIR-V |words| to |markv_binary|. |comments| context snippets of
-// disassembly and bit sequences for debugging.
-void Encode(const std::vector<uint32_t>& words,
- spv_markv_binary* markv_binary,
- std::string* comments,
- spv_target_env env = SPV_ENV_UNIVERSAL_1_2) {
- ScopedContext ctx(env);
- SetContextMessageConsumer(ctx.context, DiagnosticsMessageHandler);
-
- std::unique_ptr<spv_markv_encoder_options_t,
- std::function<void(spv_markv_encoder_options_t*)>> options(
- spvMarkvEncoderOptionsCreate(), &spvMarkvEncoderOptionsDestroy);
- spv_text spv_text_comments;
- ASSERT_EQ(SPV_SUCCESS, spvSpirvToMarkv(ctx.context, words.data(),
- words.size(), options.get(),
- markv_binary, &spv_text_comments,
- nullptr));
-
- *comments = std::string(spv_text_comments->str, spv_text_comments->length);
- spvTextDestroy(spv_text_comments);
-}
-
-// Decodes |markv_binary| to SPIR-V |words|.
-void Decode(const spv_markv_binary markv_binary,
- std::vector<uint32_t>* words,
- spv_target_env env = SPV_ENV_UNIVERSAL_1_2) {
- ScopedContext ctx(env);
- SetContextMessageConsumer(ctx.context, DiagnosticsMessageHandler);
-
- spv_binary spirv_binary = nullptr;
- std::unique_ptr<spv_markv_decoder_options_t,
- std::function<void(spv_markv_decoder_options_t*)>> options(
- spvMarkvDecoderOptionsCreate(), &spvMarkvDecoderOptionsDestroy);
- ASSERT_EQ(SPV_SUCCESS, spvMarkvToSpirv(ctx.context, markv_binary->data,
- markv_binary->length, options.get(),
- &spirv_binary, nullptr, nullptr));
-
- *words = std::vector<uint32_t>(
- spirv_binary->code, spirv_binary->code + spirv_binary->wordCount);
-
- spvBinaryDestroy(spirv_binary);
-}
-
// Encodes/decodes |original|, assembles/dissasembles |original|, then compares
// the results of the two operations.
void TestEncodeDecode(const std::string& original_text) {
+ ScopedContext ctx(SPV_ENV_UNIVERSAL_1_2);
+ std::unique_ptr<spvtools::MarkvModel> model =
+ spvtools::CreateMarkvModel(spvtools::kMarkvModelShaderDefault);
+ spvtools::MarkvEncoderOptions encoder_options;
+ spvtools::MarkvDecoderOptions decoder_options;
+
std::vector<uint32_t> expected_binary;
Compile(original_text, &expected_binary);
ASSERT_FALSE(expected_binary.empty());
SPV_TEXT_TO_BINARY_OPTION_PRESERVE_NUMERIC_IDS);
ASSERT_FALSE(binary_to_encode.empty());
- spv_markv_binary markv_binary = nullptr;
+ std::vector<uint8_t> markv;
std::string encoder_comments;
- Encode(binary_to_encode, &markv_binary, &encoder_comments);
- ASSERT_NE(nullptr, markv_binary);
-
- // std::cerr << encoder_comments << std::endl;
- // std::cerr << "SPIR-V size: " << expected_binary.size() * 4 << std::endl;
- // std::cerr << "MARK-V size: " << markv_binary->length << std::endl;
+ ASSERT_EQ(SPV_SUCCESS, spvtools::SpirvToMarkv(
+ ctx.context, binary_to_encode, encoder_options, *model,
+ DiagnosticsMessageHandler, &markv, &encoder_comments));
+ ASSERT_FALSE(markv.empty());
std::vector<uint32_t> decoded_binary;
- Decode(markv_binary, &decoded_binary);
+ ASSERT_EQ(SPV_SUCCESS, spvtools::MarkvToSpirv(
+ ctx.context, markv, decoder_options, *model,
+ DiagnosticsMessageHandler, &decoded_binary, nullptr));
ASSERT_FALSE(decoded_binary.empty());
EXPECT_EQ(expected_binary, decoded_binary) << encoder_comments;
ASSERT_FALSE(decoded_text.empty());
EXPECT_EQ(expected_text, decoded_text) << encoder_comments;
-
- spvMarkvBinaryDestroy(markv_binary);
}
void TestEncodeDecodeShaderMainBody(const std::string& body) {
spirv-cfg spirv-link)
if(SPIRV_BUILD_COMPRESSION)
- add_spvtools_tool(TARGET spirv-markv SRCS comp/markv.cpp
+ add_spvtools_tool(TARGET spirv-markv
+ SRCS comp/markv.cpp
+ comp/markv_model_factory.cpp
+ comp/markv_model_shader_default.cpp
LIBS SPIRV-Tools-comp ${SPIRV_TOOLS})
target_include_directories(spirv-markv PRIVATE ${spirv-tools_SOURCE_DIR}
${SPIRV_HEADER_INCLUDE_DIR})
#include <memory>
#include <vector>
+#include "markv_model_factory.h"
+#include "source/comp/markv.h"
#include "source/spirv_target_env.h"
#include "source/table.h"
-#include "spirv-tools/markv.h"
#include "tools/io.h"
namespace {
-h, --help Print this help.
--comments Write codec comments to stdout.
--version Display MARK-V codec version.
+ --validate Validate SPIR-V while encoding or decoding.
-o <filename> Set the output filename.
Output goes to standard output if this option is
}
bool want_comments = false;
+ bool validate_spirv_binary = false;
for (int argi = 2; argi < argc; ++argi) {
if ('-' == argv[argi][0]) {
} else if (0 == strcmp(argv[argi], "--version")) {
fprintf(stderr, "error: Not implemented\n");
return 1;
+ } else if (0 == strcmp(argv[argi], "--validate")) {
+ validate_spirv_binary = true;
} else {
print_usage(argv[0]);
return 1;
const bool write_to_stdout = output_filename == nullptr ||
0 == strcmp(output_filename, "-");
- spv_text comments = nullptr;
- spv_text* comments_ptr = want_comments ? &comments : nullptr;
+ std::string comments;
+ std::string* comments_ptr = want_comments ? &comments : nullptr;
ScopedContext ctx(SPV_ENV_UNIVERSAL_1_2);
- SetContextMessageConsumer(ctx.context, DiagnosticsMessageHandler);
+
+ std::unique_ptr<spvtools::MarkvModel> model =
+ spvtools::CreateMarkvModel(spvtools::kMarkvModelShaderDefault);
if (task == kEncode) {
- std::vector<uint32_t> contents;
- if (!ReadFile<uint32_t>(input_filename, "rb", &contents)) return 1;
+ std::vector<uint32_t> spirv;
+ if (!ReadFile<uint32_t>(input_filename, "rb", &spirv)) return 1;
+
+ spvtools::MarkvEncoderOptions options;
+ options.validate_spirv_binary = validate_spirv_binary;
- std::unique_ptr<spv_markv_encoder_options_t,
- std::function<void(spv_markv_encoder_options_t*)>> options(
- spvMarkvEncoderOptionsCreate(), &spvMarkvEncoderOptionsDestroy);
- spv_markv_binary markv_binary = nullptr;
+ std::vector<uint8_t> markv;
- if (SPV_SUCCESS !=
- spvSpirvToMarkv(ctx.context, contents.data(), contents.size(),
- options.get(), &markv_binary, comments_ptr, nullptr)) {
+ if (SPV_SUCCESS != spvtools::SpirvToMarkv(
+ ctx.context, spirv, options, *model, DiagnosticsMessageHandler,
+ &markv, comments_ptr)) {
std::cerr << "error: Failed to encode " << input_filename << " to MARK-V "
<< std::endl;
return 1;
}
if (want_comments) {
- if (!WriteFile<char>(nullptr, "w", comments->str,
- comments->length)) return 1;
+ if (!WriteFile<char>(nullptr, "w", comments.c_str(),
+ comments.length())) return 1;
}
if (!want_comments || !write_to_stdout) {
- if (!WriteFile<uint8_t>(output_filename, "wb", markv_binary->data,
- markv_binary->length)) return 1;
+ if (!WriteFile<uint8_t>(output_filename, "wb", markv.data(),
+ markv.size())) return 1;
}
} else if (task == kDecode) {
- std::vector<uint8_t> contents;
- if (!ReadFile<uint8_t>(input_filename, "rb", &contents)) return 1;
+ std::vector<uint8_t> markv;
+ if (!ReadFile<uint8_t>(input_filename, "rb", &markv)) return 1;
- std::unique_ptr<spv_markv_decoder_options_t,
- std::function<void(spv_markv_decoder_options_t*)>> options(
- spvMarkvDecoderOptionsCreate(), &spvMarkvDecoderOptionsDestroy);
- spv_binary spirv_binary = nullptr;
+ spvtools::MarkvDecoderOptions options;
+ options.validate_spirv_binary = validate_spirv_binary;
- if (SPV_SUCCESS !=
- spvMarkvToSpirv(ctx.context, contents.data(), contents.size(),
- options.get(), &spirv_binary, comments_ptr, nullptr)) {
- std::cerr << "error: Failed to encode " << input_filename << " to MARK-V "
+ std::vector<uint32_t> spirv;
+
+ if (SPV_SUCCESS != spvtools::MarkvToSpirv(
+ ctx.context, markv, options, *model, DiagnosticsMessageHandler,
+ &spirv, comments_ptr)) {
+ std::cerr << "error: Failed to decode " << input_filename << " to SPIR-V "
<< std::endl;
return 1;
}
if (want_comments) {
- if (!WriteFile<char>(nullptr, "w", comments->str,
- comments->length)) return 1;
+ if (!WriteFile<char>(nullptr, "w", comments.c_str(),
+ comments.length())) return 1;
}
if (!want_comments || !write_to_stdout) {
- if (!WriteFile<uint32_t>(output_filename, "wb", spirv_binary->code,
- spirv_binary->wordCount)) return 1;
+ if (!WriteFile<uint32_t>(output_filename, "wb", spirv.data(),
+ spirv.size())) return 1;
}
} else {
assert(false && "Unknown task");
}
- spvTextDestroy(comments);
-
return 0;
}
--- /dev/null
+// Copyright (c) 2017 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "markv_model_factory.h"
+#include "markv_model_shader_default.h"
+
+namespace spvtools {
+
+std::unique_ptr<MarkvModel> CreateMarkvModel(MarkvModelType type) {
+ std::unique_ptr<MarkvModel> model;
+ switch (type) {
+ case kMarkvModelShaderDefault: {
+ model.reset(new MarkvModelShaderDefault());
+ break;
+ }
+ }
+
+ model->SetModelType(static_cast<uint32_t>(type));
+
+ return model;
+}
+
+} // namespace spvtools
--- /dev/null
+// Copyright (c) 2017 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef SPIRV_TOOLS_COMP_MARKV_MODEL_FACTORY_H_
+#define SPIRV_TOOLS_COMP_MARKV_MODEL_FACTORY_H_
+
+#include <memory>
+
+#include "source/comp/markv_model.h"
+
+namespace spvtools {
+
+enum MarkvModelType {
+ kMarkvModelShaderDefault = 1,
+};
+
+std::unique_ptr<MarkvModel> CreateMarkvModel(MarkvModelType type);
+
+} // namespace spvtools
+
+#endif // SPIRV_TOOLS_COMP_MARKV_MODEL_FACTORY_H_
--- /dev/null
+// Copyright (c) 2017 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "markv_model_shader_default.h"
+
+#include <algorithm>
+#include <map>
+#include <memory>
+#include <vector>
+#include <unordered_map>
+#include <unordered_set>
+
+using spvutils::HuffmanCodec;
+
+namespace spvtools {
+
+namespace {
+
+// Signals that the value is not in the coding scheme and a fallback method
+// needs to be used.
+const uint64_t kMarkvNoneOfTheAbove = MarkvModel::GetMarkvNoneOfTheAbove();
+
+inline uint32_t CombineOpcodeAndNumOperands(uint32_t opcode,
+ uint32_t num_operands) {
+ return opcode | (num_operands << 16);
+}
+
+// The following file contains autogenerated statistical coding rules.
+// Generated by running spirv-stats on representative corpus of shaders with
+// flags:
+// --codegen_opcode_and_num_operands_hist
+// --codegen_opcode_and_num_operands_markov_huffman_codecs
+// --codegen_literal_string_huffman_codecs
+// --codegen_non_id_word_huffman_codecs
+// --codegen_id_descriptor_huffman_codecs
+//
+// Example:
+// find <SHADER_CORPUS_DIR> -type f -print0 | xargs -0 -s 2000000
+// ~/SPIRV-Tools/build/tools/spirv-stats -v
+// --codegen_opcode_and_num_operands_hist
+// --codegen_opcode_and_num_operands_markov_huffman_codecs
+// --codegen_literal_string_huffman_codecs --codegen_non_id_word_huffman_codecs
+// --codegen_id_descriptor_huffman_codecs -o
+// ~/SPIRV-Tools/source/comp/markv_autogen.inc
+#include "markv_model_shader_default_autogen.inc"
+
+} // namespace
+
+MarkvModelShaderDefault::MarkvModelShaderDefault() {
+ const uint16_t kVersionNumber = 0;
+ SetModelVersion(kVersionNumber);
+
+ opcode_and_num_operands_huffman_codec_.reset(
+ new HuffmanCodec<uint64_t>(GetOpcodeAndNumOperandsHist()));
+ opcode_and_num_operands_markov_huffman_codecs_ =
+ GetOpcodeAndNumOperandsMarkovHuffmanCodecs();
+ non_id_word_huffman_codecs_ = GetNonIdWordHuffmanCodecs();
+ id_descriptor_huffman_codecs_ = GetIdDescriptorHuffmanCodecs();
+ descriptors_with_coding_scheme_ = GetDescriptorsWithCodingScheme();
+ literal_string_huffman_codecs_ = GetLiteralStringHuffmanCodecs();
+
+ operand_chunk_lengths_[SPV_OPERAND_TYPE_TYPE_ID] = 4;
+ operand_chunk_lengths_[SPV_OPERAND_TYPE_RESULT_ID] = 8;
+ operand_chunk_lengths_[SPV_OPERAND_TYPE_ID] = 8;
+ operand_chunk_lengths_[SPV_OPERAND_TYPE_SCOPE_ID] = 8;
+ operand_chunk_lengths_[SPV_OPERAND_TYPE_MEMORY_SEMANTICS_ID] = 8;
+ operand_chunk_lengths_[SPV_OPERAND_TYPE_LITERAL_INTEGER] = 6;
+ operand_chunk_lengths_[SPV_OPERAND_TYPE_OPTIONAL_LITERAL_INTEGER] = 6;
+ operand_chunk_lengths_[SPV_OPERAND_TYPE_CAPABILITY] = 6;
+ operand_chunk_lengths_[SPV_OPERAND_TYPE_SOURCE_LANGUAGE] = 3;
+ operand_chunk_lengths_[SPV_OPERAND_TYPE_EXECUTION_MODEL] = 3;
+ operand_chunk_lengths_[SPV_OPERAND_TYPE_ADDRESSING_MODEL] = 2;
+ operand_chunk_lengths_[SPV_OPERAND_TYPE_MEMORY_MODEL] = 2;
+ operand_chunk_lengths_[SPV_OPERAND_TYPE_EXECUTION_MODE] = 6;
+ operand_chunk_lengths_[SPV_OPERAND_TYPE_STORAGE_CLASS] = 4;
+ operand_chunk_lengths_[SPV_OPERAND_TYPE_DIMENSIONALITY] = 3;
+ operand_chunk_lengths_[SPV_OPERAND_TYPE_SAMPLER_ADDRESSING_MODE] = 3;
+ operand_chunk_lengths_[SPV_OPERAND_TYPE_SAMPLER_FILTER_MODE] = 2;
+ operand_chunk_lengths_[SPV_OPERAND_TYPE_SAMPLER_IMAGE_FORMAT] = 6;
+ operand_chunk_lengths_[SPV_OPERAND_TYPE_FP_ROUNDING_MODE] = 2;
+ operand_chunk_lengths_[SPV_OPERAND_TYPE_LINKAGE_TYPE] = 2;
+ operand_chunk_lengths_[SPV_OPERAND_TYPE_ACCESS_QUALIFIER] = 2;
+ operand_chunk_lengths_[SPV_OPERAND_TYPE_OPTIONAL_ACCESS_QUALIFIER] = 2;
+ operand_chunk_lengths_[SPV_OPERAND_TYPE_FUNCTION_PARAMETER_ATTRIBUTE] = 3;
+ operand_chunk_lengths_[SPV_OPERAND_TYPE_DECORATION] = 6;
+ operand_chunk_lengths_[SPV_OPERAND_TYPE_BUILT_IN] = 6;
+ operand_chunk_lengths_[SPV_OPERAND_TYPE_GROUP_OPERATION] = 2;
+ operand_chunk_lengths_[SPV_OPERAND_TYPE_KERNEL_ENQ_FLAGS] = 2;
+ operand_chunk_lengths_[SPV_OPERAND_TYPE_KERNEL_PROFILING_INFO] = 2;
+ operand_chunk_lengths_[SPV_OPERAND_TYPE_FP_FAST_MATH_MODE] = 4;
+ operand_chunk_lengths_[SPV_OPERAND_TYPE_FUNCTION_CONTROL] = 4;
+ operand_chunk_lengths_[SPV_OPERAND_TYPE_LOOP_CONTROL] = 4;
+ operand_chunk_lengths_[SPV_OPERAND_TYPE_IMAGE] = 4;
+ operand_chunk_lengths_[SPV_OPERAND_TYPE_OPTIONAL_IMAGE] = 4;
+ operand_chunk_lengths_[SPV_OPERAND_TYPE_OPTIONAL_MEMORY_ACCESS] = 4;
+ operand_chunk_lengths_[SPV_OPERAND_TYPE_SELECTION_CONTROL] = 4;
+ operand_chunk_lengths_[SPV_OPERAND_TYPE_EXTENSION_INSTRUCTION_NUMBER] = 6;
+ operand_chunk_lengths_[SPV_OPERAND_TYPE_TYPED_LITERAL_NUMBER] = 6;
+}
+
+} // namespace spvtools
--- /dev/null
+// Copyright (c) 2017 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef SPIRV_TOOLS_COMP_DEFAULT_SHADER_MARKV_MODEL_H_
+#define SPIRV_TOOLS_COMP_DEFAULT_SHADER_MARKV_MODEL_H_
+
+#include "source/comp/markv_model.h"
+
+namespace spvtools {
+
+// MARK-V model designed to be a default model for shader compression.
+class MarkvModelShaderDefault : public MarkvModel {
+ public:
+ MarkvModelShaderDefault();
+};
+
+} // namespace spvtools
+
+#endif // SPIRV_TOOLS_COMP_DEFAULT_SHADER_MARKV_MODEL_H_
#include "spirv/1.2/spirv.h"
#include "source/enum_string_mapping.h"
-#include "source/comp/markv_autogen.h"
+#include "source/comp/markv_model.h"
#include "source/opcode.h"
#include "source/operand.h"
#include "source/spirv_constant.h"
// Signals that the value is not in the coding scheme and a fallback method
// needs to be used.
-const uint64_t kMarkvNoneOfTheAbove = GetMarkvNonOfTheAbove();
+const uint64_t kMarkvNoneOfTheAbove =
+ spvtools::MarkvModel::GetMarkvNoneOfTheAbove();
inline uint32_t CombineOpcodeAndNumOperands(uint32_t opcode,
uint32_t num_operands) {
projects / platform / upstream / SPIRV-Tools.git / commitdiff