* Improved vectorized sort - smaller bitonic sorters, dynamic packing/unpacking.
There are two optimizations in this PR:
- reduction of code size in the bitonic sorters: by limiting the amount of inlining in this code, we can reduce overall code size in coreclr.dll by about 180 kB.
- dynamic packing: during sorting, we can switch to 32-bit sorting as soon as the address range in a partition is less 32 GB. This will only have an impact on large heaps or machines with many processors, because we already have a similar, but static optimization where we use 32-bit sorting if the overall address range in the ephemeral region is less than 32 GB. So this additional optimization will give improvements if the overall address range is greater than 32 GB initially, but becomes less during the sort. In this case, we get about a 1.6x improvement in sorting speed.
vxsort/smallsort/bitonic_sort.AVX2.int32_t.generated.cpp
vxsort/smallsort/bitonic_sort.AVX512.int64_t.generated.cpp
vxsort/smallsort/bitonic_sort.AVX512.int32_t.generated.cpp
+ vxsort/smallsort/avx2_load_mask_tables.cpp
)
endif (CLR_CMAKE_TARGET_ARCH_AMD64 AND CLR_CMAKE_TARGET_WIN32)
if (IsSupportedInstructionSet (InstructionSet::AVX2) && (item_count > AVX2_THRESHOLD_SIZE))
{
- // is the range small enough for a 32-bit sort?
- // the 32-bit sort is almost twice as fast
- ptrdiff_t range = range_high - range_low;
- assert(sizeof(uint8_t*) == (1 << 3));
- ptrdiff_t scaled_range = range >> 3;
- if ((uint32_t)scaled_range == scaled_range)
- {
- dprintf (3, ("Sorting mark lists as 32-bit offsets"));
-
- do_pack_avx2 (item_array, item_count, range_low);
-
- int32_t* item_array_32 = (int32_t*)item_array;
-
- // use AVX512F only if the list is large enough to pay for downclocking impact
- if (IsSupportedInstructionSet (InstructionSet::AVX512F) && (item_count > AVX512F_THRESHOLD_SIZE))
- {
- do_vxsort_avx512 (item_array_32, &item_array_32[item_count - 1]);
- }
- else
- {
- do_vxsort_avx2 (item_array_32, &item_array_32[item_count - 1]);
- }
+ dprintf(3, ("Sorting mark lists"));
- do_unpack_avx2 (item_array_32, item_count, range_low);
+ // use AVX512F only if the list is large enough to pay for downclocking impact
+ if (IsSupportedInstructionSet (InstructionSet::AVX512F) && (item_count > AVX512F_THRESHOLD_SIZE))
+ {
+ do_vxsort_avx512 (item_array, &item_array[item_count - 1], range_low, range_high);
}
else
{
- dprintf(3, ("Sorting mark lists"));
-
- // use AVX512F only if the list is large enough to pay for downclocking impact
- if (IsSupportedInstructionSet (InstructionSet::AVX512F) && (item_count > AVX512F_THRESHOLD_SIZE))
- {
- do_vxsort_avx512 (item_array, &item_array[item_count - 1]);
- }
- else
- {
- do_vxsort_avx2 (item_array, &item_array[item_count - 1]);
- }
+ do_vxsort_avx2 (item_array, &item_array[item_count - 1], range_low, range_high);
}
}
else
../vxsort/smallsort/bitonic_sort.AVX2.int32_t.generated.cpp
../vxsort/smallsort/bitonic_sort.AVX512.int64_t.generated.cpp
../vxsort/smallsort/bitonic_sort.AVX512.int32_t.generated.cpp
+ ../vxsort/smallsort/avx2_load_mask_tables.cpp
)
endif (CLR_CMAKE_TARGET_ARCH_AMD64 AND CLR_CMAKE_TARGET_WIN32)
#ifndef VXSORT_ALIGNNMENT_H
#define VXSORT_ALIGNNMENT_H
-//#include <cstdint>
-
namespace vxsort {
using namespace std;
#define NOINLINE __attribute__((noinline))
#endif
+namespace std {
+template <class _Ty>
+class numeric_limits {
+ public:
+ static constexpr _Ty Max() { static_assert(sizeof(_Ty) != sizeof(_Ty), "func must be specialized!"); return _Ty(); }
+ static constexpr _Ty Min() { static_assert(sizeof(_Ty) != sizeof(_Ty), "func must be specialized!"); return _Ty(); }
+};
+
+template <>
+class numeric_limits<int32_t> {
+public:
+ static constexpr int32_t Max() { return 0x7fffffff; }
+ static constexpr int32_t Min() { return -0x7fffffff - 1; }
+};
+
+template <>
+class numeric_limits<uint32_t> {
+public:
+ static constexpr uint32_t Max() { return 0xffffffff; }
+ static constexpr uint32_t Min() { return 0; }
+};
+
+template <>
+class numeric_limits<int64_t> {
+ public:
+ static constexpr int64_t Max() { return 0x7fffffffffffffffi64; }
+
+ static constexpr int64_t Min() { return -0x7fffffffffffffffi64 - 1; }
+};
+} // namespace std
+
+#ifndef max
+template <typename T>
+T max(T a, T b) {
+ if (a > b)
+ return a;
+ else
+ return b;
+}
+#endif
+
#endif // VXSORT_DEFS_H
void InitSupportedInstructionSet (int32_t configSetting);
bool IsSupportedInstructionSet (InstructionSet instructionSet);
-void do_vxsort_avx2 (uint8_t** low, uint8_t** high);
-void do_vxsort_avx2 (int32_t* low, int32_t* high);
+void do_vxsort_avx2 (uint8_t** low, uint8_t** high, uint8_t *range_low, uint8_t *range_high);
-void do_pack_avx2 (uint8_t** mem, size_t len, uint8_t* base);
-void do_unpack_avx2 (int32_t* mem, size_t len, uint8_t* base);
-
-void do_vxsort_avx512 (uint8_t** low, uint8_t** high);
-void do_vxsort_avx512 (int32_t* low, int32_t* high);
-
-void do_pack_avx512 (uint8_t** mem, size_t len, uint8_t* base);
-void do_unpack_avx512 (int32_t* mem, size_t len, uint8_t* base);
+void do_vxsort_avx512 (uint8_t** low, uint8_t** high, uint8_t* range_low, uint8_t* range_high);
#include "vxsort_targets_enable_avx2.h"
-namespace std
-{
- template <class _Ty>
- class numeric_limits
- {
- public:
- static _Ty Max()
- {
- return _Ty();
- }
- static _Ty Min()
- {
- return _Ty();
- }
- };
- template <>
- class numeric_limits<int32_t>
- {
- public:
- static int32_t Max()
- {
- return 0x7fffffff;
- }
- static int32_t Min()
- {
- return -0x7fffffff-1;
- }
- };
- template <>
- class numeric_limits<int64_t>
- {
- public:
- static int64_t Max()
- {
- return 0x7fffffffffffffffi64;
- }
-
- static int64_t Min()
- {
- return -0x7fffffffffffffffi64-1;
- }
- };
-}
-
-#ifndef max
-template <typename T>
-T max (T a, T b)
-{
- if (a > b) return a; else return b;
-}
-#endif
#include "vxsort.h"
#include "machine_traits.avx2.h"
#include "packer.h"
-void do_vxsort_avx2 (uint8_t** low, uint8_t** high)
-{
- auto sorter = vxsort::vxsort<int64_t, vxsort::vector_machine::AVX2, 8>();
- sorter.sort ((int64_t*)low, (int64_t*)high);
-}
-
-void do_vxsort_avx2 (int32_t* low, int32_t* high)
-{
- auto sorter = vxsort::vxsort<int32_t, vxsort::vector_machine::AVX2, 8>();
- sorter.sort (low, high);
-}
-
-void do_pack_avx2 (uint8_t** mem, size_t len, uint8_t* base)
-{
- auto packer = vxsort::packer<int64_t, int32_t, vxsort::vector_machine::AVX2, 3>();
- packer.pack ((int64_t*)mem, len, (int64_t)base);
-}
-
-void do_unpack_avx2 (int32_t* mem, size_t len, uint8_t* base)
+void do_vxsort_avx2 (uint8_t** low, uint8_t** high, uint8_t* range_low, uint8_t* range_high)
{
- auto packer = vxsort::packer<int64_t, int32_t, vxsort::vector_machine::AVX2, 3>();
- packer.unpack (mem, len, (int64_t)base);
+ const int shift = 3;
+ assert((1 << shift) == sizeof(size_t));
+ auto sorter = vxsort::vxsort<int64_t, vxsort::vector_machine::AVX2, 8, shift>();
+ sorter.sort ((int64_t*)low, (int64_t*)high, (int64_t)range_low, (int64_t)(range_high+sizeof(uint8_t*)));
}
#include "vxsort_targets_disable.h"
#include "vxsort_targets_enable_avx512.h"
-namespace std
-{
- template <class _Ty>
- class numeric_limits
- {
- public:
- static _Ty Max()
- {
- return _Ty();
- }
- static _Ty Min()
- {
- return _Ty();
- }
- };
- template <>
- class numeric_limits<int32_t>
- {
- public:
- static int32_t Max()
- {
- return 0x7fffffff;
- }
- static int32_t Min()
- {
- return -0x7fffffff - 1;
- }
- };
- template <>
- class numeric_limits<int64_t>
- {
- public:
- static int64_t Max()
- {
- return 0x7fffffffffffffffi64;
- }
-
- static int64_t Min()
- {
- return -0x7fffffffffffffffi64 - 1;
- }
- };
-}
-
-#ifndef max
-template <typename T>
-T max (T a, T b)
-{
- if (a > b) return a; else return b;
-}
-#endif
-
#include "vxsort.h"
#include "machine_traits.avx512.h"
-void do_vxsort_avx512 (uint8_t** low, uint8_t** high)
+void do_vxsort_avx512 (uint8_t** low, uint8_t** high, uint8_t* range_low, uint8_t* range_high)
{
- auto sorter = vxsort::vxsort<int64_t, vxsort::vector_machine::AVX512, 8>();
- sorter.sort ((int64_t*)low, (int64_t*)high);
+ const int shift = 3;
+ assert((1 << shift) == sizeof(size_t));
+ auto sorter = vxsort::vxsort<int64_t, vxsort::vector_machine::AVX512, 8, shift>();
+ sorter.sort ((int64_t*)low, (int64_t*)high, (int64_t)range_low, (int64_t)(range_high+sizeof(uint8_t*)));
}
-
-void do_vxsort_avx512 (int32_t* low, int32_t* high)
-{
- auto sorter = vxsort::vxsort<int32_t, vxsort::vector_machine::AVX512, 8>();
- sorter.sort (low, high);
-}
-
#include "vxsort_targets_disable.h"
// bit definitions to make code more readable
enum bits
{
- OCXSAVE = 1<<27,
- AVX = 1<<28,
- AVX2 = 1<<5,
- AVX512F=1<<16,
+ OCXSAVE = 1<<27,
+ AVX = 1<<28,
+ AVX2 = 1<< 5,
+ AVX512F = 1<<16,
+ AVX512DQ = 1<<17,
};
int reg[COUNT];
// get processor extended feature flag info
__cpuid(reg, 7);
- // check if both AVX2 and AVX512F are supported by both processor and OS
- if ((reg[EBX] & (AVX2 | AVX512F)) == (AVX2 | AVX512F) &&
+ // check if all of AVX2, AVX512F and AVX512DQ are supported by both processor and OS
+ if ((reg[EBX] & (AVX2 | AVX512F | AVX512DQ)) == (AVX2 | AVX512F | AVX512DQ) &&
(xcr0 & 0xe6) == 0xe6 &&
(FeatureMask & (XSTATE_MASK_AVX | XSTATE_MASK_AVX512)) == (XSTATE_MASK_AVX | XSTATE_MASK_AVX512))
{
// The .NET Foundation licenses this file to you under the MIT license.
#include "common.h"
-//#include <cstdint>
-
#include "machine_traits.avx2.h"
namespace vxsort {
#include "vxsort_targets_enable_avx2.h"
#include <immintrin.h>
-//#include <stdexcept>
#include <assert.h>
-
+#include <inttypes.h>
#include "defs.h"
#include "machine_traits.h"
// in _DEBUG, we #define return to be something more complicated,
// containing a statement, so #define away constexpr for _DEBUG
#define constexpr
-#endif //_DEBUG
+#endif //_DEBUG
template <>
class vxsort_machine_traits<int32_t, AVX2> {
public:
+ typedef int32_t T;
typedef __m256i TV;
typedef uint32_t TMASK;
+ typedef int32_t TPACK;
+ typedef typename std::make_unsigned<T>::type TU;
static constexpr bool supports_compress_writes() { return false; }
+ static constexpr bool supports_packing() { return false; }
+
+ template <int Shift>
+ static constexpr bool can_pack(T span) { return false; }
+
static INLINE TV load_vec(TV* p) { return _mm256_lddqu_si256(p); }
static INLINE void store_vec(TV* ptr, TV v) { _mm256_storeu_si256(ptr, v); }
static INLINE TV partition_vector(TV v, int mask) {
assert(mask >= 0);
assert(mask <= 255);
- return s2i(_mm256_permutevar8x32_ps(i2s(v), _mm256_cvtepu8_epi32(_mm_loadu_si128((__m128i*)(perm_table_32 + mask * 8)))));
+ return s2i(_mm256_permutevar8x32_ps(i2s(v), _mm256_cvtepi8_epi32(_mm_loadu_si128((__m128i*)(perm_table_32 + mask * 8)))));
}
static INLINE TV broadcast(int32_t pivot) { return _mm256_set1_epi32(pivot); }
static INLINE TV add(TV a, TV b) { return _mm256_add_epi32(a, b); }
static INLINE TV sub(TV a, TV b) { return _mm256_sub_epi32(a, b); };
-};
-
-template <>
-class vxsort_machine_traits<uint32_t, AVX2> {
- public:
- typedef __m256i TV;
- typedef uint32_t TMASK;
-
- static constexpr bool supports_compress_writes() { return false; }
-
- static INLINE TV load_vec(TV* p) { return _mm256_lddqu_si256(p); }
- static INLINE void store_vec(TV* ptr, TV v) { _mm256_storeu_si256(ptr, v); }
-
- static void store_compress_vec(TV* ptr, TV v, TMASK mask) { not_supported(); }
+ static INLINE TV pack_ordered(TV a, TV b) { return a; }
+ static INLINE TV pack_unordered(TV a, TV b) { return a; }
+ static INLINE void unpack_ordered(TV p, TV& u1, TV& u2) { }
- static INLINE TV partition_vector(TV v, int mask) {
- assert(mask >= 0);
- assert(mask <= 255);
- return s2i(_mm256_permutevar8x32_ps(i2s(v), _mm256_cvtepu8_epi32(_mm_loadu_si128((__m128i*)(perm_table_32 + mask * 8)))));
+ template <int Shift>
+ static T shift_n_sub(T v, T sub) {
+ if (Shift > 0)
+ v >>= Shift;
+ v -= sub;
+ return v;
}
- static INLINE TV broadcast(uint32_t pivot) { return _mm256_set1_epi32(pivot); }
- static INLINE TMASK get_cmpgt_mask(TV a, TV b) {
- __m256i top_bit = _mm256_set1_epi32(1U << 31);
- return _mm256_movemask_ps(i2s(_mm256_cmpgt_epi32(_mm256_xor_si256(top_bit, a), _mm256_xor_si256(top_bit, b))));
+ template <int Shift>
+ static T unshift_and_add(TPACK from, T add) {
+ add += from;
+ if (Shift > 0)
+ add = (T) (((TU) add) << Shift);
+ return add;
}
-
- static TV shift_right(TV v, int i) { return _mm256_srli_epi32(v, i); }
- static TV shift_left(TV v, int i) { return _mm256_slli_epi32(v, i); }
-
- static INLINE TV add(TV a, TV b) { return _mm256_add_epi32(a, b); }
- static INLINE TV sub(TV a, TV b) { return _mm256_sub_epi32(a, b); };
};
template <>
-class vxsort_machine_traits<float, AVX2> {
+class vxsort_machine_traits<int64_t, AVX2> {
public:
- typedef __m256 TV;
+ typedef int64_t T;
+ typedef __m256i TV;
typedef uint32_t TMASK;
+ typedef int32_t TPACK;
+ typedef typename std::make_unsigned<T>::type TU;
static constexpr bool supports_compress_writes() { return false; }
- static INLINE TV load_vec(TV* p) { return _mm256_loadu_ps((float*)p); }
+ static constexpr bool supports_packing() { return true; }
- static INLINE void store_vec(TV* ptr, TV v) { _mm256_storeu_ps((float*)ptr, v); }
-
- static void store_compress_vec(TV* ptr, TV v, TMASK mask) { not_supported(); }
-
- static INLINE TV partition_vector(TV v, int mask) {
- assert(mask >= 0);
- assert(mask <= 255);
- return _mm256_permutevar8x32_ps(v, _mm256_cvtepu8_epi32(_mm_loadu_si128((__m128i*)(perm_table_32 + mask * 8))));
- }
-
- static INLINE TV broadcast(float pivot) { return _mm256_set1_ps(pivot); }
-
- static INLINE TMASK get_cmpgt_mask(TV a, TV b) {
- /// 0x0E: Greater-than (ordered, signaling) \n
- /// 0x1E: Greater-than (ordered, non-signaling)
- return _mm256_movemask_ps(_mm256_cmp_ps(a, b, _CMP_GT_OS));
+ template <int Shift>
+ static constexpr bool can_pack(T span) {
+ const auto PACK_LIMIT = (((TU) std::numeric_limits<uint32_t>::Max() + 1)) << Shift;
+ return ((TU) span) < PACK_LIMIT;
}
- static INLINE TV add(TV a, TV b) { return _mm256_add_ps(a, b); }
- static INLINE TV sub(TV a, TV b) { return _mm256_sub_ps(a, b); };
-
-};
-
-template <>
-class vxsort_machine_traits<int64_t, AVX2> {
- public:
- typedef __m256i TV;
- typedef uint32_t TMASK;
-
- static constexpr bool supports_compress_writes() { return false; }
-
static INLINE TV load_vec(TV* p) { return _mm256_lddqu_si256(p); }
static INLINE void store_vec(TV* ptr, TV v) { _mm256_storeu_si256(ptr, v); }
static INLINE TV add(TV a, TV b) { return _mm256_add_epi64(a, b); }
static INLINE TV sub(TV a, TV b) { return _mm256_sub_epi64(a, b); };
-
-
static INLINE TV pack_ordered(TV a, TV b) {
a = _mm256_permute4x64_epi64(_mm256_shuffle_epi32(a, _MM_PERM_DBCA), _MM_PERM_DBCA);
b = _mm256_permute4x64_epi64(_mm256_shuffle_epi32(b, _MM_PERM_DBCA), _MM_PERM_CADB);
return _mm256_blend_epi32(a, b, 0b10101010);
}
- static INLINE void unpack_ordered_signed(TV p, TV& u1, TV& u2) {
+ static INLINE void unpack_ordered(TV p, TV& u1, TV& u2) {
auto p01 = _mm256_extracti128_si256(p, 0);
auto p02 = _mm256_extracti128_si256(p, 1);
u1 = _mm256_cvtepi32_epi64(p01);
u2 = _mm256_cvtepi32_epi64(p02);
-
- }
-
- static INLINE void unpack_ordered_unsigned(TV p, TV& u1, TV& u2) {
- auto p01 = _mm256_extracti128_si256(p, 0);
- auto p02 = _mm256_extracti128_si256(p, 1);
-
- u1 = _mm256_cvtepu32_epi64(p01);
- u2 = _mm256_cvtepu32_epi64(p02);
-
}
-/*
- template<>
- static INLINE TV pack_ordered<int32_t>(TV a, TV b) {
- a = _mm256_permute4x64_epi64(_mm256_shuffle_epi32(a, _MM_PERM_DBCA), _MM_PERM_DBCA);
- b = _mm256_permute4x64_epi64(_mm256_shuffle_epi32(b, _MM_PERM_DBCA), _MM_PERM_CADB);
- return _mm256_blend_epi32(a, b, 0b11110000);
+ template <int Shift>
+ static T shift_n_sub(T v, T sub) {
+ if (Shift > 0)
+ v >>= Shift;
+ v -= sub;
+ return v;
}
- template<>
- static INLINE typename vxsort_machine_traits<int32_t, AVX2>::TV pack_unordered<int32_t>(TV a, TV b) {
- b = _mm256_shuffle_epi32(b, _MM_PERM_CDAB);
- return _mm256_blend_epi32(a, b, 0b10101010);
+ template <int Shift>
+ static T unshift_and_add(TPACK from, T add) {
+ add += from;
+ if (Shift > 0)
+ add = (T) (((TU) add) << Shift);
+ return add;
}
-
- */
-
-
-
};
-template <>
-class vxsort_machine_traits<uint64_t, AVX2> {
- public:
- typedef __m256i TV;
- typedef uint32_t TMASK;
-
- static constexpr bool supports_compress_writes() { return false; }
-
- static INLINE TV load_vec(TV* p) { return _mm256_lddqu_si256(p); }
-
- static INLINE void store_vec(TV* ptr, TV v) { _mm256_storeu_si256(ptr, v); }
-
- static void store_compress_vec(TV* ptr, TV v, TMASK mask) { not_supported(); }
-
- static INLINE TV partition_vector(TV v, int mask) {
- assert(mask >= 0);
- assert(mask <= 15);
- return s2i(_mm256_permutevar8x32_ps(i2s(v), _mm256_cvtepu8_epi32(_mm_loadu_si128((__m128i*)(perm_table_64 + mask * 8)))));
- }
- static INLINE TV broadcast(int64_t pivot) { return _mm256_set1_epi64x(pivot); }
- static INLINE TMASK get_cmpgt_mask(TV a, TV b) {
- __m256i top_bit = _mm256_set1_epi64x(1LLU << 63);
- return _mm256_movemask_pd(i2d(_mm256_cmpgt_epi64(_mm256_xor_si256(top_bit, a), _mm256_xor_si256(top_bit, b))));
- }
-
- static INLINE TV shift_right(TV v, int i) { return _mm256_srli_epi64(v, i); }
- static INLINE TV shift_left(TV v, int i) { return _mm256_slli_epi64(v, i); }
-
- static INLINE TV add(TV a, TV b) { return _mm256_add_epi64(a, b); }
- static INLINE TV sub(TV a, TV b) { return _mm256_sub_epi64(a, b); };
-};
-
-template <>
-class vxsort_machine_traits<double, AVX2> {
- public:
- typedef __m256d TV;
- typedef uint32_t TMASK;
-
- static constexpr bool supports_compress_writes() { return false; }
-
- static INLINE TV load_vec(TV* p) { return _mm256_loadu_pd((double*)p); }
-
- static INLINE void store_vec(TV* ptr, TV v) { _mm256_storeu_pd((double*)ptr, v); }
-
- static void store_compress_vec(TV* ptr, TV v, TMASK mask) { not_supported(); }
-
- static INLINE TV partition_vector(TV v, int mask) {
- assert(mask >= 0);
- assert(mask <= 15);
- return s2d(_mm256_permutevar8x32_ps(d2s(v), _mm256_cvtepu8_epi32(_mm_loadu_si128((__m128i*)(perm_table_64 + mask * 8)))));
- }
-
- static INLINE TV broadcast(double pivot) { return _mm256_set1_pd(pivot); }
- static INLINE TMASK get_cmpgt_mask(TV a, TV b) {
- /// 0x0E: Greater-than (ordered, signaling) \n
- /// 0x1E: Greater-than (ordered, non-signaling)
- return _mm256_movemask_pd(_mm256_cmp_pd(a, b, _CMP_GT_OS));
- }
-
- static INLINE TV add(TV a, TV b) { return _mm256_add_pd(a, b); }
- static INLINE TV sub(TV a, TV b) { return _mm256_sub_pd(a, b); };
-};
}
// in _DEBUG, we #define return to be something more complicated,
// containing a statement, so #define away constexpr for _DEBUG
#define constexpr
-#endif //_DEBUG
+#endif //_DEBUG
namespace vxsort {
template <>
class vxsort_machine_traits<int32_t, AVX512> {
- public:
- typedef __m512i TV;
- typedef __mmask16 TMASK;
-
- static constexpr bool supports_compress_writes() { return true; }
+ public:
+ typedef int32_t T;
+ typedef __m512i TV;
+ typedef __mmask16 TMASK;
+ typedef int32_t TPACK;
+ typedef typename std::make_unsigned<T>::type TU;
- static INLINE TV load_vec(TV* p) {
- return _mm512_loadu_si512(p);
- }
+ static constexpr bool supports_compress_writes() { return true; }
- static INLINE void store_vec(TV* ptr, TV v) {
- _mm512_storeu_si512(ptr, v);
- }
+ static constexpr bool supports_packing() { return false; }
- // Will never be called
- static INLINE TV partition_vector(TV v, int mask) { return v; }
+ template <int Shift>
+ static constexpr bool can_pack(T span) { return false; }
+ static INLINE TV load_vec(TV* p) { return _mm512_loadu_si512(p); }
- static void store_compress_vec(TV *ptr, TV v, TMASK mask) {
- _mm512_mask_compressstoreu_epi32(ptr, mask, v);
- }
+ static INLINE void store_vec(TV* ptr, TV v) { _mm512_storeu_si512(ptr, v); }
- static INLINE TV broadcast(int32_t pivot) {
- return _mm512_set1_epi32(pivot);
- }
+ // Will never be called
+ static INLINE TV partition_vector(TV v, int mask) { return v; }
- static INLINE TMASK get_cmpgt_mask(TV a, TV b) {
- return _mm512_cmp_epi32_mask(a, b, _MM_CMPINT_GT);
- }
-};
+ static void store_compress_vec(TV* ptr, TV v, TMASK mask) { _mm512_mask_compressstoreu_epi32(ptr, mask, v); }
-template <>
-class vxsort_machine_traits<uint32_t, AVX512> {
- public:
- typedef __m512i TV;
- typedef __mmask16 TMASK;
+ static INLINE TV broadcast(int32_t pivot) { return _mm512_set1_epi32(pivot); }
- static constexpr bool supports_compress_writes() { return true; }
+ static INLINE TMASK get_cmpgt_mask(TV a, TV b) { return _mm512_cmp_epi32_mask(a, b, _MM_CMPINT_GT); }
- static INLINE TV load_vec(TV* p) {
- return _mm512_loadu_si512(p);
- }
+ static TV shift_right(TV v, int i) { return _mm512_srli_epi32(v, i); }
+ static TV shift_left(TV v, int i) { return _mm512_slli_epi32(v, i); }
- static INLINE void store_vec(TV* ptr, TV v) {
- _mm512_storeu_si512(ptr, v);
- }
+ static INLINE TV add(TV a, TV b) { return _mm512_add_epi32(a, b); }
+ static INLINE TV sub(TV a, TV b) { return _mm512_sub_epi32(a, b); };
- // Will never be called
- static INLINE TV partition_vector(TV v, int mask) { return v; }
+ static INLINE TV pack_ordered(TV a, TV b) { return a; }
+ static INLINE TV pack_unordered(TV a, TV b) { return a; }
+ static INLINE void unpack_ordered(TV p, TV& u1, TV& u2) { }
+ template <int Shift>
+ static T shift_n_sub(T v, T sub) {
+ if (Shift > 0)
+ v >>= Shift;
+ v -= sub;
+ return v;
+ }
- static void store_compress_vec(TV *ptr, TV v, TMASK mask) {
- _mm512_mask_compressstoreu_epi32(ptr, mask, v);
- }
-
- static INLINE TV broadcast(uint32_t pivot) {
- return _mm512_set1_epi32(pivot);
- }
-
- static INLINE TMASK get_cmpgt_mask(TV a, TV b) {
- return _mm512_cmp_epu32_mask(a, b, _MM_CMPINT_GT);
- }
-};
-
-template <>
-class vxsort_machine_traits<float, AVX512> {
- public:
- typedef __m512 TV;
- typedef __mmask16 TMASK;
-
- static constexpr bool supports_compress_writes() { return true; }
-
- static INLINE TV load_vec(TV* p) {
- return _mm512_loadu_ps(p);
- }
-
- static INLINE void store_vec(TV* ptr, TV v) {
- _mm512_storeu_ps(ptr, v);
- }
-
- // Will never be called
- static INLINE TV partition_vector(TV v, int mask) { return v; }
-
-
- static void store_compress_vec(TV *ptr, TV v, TMASK mask) {
- _mm512_mask_compressstoreu_ps(ptr, mask, v);
- }
-
- static INLINE TV broadcast(float pivot) {
- return _mm512_set1_ps(pivot);
- }
-
- static INLINE TMASK get_cmpgt_mask(TV a, TV b) {
- return _mm512_cmp_ps_mask(a, b, _CMP_GT_OS);
- }
+ template <int Shift>
+ static T unshift_and_add(TPACK from, T add) {
+ add += from;
+ if (Shift > 0)
+ add = (T) (((TU) add) << Shift);
+ return add;
+ }
};
template <>
class vxsort_machine_traits<int64_t, AVX512> {
- public:
- typedef __m512i TV;
- typedef __mmask8 TMASK;
-
- static bool supports_compress_writes() { return true; }
+ public:
+ typedef int64_t T;
+ typedef __m512i TV;
+ typedef __mmask8 TMASK;
+ typedef int32_t TPACK;
+ typedef typename std::make_unsigned<T>::type TU;
- static INLINE TV load_vec(TV* p) {
- return _mm512_loadu_si512(p);
- }
+ static constexpr bool supports_compress_writes() { return true; }
- static INLINE void store_vec(TV* ptr, TV v) {
- _mm512_storeu_si512(ptr, v);
- }
+ static constexpr bool supports_packing() { return true; }
- // Will never be called
- static INLINE TV partition_vector(TV v, int mask) { return v; }
+ template <int Shift>
+ static constexpr bool can_pack(T span) {
+ const auto PACK_LIMIT = (((TU) std::numeric_limits<uint32_t>::Max() + 1)) << Shift;
+ return ((TU) span) < PACK_LIMIT;
+ }
+ static INLINE TV load_vec(TV* p) { return _mm512_loadu_si512(p); }
- static void store_compress_vec(TV *ptr, TV v, TMASK mask) {
- _mm512_mask_compressstoreu_epi64(ptr, mask, v);
- }
+ static INLINE void store_vec(TV* ptr, TV v) { _mm512_storeu_si512(ptr, v); }
- static INLINE TV broadcast(int64_t pivot) {
- return _mm512_set1_epi64(pivot);
- }
+ // Will never be called
+ static INLINE TV partition_vector(TV v, int mask) { return v; }
- static INLINE TMASK get_cmpgt_mask(TV a, TV b) {
- return _mm512_cmp_epi64_mask(a, b, _MM_CMPINT_GT);
- }
-};
-
-template <>
-class vxsort_machine_traits<uint64_t, AVX512> {
- public:
- typedef __m512i TV;
- typedef __mmask8 TMASK;
+ static void store_compress_vec(TV* ptr, TV v, TMASK mask) { _mm512_mask_compressstoreu_epi64(ptr, mask, v); }
- static constexpr bool supports_compress_writes() { return true; }
+ static INLINE TV broadcast(int64_t pivot) { return _mm512_set1_epi64(pivot); }
- static INLINE TV load_vec(TV* p) {
- return _mm512_loadu_si512(p);
- }
+ static INLINE TMASK get_cmpgt_mask(TV a, TV b) { return _mm512_cmp_epi64_mask(a, b, _MM_CMPINT_GT); }
- static INLINE void store_vec(TV* ptr, TV v) {
- _mm512_storeu_si512(ptr, v);
- }
+ static TV shift_right(TV v, int i) { return _mm512_srli_epi64(v, i); }
+ static TV shift_left(TV v, int i) { return _mm512_slli_epi64(v, i); }
- // Will never be called
- static INLINE TV partition_vector(TV v, int mask) { return v; }
+ static INLINE TV add(TV a, TV b) { return _mm512_add_epi64(a, b); }
+ static INLINE TV sub(TV a, TV b) { return _mm512_sub_epi64(a, b); };
+ static INLINE TV pack_ordered(TV a, TV b) {
+ a = _mm512_permutex_epi64(_mm512_shuffle_epi32(a, _MM_PERM_DBCA), _MM_PERM_DBCA);
+ b = _mm512_permutex_epi64(_mm512_shuffle_epi32(b, _MM_PERM_DBCA), _MM_PERM_CADB);
+ return _mm512_shuffle_i64x2(a, b, _MM_PERM_DBCA);
+ }
- static void store_compress_vec(TV *ptr, TV v, TMASK mask) {
- _mm512_mask_compressstoreu_epi64(ptr, mask, v);
- }
-
- static INLINE TV broadcast(uint64_t pivot) {
- return _mm512_set1_epi64(pivot);
- }
-
- static INLINE TMASK get_cmpgt_mask(TV a, TV b) {
- return _mm512_cmp_epu64_mask(a, b, _MM_CMPINT_GT);
- }
-};
-
-template <>
-class vxsort_machine_traits<double, AVX512> {
- public:
- typedef __m512d TV;
- typedef __mmask8 TMASK;
-
- static constexpr bool supports_compress_writes() { return true; }
+ static INLINE TV pack_unordered(TV a, TV b) { return _mm512_mask_shuffle_epi32(a, 0b1010101010101010, b, _MM_PERM_CDAB); }
- static INLINE TV load_vec(TV* p) {
- return _mm512_loadu_pd(p);
- }
+ static INLINE void unpack_ordered(TV p, TV& u1, TV& u2) {
+ auto p01 = _mm512_extracti32x8_epi32(p, 0);
+ auto p02 = _mm512_extracti32x8_epi32(p, 1);
- static INLINE void store_vec(TV* ptr, TV v) {
- _mm512_storeu_pd(ptr, v);
- }
+ u1 = _mm512_cvtepi32_epi64(p01);
+ u2 = _mm512_cvtepi32_epi64(p02);
+ }
- // Will never be called
- static INLINE TV partition_vector(TV v, int mask) { return v; }
+ template <int Shift>
+ static T shift_n_sub(T v, T sub) {
+ if (Shift > 0)
+ v >>= Shift;
+ v -= sub;
+ return v;
+ }
+ template <int Shift>
+ static T unshift_and_add(TPACK from, T add) {
+ add += from;
- static void store_compress_vec(TV *ptr, TV v, TMASK mask) {
- _mm512_mask_compressstoreu_pd(ptr, mask, v);
- }
+ if (Shift > 0)
+ add = (T) (((TU) add) << Shift);
- static INLINE TV broadcast(double pivot) {
- return _mm512_set1_pd(pivot);
- }
+ return add;
+ }
- static INLINE TMASK get_cmpgt_mask(TV a, TV b) {
- return _mm512_cmp_pd_mask(a, b, _CMP_GT_OS);
- }
};
}
#ifndef VXSORT_MACHINE_TRAITS_H
#define VXSORT_MACHINE_TRAITS_H
-//#include <cstdint>
-
namespace vxsort {
enum vector_machine {
template <typename T, vector_machine M>
struct vxsort_machine_traits {
public:
- typedef int TV;
- typedef int TMASK;
+ typedef T TV;
+ typedef T TMASK;
+ typedef T TPACK;
+
+ static constexpr bool supports_compress_writes() {
+ static_assert(sizeof(TV) != sizeof(TV), "func must be specialized!");
+ return false;
+ }
+
+ static constexpr bool supports_packing() {
+ static_assert(sizeof(TV) != sizeof(TV), "func must be specialized!");
+ return false;
+ }
- static constexpr bool supports_compress_writes();
+ template <int Shift>
+ static constexpr bool can_pack(T span) {
+ static_assert(sizeof(TV) != sizeof(TV), "func must be specialized!");
+ return false;
+ }
- static TV load_vec(TV* ptr);
- static void store_vec(TV* ptr, TV v);
- static void store_compress_vec(TV* ptr, TV v, TMASK mask);
+ static TV load_vec(TV* ptr) {
+ static_assert(sizeof(TV) != sizeof(TV), "func must be specialized!");
+ }
+ static void store_vec(TV* ptr, TV v) {
+ static_assert(sizeof(TV) != sizeof(TV), "func must be specialized!");
+ }
+ static void store_compress_vec(TV* ptr, TV v, TMASK mask) {
+ static_assert(sizeof(TV) != sizeof(TV), "func must be specialized!");
+ }
static TV partition_vector(TV v, int mask);
static TV broadcast(T pivot);
static TMASK get_cmpgt_mask(TV a, TV b);
static TV pack_ordered(TV a, TV b);
static TV pack_unordered(TV a, TV b);
- static void unpack_ordered_signed(TV p, TV& u1, TV& u2);
- static void unpack_ordered_unsigned(TV p, TV& u1, TV& u2);
+ static void unpack_ordered(TV p, TV& u1, TV& u2) {
+ static_assert(sizeof(TV) != sizeof(TV), "func must be specialized!");
+ }
+ template <int Shift>
+ static T shift_n_sub(T v, T sub) {
+ static_assert(sizeof(TV) != sizeof(TV), "func must be specialized!");
+ return v;
+ }
+ template <int Shift>
+ static T unshift_and_add(TPACK from, T add) {
+ static_assert(sizeof(TV) != sizeof(TV), "func must be specialized!");
+ return add;
+ }
};
+
}
+
+
#endif // VXSORT_MACHINE_TRAITS_H
#ifndef VXSORT_PACKER_H
#define VXSORT_PACKER_H
-#include "vxsort_targets_enable_avx2.h"
-
-//#include <cstdint>
-//#include <limits>
-//#include <type_traits>
-#//include <cassert>
+#include "defs.h"
#include "alignment.h"
#include "machine_traits.h"
-#include "machine_traits.avx2.h"
-#include "machine_traits.avx512.h"
#include <immintrin.h>
-//#include <cstdio>
namespace vxsort {
-template <typename TFrom, typename TTo, vector_machine M, int Shift = 0, int MinLength=1, bool RespectPackingOrder=false>
+template<typename TFrom, typename TTo, vector_machine M, int Shift = 0, int Unroll = 1, int MinLength = 1, bool RespectPackingOrder = false>
class packer {
- static_assert(Shift <= 31, "Shift must be in the range 0..31");
- using MT = vxsort_machine_traits<TFrom, M>;
- typedef typename MT::TV TV;
- typedef typename std::make_unsigned<TFrom>::type TU;
- static const int N = sizeof(TV) / sizeof(TFrom);
- typedef alignment_hint<sizeof(TV)> AH;
-
- static const size_t ALIGN = AH::ALIGN;
- static const size_t ALIGN_MASK = ALIGN - 1;
-
- static INLINE void pack_scalar(const TFrom offset, TFrom*& mem_read, TTo*& mem_write) {
- auto d = *(mem_read++);
- if (Shift > 0)
- d >>= Shift;
- d -= offset;
- *(mem_write++) = (TTo) d;
- }
-
- static INLINE void unpack_scalar(const TFrom offset, TTo*& mem_read, TFrom*& mem_write) {
- TFrom d = *(--mem_read);
-
- d += offset;
-
- if (Shift > 0)
- d = (TFrom) (((TU) d) << Shift);
-
- *(--mem_write) = d;
- }
-
- public:
-
- static void pack(TFrom *mem, size_t len, TFrom base) {
- TFrom offset = (base >> Shift) - std::numeric_limits<TTo>::Min();
- auto baseVec = MT::broadcast(offset);
-
- auto pre_aligned_mem = reinterpret_cast<TFrom*>(reinterpret_cast<size_t>(mem) & ~ALIGN_MASK);
-
- auto mem_read = mem;
- auto mem_write = (TTo *) mem;
-
- // Include a "special" pass to handle very short scalar
- // passes
- if (MinLength < N && len < N) {
- while (len--) {
- pack_scalar(offset, mem_read, mem_write);
- }
- return;
+ static_assert(Shift <= 31, "Shift must be in the range 0..31");
+ static_assert(Unroll >= 1, "Unroll can be in the range 1..4");
+ static_assert(Unroll <= 4, "Unroll can be in the range 1..4");
+
+ using MT = vxsort_machine_traits<TFrom, M>;
+ typedef typename MT::TV TV;
+ static const int N = sizeof(TV) / sizeof(TFrom);
+ typedef alignment_hint<sizeof(TV)> AH;
+
+ static const size_t ALIGN = AH::ALIGN;
+ static const size_t ALIGN_MASK = ALIGN - 1;
+
+
+ static INLINE TV pack_vectorized(const TV baseVec, TV d01, TV d02) {
+ if (Shift > 0) { // This is statically compiled in/out
+ d01 = MT::shift_right(d01, Shift);
+ d02 = MT::shift_right(d02, Shift);
+ }
+ d01 = MT::sub(d01, baseVec);
+ d02 = MT::sub(d02, baseVec);
+
+ auto packed_data = RespectPackingOrder ?
+ MT::pack_ordered(d01, d02) :
+ MT::pack_unordered(d01, d02);
+ return packed_data;
}
- // We have at least
- // one vector worth of data to handle
- // Let's try to align to vector size first
-
- if (pre_aligned_mem < mem) {
- const auto alignment_point = pre_aligned_mem + N;
- len -= (alignment_point - mem_read);
- while (mem_read < alignment_point) {
- pack_scalar(offset, mem_read, mem_write);
- }
- }
-
- assert(AH::is_aligned(mem_read));
-
- auto memv_read = (TV *) mem_read;
- auto memv_write = (TV *) mem_write;
+ static NOINLINE void unpack_vectorized(const TV baseVec, TV d01, TV& u01, TV& u02) {
+ MT::unpack_ordered(d01, u01, u02);
- auto lenv = len / N;
- len -= (lenv * N);
+ u01 = MT::add(u01, baseVec);
+ u02 = MT::add(u02, baseVec);
- while (lenv >= 2) {
- assert(memv_read >= memv_write);
-
- auto d01 = MT::load_vec(memv_read);
- auto d02 = MT::load_vec(memv_read + 1);
- if (Shift > 0) { // This is statically compiled in/out
- d01 = MT::shift_right(d01, Shift);
- d02 = MT::shift_right(d02, Shift);
- }
- d01 = MT::sub(d01, baseVec);
- d02 = MT::sub(d02, baseVec);
-
- auto packed_data = RespectPackingOrder ?
- MT::pack_ordered(d01, d02) :
- MT::pack_unordered(d01, d02);
-
- MT::store_vec(memv_write, packed_data);
-
- memv_read += 2;
- memv_write++;
- lenv -= 2;
+ if (Shift > 0) { // This is statically compiled in/out
+ u01 = MT::shift_left(u01, Shift);
+ u02 = MT::shift_left(u02, Shift);
+ }
}
- len += lenv * N;
-
- mem_read = (TFrom *) memv_read;
- mem_write = (TTo *) memv_write;
-
- while (len-- > 0) {
- pack_scalar(offset, mem_read, mem_write);
- }
- }
-
- static void unpack(TTo *mem, size_t len, TFrom base) {
- TFrom offset = (base >> Shift) - std::numeric_limits<TTo>::Min();
- auto baseVec = MT::broadcast(offset);
-
- auto mem_read = mem + len;
- auto mem_write = ((TFrom *) mem) + len;
-
-
- // Include a "special" pass to handle very short scalar
- // passers
- if (MinLength < 2*N && len < 2*N) {
- while (len--) {
- unpack_scalar(offset, mem_read, mem_write);
- }
- return;
- }
-
- auto pre_aligned_mem = reinterpret_cast<TTo*>(reinterpret_cast<size_t>(mem_read) & ~ALIGN_MASK);
-
- if (pre_aligned_mem < mem_read) {
- len -= (mem_read - pre_aligned_mem);
- while (mem_read > pre_aligned_mem) {
- unpack_scalar(offset, mem_read, mem_write);
- }
- }
-
- assert(AH::is_aligned(mem_read));
-
- auto lenv = len / (N*2);
- auto memv_read = ((TV *) mem_read) - 1;
- auto memv_write = ((TV *) mem_write) - 2;
- len -= lenv * N * 2;
-
- while (lenv > 0) {
- assert(memv_read <= memv_write);
- TV d01, d02;
-
- if (std::numeric_limits<TTo>::Min() < 0)
- MT::unpack_ordered_signed(MT::load_vec(memv_read), d01, d02);
- else
- MT::unpack_ordered_unsigned(MT::load_vec(memv_read), d01, d02);
-
- d01 = MT::add(d01, baseVec);
- d02 = MT::add(d02, baseVec);
-
- if (Shift > 0) { // This is statically compiled in/out
- d01 = MT::shift_left(d01, Shift);
- d02 = MT::shift_left(d02, Shift);
- }
-
- MT::store_vec(memv_write, d01);
- MT::store_vec(memv_write + 1, d02);
-
- memv_read -= 1;
- memv_write -= 2;
- lenv--;
+ public:
+
+ static void pack(TFrom *mem, size_t len, TFrom base) {
+ TFrom offset = MT::template shift_n_sub<Shift>(base, (TFrom) std::numeric_limits<TTo>::Min());
+ auto baseVec = MT::broadcast(offset);
+
+ auto pre_aligned_mem = reinterpret_cast<TFrom *>(reinterpret_cast<size_t>(mem) & ~ALIGN_MASK);
+
+ auto mem_read = mem;
+ auto mem_write = (TTo *) mem;
+
+ // Include a "special" pass to handle very short scalar
+ // passes
+ if (MinLength < N && len < N) {
+ while (len--) {
+ *(mem_write++) = (TTo) MT::template shift_n_sub<Shift>(*(mem_read++), offset);
+ }
+ return;
+ }
+
+ // We have at least
+ // one vector worth of data to handle
+ // Let's try to align to vector size first
+
+ if (pre_aligned_mem < mem) {
+ const auto alignment_point = pre_aligned_mem + N;
+ len -= (alignment_point - mem_read);
+ while (mem_read < alignment_point) {
+ *(mem_write++) = (TTo) MT::template shift_n_sub<Shift>(*(mem_read++), offset);
+ }
+ }
+
+ assert(AH::is_aligned(mem_read));
+
+ auto memv_read = (TV *) mem_read;
+ auto memv_write = (TV *) mem_write;
+
+ auto lenv = len / N;
+ len -= (lenv * N);
+
+ while (lenv >= 2 * Unroll) {
+ assert(memv_read >= memv_write);
+
+ TV d01, d02, d03, d04, d05, d06, d07, d08;
+
+ do {
+ d01 = MT::load_vec(memv_read + 0);
+ d02 = MT::load_vec(memv_read + 1);
+ if (Unroll == 1) break;
+ d03 = MT::load_vec(memv_read + 2);
+ d04 = MT::load_vec(memv_read + 3);
+ if (Unroll == 2) break;
+ d05 = MT::load_vec(memv_read + 4);
+ d06 = MT::load_vec(memv_read + 5);
+ if (Unroll == 3) break;
+ d07 = MT::load_vec(memv_read + 6);
+ d08 = MT::load_vec(memv_read + 7);
+ break;
+ } while (true);
+
+ do {
+ MT::store_vec(memv_write + 0, pack_vectorized(baseVec, d01, d02));
+ if (Unroll == 1) break;
+ MT::store_vec(memv_write + 1, pack_vectorized(baseVec, d03, d04));
+ if (Unroll == 2) break;
+ MT::store_vec(memv_write + 2, pack_vectorized(baseVec, d05, d06));
+ if (Unroll == 3) break;
+ MT::store_vec(memv_write + 3, pack_vectorized(baseVec, d07, d08));
+ break;
+ } while(true);
+
+ memv_read += 2*Unroll;
+ memv_write += Unroll;
+ lenv -= 2*Unroll;
+ }
+
+ if (Unroll > 1) {
+ while (lenv >= 2) {
+ assert(memv_read >= memv_write);
+ TV d01, d02;
+
+ d01 = MT::load_vec(memv_read + 0);
+ d02 = MT::load_vec(memv_read + 1);
+
+ MT::store_vec(memv_write + 0, pack_vectorized(baseVec, d01, d02));
+ memv_read += 2;
+ memv_write++;
+ lenv -= 2;
+ }
+ }
+
+ len += lenv * N;
+
+ mem_read = (TFrom *) memv_read;
+ mem_write = (TTo *) memv_write;
+
+ while (len-- > 0) {
+ *(mem_write++) = (TTo) MT::template shift_n_sub<Shift>(*(mem_read++), offset);
+ }
}
- mem_read = (TTo *) (memv_read + 1);
- mem_write = (TFrom *) (memv_write + 2);
- while (len-- > 0) {
- unpack_scalar(offset, mem_read, mem_write);
+ static void unpack(TTo *mem, size_t len, TFrom base) {
+ TFrom offset = MT::template shift_n_sub<Shift>(base, (TFrom) std::numeric_limits<TTo>::Min());
+ auto baseVec = MT::broadcast(offset);
+
+ auto mem_read = mem + len;
+ auto mem_write = ((TFrom *) mem) + len;
+
+
+ // Include a "special" pass to handle very short scalar
+ // passers
+ if (MinLength < 2 * N && len < 2 * N) {
+ while (len--) {
+ *(--mem_write) = MT::template unshift_and_add<Shift>(*(--mem_read), offset);
+ }
+ return;
+ }
+
+ auto pre_aligned_mem = reinterpret_cast<TTo *>(reinterpret_cast<size_t>(mem_read) & ~ALIGN_MASK);
+
+ if (pre_aligned_mem < mem_read) {
+ len -= (mem_read - pre_aligned_mem);
+ while (mem_read > pre_aligned_mem) {
+ *(--mem_write) = MT::template unshift_and_add<Shift>(*(--mem_read), offset);
+ }
+ }
+
+ assert(AH::is_aligned(mem_read));
+
+ auto lenv = len / (N * 2);
+ auto memv_read = ((TV *) mem_read) - 1;
+ auto memv_write = ((TV *) mem_write) - 2;
+ len -= lenv * N * 2;
+
+ while (lenv >= Unroll) {
+ assert(memv_read <= memv_write);
+
+ TV d01, d02, d03, d04;
+ TV u01, u02, u03, u04, u05, u06, u07, u08;
+
+ do {
+ d01 = MT::load_vec(memv_read + 0);
+ if (Unroll == 1) break;
+ d02 = MT::load_vec(memv_read - 1);
+ if (Unroll == 2) break;
+ d03 = MT::load_vec(memv_read - 2);
+ if (Unroll == 3) break;
+ d04 = MT::load_vec(memv_read - 3);
+ break;
+ } while(true);
+
+ do {
+ unpack_vectorized(baseVec, d01, u01, u02);
+ MT::store_vec(memv_write + 0, u01);
+ MT::store_vec(memv_write + 1, u02);
+ if (Unroll == 1) break;
+ unpack_vectorized(baseVec, d02, u03, u04);
+ MT::store_vec(memv_write - 2, u03);
+ MT::store_vec(memv_write - 1, u04);
+ if (Unroll == 2) break;
+ unpack_vectorized(baseVec, d03, u05, u06);
+ MT::store_vec(memv_write - 4, u05);
+ MT::store_vec(memv_write - 3, u06);
+ if (Unroll == 3) break;
+ unpack_vectorized(baseVec, d04, u07, u08);
+ MT::store_vec(memv_write - 6, u07);
+ MT::store_vec(memv_write - 5, u08);
+ break;
+ } while(true);
+
+ memv_read -= Unroll;
+ memv_write -= 2 * Unroll;
+ lenv -= Unroll;
+ }
+
+ if (Unroll > 1) {
+ while (lenv >= 1) {
+ assert(memv_read <= memv_write);
+
+ TV d01;
+ TV u01, u02;
+
+ d01 = MT::load_vec(memv_read + 0);
+
+ unpack_vectorized(baseVec, d01, u01, u02);
+ MT::store_vec(memv_write + 0, u01);
+ MT::store_vec(memv_write + 1, u02);
+
+ memv_read--;
+ memv_write -= 2;
+ lenv--;
+ }
+ }
+
+ mem_read = (TTo *) (memv_read + 1);
+ mem_write = (TFrom *) (memv_write + 2);
+
+ while (len-- > 0) {
+ *(--mem_write) = MT::template unshift_and_add<Shift>(*(--mem_read), offset);
+ }
}
- }
};
-
}
-#include "vxsort_targets_disable.h"
-
#endif // VXSORT_PACKER_H
--- /dev/null
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+#include "common.h"
+
+#include "bitonic_sort.h"
+
+namespace vxsort {
+ namespace smallsort {
+
+ extern "C" alignas(16) const uint8_t mask_table_4[M4_SIZE] = {
+ 0xFF, 0xFF, 0xFF, 0xFF, // 0b0000 (0)
+ 0xFF, 0x00, 0x00, 0x00, // 0b0001 (1)
+ 0xFF, 0xFF, 0x00, 0x00, // 0b0011 (3)
+ 0xFF, 0xFF, 0xFF, 0x00, // 0b0111 (7)
+ #if defined(__has_feature)
+ #if __has_feature(address_sanitizer)
+ 0xCC, 0xCC, 0xCC, 0xCC, // Garbage to make ASAN happy
+ 0xCC, 0xCC, 0xCC, 0xCC, // Garbage to make ASAN happy
+ 0xCC, 0xCC, 0xCC, 0xCC, // Garbage to make ASAN happy
+ #endif
+ #endif
+ };
+
+ extern "C" alignas(128) const uint8_t mask_table_8[M8_SIZE] = {
+ 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 0b00000000 ( 0)
+ 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0b00000001 ( 1)
+ 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0b00000011 ( 3)
+ 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, // 0b00000111 ( 7)
+ 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, // 0b00001111 ( 15)
+ 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, // 0b00011111 ( 31)
+ 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, // 0b00111111 ( 63)
+ 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, // 0b01111111 (127)
+ #if defined(__has_feature)
+ #if __has_feature(address_sanitizer)
+ 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, // Garbage to make ASAN happy
+ #endif
+ #endif
+ };
+ }
+}
using namespace vxsort;
void vxsort::smallsort::bitonic<int32_t, vector_machine::AVX2 >::sort(int32_t *ptr, size_t length) {
- const int N = 8;
-
- switch(length / N) {
- case 1: sort_01v(ptr); break;
- case 2: sort_02v(ptr); break;
- case 3: sort_03v(ptr); break;
- case 4: sort_04v(ptr); break;
- case 5: sort_05v(ptr); break;
- case 6: sort_06v(ptr); break;
- case 7: sort_07v(ptr); break;
- case 8: sort_08v(ptr); break;
- case 9: sort_09v(ptr); break;
- case 10: sort_10v(ptr); break;
- case 11: sort_11v(ptr); break;
- case 12: sort_12v(ptr); break;
- case 13: sort_13v(ptr); break;
- case 14: sort_14v(ptr); break;
- case 15: sort_15v(ptr); break;
- case 16: sort_16v(ptr); break;
+ const auto fullvlength = length / N;
+ const int remainder = (int) (length - fullvlength * N);
+ const auto v = fullvlength + ((remainder > 0) ? 1 : 0);
+ switch(v) {
+ case 1: sort_01v_alt(ptr, remainder); break;
+ case 2: sort_02v_alt(ptr, remainder); break;
+ case 3: sort_03v_alt(ptr, remainder); break;
+ case 4: sort_04v_alt(ptr, remainder); break;
+ case 5: sort_05v_alt(ptr, remainder); break;
+ case 6: sort_06v_alt(ptr, remainder); break;
+ case 7: sort_07v_alt(ptr, remainder); break;
+ case 8: sort_08v_alt(ptr, remainder); break;
+ case 9: sort_09v_alt(ptr, remainder); break;
+ case 10: sort_10v_alt(ptr, remainder); break;
+ case 11: sort_11v_alt(ptr, remainder); break;
+ case 12: sort_12v_alt(ptr, remainder); break;
+ case 13: sort_13v_alt(ptr, remainder); break;
+ case 14: sort_14v_alt(ptr, remainder); break;
+ case 15: sort_15v_alt(ptr, remainder); break;
+ case 16: sort_16v_alt(ptr, remainder); break;
}
}
/////////////////////////////////////////////////////////////////////////////
////
-// This file was auto-generated by a tool at 2020-06-22 05:27:48
+// This file was auto-generated by a tool at 2020-07-21 14:05:39
//
// It is recommended you DO NOT directly edit this file but instead edit
// the code-generator that generated this source file instead.
namespace vxsort {
namespace smallsort {
+
+extern "C" const uint8_t mask_table_4[16];
+extern "C" const uint8_t mask_table_8[64];
+
template<> struct bitonic<int32_t, AVX2> {
+ static const int N = 8;
+ static constexpr int32_t MAX = std::numeric_limits<int32_t>::Max();
public:
static INLINE void sort_01v_ascending(__m256i& d01) {
sort_02v_merge_descending(d01, d02);
sort_01v_merge_descending(d03);
}
- static INLINE void sort_04v_ascending(__m256i& d01, __m256i& d02, __m256i& d03, __m256i& d04) {
+ static NOINLINE void sort_04v_ascending(__m256i& d01, __m256i& d02, __m256i& d03, __m256i& d04) {
__m256i tmp;
sort_02v_ascending(d01, d02);
sort_02v_merge_ascending(d01, d02);
sort_02v_merge_ascending(d03, d04);
}
- static INLINE void sort_04v_descending(__m256i& d01, __m256i& d02, __m256i& d03, __m256i& d04) {
+ static NOINLINE void sort_04v_descending(__m256i& d01, __m256i& d02, __m256i& d03, __m256i& d04) {
__m256i tmp;
sort_02v_descending(d01, d02);
sort_02v_merge_descending(d01, d02);
sort_02v_merge_descending(d03, d04);
}
- static INLINE void sort_04v_merge_ascending(__m256i& d01, __m256i& d02, __m256i& d03, __m256i& d04) {
+ static NOINLINE void sort_04v_merge_ascending(__m256i& d01, __m256i& d02, __m256i& d03, __m256i& d04) {
__m256i tmp;
tmp = d01;
sort_02v_merge_ascending(d01, d02);
sort_02v_merge_ascending(d03, d04);
}
- static INLINE void sort_04v_merge_descending(__m256i& d01, __m256i& d02, __m256i& d03, __m256i& d04) {
+ static NOINLINE void sort_04v_merge_descending(__m256i& d01, __m256i& d02, __m256i& d03, __m256i& d04) {
__m256i tmp;
tmp = d01;
sort_04v_merge_descending(d01, d02, d03, d04);
sort_03v_merge_descending(d05, d06, d07);
}
- static INLINE void sort_08v_ascending(__m256i& d01, __m256i& d02, __m256i& d03, __m256i& d04, __m256i& d05, __m256i& d06, __m256i& d07, __m256i& d08) {
+ static NOINLINE void sort_08v_ascending(__m256i& d01, __m256i& d02, __m256i& d03, __m256i& d04, __m256i& d05, __m256i& d06, __m256i& d07, __m256i& d08) {
__m256i tmp;
sort_04v_ascending(d01, d02, d03, d04);
sort_04v_merge_ascending(d01, d02, d03, d04);
sort_04v_merge_ascending(d05, d06, d07, d08);
}
- static INLINE void sort_08v_descending(__m256i& d01, __m256i& d02, __m256i& d03, __m256i& d04, __m256i& d05, __m256i& d06, __m256i& d07, __m256i& d08) {
+ static NOINLINE void sort_08v_descending(__m256i& d01, __m256i& d02, __m256i& d03, __m256i& d04, __m256i& d05, __m256i& d06, __m256i& d07, __m256i& d08) {
__m256i tmp;
sort_04v_descending(d01, d02, d03, d04);
sort_04v_merge_descending(d01, d02, d03, d04);
sort_04v_merge_descending(d05, d06, d07, d08);
}
- static INLINE void sort_08v_merge_ascending(__m256i& d01, __m256i& d02, __m256i& d03, __m256i& d04, __m256i& d05, __m256i& d06, __m256i& d07, __m256i& d08) {
+ static NOINLINE void sort_08v_merge_ascending(__m256i& d01, __m256i& d02, __m256i& d03, __m256i& d04, __m256i& d05, __m256i& d06, __m256i& d07, __m256i& d08) {
__m256i tmp;
tmp = d01;
sort_04v_merge_ascending(d01, d02, d03, d04);
sort_04v_merge_ascending(d05, d06, d07, d08);
}
- static INLINE void sort_08v_merge_descending(__m256i& d01, __m256i& d02, __m256i& d03, __m256i& d04, __m256i& d05, __m256i& d06, __m256i& d07, __m256i& d08) {
+ static NOINLINE void sort_08v_merge_descending(__m256i& d01, __m256i& d02, __m256i& d03, __m256i& d04, __m256i& d05, __m256i& d06, __m256i& d07, __m256i& d08) {
__m256i tmp;
tmp = d01;
sort_08v_merge_descending(d01, d02, d03, d04, d05, d06, d07, d08);
sort_03v_merge_descending(d09, d10, d11);
}
- static INLINE void sort_12v_ascending(__m256i& d01, __m256i& d02, __m256i& d03, __m256i& d04, __m256i& d05, __m256i& d06, __m256i& d07, __m256i& d08, __m256i& d09, __m256i& d10, __m256i& d11, __m256i& d12) {
+ static NOINLINE void sort_12v_ascending(__m256i& d01, __m256i& d02, __m256i& d03, __m256i& d04, __m256i& d05, __m256i& d06, __m256i& d07, __m256i& d08, __m256i& d09, __m256i& d10, __m256i& d11, __m256i& d12) {
__m256i tmp;
sort_08v_ascending(d01, d02, d03, d04, d05, d06, d07, d08);
sort_08v_merge_ascending(d01, d02, d03, d04, d05, d06, d07, d08);
sort_04v_merge_ascending(d09, d10, d11, d12);
}
- static INLINE void sort_12v_descending(__m256i& d01, __m256i& d02, __m256i& d03, __m256i& d04, __m256i& d05, __m256i& d06, __m256i& d07, __m256i& d08, __m256i& d09, __m256i& d10, __m256i& d11, __m256i& d12) {
+ static NOINLINE void sort_12v_descending(__m256i& d01, __m256i& d02, __m256i& d03, __m256i& d04, __m256i& d05, __m256i& d06, __m256i& d07, __m256i& d08, __m256i& d09, __m256i& d10, __m256i& d11, __m256i& d12) {
__m256i tmp;
sort_08v_descending(d01, d02, d03, d04, d05, d06, d07, d08);
sort_08v_merge_descending(d01, d02, d03, d04, d05, d06, d07, d08);
sort_07v_merge_descending(d09, d10, d11, d12, d13, d14, d15);
}
- static INLINE void sort_16v_ascending(__m256i& d01, __m256i& d02, __m256i& d03, __m256i& d04, __m256i& d05, __m256i& d06, __m256i& d07, __m256i& d08, __m256i& d09, __m256i& d10, __m256i& d11, __m256i& d12, __m256i& d13, __m256i& d14, __m256i& d15, __m256i& d16) {
+ static NOINLINE void sort_16v_ascending(__m256i& d01, __m256i& d02, __m256i& d03, __m256i& d04, __m256i& d05, __m256i& d06, __m256i& d07, __m256i& d08, __m256i& d09, __m256i& d10, __m256i& d11, __m256i& d12, __m256i& d13, __m256i& d14, __m256i& d15, __m256i& d16) {
__m256i tmp;
sort_08v_ascending(d01, d02, d03, d04, d05, d06, d07, d08);
sort_08v_merge_ascending(d01, d02, d03, d04, d05, d06, d07, d08);
sort_08v_merge_ascending(d09, d10, d11, d12, d13, d14, d15, d16);
}
- static INLINE void sort_16v_descending(__m256i& d01, __m256i& d02, __m256i& d03, __m256i& d04, __m256i& d05, __m256i& d06, __m256i& d07, __m256i& d08, __m256i& d09, __m256i& d10, __m256i& d11, __m256i& d12, __m256i& d13, __m256i& d14, __m256i& d15, __m256i& d16) {
+ static NOINLINE void sort_16v_descending(__m256i& d01, __m256i& d02, __m256i& d03, __m256i& d04, __m256i& d05, __m256i& d06, __m256i& d07, __m256i& d08, __m256i& d09, __m256i& d10, __m256i& d11, __m256i& d12, __m256i& d13, __m256i& d14, __m256i& d15, __m256i& d16) {
__m256i tmp;
sort_08v_descending(d01, d02, d03, d04, d05, d06, d07, d08);
sort_08v_merge_descending(d09, d10, d11, d12, d13, d14, d15, d16);
}
- static NOINLINE void sort_01v(int32_t *ptr) {
- __m256i d01 = _mm256_lddqu_si256((__m256i const *) ptr + 0);;
+ static NOINLINE void sort_01v_alt(int32_t *ptr, int remainder) {
+ const auto mask = _mm256_cvtepi8_epi32(_mm_loadu_si128((__m128i*)(mask_table_8 + remainder * N)));
+
+ __m256i d01 = _mm256_or_si256(_mm256_maskload_epi32((int32_t const *) ((__m256i const *) ptr + 0), mask), _mm256_andnot_si256(mask, _mm256_set1_epi32(MAX)));
sort_01v_ascending(d01);
- _mm256_storeu_si256((__m256i *) ptr + 0, d01);
-}
+ _mm256_maskstore_epi32((int32_t *) ((__m256i *) ptr + 0), mask, d01);
+ }
+
+ static NOINLINE void sort_02v_alt(int32_t *ptr, int remainder) {
+ const auto mask = _mm256_cvtepi8_epi32(_mm_loadu_si128((__m128i*)(mask_table_8 + remainder * N)));
- static NOINLINE void sort_02v(int32_t *ptr) {
__m256i d01 = _mm256_lddqu_si256((__m256i const *) ptr + 0);;
- __m256i d02 = _mm256_lddqu_si256((__m256i const *) ptr + 1);;
+ __m256i d02 = _mm256_or_si256(_mm256_maskload_epi32((int32_t const *) ((__m256i const *) ptr + 1), mask), _mm256_andnot_si256(mask, _mm256_set1_epi32(MAX)));
sort_02v_ascending(d01, d02);
_mm256_storeu_si256((__m256i *) ptr + 0, d01);
- _mm256_storeu_si256((__m256i *) ptr + 1, d02);
-}
+ _mm256_maskstore_epi32((int32_t *) ((__m256i *) ptr + 1), mask, d02);
+ }
+
+ static NOINLINE void sort_03v_alt(int32_t *ptr, int remainder) {
+ const auto mask = _mm256_cvtepi8_epi32(_mm_loadu_si128((__m128i*)(mask_table_8 + remainder * N)));
- static NOINLINE void sort_03v(int32_t *ptr) {
__m256i d01 = _mm256_lddqu_si256((__m256i const *) ptr + 0);;
__m256i d02 = _mm256_lddqu_si256((__m256i const *) ptr + 1);;
- __m256i d03 = _mm256_lddqu_si256((__m256i const *) ptr + 2);;
+ __m256i d03 = _mm256_or_si256(_mm256_maskload_epi32((int32_t const *) ((__m256i const *) ptr + 2), mask), _mm256_andnot_si256(mask, _mm256_set1_epi32(MAX)));
sort_03v_ascending(d01, d02, d03);
_mm256_storeu_si256((__m256i *) ptr + 0, d01);
_mm256_storeu_si256((__m256i *) ptr + 1, d02);
- _mm256_storeu_si256((__m256i *) ptr + 2, d03);
-}
+ _mm256_maskstore_epi32((int32_t *) ((__m256i *) ptr + 2), mask, d03);
+ }
+
+ static NOINLINE void sort_04v_alt(int32_t *ptr, int remainder) {
+ const auto mask = _mm256_cvtepi8_epi32(_mm_loadu_si128((__m128i*)(mask_table_8 + remainder * N)));
- static NOINLINE void sort_04v(int32_t *ptr) {
__m256i d01 = _mm256_lddqu_si256((__m256i const *) ptr + 0);;
__m256i d02 = _mm256_lddqu_si256((__m256i const *) ptr + 1);;
__m256i d03 = _mm256_lddqu_si256((__m256i const *) ptr + 2);;
- __m256i d04 = _mm256_lddqu_si256((__m256i const *) ptr + 3);;
+ __m256i d04 = _mm256_or_si256(_mm256_maskload_epi32((int32_t const *) ((__m256i const *) ptr + 3), mask), _mm256_andnot_si256(mask, _mm256_set1_epi32(MAX)));
sort_04v_ascending(d01, d02, d03, d04);
_mm256_storeu_si256((__m256i *) ptr + 0, d01);
_mm256_storeu_si256((__m256i *) ptr + 1, d02);
_mm256_storeu_si256((__m256i *) ptr + 2, d03);
- _mm256_storeu_si256((__m256i *) ptr + 3, d04);
-}
+ _mm256_maskstore_epi32((int32_t *) ((__m256i *) ptr + 3), mask, d04);
+ }
+
+ static NOINLINE void sort_05v_alt(int32_t *ptr, int remainder) {
+ const auto mask = _mm256_cvtepi8_epi32(_mm_loadu_si128((__m128i*)(mask_table_8 + remainder * N)));
- static NOINLINE void sort_05v(int32_t *ptr) {
__m256i d01 = _mm256_lddqu_si256((__m256i const *) ptr + 0);;
__m256i d02 = _mm256_lddqu_si256((__m256i const *) ptr + 1);;
__m256i d03 = _mm256_lddqu_si256((__m256i const *) ptr + 2);;
__m256i d04 = _mm256_lddqu_si256((__m256i const *) ptr + 3);;
- __m256i d05 = _mm256_lddqu_si256((__m256i const *) ptr + 4);;
+ __m256i d05 = _mm256_or_si256(_mm256_maskload_epi32((int32_t const *) ((__m256i const *) ptr + 4), mask), _mm256_andnot_si256(mask, _mm256_set1_epi32(MAX)));
sort_05v_ascending(d01, d02, d03, d04, d05);
_mm256_storeu_si256((__m256i *) ptr + 0, d01);
_mm256_storeu_si256((__m256i *) ptr + 1, d02);
_mm256_storeu_si256((__m256i *) ptr + 2, d03);
_mm256_storeu_si256((__m256i *) ptr + 3, d04);
- _mm256_storeu_si256((__m256i *) ptr + 4, d05);
-}
+ _mm256_maskstore_epi32((int32_t *) ((__m256i *) ptr + 4), mask, d05);
+ }
+
+ static NOINLINE void sort_06v_alt(int32_t *ptr, int remainder) {
+ const auto mask = _mm256_cvtepi8_epi32(_mm_loadu_si128((__m128i*)(mask_table_8 + remainder * N)));
- static NOINLINE void sort_06v(int32_t *ptr) {
__m256i d01 = _mm256_lddqu_si256((__m256i const *) ptr + 0);;
__m256i d02 = _mm256_lddqu_si256((__m256i const *) ptr + 1);;
__m256i d03 = _mm256_lddqu_si256((__m256i const *) ptr + 2);;
__m256i d04 = _mm256_lddqu_si256((__m256i const *) ptr + 3);;
__m256i d05 = _mm256_lddqu_si256((__m256i const *) ptr + 4);;
- __m256i d06 = _mm256_lddqu_si256((__m256i const *) ptr + 5);;
+ __m256i d06 = _mm256_or_si256(_mm256_maskload_epi32((int32_t const *) ((__m256i const *) ptr + 5), mask), _mm256_andnot_si256(mask, _mm256_set1_epi32(MAX)));
sort_06v_ascending(d01, d02, d03, d04, d05, d06);
_mm256_storeu_si256((__m256i *) ptr + 0, d01);
_mm256_storeu_si256((__m256i *) ptr + 1, d02);
_mm256_storeu_si256((__m256i *) ptr + 2, d03);
_mm256_storeu_si256((__m256i *) ptr + 3, d04);
_mm256_storeu_si256((__m256i *) ptr + 4, d05);
- _mm256_storeu_si256((__m256i *) ptr + 5, d06);
-}
+ _mm256_maskstore_epi32((int32_t *) ((__m256i *) ptr + 5), mask, d06);
+ }
+
+ static NOINLINE void sort_07v_alt(int32_t *ptr, int remainder) {
+ const auto mask = _mm256_cvtepi8_epi32(_mm_loadu_si128((__m128i*)(mask_table_8 + remainder * N)));
- static NOINLINE void sort_07v(int32_t *ptr) {
__m256i d01 = _mm256_lddqu_si256((__m256i const *) ptr + 0);;
__m256i d02 = _mm256_lddqu_si256((__m256i const *) ptr + 1);;
__m256i d03 = _mm256_lddqu_si256((__m256i const *) ptr + 2);;
__m256i d04 = _mm256_lddqu_si256((__m256i const *) ptr + 3);;
__m256i d05 = _mm256_lddqu_si256((__m256i const *) ptr + 4);;
__m256i d06 = _mm256_lddqu_si256((__m256i const *) ptr + 5);;
- __m256i d07 = _mm256_lddqu_si256((__m256i const *) ptr + 6);;
+ __m256i d07 = _mm256_or_si256(_mm256_maskload_epi32((int32_t const *) ((__m256i const *) ptr + 6), mask), _mm256_andnot_si256(mask, _mm256_set1_epi32(MAX)));
sort_07v_ascending(d01, d02, d03, d04, d05, d06, d07);
_mm256_storeu_si256((__m256i *) ptr + 0, d01);
_mm256_storeu_si256((__m256i *) ptr + 1, d02);
_mm256_storeu_si256((__m256i *) ptr + 3, d04);
_mm256_storeu_si256((__m256i *) ptr + 4, d05);
_mm256_storeu_si256((__m256i *) ptr + 5, d06);
- _mm256_storeu_si256((__m256i *) ptr + 6, d07);
-}
+ _mm256_maskstore_epi32((int32_t *) ((__m256i *) ptr + 6), mask, d07);
+ }
+
+ static NOINLINE void sort_08v_alt(int32_t *ptr, int remainder) {
+ const auto mask = _mm256_cvtepi8_epi32(_mm_loadu_si128((__m128i*)(mask_table_8 + remainder * N)));
- static NOINLINE void sort_08v(int32_t *ptr) {
__m256i d01 = _mm256_lddqu_si256((__m256i const *) ptr + 0);;
__m256i d02 = _mm256_lddqu_si256((__m256i const *) ptr + 1);;
__m256i d03 = _mm256_lddqu_si256((__m256i const *) ptr + 2);;
__m256i d05 = _mm256_lddqu_si256((__m256i const *) ptr + 4);;
__m256i d06 = _mm256_lddqu_si256((__m256i const *) ptr + 5);;
__m256i d07 = _mm256_lddqu_si256((__m256i const *) ptr + 6);;
- __m256i d08 = _mm256_lddqu_si256((__m256i const *) ptr + 7);;
+ __m256i d08 = _mm256_or_si256(_mm256_maskload_epi32((int32_t const *) ((__m256i const *) ptr + 7), mask), _mm256_andnot_si256(mask, _mm256_set1_epi32(MAX)));
sort_08v_ascending(d01, d02, d03, d04, d05, d06, d07, d08);
_mm256_storeu_si256((__m256i *) ptr + 0, d01);
_mm256_storeu_si256((__m256i *) ptr + 1, d02);
_mm256_storeu_si256((__m256i *) ptr + 4, d05);
_mm256_storeu_si256((__m256i *) ptr + 5, d06);
_mm256_storeu_si256((__m256i *) ptr + 6, d07);
- _mm256_storeu_si256((__m256i *) ptr + 7, d08);
-}
+ _mm256_maskstore_epi32((int32_t *) ((__m256i *) ptr + 7), mask, d08);
+ }
+
+ static NOINLINE void sort_09v_alt(int32_t *ptr, int remainder) {
+ const auto mask = _mm256_cvtepi8_epi32(_mm_loadu_si128((__m128i*)(mask_table_8 + remainder * N)));
- static NOINLINE void sort_09v(int32_t *ptr) {
__m256i d01 = _mm256_lddqu_si256((__m256i const *) ptr + 0);;
__m256i d02 = _mm256_lddqu_si256((__m256i const *) ptr + 1);;
__m256i d03 = _mm256_lddqu_si256((__m256i const *) ptr + 2);;
__m256i d06 = _mm256_lddqu_si256((__m256i const *) ptr + 5);;
__m256i d07 = _mm256_lddqu_si256((__m256i const *) ptr + 6);;
__m256i d08 = _mm256_lddqu_si256((__m256i const *) ptr + 7);;
- __m256i d09 = _mm256_lddqu_si256((__m256i const *) ptr + 8);;
+ __m256i d09 = _mm256_or_si256(_mm256_maskload_epi32((int32_t const *) ((__m256i const *) ptr + 8), mask), _mm256_andnot_si256(mask, _mm256_set1_epi32(MAX)));
sort_09v_ascending(d01, d02, d03, d04, d05, d06, d07, d08, d09);
_mm256_storeu_si256((__m256i *) ptr + 0, d01);
_mm256_storeu_si256((__m256i *) ptr + 1, d02);
_mm256_storeu_si256((__m256i *) ptr + 5, d06);
_mm256_storeu_si256((__m256i *) ptr + 6, d07);
_mm256_storeu_si256((__m256i *) ptr + 7, d08);
- _mm256_storeu_si256((__m256i *) ptr + 8, d09);
-}
+ _mm256_maskstore_epi32((int32_t *) ((__m256i *) ptr + 8), mask, d09);
+ }
+
+ static NOINLINE void sort_10v_alt(int32_t *ptr, int remainder) {
+ const auto mask = _mm256_cvtepi8_epi32(_mm_loadu_si128((__m128i*)(mask_table_8 + remainder * N)));
- static NOINLINE void sort_10v(int32_t *ptr) {
__m256i d01 = _mm256_lddqu_si256((__m256i const *) ptr + 0);;
__m256i d02 = _mm256_lddqu_si256((__m256i const *) ptr + 1);;
__m256i d03 = _mm256_lddqu_si256((__m256i const *) ptr + 2);;
__m256i d07 = _mm256_lddqu_si256((__m256i const *) ptr + 6);;
__m256i d08 = _mm256_lddqu_si256((__m256i const *) ptr + 7);;
__m256i d09 = _mm256_lddqu_si256((__m256i const *) ptr + 8);;
- __m256i d10 = _mm256_lddqu_si256((__m256i const *) ptr + 9);;
+ __m256i d10 = _mm256_or_si256(_mm256_maskload_epi32((int32_t const *) ((__m256i const *) ptr + 9), mask), _mm256_andnot_si256(mask, _mm256_set1_epi32(MAX)));
sort_10v_ascending(d01, d02, d03, d04, d05, d06, d07, d08, d09, d10);
_mm256_storeu_si256((__m256i *) ptr + 0, d01);
_mm256_storeu_si256((__m256i *) ptr + 1, d02);
_mm256_storeu_si256((__m256i *) ptr + 6, d07);
_mm256_storeu_si256((__m256i *) ptr + 7, d08);
_mm256_storeu_si256((__m256i *) ptr + 8, d09);
- _mm256_storeu_si256((__m256i *) ptr + 9, d10);
-}
+ _mm256_maskstore_epi32((int32_t *) ((__m256i *) ptr + 9), mask, d10);
+ }
+
+ static NOINLINE void sort_11v_alt(int32_t *ptr, int remainder) {
+ const auto mask = _mm256_cvtepi8_epi32(_mm_loadu_si128((__m128i*)(mask_table_8 + remainder * N)));
- static NOINLINE void sort_11v(int32_t *ptr) {
__m256i d01 = _mm256_lddqu_si256((__m256i const *) ptr + 0);;
__m256i d02 = _mm256_lddqu_si256((__m256i const *) ptr + 1);;
__m256i d03 = _mm256_lddqu_si256((__m256i const *) ptr + 2);;
__m256i d08 = _mm256_lddqu_si256((__m256i const *) ptr + 7);;
__m256i d09 = _mm256_lddqu_si256((__m256i const *) ptr + 8);;
__m256i d10 = _mm256_lddqu_si256((__m256i const *) ptr + 9);;
- __m256i d11 = _mm256_lddqu_si256((__m256i const *) ptr + 10);;
+ __m256i d11 = _mm256_or_si256(_mm256_maskload_epi32((int32_t const *) ((__m256i const *) ptr + 10), mask), _mm256_andnot_si256(mask, _mm256_set1_epi32(MAX)));
sort_11v_ascending(d01, d02, d03, d04, d05, d06, d07, d08, d09, d10, d11);
_mm256_storeu_si256((__m256i *) ptr + 0, d01);
_mm256_storeu_si256((__m256i *) ptr + 1, d02);
_mm256_storeu_si256((__m256i *) ptr + 7, d08);
_mm256_storeu_si256((__m256i *) ptr + 8, d09);
_mm256_storeu_si256((__m256i *) ptr + 9, d10);
- _mm256_storeu_si256((__m256i *) ptr + 10, d11);
-}
+ _mm256_maskstore_epi32((int32_t *) ((__m256i *) ptr + 10), mask, d11);
+ }
+
+ static NOINLINE void sort_12v_alt(int32_t *ptr, int remainder) {
+ const auto mask = _mm256_cvtepi8_epi32(_mm_loadu_si128((__m128i*)(mask_table_8 + remainder * N)));
- static NOINLINE void sort_12v(int32_t *ptr) {
__m256i d01 = _mm256_lddqu_si256((__m256i const *) ptr + 0);;
__m256i d02 = _mm256_lddqu_si256((__m256i const *) ptr + 1);;
__m256i d03 = _mm256_lddqu_si256((__m256i const *) ptr + 2);;
__m256i d09 = _mm256_lddqu_si256((__m256i const *) ptr + 8);;
__m256i d10 = _mm256_lddqu_si256((__m256i const *) ptr + 9);;
__m256i d11 = _mm256_lddqu_si256((__m256i const *) ptr + 10);;
- __m256i d12 = _mm256_lddqu_si256((__m256i const *) ptr + 11);;
+ __m256i d12 = _mm256_or_si256(_mm256_maskload_epi32((int32_t const *) ((__m256i const *) ptr + 11), mask), _mm256_andnot_si256(mask, _mm256_set1_epi32(MAX)));
sort_12v_ascending(d01, d02, d03, d04, d05, d06, d07, d08, d09, d10, d11, d12);
_mm256_storeu_si256((__m256i *) ptr + 0, d01);
_mm256_storeu_si256((__m256i *) ptr + 1, d02);
_mm256_storeu_si256((__m256i *) ptr + 8, d09);
_mm256_storeu_si256((__m256i *) ptr + 9, d10);
_mm256_storeu_si256((__m256i *) ptr + 10, d11);
- _mm256_storeu_si256((__m256i *) ptr + 11, d12);
-}
+ _mm256_maskstore_epi32((int32_t *) ((__m256i *) ptr + 11), mask, d12);
+ }
+
+ static NOINLINE void sort_13v_alt(int32_t *ptr, int remainder) {
+ const auto mask = _mm256_cvtepi8_epi32(_mm_loadu_si128((__m128i*)(mask_table_8 + remainder * N)));
- static NOINLINE void sort_13v(int32_t *ptr) {
__m256i d01 = _mm256_lddqu_si256((__m256i const *) ptr + 0);;
__m256i d02 = _mm256_lddqu_si256((__m256i const *) ptr + 1);;
__m256i d03 = _mm256_lddqu_si256((__m256i const *) ptr + 2);;
__m256i d10 = _mm256_lddqu_si256((__m256i const *) ptr + 9);;
__m256i d11 = _mm256_lddqu_si256((__m256i const *) ptr + 10);;
__m256i d12 = _mm256_lddqu_si256((__m256i const *) ptr + 11);;
- __m256i d13 = _mm256_lddqu_si256((__m256i const *) ptr + 12);;
+ __m256i d13 = _mm256_or_si256(_mm256_maskload_epi32((int32_t const *) ((__m256i const *) ptr + 12), mask), _mm256_andnot_si256(mask, _mm256_set1_epi32(MAX)));
sort_13v_ascending(d01, d02, d03, d04, d05, d06, d07, d08, d09, d10, d11, d12, d13);
_mm256_storeu_si256((__m256i *) ptr + 0, d01);
_mm256_storeu_si256((__m256i *) ptr + 1, d02);
_mm256_storeu_si256((__m256i *) ptr + 9, d10);
_mm256_storeu_si256((__m256i *) ptr + 10, d11);
_mm256_storeu_si256((__m256i *) ptr + 11, d12);
- _mm256_storeu_si256((__m256i *) ptr + 12, d13);
-}
+ _mm256_maskstore_epi32((int32_t *) ((__m256i *) ptr + 12), mask, d13);
+ }
+
+ static NOINLINE void sort_14v_alt(int32_t *ptr, int remainder) {
+ const auto mask = _mm256_cvtepi8_epi32(_mm_loadu_si128((__m128i*)(mask_table_8 + remainder * N)));
- static NOINLINE void sort_14v(int32_t *ptr) {
__m256i d01 = _mm256_lddqu_si256((__m256i const *) ptr + 0);;
__m256i d02 = _mm256_lddqu_si256((__m256i const *) ptr + 1);;
__m256i d03 = _mm256_lddqu_si256((__m256i const *) ptr + 2);;
__m256i d11 = _mm256_lddqu_si256((__m256i const *) ptr + 10);;
__m256i d12 = _mm256_lddqu_si256((__m256i const *) ptr + 11);;
__m256i d13 = _mm256_lddqu_si256((__m256i const *) ptr + 12);;
- __m256i d14 = _mm256_lddqu_si256((__m256i const *) ptr + 13);;
+ __m256i d14 = _mm256_or_si256(_mm256_maskload_epi32((int32_t const *) ((__m256i const *) ptr + 13), mask), _mm256_andnot_si256(mask, _mm256_set1_epi32(MAX)));
sort_14v_ascending(d01, d02, d03, d04, d05, d06, d07, d08, d09, d10, d11, d12, d13, d14);
_mm256_storeu_si256((__m256i *) ptr + 0, d01);
_mm256_storeu_si256((__m256i *) ptr + 1, d02);
_mm256_storeu_si256((__m256i *) ptr + 10, d11);
_mm256_storeu_si256((__m256i *) ptr + 11, d12);
_mm256_storeu_si256((__m256i *) ptr + 12, d13);
- _mm256_storeu_si256((__m256i *) ptr + 13, d14);
-}
+ _mm256_maskstore_epi32((int32_t *) ((__m256i *) ptr + 13), mask, d14);
+ }
+
+ static NOINLINE void sort_15v_alt(int32_t *ptr, int remainder) {
+ const auto mask = _mm256_cvtepi8_epi32(_mm_loadu_si128((__m128i*)(mask_table_8 + remainder * N)));
- static NOINLINE void sort_15v(int32_t *ptr) {
__m256i d01 = _mm256_lddqu_si256((__m256i const *) ptr + 0);;
__m256i d02 = _mm256_lddqu_si256((__m256i const *) ptr + 1);;
__m256i d03 = _mm256_lddqu_si256((__m256i const *) ptr + 2);;
__m256i d12 = _mm256_lddqu_si256((__m256i const *) ptr + 11);;
__m256i d13 = _mm256_lddqu_si256((__m256i const *) ptr + 12);;
__m256i d14 = _mm256_lddqu_si256((__m256i const *) ptr + 13);;
- __m256i d15 = _mm256_lddqu_si256((__m256i const *) ptr + 14);;
+ __m256i d15 = _mm256_or_si256(_mm256_maskload_epi32((int32_t const *) ((__m256i const *) ptr + 14), mask), _mm256_andnot_si256(mask, _mm256_set1_epi32(MAX)));
sort_15v_ascending(d01, d02, d03, d04, d05, d06, d07, d08, d09, d10, d11, d12, d13, d14, d15);
_mm256_storeu_si256((__m256i *) ptr + 0, d01);
_mm256_storeu_si256((__m256i *) ptr + 1, d02);
_mm256_storeu_si256((__m256i *) ptr + 11, d12);
_mm256_storeu_si256((__m256i *) ptr + 12, d13);
_mm256_storeu_si256((__m256i *) ptr + 13, d14);
- _mm256_storeu_si256((__m256i *) ptr + 14, d15);
-}
+ _mm256_maskstore_epi32((int32_t *) ((__m256i *) ptr + 14), mask, d15);
+ }
+
+ static NOINLINE void sort_16v_alt(int32_t *ptr, int remainder) {
+ const auto mask = _mm256_cvtepi8_epi32(_mm_loadu_si128((__m128i*)(mask_table_8 + remainder * N)));
- static NOINLINE void sort_16v(int32_t *ptr) {
__m256i d01 = _mm256_lddqu_si256((__m256i const *) ptr + 0);;
__m256i d02 = _mm256_lddqu_si256((__m256i const *) ptr + 1);;
__m256i d03 = _mm256_lddqu_si256((__m256i const *) ptr + 2);;
__m256i d13 = _mm256_lddqu_si256((__m256i const *) ptr + 12);;
__m256i d14 = _mm256_lddqu_si256((__m256i const *) ptr + 13);;
__m256i d15 = _mm256_lddqu_si256((__m256i const *) ptr + 14);;
- __m256i d16 = _mm256_lddqu_si256((__m256i const *) ptr + 15);;
+ __m256i d16 = _mm256_or_si256(_mm256_maskload_epi32((int32_t const *) ((__m256i const *) ptr + 15), mask), _mm256_andnot_si256(mask, _mm256_set1_epi32(MAX)));
sort_16v_ascending(d01, d02, d03, d04, d05, d06, d07, d08, d09, d10, d11, d12, d13, d14, d15, d16);
_mm256_storeu_si256((__m256i *) ptr + 0, d01);
_mm256_storeu_si256((__m256i *) ptr + 1, d02);
_mm256_storeu_si256((__m256i *) ptr + 12, d13);
_mm256_storeu_si256((__m256i *) ptr + 13, d14);
_mm256_storeu_si256((__m256i *) ptr + 14, d15);
- _mm256_storeu_si256((__m256i *) ptr + 15, d16);
-}
+ _mm256_maskstore_epi32((int32_t *) ((__m256i *) ptr + 15), mask, d16);
+ }
static void sort(int32_t *ptr, size_t length);
};
using namespace vxsort;
void vxsort::smallsort::bitonic<int64_t, vector_machine::AVX2 >::sort(int64_t *ptr, size_t length) {
- const int N = 4;
-
- switch(length / N) {
- case 1: sort_01v(ptr); break;
- case 2: sort_02v(ptr); break;
- case 3: sort_03v(ptr); break;
- case 4: sort_04v(ptr); break;
- case 5: sort_05v(ptr); break;
- case 6: sort_06v(ptr); break;
- case 7: sort_07v(ptr); break;
- case 8: sort_08v(ptr); break;
- case 9: sort_09v(ptr); break;
- case 10: sort_10v(ptr); break;
- case 11: sort_11v(ptr); break;
- case 12: sort_12v(ptr); break;
- case 13: sort_13v(ptr); break;
- case 14: sort_14v(ptr); break;
- case 15: sort_15v(ptr); break;
- case 16: sort_16v(ptr); break;
+ const auto fullvlength = length / N;
+ const int remainder = (int) (length - fullvlength * N);
+ const auto v = fullvlength + ((remainder > 0) ? 1 : 0);
+ switch(v) {
+ case 1: sort_01v_alt(ptr, remainder); break;
+ case 2: sort_02v_alt(ptr, remainder); break;
+ case 3: sort_03v_alt(ptr, remainder); break;
+ case 4: sort_04v_alt(ptr, remainder); break;
+ case 5: sort_05v_alt(ptr, remainder); break;
+ case 6: sort_06v_alt(ptr, remainder); break;
+ case 7: sort_07v_alt(ptr, remainder); break;
+ case 8: sort_08v_alt(ptr, remainder); break;
+ case 9: sort_09v_alt(ptr, remainder); break;
+ case 10: sort_10v_alt(ptr, remainder); break;
+ case 11: sort_11v_alt(ptr, remainder); break;
+ case 12: sort_12v_alt(ptr, remainder); break;
+ case 13: sort_13v_alt(ptr, remainder); break;
+ case 14: sort_14v_alt(ptr, remainder); break;
+ case 15: sort_15v_alt(ptr, remainder); break;
+ case 16: sort_16v_alt(ptr, remainder); break;
}
}
/////////////////////////////////////////////////////////////////////////////
////
-// This file was auto-generated by a tool at 2020-06-22 05:27:48
+// This file was auto-generated by a tool at 2020-07-21 14:05:39
//
// It is recommended you DO NOT directly edit this file but instead edit
// the code-generator that generated this source file instead.
namespace vxsort {
namespace smallsort {
+
+extern "C" const uint8_t mask_table_4[16];
+extern "C" const uint8_t mask_table_8[64];
+
template<> struct bitonic<int64_t, AVX2> {
+ static const int N = 4;
+ static constexpr int64_t MAX = std::numeric_limits<int64_t>::Max();
public:
static INLINE void sort_01v_ascending(__m256i& d01) {
sort_02v_merge_descending(d01, d02);
sort_01v_merge_descending(d03);
}
- static INLINE void sort_04v_ascending(__m256i& d01, __m256i& d02, __m256i& d03, __m256i& d04) {
+ static NOINLINE void sort_04v_ascending(__m256i& d01, __m256i& d02, __m256i& d03, __m256i& d04) {
__m256i tmp, cmp;
sort_02v_ascending(d01, d02);
sort_02v_merge_ascending(d01, d02);
sort_02v_merge_ascending(d03, d04);
}
- static INLINE void sort_04v_descending(__m256i& d01, __m256i& d02, __m256i& d03, __m256i& d04) {
+ static NOINLINE void sort_04v_descending(__m256i& d01, __m256i& d02, __m256i& d03, __m256i& d04) {
__m256i tmp, cmp;
sort_02v_descending(d01, d02);
sort_02v_merge_descending(d01, d02);
sort_02v_merge_descending(d03, d04);
}
- static INLINE void sort_04v_merge_ascending(__m256i& d01, __m256i& d02, __m256i& d03, __m256i& d04) {
+ static NOINLINE void sort_04v_merge_ascending(__m256i& d01, __m256i& d02, __m256i& d03, __m256i& d04) {
__m256i tmp, cmp;
tmp = d01;
sort_02v_merge_ascending(d01, d02);
sort_02v_merge_ascending(d03, d04);
}
- static INLINE void sort_04v_merge_descending(__m256i& d01, __m256i& d02, __m256i& d03, __m256i& d04) {
+ static NOINLINE void sort_04v_merge_descending(__m256i& d01, __m256i& d02, __m256i& d03, __m256i& d04) {
__m256i tmp, cmp;
tmp = d01;
sort_04v_merge_descending(d01, d02, d03, d04);
sort_03v_merge_descending(d05, d06, d07);
}
- static INLINE void sort_08v_ascending(__m256i& d01, __m256i& d02, __m256i& d03, __m256i& d04, __m256i& d05, __m256i& d06, __m256i& d07, __m256i& d08) {
+ static NOINLINE void sort_08v_ascending(__m256i& d01, __m256i& d02, __m256i& d03, __m256i& d04, __m256i& d05, __m256i& d06, __m256i& d07, __m256i& d08) {
__m256i tmp, cmp;
sort_04v_ascending(d01, d02, d03, d04);
sort_04v_merge_ascending(d01, d02, d03, d04);
sort_04v_merge_ascending(d05, d06, d07, d08);
}
- static INLINE void sort_08v_descending(__m256i& d01, __m256i& d02, __m256i& d03, __m256i& d04, __m256i& d05, __m256i& d06, __m256i& d07, __m256i& d08) {
+ static NOINLINE void sort_08v_descending(__m256i& d01, __m256i& d02, __m256i& d03, __m256i& d04, __m256i& d05, __m256i& d06, __m256i& d07, __m256i& d08) {
__m256i tmp, cmp;
sort_04v_descending(d01, d02, d03, d04);
sort_04v_merge_descending(d01, d02, d03, d04);
sort_04v_merge_descending(d05, d06, d07, d08);
}
- static INLINE void sort_08v_merge_ascending(__m256i& d01, __m256i& d02, __m256i& d03, __m256i& d04, __m256i& d05, __m256i& d06, __m256i& d07, __m256i& d08) {
+ static NOINLINE void sort_08v_merge_ascending(__m256i& d01, __m256i& d02, __m256i& d03, __m256i& d04, __m256i& d05, __m256i& d06, __m256i& d07, __m256i& d08) {
__m256i tmp, cmp;
tmp = d01;
sort_04v_merge_ascending(d01, d02, d03, d04);
sort_04v_merge_ascending(d05, d06, d07, d08);
}
- static INLINE void sort_08v_merge_descending(__m256i& d01, __m256i& d02, __m256i& d03, __m256i& d04, __m256i& d05, __m256i& d06, __m256i& d07, __m256i& d08) {
+ static NOINLINE void sort_08v_merge_descending(__m256i& d01, __m256i& d02, __m256i& d03, __m256i& d04, __m256i& d05, __m256i& d06, __m256i& d07, __m256i& d08) {
__m256i tmp, cmp;
tmp = d01;
sort_08v_merge_descending(d01, d02, d03, d04, d05, d06, d07, d08);
sort_03v_merge_descending(d09, d10, d11);
}
- static INLINE void sort_12v_ascending(__m256i& d01, __m256i& d02, __m256i& d03, __m256i& d04, __m256i& d05, __m256i& d06, __m256i& d07, __m256i& d08, __m256i& d09, __m256i& d10, __m256i& d11, __m256i& d12) {
+ static NOINLINE void sort_12v_ascending(__m256i& d01, __m256i& d02, __m256i& d03, __m256i& d04, __m256i& d05, __m256i& d06, __m256i& d07, __m256i& d08, __m256i& d09, __m256i& d10, __m256i& d11, __m256i& d12) {
__m256i tmp, cmp;
sort_08v_ascending(d01, d02, d03, d04, d05, d06, d07, d08);
sort_08v_merge_ascending(d01, d02, d03, d04, d05, d06, d07, d08);
sort_04v_merge_ascending(d09, d10, d11, d12);
}
- static INLINE void sort_12v_descending(__m256i& d01, __m256i& d02, __m256i& d03, __m256i& d04, __m256i& d05, __m256i& d06, __m256i& d07, __m256i& d08, __m256i& d09, __m256i& d10, __m256i& d11, __m256i& d12) {
+ static NOINLINE void sort_12v_descending(__m256i& d01, __m256i& d02, __m256i& d03, __m256i& d04, __m256i& d05, __m256i& d06, __m256i& d07, __m256i& d08, __m256i& d09, __m256i& d10, __m256i& d11, __m256i& d12) {
__m256i tmp, cmp;
sort_08v_descending(d01, d02, d03, d04, d05, d06, d07, d08);
sort_08v_merge_descending(d01, d02, d03, d04, d05, d06, d07, d08);
sort_07v_merge_descending(d09, d10, d11, d12, d13, d14, d15);
}
- static INLINE void sort_16v_ascending(__m256i& d01, __m256i& d02, __m256i& d03, __m256i& d04, __m256i& d05, __m256i& d06, __m256i& d07, __m256i& d08, __m256i& d09, __m256i& d10, __m256i& d11, __m256i& d12, __m256i& d13, __m256i& d14, __m256i& d15, __m256i& d16) {
+ static NOINLINE void sort_16v_ascending(__m256i& d01, __m256i& d02, __m256i& d03, __m256i& d04, __m256i& d05, __m256i& d06, __m256i& d07, __m256i& d08, __m256i& d09, __m256i& d10, __m256i& d11, __m256i& d12, __m256i& d13, __m256i& d14, __m256i& d15, __m256i& d16) {
__m256i tmp, cmp;
sort_08v_ascending(d01, d02, d03, d04, d05, d06, d07, d08);
sort_08v_merge_ascending(d01, d02, d03, d04, d05, d06, d07, d08);
sort_08v_merge_ascending(d09, d10, d11, d12, d13, d14, d15, d16);
}
- static INLINE void sort_16v_descending(__m256i& d01, __m256i& d02, __m256i& d03, __m256i& d04, __m256i& d05, __m256i& d06, __m256i& d07, __m256i& d08, __m256i& d09, __m256i& d10, __m256i& d11, __m256i& d12, __m256i& d13, __m256i& d14, __m256i& d15, __m256i& d16) {
+ static NOINLINE void sort_16v_descending(__m256i& d01, __m256i& d02, __m256i& d03, __m256i& d04, __m256i& d05, __m256i& d06, __m256i& d07, __m256i& d08, __m256i& d09, __m256i& d10, __m256i& d11, __m256i& d12, __m256i& d13, __m256i& d14, __m256i& d15, __m256i& d16) {
__m256i tmp, cmp;
sort_08v_descending(d01, d02, d03, d04, d05, d06, d07, d08);
sort_08v_merge_descending(d09, d10, d11, d12, d13, d14, d15, d16);
}
- static NOINLINE void sort_01v(int64_t *ptr) {
- __m256i d01 = _mm256_lddqu_si256((__m256i const *) ptr + 0);;
+ static NOINLINE void sort_01v_alt(int64_t *ptr, int remainder) {
+ const auto mask = _mm256_cvtepi8_epi64(_mm_loadu_si128((__m128i*)(mask_table_4 + remainder * N)));
+
+ __m256i d01 = _mm256_or_si256(_mm256_maskload_epi64((long long const *) ((__m256i const *) ptr + 0), mask), _mm256_andnot_si256(mask, _mm256_set1_epi64x(MAX)));
sort_01v_ascending(d01);
- _mm256_storeu_si256((__m256i *) ptr + 0, d01);
-}
+ _mm256_maskstore_epi64((long long *) ((__m256i *) ptr + 0), mask, d01);
+ }
+
+ static NOINLINE void sort_02v_alt(int64_t *ptr, int remainder) {
+ const auto mask = _mm256_cvtepi8_epi64(_mm_loadu_si128((__m128i*)(mask_table_4 + remainder * N)));
- static NOINLINE void sort_02v(int64_t *ptr) {
__m256i d01 = _mm256_lddqu_si256((__m256i const *) ptr + 0);;
- __m256i d02 = _mm256_lddqu_si256((__m256i const *) ptr + 1);;
+ __m256i d02 = _mm256_or_si256(_mm256_maskload_epi64((long long const *) ((__m256i const *) ptr + 1), mask), _mm256_andnot_si256(mask, _mm256_set1_epi64x(MAX)));
sort_02v_ascending(d01, d02);
_mm256_storeu_si256((__m256i *) ptr + 0, d01);
- _mm256_storeu_si256((__m256i *) ptr + 1, d02);
-}
+ _mm256_maskstore_epi64((long long *) ((__m256i *) ptr + 1), mask, d02);
+ }
+
+ static NOINLINE void sort_03v_alt(int64_t *ptr, int remainder) {
+ const auto mask = _mm256_cvtepi8_epi64(_mm_loadu_si128((__m128i*)(mask_table_4 + remainder * N)));
- static NOINLINE void sort_03v(int64_t *ptr) {
__m256i d01 = _mm256_lddqu_si256((__m256i const *) ptr + 0);;
__m256i d02 = _mm256_lddqu_si256((__m256i const *) ptr + 1);;
- __m256i d03 = _mm256_lddqu_si256((__m256i const *) ptr + 2);;
+ __m256i d03 = _mm256_or_si256(_mm256_maskload_epi64((long long const *) ((__m256i const *) ptr + 2), mask), _mm256_andnot_si256(mask, _mm256_set1_epi64x(MAX)));
sort_03v_ascending(d01, d02, d03);
_mm256_storeu_si256((__m256i *) ptr + 0, d01);
_mm256_storeu_si256((__m256i *) ptr + 1, d02);
- _mm256_storeu_si256((__m256i *) ptr + 2, d03);
-}
+ _mm256_maskstore_epi64((long long *) ((__m256i *) ptr + 2), mask, d03);
+ }
+
+ static NOINLINE void sort_04v_alt(int64_t *ptr, int remainder) {
+ const auto mask = _mm256_cvtepi8_epi64(_mm_loadu_si128((__m128i*)(mask_table_4 + remainder * N)));
- static NOINLINE void sort_04v(int64_t *ptr) {
__m256i d01 = _mm256_lddqu_si256((__m256i const *) ptr + 0);;
__m256i d02 = _mm256_lddqu_si256((__m256i const *) ptr + 1);;
__m256i d03 = _mm256_lddqu_si256((__m256i const *) ptr + 2);;
- __m256i d04 = _mm256_lddqu_si256((__m256i const *) ptr + 3);;
+ __m256i d04 = _mm256_or_si256(_mm256_maskload_epi64((long long const *) ((__m256i const *) ptr + 3), mask), _mm256_andnot_si256(mask, _mm256_set1_epi64x(MAX)));
sort_04v_ascending(d01, d02, d03, d04);
_mm256_storeu_si256((__m256i *) ptr + 0, d01);
_mm256_storeu_si256((__m256i *) ptr + 1, d02);
_mm256_storeu_si256((__m256i *) ptr + 2, d03);
- _mm256_storeu_si256((__m256i *) ptr + 3, d04);
-}
+ _mm256_maskstore_epi64((long long *) ((__m256i *) ptr + 3), mask, d04);
+ }
+
+ static NOINLINE void sort_05v_alt(int64_t *ptr, int remainder) {
+ const auto mask = _mm256_cvtepi8_epi64(_mm_loadu_si128((__m128i*)(mask_table_4 + remainder * N)));
- static NOINLINE void sort_05v(int64_t *ptr) {
__m256i d01 = _mm256_lddqu_si256((__m256i const *) ptr + 0);;
__m256i d02 = _mm256_lddqu_si256((__m256i const *) ptr + 1);;
__m256i d03 = _mm256_lddqu_si256((__m256i const *) ptr + 2);;
__m256i d04 = _mm256_lddqu_si256((__m256i const *) ptr + 3);;
- __m256i d05 = _mm256_lddqu_si256((__m256i const *) ptr + 4);;
+ __m256i d05 = _mm256_or_si256(_mm256_maskload_epi64((long long const *) ((__m256i const *) ptr + 4), mask), _mm256_andnot_si256(mask, _mm256_set1_epi64x(MAX)));
sort_05v_ascending(d01, d02, d03, d04, d05);
_mm256_storeu_si256((__m256i *) ptr + 0, d01);
_mm256_storeu_si256((__m256i *) ptr + 1, d02);
_mm256_storeu_si256((__m256i *) ptr + 2, d03);
_mm256_storeu_si256((__m256i *) ptr + 3, d04);
- _mm256_storeu_si256((__m256i *) ptr + 4, d05);
-}
+ _mm256_maskstore_epi64((long long *) ((__m256i *) ptr + 4), mask, d05);
+ }
+
+ static NOINLINE void sort_06v_alt(int64_t *ptr, int remainder) {
+ const auto mask = _mm256_cvtepi8_epi64(_mm_loadu_si128((__m128i*)(mask_table_4 + remainder * N)));
- static NOINLINE void sort_06v(int64_t *ptr) {
__m256i d01 = _mm256_lddqu_si256((__m256i const *) ptr + 0);;
__m256i d02 = _mm256_lddqu_si256((__m256i const *) ptr + 1);;
__m256i d03 = _mm256_lddqu_si256((__m256i const *) ptr + 2);;
__m256i d04 = _mm256_lddqu_si256((__m256i const *) ptr + 3);;
__m256i d05 = _mm256_lddqu_si256((__m256i const *) ptr + 4);;
- __m256i d06 = _mm256_lddqu_si256((__m256i const *) ptr + 5);;
+ __m256i d06 = _mm256_or_si256(_mm256_maskload_epi64((long long const *) ((__m256i const *) ptr + 5), mask), _mm256_andnot_si256(mask, _mm256_set1_epi64x(MAX)));
sort_06v_ascending(d01, d02, d03, d04, d05, d06);
_mm256_storeu_si256((__m256i *) ptr + 0, d01);
_mm256_storeu_si256((__m256i *) ptr + 1, d02);
_mm256_storeu_si256((__m256i *) ptr + 2, d03);
_mm256_storeu_si256((__m256i *) ptr + 3, d04);
_mm256_storeu_si256((__m256i *) ptr + 4, d05);
- _mm256_storeu_si256((__m256i *) ptr + 5, d06);
-}
+ _mm256_maskstore_epi64((long long *) ((__m256i *) ptr + 5), mask, d06);
+ }
+
+ static NOINLINE void sort_07v_alt(int64_t *ptr, int remainder) {
+ const auto mask = _mm256_cvtepi8_epi64(_mm_loadu_si128((__m128i*)(mask_table_4 + remainder * N)));
- static NOINLINE void sort_07v(int64_t *ptr) {
__m256i d01 = _mm256_lddqu_si256((__m256i const *) ptr + 0);;
__m256i d02 = _mm256_lddqu_si256((__m256i const *) ptr + 1);;
__m256i d03 = _mm256_lddqu_si256((__m256i const *) ptr + 2);;
__m256i d04 = _mm256_lddqu_si256((__m256i const *) ptr + 3);;
__m256i d05 = _mm256_lddqu_si256((__m256i const *) ptr + 4);;
__m256i d06 = _mm256_lddqu_si256((__m256i const *) ptr + 5);;
- __m256i d07 = _mm256_lddqu_si256((__m256i const *) ptr + 6);;
+ __m256i d07 = _mm256_or_si256(_mm256_maskload_epi64((long long const *) ((__m256i const *) ptr + 6), mask), _mm256_andnot_si256(mask, _mm256_set1_epi64x(MAX)));
sort_07v_ascending(d01, d02, d03, d04, d05, d06, d07);
_mm256_storeu_si256((__m256i *) ptr + 0, d01);
_mm256_storeu_si256((__m256i *) ptr + 1, d02);
_mm256_storeu_si256((__m256i *) ptr + 3, d04);
_mm256_storeu_si256((__m256i *) ptr + 4, d05);
_mm256_storeu_si256((__m256i *) ptr + 5, d06);
- _mm256_storeu_si256((__m256i *) ptr + 6, d07);
-}
+ _mm256_maskstore_epi64((long long *) ((__m256i *) ptr + 6), mask, d07);
+ }
+
+ static NOINLINE void sort_08v_alt(int64_t *ptr, int remainder) {
+ const auto mask = _mm256_cvtepi8_epi64(_mm_loadu_si128((__m128i*)(mask_table_4 + remainder * N)));
- static NOINLINE void sort_08v(int64_t *ptr) {
__m256i d01 = _mm256_lddqu_si256((__m256i const *) ptr + 0);;
__m256i d02 = _mm256_lddqu_si256((__m256i const *) ptr + 1);;
__m256i d03 = _mm256_lddqu_si256((__m256i const *) ptr + 2);;
__m256i d05 = _mm256_lddqu_si256((__m256i const *) ptr + 4);;
__m256i d06 = _mm256_lddqu_si256((__m256i const *) ptr + 5);;
__m256i d07 = _mm256_lddqu_si256((__m256i const *) ptr + 6);;
- __m256i d08 = _mm256_lddqu_si256((__m256i const *) ptr + 7);;
+ __m256i d08 = _mm256_or_si256(_mm256_maskload_epi64((long long const *) ((__m256i const *) ptr + 7), mask), _mm256_andnot_si256(mask, _mm256_set1_epi64x(MAX)));
sort_08v_ascending(d01, d02, d03, d04, d05, d06, d07, d08);
_mm256_storeu_si256((__m256i *) ptr + 0, d01);
_mm256_storeu_si256((__m256i *) ptr + 1, d02);
_mm256_storeu_si256((__m256i *) ptr + 4, d05);
_mm256_storeu_si256((__m256i *) ptr + 5, d06);
_mm256_storeu_si256((__m256i *) ptr + 6, d07);
- _mm256_storeu_si256((__m256i *) ptr + 7, d08);
-}
+ _mm256_maskstore_epi64((long long *) ((__m256i *) ptr + 7), mask, d08);
+ }
+
+ static NOINLINE void sort_09v_alt(int64_t *ptr, int remainder) {
+ const auto mask = _mm256_cvtepi8_epi64(_mm_loadu_si128((__m128i*)(mask_table_4 + remainder * N)));
- static NOINLINE void sort_09v(int64_t *ptr) {
__m256i d01 = _mm256_lddqu_si256((__m256i const *) ptr + 0);;
__m256i d02 = _mm256_lddqu_si256((__m256i const *) ptr + 1);;
__m256i d03 = _mm256_lddqu_si256((__m256i const *) ptr + 2);;
__m256i d06 = _mm256_lddqu_si256((__m256i const *) ptr + 5);;
__m256i d07 = _mm256_lddqu_si256((__m256i const *) ptr + 6);;
__m256i d08 = _mm256_lddqu_si256((__m256i const *) ptr + 7);;
- __m256i d09 = _mm256_lddqu_si256((__m256i const *) ptr + 8);;
+ __m256i d09 = _mm256_or_si256(_mm256_maskload_epi64((long long const *) ((__m256i const *) ptr + 8), mask), _mm256_andnot_si256(mask, _mm256_set1_epi64x(MAX)));
sort_09v_ascending(d01, d02, d03, d04, d05, d06, d07, d08, d09);
_mm256_storeu_si256((__m256i *) ptr + 0, d01);
_mm256_storeu_si256((__m256i *) ptr + 1, d02);
_mm256_storeu_si256((__m256i *) ptr + 5, d06);
_mm256_storeu_si256((__m256i *) ptr + 6, d07);
_mm256_storeu_si256((__m256i *) ptr + 7, d08);
- _mm256_storeu_si256((__m256i *) ptr + 8, d09);
-}
+ _mm256_maskstore_epi64((long long *) ((__m256i *) ptr + 8), mask, d09);
+ }
+
+ static NOINLINE void sort_10v_alt(int64_t *ptr, int remainder) {
+ const auto mask = _mm256_cvtepi8_epi64(_mm_loadu_si128((__m128i*)(mask_table_4 + remainder * N)));
- static NOINLINE void sort_10v(int64_t *ptr) {
__m256i d01 = _mm256_lddqu_si256((__m256i const *) ptr + 0);;
__m256i d02 = _mm256_lddqu_si256((__m256i const *) ptr + 1);;
__m256i d03 = _mm256_lddqu_si256((__m256i const *) ptr + 2);;
__m256i d07 = _mm256_lddqu_si256((__m256i const *) ptr + 6);;
__m256i d08 = _mm256_lddqu_si256((__m256i const *) ptr + 7);;
__m256i d09 = _mm256_lddqu_si256((__m256i const *) ptr + 8);;
- __m256i d10 = _mm256_lddqu_si256((__m256i const *) ptr + 9);;
+ __m256i d10 = _mm256_or_si256(_mm256_maskload_epi64((long long const *) ((__m256i const *) ptr + 9), mask), _mm256_andnot_si256(mask, _mm256_set1_epi64x(MAX)));
sort_10v_ascending(d01, d02, d03, d04, d05, d06, d07, d08, d09, d10);
_mm256_storeu_si256((__m256i *) ptr + 0, d01);
_mm256_storeu_si256((__m256i *) ptr + 1, d02);
_mm256_storeu_si256((__m256i *) ptr + 6, d07);
_mm256_storeu_si256((__m256i *) ptr + 7, d08);
_mm256_storeu_si256((__m256i *) ptr + 8, d09);
- _mm256_storeu_si256((__m256i *) ptr + 9, d10);
-}
+ _mm256_maskstore_epi64((long long *) ((__m256i *) ptr + 9), mask, d10);
+ }
+
+ static NOINLINE void sort_11v_alt(int64_t *ptr, int remainder) {
+ const auto mask = _mm256_cvtepi8_epi64(_mm_loadu_si128((__m128i*)(mask_table_4 + remainder * N)));
- static NOINLINE void sort_11v(int64_t *ptr) {
__m256i d01 = _mm256_lddqu_si256((__m256i const *) ptr + 0);;
__m256i d02 = _mm256_lddqu_si256((__m256i const *) ptr + 1);;
__m256i d03 = _mm256_lddqu_si256((__m256i const *) ptr + 2);;
__m256i d08 = _mm256_lddqu_si256((__m256i const *) ptr + 7);;
__m256i d09 = _mm256_lddqu_si256((__m256i const *) ptr + 8);;
__m256i d10 = _mm256_lddqu_si256((__m256i const *) ptr + 9);;
- __m256i d11 = _mm256_lddqu_si256((__m256i const *) ptr + 10);;
+ __m256i d11 = _mm256_or_si256(_mm256_maskload_epi64((long long const *) ((__m256i const *) ptr + 10), mask), _mm256_andnot_si256(mask, _mm256_set1_epi64x(MAX)));
sort_11v_ascending(d01, d02, d03, d04, d05, d06, d07, d08, d09, d10, d11);
_mm256_storeu_si256((__m256i *) ptr + 0, d01);
_mm256_storeu_si256((__m256i *) ptr + 1, d02);
_mm256_storeu_si256((__m256i *) ptr + 7, d08);
_mm256_storeu_si256((__m256i *) ptr + 8, d09);
_mm256_storeu_si256((__m256i *) ptr + 9, d10);
- _mm256_storeu_si256((__m256i *) ptr + 10, d11);
-}
+ _mm256_maskstore_epi64((long long *) ((__m256i *) ptr + 10), mask, d11);
+ }
+
+ static NOINLINE void sort_12v_alt(int64_t *ptr, int remainder) {
+ const auto mask = _mm256_cvtepi8_epi64(_mm_loadu_si128((__m128i*)(mask_table_4 + remainder * N)));
- static NOINLINE void sort_12v(int64_t *ptr) {
__m256i d01 = _mm256_lddqu_si256((__m256i const *) ptr + 0);;
__m256i d02 = _mm256_lddqu_si256((__m256i const *) ptr + 1);;
__m256i d03 = _mm256_lddqu_si256((__m256i const *) ptr + 2);;
__m256i d09 = _mm256_lddqu_si256((__m256i const *) ptr + 8);;
__m256i d10 = _mm256_lddqu_si256((__m256i const *) ptr + 9);;
__m256i d11 = _mm256_lddqu_si256((__m256i const *) ptr + 10);;
- __m256i d12 = _mm256_lddqu_si256((__m256i const *) ptr + 11);;
+ __m256i d12 = _mm256_or_si256(_mm256_maskload_epi64((long long const *) ((__m256i const *) ptr + 11), mask), _mm256_andnot_si256(mask, _mm256_set1_epi64x(MAX)));
sort_12v_ascending(d01, d02, d03, d04, d05, d06, d07, d08, d09, d10, d11, d12);
_mm256_storeu_si256((__m256i *) ptr + 0, d01);
_mm256_storeu_si256((__m256i *) ptr + 1, d02);
_mm256_storeu_si256((__m256i *) ptr + 8, d09);
_mm256_storeu_si256((__m256i *) ptr + 9, d10);
_mm256_storeu_si256((__m256i *) ptr + 10, d11);
- _mm256_storeu_si256((__m256i *) ptr + 11, d12);
-}
+ _mm256_maskstore_epi64((long long *) ((__m256i *) ptr + 11), mask, d12);
+ }
+
+ static NOINLINE void sort_13v_alt(int64_t *ptr, int remainder) {
+ const auto mask = _mm256_cvtepi8_epi64(_mm_loadu_si128((__m128i*)(mask_table_4 + remainder * N)));
- static NOINLINE void sort_13v(int64_t *ptr) {
__m256i d01 = _mm256_lddqu_si256((__m256i const *) ptr + 0);;
__m256i d02 = _mm256_lddqu_si256((__m256i const *) ptr + 1);;
__m256i d03 = _mm256_lddqu_si256((__m256i const *) ptr + 2);;
__m256i d10 = _mm256_lddqu_si256((__m256i const *) ptr + 9);;
__m256i d11 = _mm256_lddqu_si256((__m256i const *) ptr + 10);;
__m256i d12 = _mm256_lddqu_si256((__m256i const *) ptr + 11);;
- __m256i d13 = _mm256_lddqu_si256((__m256i const *) ptr + 12);;
+ __m256i d13 = _mm256_or_si256(_mm256_maskload_epi64((long long const *) ((__m256i const *) ptr + 12), mask), _mm256_andnot_si256(mask, _mm256_set1_epi64x(MAX)));
sort_13v_ascending(d01, d02, d03, d04, d05, d06, d07, d08, d09, d10, d11, d12, d13);
_mm256_storeu_si256((__m256i *) ptr + 0, d01);
_mm256_storeu_si256((__m256i *) ptr + 1, d02);
_mm256_storeu_si256((__m256i *) ptr + 9, d10);
_mm256_storeu_si256((__m256i *) ptr + 10, d11);
_mm256_storeu_si256((__m256i *) ptr + 11, d12);
- _mm256_storeu_si256((__m256i *) ptr + 12, d13);
-}
+ _mm256_maskstore_epi64((long long *) ((__m256i *) ptr + 12), mask, d13);
+ }
+
+ static NOINLINE void sort_14v_alt(int64_t *ptr, int remainder) {
+ const auto mask = _mm256_cvtepi8_epi64(_mm_loadu_si128((__m128i*)(mask_table_4 + remainder * N)));
- static NOINLINE void sort_14v(int64_t *ptr) {
__m256i d01 = _mm256_lddqu_si256((__m256i const *) ptr + 0);;
__m256i d02 = _mm256_lddqu_si256((__m256i const *) ptr + 1);;
__m256i d03 = _mm256_lddqu_si256((__m256i const *) ptr + 2);;
__m256i d11 = _mm256_lddqu_si256((__m256i const *) ptr + 10);;
__m256i d12 = _mm256_lddqu_si256((__m256i const *) ptr + 11);;
__m256i d13 = _mm256_lddqu_si256((__m256i const *) ptr + 12);;
- __m256i d14 = _mm256_lddqu_si256((__m256i const *) ptr + 13);;
+ __m256i d14 = _mm256_or_si256(_mm256_maskload_epi64((long long const *) ((__m256i const *) ptr + 13), mask), _mm256_andnot_si256(mask, _mm256_set1_epi64x(MAX)));
sort_14v_ascending(d01, d02, d03, d04, d05, d06, d07, d08, d09, d10, d11, d12, d13, d14);
_mm256_storeu_si256((__m256i *) ptr + 0, d01);
_mm256_storeu_si256((__m256i *) ptr + 1, d02);
_mm256_storeu_si256((__m256i *) ptr + 10, d11);
_mm256_storeu_si256((__m256i *) ptr + 11, d12);
_mm256_storeu_si256((__m256i *) ptr + 12, d13);
- _mm256_storeu_si256((__m256i *) ptr + 13, d14);
-}
+ _mm256_maskstore_epi64((long long *) ((__m256i *) ptr + 13), mask, d14);
+ }
+
+ static NOINLINE void sort_15v_alt(int64_t *ptr, int remainder) {
+ const auto mask = _mm256_cvtepi8_epi64(_mm_loadu_si128((__m128i*)(mask_table_4 + remainder * N)));
- static NOINLINE void sort_15v(int64_t *ptr) {
__m256i d01 = _mm256_lddqu_si256((__m256i const *) ptr + 0);;
__m256i d02 = _mm256_lddqu_si256((__m256i const *) ptr + 1);;
__m256i d03 = _mm256_lddqu_si256((__m256i const *) ptr + 2);;
__m256i d12 = _mm256_lddqu_si256((__m256i const *) ptr + 11);;
__m256i d13 = _mm256_lddqu_si256((__m256i const *) ptr + 12);;
__m256i d14 = _mm256_lddqu_si256((__m256i const *) ptr + 13);;
- __m256i d15 = _mm256_lddqu_si256((__m256i const *) ptr + 14);;
+ __m256i d15 = _mm256_or_si256(_mm256_maskload_epi64((long long const *) ((__m256i const *) ptr + 14), mask), _mm256_andnot_si256(mask, _mm256_set1_epi64x(MAX)));
sort_15v_ascending(d01, d02, d03, d04, d05, d06, d07, d08, d09, d10, d11, d12, d13, d14, d15);
_mm256_storeu_si256((__m256i *) ptr + 0, d01);
_mm256_storeu_si256((__m256i *) ptr + 1, d02);
_mm256_storeu_si256((__m256i *) ptr + 11, d12);
_mm256_storeu_si256((__m256i *) ptr + 12, d13);
_mm256_storeu_si256((__m256i *) ptr + 13, d14);
- _mm256_storeu_si256((__m256i *) ptr + 14, d15);
-}
+ _mm256_maskstore_epi64((long long *) ((__m256i *) ptr + 14), mask, d15);
+ }
+
+ static NOINLINE void sort_16v_alt(int64_t *ptr, int remainder) {
+ const auto mask = _mm256_cvtepi8_epi64(_mm_loadu_si128((__m128i*)(mask_table_4 + remainder * N)));
- static NOINLINE void sort_16v(int64_t *ptr) {
__m256i d01 = _mm256_lddqu_si256((__m256i const *) ptr + 0);;
__m256i d02 = _mm256_lddqu_si256((__m256i const *) ptr + 1);;
__m256i d03 = _mm256_lddqu_si256((__m256i const *) ptr + 2);;
__m256i d13 = _mm256_lddqu_si256((__m256i const *) ptr + 12);;
__m256i d14 = _mm256_lddqu_si256((__m256i const *) ptr + 13);;
__m256i d15 = _mm256_lddqu_si256((__m256i const *) ptr + 14);;
- __m256i d16 = _mm256_lddqu_si256((__m256i const *) ptr + 15);;
+ __m256i d16 = _mm256_or_si256(_mm256_maskload_epi64((long long const *) ((__m256i const *) ptr + 15), mask), _mm256_andnot_si256(mask, _mm256_set1_epi64x(MAX)));
sort_16v_ascending(d01, d02, d03, d04, d05, d06, d07, d08, d09, d10, d11, d12, d13, d14, d15, d16);
_mm256_storeu_si256((__m256i *) ptr + 0, d01);
_mm256_storeu_si256((__m256i *) ptr + 1, d02);
_mm256_storeu_si256((__m256i *) ptr + 12, d13);
_mm256_storeu_si256((__m256i *) ptr + 13, d14);
_mm256_storeu_si256((__m256i *) ptr + 14, d15);
- _mm256_storeu_si256((__m256i *) ptr + 15, d16);
-}
+ _mm256_maskstore_epi64((long long *) ((__m256i *) ptr + 15), mask, d16);
+ }
static void sort(int64_t *ptr, size_t length);
};
using namespace vxsort;
void vxsort::smallsort::bitonic<int32_t, vector_machine::AVX512 >::sort(int32_t *ptr, size_t length) {
- const int N = 16;
-
- switch(length / N) {
- case 1: sort_01v(ptr); break;
- case 2: sort_02v(ptr); break;
- case 3: sort_03v(ptr); break;
- case 4: sort_04v(ptr); break;
- case 5: sort_05v(ptr); break;
- case 6: sort_06v(ptr); break;
- case 7: sort_07v(ptr); break;
- case 8: sort_08v(ptr); break;
- case 9: sort_09v(ptr); break;
- case 10: sort_10v(ptr); break;
- case 11: sort_11v(ptr); break;
- case 12: sort_12v(ptr); break;
- case 13: sort_13v(ptr); break;
- case 14: sort_14v(ptr); break;
- case 15: sort_15v(ptr); break;
- case 16: sort_16v(ptr); break;
+ const auto fullvlength = length / N;
+ const int remainder = (int) (length - fullvlength * N);
+ const auto v = fullvlength + ((remainder > 0) ? 1 : 0);
+ switch(v) {
+ case 1: sort_01v_alt(ptr, remainder); break;
+ case 2: sort_02v_alt(ptr, remainder); break;
+ case 3: sort_03v_alt(ptr, remainder); break;
+ case 4: sort_04v_alt(ptr, remainder); break;
+ case 5: sort_05v_alt(ptr, remainder); break;
+ case 6: sort_06v_alt(ptr, remainder); break;
+ case 7: sort_07v_alt(ptr, remainder); break;
+ case 8: sort_08v_alt(ptr, remainder); break;
+ case 9: sort_09v_alt(ptr, remainder); break;
+ case 10: sort_10v_alt(ptr, remainder); break;
+ case 11: sort_11v_alt(ptr, remainder); break;
+ case 12: sort_12v_alt(ptr, remainder); break;
+ case 13: sort_13v_alt(ptr, remainder); break;
+ case 14: sort_14v_alt(ptr, remainder); break;
+ case 15: sort_15v_alt(ptr, remainder); break;
+ case 16: sort_16v_alt(ptr, remainder); break;
}
}
/////////////////////////////////////////////////////////////////////////////
////
-// This file was auto-generated by a tool at 2020-06-22 05:27:48
+// This file was auto-generated by a tool at 2020-07-21 14:05:39
//
// It is recommended you DO NOT directly edit this file but instead edit
// the code-generator that generated this source file instead.
namespace vxsort {
namespace smallsort {
template<> struct bitonic<int32_t, AVX512> {
+ static const int N = 16;
+ static constexpr int32_t MAX = std::numeric_limits<int32_t>::Max();
public:
static INLINE void sort_01v_ascending(__m512i& d01) {
sort_02v_merge_descending(d01, d02);
sort_01v_merge_descending(d03);
}
- static INLINE void sort_04v_ascending(__m512i& d01, __m512i& d02, __m512i& d03, __m512i& d04) {
+ static NOINLINE void sort_04v_ascending(__m512i& d01, __m512i& d02, __m512i& d03, __m512i& d04) {
__m512i tmp;
sort_02v_ascending(d01, d02);
sort_02v_merge_ascending(d01, d02);
sort_02v_merge_ascending(d03, d04);
}
- static INLINE void sort_04v_descending(__m512i& d01, __m512i& d02, __m512i& d03, __m512i& d04) {
+ static NOINLINE void sort_04v_descending(__m512i& d01, __m512i& d02, __m512i& d03, __m512i& d04) {
__m512i tmp;
sort_02v_descending(d01, d02);
sort_02v_merge_descending(d01, d02);
sort_02v_merge_descending(d03, d04);
}
- static INLINE void sort_04v_merge_ascending(__m512i& d01, __m512i& d02, __m512i& d03, __m512i& d04) {
+ static NOINLINE void sort_04v_merge_ascending(__m512i& d01, __m512i& d02, __m512i& d03, __m512i& d04) {
__m512i tmp;
tmp = d01;
sort_02v_merge_ascending(d01, d02);
sort_02v_merge_ascending(d03, d04);
}
- static INLINE void sort_04v_merge_descending(__m512i& d01, __m512i& d02, __m512i& d03, __m512i& d04) {
+ static NOINLINE void sort_04v_merge_descending(__m512i& d01, __m512i& d02, __m512i& d03, __m512i& d04) {
__m512i tmp;
tmp = d01;
sort_04v_merge_descending(d01, d02, d03, d04);
sort_03v_merge_descending(d05, d06, d07);
}
- static INLINE void sort_08v_ascending(__m512i& d01, __m512i& d02, __m512i& d03, __m512i& d04, __m512i& d05, __m512i& d06, __m512i& d07, __m512i& d08) {
+ static NOINLINE void sort_08v_ascending(__m512i& d01, __m512i& d02, __m512i& d03, __m512i& d04, __m512i& d05, __m512i& d06, __m512i& d07, __m512i& d08) {
__m512i tmp;
sort_04v_ascending(d01, d02, d03, d04);
sort_04v_merge_ascending(d01, d02, d03, d04);
sort_04v_merge_ascending(d05, d06, d07, d08);
}
- static INLINE void sort_08v_descending(__m512i& d01, __m512i& d02, __m512i& d03, __m512i& d04, __m512i& d05, __m512i& d06, __m512i& d07, __m512i& d08) {
+ static NOINLINE void sort_08v_descending(__m512i& d01, __m512i& d02, __m512i& d03, __m512i& d04, __m512i& d05, __m512i& d06, __m512i& d07, __m512i& d08) {
__m512i tmp;
sort_04v_descending(d01, d02, d03, d04);
sort_04v_merge_descending(d01, d02, d03, d04);
sort_04v_merge_descending(d05, d06, d07, d08);
}
- static INLINE void sort_08v_merge_ascending(__m512i& d01, __m512i& d02, __m512i& d03, __m512i& d04, __m512i& d05, __m512i& d06, __m512i& d07, __m512i& d08) {
+ static NOINLINE void sort_08v_merge_ascending(__m512i& d01, __m512i& d02, __m512i& d03, __m512i& d04, __m512i& d05, __m512i& d06, __m512i& d07, __m512i& d08) {
__m512i tmp;
tmp = d01;
sort_04v_merge_ascending(d01, d02, d03, d04);
sort_04v_merge_ascending(d05, d06, d07, d08);
}
- static INLINE void sort_08v_merge_descending(__m512i& d01, __m512i& d02, __m512i& d03, __m512i& d04, __m512i& d05, __m512i& d06, __m512i& d07, __m512i& d08) {
+ static NOINLINE void sort_08v_merge_descending(__m512i& d01, __m512i& d02, __m512i& d03, __m512i& d04, __m512i& d05, __m512i& d06, __m512i& d07, __m512i& d08) {
__m512i tmp;
tmp = d01;
sort_08v_merge_descending(d01, d02, d03, d04, d05, d06, d07, d08);
sort_03v_merge_descending(d09, d10, d11);
}
- static INLINE void sort_12v_ascending(__m512i& d01, __m512i& d02, __m512i& d03, __m512i& d04, __m512i& d05, __m512i& d06, __m512i& d07, __m512i& d08, __m512i& d09, __m512i& d10, __m512i& d11, __m512i& d12) {
+ static NOINLINE void sort_12v_ascending(__m512i& d01, __m512i& d02, __m512i& d03, __m512i& d04, __m512i& d05, __m512i& d06, __m512i& d07, __m512i& d08, __m512i& d09, __m512i& d10, __m512i& d11, __m512i& d12) {
__m512i tmp;
sort_08v_ascending(d01, d02, d03, d04, d05, d06, d07, d08);
sort_08v_merge_ascending(d01, d02, d03, d04, d05, d06, d07, d08);
sort_04v_merge_ascending(d09, d10, d11, d12);
}
- static INLINE void sort_12v_descending(__m512i& d01, __m512i& d02, __m512i& d03, __m512i& d04, __m512i& d05, __m512i& d06, __m512i& d07, __m512i& d08, __m512i& d09, __m512i& d10, __m512i& d11, __m512i& d12) {
+ static NOINLINE void sort_12v_descending(__m512i& d01, __m512i& d02, __m512i& d03, __m512i& d04, __m512i& d05, __m512i& d06, __m512i& d07, __m512i& d08, __m512i& d09, __m512i& d10, __m512i& d11, __m512i& d12) {
__m512i tmp;
sort_08v_descending(d01, d02, d03, d04, d05, d06, d07, d08);
sort_08v_merge_descending(d01, d02, d03, d04, d05, d06, d07, d08);
sort_07v_merge_descending(d09, d10, d11, d12, d13, d14, d15);
}
- static INLINE void sort_16v_ascending(__m512i& d01, __m512i& d02, __m512i& d03, __m512i& d04, __m512i& d05, __m512i& d06, __m512i& d07, __m512i& d08, __m512i& d09, __m512i& d10, __m512i& d11, __m512i& d12, __m512i& d13, __m512i& d14, __m512i& d15, __m512i& d16) {
+ static NOINLINE void sort_16v_ascending(__m512i& d01, __m512i& d02, __m512i& d03, __m512i& d04, __m512i& d05, __m512i& d06, __m512i& d07, __m512i& d08, __m512i& d09, __m512i& d10, __m512i& d11, __m512i& d12, __m512i& d13, __m512i& d14, __m512i& d15, __m512i& d16) {
__m512i tmp;
sort_08v_ascending(d01, d02, d03, d04, d05, d06, d07, d08);
sort_08v_merge_ascending(d01, d02, d03, d04, d05, d06, d07, d08);
sort_08v_merge_ascending(d09, d10, d11, d12, d13, d14, d15, d16);
}
- static INLINE void sort_16v_descending(__m512i& d01, __m512i& d02, __m512i& d03, __m512i& d04, __m512i& d05, __m512i& d06, __m512i& d07, __m512i& d08, __m512i& d09, __m512i& d10, __m512i& d11, __m512i& d12, __m512i& d13, __m512i& d14, __m512i& d15, __m512i& d16) {
+ static NOINLINE void sort_16v_descending(__m512i& d01, __m512i& d02, __m512i& d03, __m512i& d04, __m512i& d05, __m512i& d06, __m512i& d07, __m512i& d08, __m512i& d09, __m512i& d10, __m512i& d11, __m512i& d12, __m512i& d13, __m512i& d14, __m512i& d15, __m512i& d16) {
__m512i tmp;
sort_08v_descending(d01, d02, d03, d04, d05, d06, d07, d08);
sort_08v_merge_descending(d09, d10, d11, d12, d13, d14, d15, d16);
}
- static NOINLINE void sort_01v(int32_t *ptr) {
- __m512i d01 = _mm512_loadu_si512((__m512i const *) ptr + 0);;
+ static NOINLINE void sort_01v_alt(int32_t *ptr, int remainder) {
+ const auto mask = 0xFFFF >> ((N - remainder) & (N-1));
+
+ __m512i d01 = _mm512_mask_loadu_epi32(_mm512_set1_epi32(MAX),
+ mask,
+ (int32_t const *) ((__m512i const *) ptr + 0));
sort_01v_ascending(d01);
- _mm512_storeu_si512((__m512i *) ptr + 0, d01);
-}
+ _mm512_mask_storeu_epi32((__m512i *) ptr + 0, mask, d01);
+ }
+
+ static NOINLINE void sort_02v_alt(int32_t *ptr, int remainder) {
+ const auto mask = 0xFFFF >> ((N - remainder) & (N-1));
- static NOINLINE void sort_02v(int32_t *ptr) {
__m512i d01 = _mm512_loadu_si512((__m512i const *) ptr + 0);;
- __m512i d02 = _mm512_loadu_si512((__m512i const *) ptr + 1);;
+ __m512i d02 = _mm512_mask_loadu_epi32(_mm512_set1_epi32(MAX),
+ mask,
+ (int32_t const *) ((__m512i const *) ptr + 1));
sort_02v_ascending(d01, d02);
_mm512_storeu_si512((__m512i *) ptr + 0, d01);
- _mm512_storeu_si512((__m512i *) ptr + 1, d02);
-}
+ _mm512_mask_storeu_epi32((__m512i *) ptr + 1, mask, d02);
+ }
+
+ static NOINLINE void sort_03v_alt(int32_t *ptr, int remainder) {
+ const auto mask = 0xFFFF >> ((N - remainder) & (N-1));
- static NOINLINE void sort_03v(int32_t *ptr) {
__m512i d01 = _mm512_loadu_si512((__m512i const *) ptr + 0);;
__m512i d02 = _mm512_loadu_si512((__m512i const *) ptr + 1);;
- __m512i d03 = _mm512_loadu_si512((__m512i const *) ptr + 2);;
+ __m512i d03 = _mm512_mask_loadu_epi32(_mm512_set1_epi32(MAX),
+ mask,
+ (int32_t const *) ((__m512i const *) ptr + 2));
sort_03v_ascending(d01, d02, d03);
_mm512_storeu_si512((__m512i *) ptr + 0, d01);
_mm512_storeu_si512((__m512i *) ptr + 1, d02);
- _mm512_storeu_si512((__m512i *) ptr + 2, d03);
-}
+ _mm512_mask_storeu_epi32((__m512i *) ptr + 2, mask, d03);
+ }
+
+ static NOINLINE void sort_04v_alt(int32_t *ptr, int remainder) {
+ const auto mask = 0xFFFF >> ((N - remainder) & (N-1));
- static NOINLINE void sort_04v(int32_t *ptr) {
__m512i d01 = _mm512_loadu_si512((__m512i const *) ptr + 0);;
__m512i d02 = _mm512_loadu_si512((__m512i const *) ptr + 1);;
__m512i d03 = _mm512_loadu_si512((__m512i const *) ptr + 2);;
- __m512i d04 = _mm512_loadu_si512((__m512i const *) ptr + 3);;
+ __m512i d04 = _mm512_mask_loadu_epi32(_mm512_set1_epi32(MAX),
+ mask,
+ (int32_t const *) ((__m512i const *) ptr + 3));
sort_04v_ascending(d01, d02, d03, d04);
_mm512_storeu_si512((__m512i *) ptr + 0, d01);
_mm512_storeu_si512((__m512i *) ptr + 1, d02);
_mm512_storeu_si512((__m512i *) ptr + 2, d03);
- _mm512_storeu_si512((__m512i *) ptr + 3, d04);
-}
+ _mm512_mask_storeu_epi32((__m512i *) ptr + 3, mask, d04);
+ }
+
+ static NOINLINE void sort_05v_alt(int32_t *ptr, int remainder) {
+ const auto mask = 0xFFFF >> ((N - remainder) & (N-1));
- static NOINLINE void sort_05v(int32_t *ptr) {
__m512i d01 = _mm512_loadu_si512((__m512i const *) ptr + 0);;
__m512i d02 = _mm512_loadu_si512((__m512i const *) ptr + 1);;
__m512i d03 = _mm512_loadu_si512((__m512i const *) ptr + 2);;
__m512i d04 = _mm512_loadu_si512((__m512i const *) ptr + 3);;
- __m512i d05 = _mm512_loadu_si512((__m512i const *) ptr + 4);;
+ __m512i d05 = _mm512_mask_loadu_epi32(_mm512_set1_epi32(MAX),
+ mask,
+ (int32_t const *) ((__m512i const *) ptr + 4));
sort_05v_ascending(d01, d02, d03, d04, d05);
_mm512_storeu_si512((__m512i *) ptr + 0, d01);
_mm512_storeu_si512((__m512i *) ptr + 1, d02);
_mm512_storeu_si512((__m512i *) ptr + 2, d03);
_mm512_storeu_si512((__m512i *) ptr + 3, d04);
- _mm512_storeu_si512((__m512i *) ptr + 4, d05);
-}
+ _mm512_mask_storeu_epi32((__m512i *) ptr + 4, mask, d05);
+ }
+
+ static NOINLINE void sort_06v_alt(int32_t *ptr, int remainder) {
+ const auto mask = 0xFFFF >> ((N - remainder) & (N-1));
- static NOINLINE void sort_06v(int32_t *ptr) {
__m512i d01 = _mm512_loadu_si512((__m512i const *) ptr + 0);;
__m512i d02 = _mm512_loadu_si512((__m512i const *) ptr + 1);;
__m512i d03 = _mm512_loadu_si512((__m512i const *) ptr + 2);;
__m512i d04 = _mm512_loadu_si512((__m512i const *) ptr + 3);;
__m512i d05 = _mm512_loadu_si512((__m512i const *) ptr + 4);;
- __m512i d06 = _mm512_loadu_si512((__m512i const *) ptr + 5);;
+ __m512i d06 = _mm512_mask_loadu_epi32(_mm512_set1_epi32(MAX),
+ mask,
+ (int32_t const *) ((__m512i const *) ptr + 5));
sort_06v_ascending(d01, d02, d03, d04, d05, d06);
_mm512_storeu_si512((__m512i *) ptr + 0, d01);
_mm512_storeu_si512((__m512i *) ptr + 1, d02);
_mm512_storeu_si512((__m512i *) ptr + 2, d03);
_mm512_storeu_si512((__m512i *) ptr + 3, d04);
_mm512_storeu_si512((__m512i *) ptr + 4, d05);
- _mm512_storeu_si512((__m512i *) ptr + 5, d06);
-}
+ _mm512_mask_storeu_epi32((__m512i *) ptr + 5, mask, d06);
+ }
+
+ static NOINLINE void sort_07v_alt(int32_t *ptr, int remainder) {
+ const auto mask = 0xFFFF >> ((N - remainder) & (N-1));
- static NOINLINE void sort_07v(int32_t *ptr) {
__m512i d01 = _mm512_loadu_si512((__m512i const *) ptr + 0);;
__m512i d02 = _mm512_loadu_si512((__m512i const *) ptr + 1);;
__m512i d03 = _mm512_loadu_si512((__m512i const *) ptr + 2);;
__m512i d04 = _mm512_loadu_si512((__m512i const *) ptr + 3);;
__m512i d05 = _mm512_loadu_si512((__m512i const *) ptr + 4);;
__m512i d06 = _mm512_loadu_si512((__m512i const *) ptr + 5);;
- __m512i d07 = _mm512_loadu_si512((__m512i const *) ptr + 6);;
+ __m512i d07 = _mm512_mask_loadu_epi32(_mm512_set1_epi32(MAX),
+ mask,
+ (int32_t const *) ((__m512i const *) ptr + 6));
sort_07v_ascending(d01, d02, d03, d04, d05, d06, d07);
_mm512_storeu_si512((__m512i *) ptr + 0, d01);
_mm512_storeu_si512((__m512i *) ptr + 1, d02);
_mm512_storeu_si512((__m512i *) ptr + 3, d04);
_mm512_storeu_si512((__m512i *) ptr + 4, d05);
_mm512_storeu_si512((__m512i *) ptr + 5, d06);
- _mm512_storeu_si512((__m512i *) ptr + 6, d07);
-}
+ _mm512_mask_storeu_epi32((__m512i *) ptr + 6, mask, d07);
+ }
+
+ static NOINLINE void sort_08v_alt(int32_t *ptr, int remainder) {
+ const auto mask = 0xFFFF >> ((N - remainder) & (N-1));
- static NOINLINE void sort_08v(int32_t *ptr) {
__m512i d01 = _mm512_loadu_si512((__m512i const *) ptr + 0);;
__m512i d02 = _mm512_loadu_si512((__m512i const *) ptr + 1);;
__m512i d03 = _mm512_loadu_si512((__m512i const *) ptr + 2);;
__m512i d05 = _mm512_loadu_si512((__m512i const *) ptr + 4);;
__m512i d06 = _mm512_loadu_si512((__m512i const *) ptr + 5);;
__m512i d07 = _mm512_loadu_si512((__m512i const *) ptr + 6);;
- __m512i d08 = _mm512_loadu_si512((__m512i const *) ptr + 7);;
+ __m512i d08 = _mm512_mask_loadu_epi32(_mm512_set1_epi32(MAX),
+ mask,
+ (int32_t const *) ((__m512i const *) ptr + 7));
sort_08v_ascending(d01, d02, d03, d04, d05, d06, d07, d08);
_mm512_storeu_si512((__m512i *) ptr + 0, d01);
_mm512_storeu_si512((__m512i *) ptr + 1, d02);
_mm512_storeu_si512((__m512i *) ptr + 4, d05);
_mm512_storeu_si512((__m512i *) ptr + 5, d06);
_mm512_storeu_si512((__m512i *) ptr + 6, d07);
- _mm512_storeu_si512((__m512i *) ptr + 7, d08);
-}
+ _mm512_mask_storeu_epi32((__m512i *) ptr + 7, mask, d08);
+ }
+
+ static NOINLINE void sort_09v_alt(int32_t *ptr, int remainder) {
+ const auto mask = 0xFFFF >> ((N - remainder) & (N-1));
- static NOINLINE void sort_09v(int32_t *ptr) {
__m512i d01 = _mm512_loadu_si512((__m512i const *) ptr + 0);;
__m512i d02 = _mm512_loadu_si512((__m512i const *) ptr + 1);;
__m512i d03 = _mm512_loadu_si512((__m512i const *) ptr + 2);;
__m512i d06 = _mm512_loadu_si512((__m512i const *) ptr + 5);;
__m512i d07 = _mm512_loadu_si512((__m512i const *) ptr + 6);;
__m512i d08 = _mm512_loadu_si512((__m512i const *) ptr + 7);;
- __m512i d09 = _mm512_loadu_si512((__m512i const *) ptr + 8);;
+ __m512i d09 = _mm512_mask_loadu_epi32(_mm512_set1_epi32(MAX),
+ mask,
+ (int32_t const *) ((__m512i const *) ptr + 8));
sort_09v_ascending(d01, d02, d03, d04, d05, d06, d07, d08, d09);
_mm512_storeu_si512((__m512i *) ptr + 0, d01);
_mm512_storeu_si512((__m512i *) ptr + 1, d02);
_mm512_storeu_si512((__m512i *) ptr + 5, d06);
_mm512_storeu_si512((__m512i *) ptr + 6, d07);
_mm512_storeu_si512((__m512i *) ptr + 7, d08);
- _mm512_storeu_si512((__m512i *) ptr + 8, d09);
-}
+ _mm512_mask_storeu_epi32((__m512i *) ptr + 8, mask, d09);
+ }
+
+ static NOINLINE void sort_10v_alt(int32_t *ptr, int remainder) {
+ const auto mask = 0xFFFF >> ((N - remainder) & (N-1));
- static NOINLINE void sort_10v(int32_t *ptr) {
__m512i d01 = _mm512_loadu_si512((__m512i const *) ptr + 0);;
__m512i d02 = _mm512_loadu_si512((__m512i const *) ptr + 1);;
__m512i d03 = _mm512_loadu_si512((__m512i const *) ptr + 2);;
__m512i d07 = _mm512_loadu_si512((__m512i const *) ptr + 6);;
__m512i d08 = _mm512_loadu_si512((__m512i const *) ptr + 7);;
__m512i d09 = _mm512_loadu_si512((__m512i const *) ptr + 8);;
- __m512i d10 = _mm512_loadu_si512((__m512i const *) ptr + 9);;
+ __m512i d10 = _mm512_mask_loadu_epi32(_mm512_set1_epi32(MAX),
+ mask,
+ (int32_t const *) ((__m512i const *) ptr + 9));
sort_10v_ascending(d01, d02, d03, d04, d05, d06, d07, d08, d09, d10);
_mm512_storeu_si512((__m512i *) ptr + 0, d01);
_mm512_storeu_si512((__m512i *) ptr + 1, d02);
_mm512_storeu_si512((__m512i *) ptr + 6, d07);
_mm512_storeu_si512((__m512i *) ptr + 7, d08);
_mm512_storeu_si512((__m512i *) ptr + 8, d09);
- _mm512_storeu_si512((__m512i *) ptr + 9, d10);
-}
+ _mm512_mask_storeu_epi32((__m512i *) ptr + 9, mask, d10);
+ }
+
+ static NOINLINE void sort_11v_alt(int32_t *ptr, int remainder) {
+ const auto mask = 0xFFFF >> ((N - remainder) & (N-1));
- static NOINLINE void sort_11v(int32_t *ptr) {
__m512i d01 = _mm512_loadu_si512((__m512i const *) ptr + 0);;
__m512i d02 = _mm512_loadu_si512((__m512i const *) ptr + 1);;
__m512i d03 = _mm512_loadu_si512((__m512i const *) ptr + 2);;
__m512i d08 = _mm512_loadu_si512((__m512i const *) ptr + 7);;
__m512i d09 = _mm512_loadu_si512((__m512i const *) ptr + 8);;
__m512i d10 = _mm512_loadu_si512((__m512i const *) ptr + 9);;
- __m512i d11 = _mm512_loadu_si512((__m512i const *) ptr + 10);;
+ __m512i d11 = _mm512_mask_loadu_epi32(_mm512_set1_epi32(MAX),
+ mask,
+ (int32_t const *) ((__m512i const *) ptr + 10));
sort_11v_ascending(d01, d02, d03, d04, d05, d06, d07, d08, d09, d10, d11);
_mm512_storeu_si512((__m512i *) ptr + 0, d01);
_mm512_storeu_si512((__m512i *) ptr + 1, d02);
_mm512_storeu_si512((__m512i *) ptr + 7, d08);
_mm512_storeu_si512((__m512i *) ptr + 8, d09);
_mm512_storeu_si512((__m512i *) ptr + 9, d10);
- _mm512_storeu_si512((__m512i *) ptr + 10, d11);
-}
+ _mm512_mask_storeu_epi32((__m512i *) ptr + 10, mask, d11);
+ }
+
+ static NOINLINE void sort_12v_alt(int32_t *ptr, int remainder) {
+ const auto mask = 0xFFFF >> ((N - remainder) & (N-1));
- static NOINLINE void sort_12v(int32_t *ptr) {
__m512i d01 = _mm512_loadu_si512((__m512i const *) ptr + 0);;
__m512i d02 = _mm512_loadu_si512((__m512i const *) ptr + 1);;
__m512i d03 = _mm512_loadu_si512((__m512i const *) ptr + 2);;
__m512i d09 = _mm512_loadu_si512((__m512i const *) ptr + 8);;
__m512i d10 = _mm512_loadu_si512((__m512i const *) ptr + 9);;
__m512i d11 = _mm512_loadu_si512((__m512i const *) ptr + 10);;
- __m512i d12 = _mm512_loadu_si512((__m512i const *) ptr + 11);;
+ __m512i d12 = _mm512_mask_loadu_epi32(_mm512_set1_epi32(MAX),
+ mask,
+ (int32_t const *) ((__m512i const *) ptr + 11));
sort_12v_ascending(d01, d02, d03, d04, d05, d06, d07, d08, d09, d10, d11, d12);
_mm512_storeu_si512((__m512i *) ptr + 0, d01);
_mm512_storeu_si512((__m512i *) ptr + 1, d02);
_mm512_storeu_si512((__m512i *) ptr + 8, d09);
_mm512_storeu_si512((__m512i *) ptr + 9, d10);
_mm512_storeu_si512((__m512i *) ptr + 10, d11);
- _mm512_storeu_si512((__m512i *) ptr + 11, d12);
-}
+ _mm512_mask_storeu_epi32((__m512i *) ptr + 11, mask, d12);
+ }
+
+ static NOINLINE void sort_13v_alt(int32_t *ptr, int remainder) {
+ const auto mask = 0xFFFF >> ((N - remainder) & (N-1));
- static NOINLINE void sort_13v(int32_t *ptr) {
__m512i d01 = _mm512_loadu_si512((__m512i const *) ptr + 0);;
__m512i d02 = _mm512_loadu_si512((__m512i const *) ptr + 1);;
__m512i d03 = _mm512_loadu_si512((__m512i const *) ptr + 2);;
__m512i d10 = _mm512_loadu_si512((__m512i const *) ptr + 9);;
__m512i d11 = _mm512_loadu_si512((__m512i const *) ptr + 10);;
__m512i d12 = _mm512_loadu_si512((__m512i const *) ptr + 11);;
- __m512i d13 = _mm512_loadu_si512((__m512i const *) ptr + 12);;
+ __m512i d13 = _mm512_mask_loadu_epi32(_mm512_set1_epi32(MAX),
+ mask,
+ (int32_t const *) ((__m512i const *) ptr + 12));
sort_13v_ascending(d01, d02, d03, d04, d05, d06, d07, d08, d09, d10, d11, d12, d13);
_mm512_storeu_si512((__m512i *) ptr + 0, d01);
_mm512_storeu_si512((__m512i *) ptr + 1, d02);
_mm512_storeu_si512((__m512i *) ptr + 9, d10);
_mm512_storeu_si512((__m512i *) ptr + 10, d11);
_mm512_storeu_si512((__m512i *) ptr + 11, d12);
- _mm512_storeu_si512((__m512i *) ptr + 12, d13);
-}
+ _mm512_mask_storeu_epi32((__m512i *) ptr + 12, mask, d13);
+ }
+
+ static NOINLINE void sort_14v_alt(int32_t *ptr, int remainder) {
+ const auto mask = 0xFFFF >> ((N - remainder) & (N-1));
- static NOINLINE void sort_14v(int32_t *ptr) {
__m512i d01 = _mm512_loadu_si512((__m512i const *) ptr + 0);;
__m512i d02 = _mm512_loadu_si512((__m512i const *) ptr + 1);;
__m512i d03 = _mm512_loadu_si512((__m512i const *) ptr + 2);;
__m512i d11 = _mm512_loadu_si512((__m512i const *) ptr + 10);;
__m512i d12 = _mm512_loadu_si512((__m512i const *) ptr + 11);;
__m512i d13 = _mm512_loadu_si512((__m512i const *) ptr + 12);;
- __m512i d14 = _mm512_loadu_si512((__m512i const *) ptr + 13);;
+ __m512i d14 = _mm512_mask_loadu_epi32(_mm512_set1_epi32(MAX),
+ mask,
+ (int32_t const *) ((__m512i const *) ptr + 13));
sort_14v_ascending(d01, d02, d03, d04, d05, d06, d07, d08, d09, d10, d11, d12, d13, d14);
_mm512_storeu_si512((__m512i *) ptr + 0, d01);
_mm512_storeu_si512((__m512i *) ptr + 1, d02);
_mm512_storeu_si512((__m512i *) ptr + 10, d11);
_mm512_storeu_si512((__m512i *) ptr + 11, d12);
_mm512_storeu_si512((__m512i *) ptr + 12, d13);
- _mm512_storeu_si512((__m512i *) ptr + 13, d14);
-}
+ _mm512_mask_storeu_epi32((__m512i *) ptr + 13, mask, d14);
+ }
+
+ static NOINLINE void sort_15v_alt(int32_t *ptr, int remainder) {
+ const auto mask = 0xFFFF >> ((N - remainder) & (N-1));
- static NOINLINE void sort_15v(int32_t *ptr) {
__m512i d01 = _mm512_loadu_si512((__m512i const *) ptr + 0);;
__m512i d02 = _mm512_loadu_si512((__m512i const *) ptr + 1);;
__m512i d03 = _mm512_loadu_si512((__m512i const *) ptr + 2);;
__m512i d12 = _mm512_loadu_si512((__m512i const *) ptr + 11);;
__m512i d13 = _mm512_loadu_si512((__m512i const *) ptr + 12);;
__m512i d14 = _mm512_loadu_si512((__m512i const *) ptr + 13);;
- __m512i d15 = _mm512_loadu_si512((__m512i const *) ptr + 14);;
+ __m512i d15 = _mm512_mask_loadu_epi32(_mm512_set1_epi32(MAX),
+ mask,
+ (int32_t const *) ((__m512i const *) ptr + 14));
sort_15v_ascending(d01, d02, d03, d04, d05, d06, d07, d08, d09, d10, d11, d12, d13, d14, d15);
_mm512_storeu_si512((__m512i *) ptr + 0, d01);
_mm512_storeu_si512((__m512i *) ptr + 1, d02);
_mm512_storeu_si512((__m512i *) ptr + 11, d12);
_mm512_storeu_si512((__m512i *) ptr + 12, d13);
_mm512_storeu_si512((__m512i *) ptr + 13, d14);
- _mm512_storeu_si512((__m512i *) ptr + 14, d15);
-}
+ _mm512_mask_storeu_epi32((__m512i *) ptr + 14, mask, d15);
+ }
+
+ static NOINLINE void sort_16v_alt(int32_t *ptr, int remainder) {
+ const auto mask = 0xFFFF >> ((N - remainder) & (N-1));
- static NOINLINE void sort_16v(int32_t *ptr) {
__m512i d01 = _mm512_loadu_si512((__m512i const *) ptr + 0);;
__m512i d02 = _mm512_loadu_si512((__m512i const *) ptr + 1);;
__m512i d03 = _mm512_loadu_si512((__m512i const *) ptr + 2);;
__m512i d13 = _mm512_loadu_si512((__m512i const *) ptr + 12);;
__m512i d14 = _mm512_loadu_si512((__m512i const *) ptr + 13);;
__m512i d15 = _mm512_loadu_si512((__m512i const *) ptr + 14);;
- __m512i d16 = _mm512_loadu_si512((__m512i const *) ptr + 15);;
+ __m512i d16 = _mm512_mask_loadu_epi32(_mm512_set1_epi32(MAX),
+ mask,
+ (int32_t const *) ((__m512i const *) ptr + 15));
sort_16v_ascending(d01, d02, d03, d04, d05, d06, d07, d08, d09, d10, d11, d12, d13, d14, d15, d16);
_mm512_storeu_si512((__m512i *) ptr + 0, d01);
_mm512_storeu_si512((__m512i *) ptr + 1, d02);
_mm512_storeu_si512((__m512i *) ptr + 12, d13);
_mm512_storeu_si512((__m512i *) ptr + 13, d14);
_mm512_storeu_si512((__m512i *) ptr + 14, d15);
- _mm512_storeu_si512((__m512i *) ptr + 15, d16);
-}
+ _mm512_mask_storeu_epi32((__m512i *) ptr + 15, mask, d16);
+ }
static void sort(int32_t *ptr, size_t length);
};
using namespace vxsort;
void vxsort::smallsort::bitonic<int64_t, vector_machine::AVX512 >::sort(int64_t *ptr, size_t length) {
- const int N = 8;
-
- switch(length / N) {
- case 1: sort_01v(ptr); break;
- case 2: sort_02v(ptr); break;
- case 3: sort_03v(ptr); break;
- case 4: sort_04v(ptr); break;
- case 5: sort_05v(ptr); break;
- case 6: sort_06v(ptr); break;
- case 7: sort_07v(ptr); break;
- case 8: sort_08v(ptr); break;
- case 9: sort_09v(ptr); break;
- case 10: sort_10v(ptr); break;
- case 11: sort_11v(ptr); break;
- case 12: sort_12v(ptr); break;
- case 13: sort_13v(ptr); break;
- case 14: sort_14v(ptr); break;
- case 15: sort_15v(ptr); break;
- case 16: sort_16v(ptr); break;
+ const auto fullvlength = length / N;
+ const int remainder = (int) (length - fullvlength * N);
+ const auto v = fullvlength + ((remainder > 0) ? 1 : 0);
+ switch(v) {
+ case 1: sort_01v_alt(ptr, remainder); break;
+ case 2: sort_02v_alt(ptr, remainder); break;
+ case 3: sort_03v_alt(ptr, remainder); break;
+ case 4: sort_04v_alt(ptr, remainder); break;
+ case 5: sort_05v_alt(ptr, remainder); break;
+ case 6: sort_06v_alt(ptr, remainder); break;
+ case 7: sort_07v_alt(ptr, remainder); break;
+ case 8: sort_08v_alt(ptr, remainder); break;
+ case 9: sort_09v_alt(ptr, remainder); break;
+ case 10: sort_10v_alt(ptr, remainder); break;
+ case 11: sort_11v_alt(ptr, remainder); break;
+ case 12: sort_12v_alt(ptr, remainder); break;
+ case 13: sort_13v_alt(ptr, remainder); break;
+ case 14: sort_14v_alt(ptr, remainder); break;
+ case 15: sort_15v_alt(ptr, remainder); break;
+ case 16: sort_16v_alt(ptr, remainder); break;
}
}
/////////////////////////////////////////////////////////////////////////////
////
-// This file was auto-generated by a tool at 2020-06-22 05:27:48
+// This file was auto-generated by a tool at 2020-07-21 14:05:39
//
// It is recommended you DO NOT directly edit this file but instead edit
// the code-generator that generated this source file instead.
namespace vxsort {
namespace smallsort {
template<> struct bitonic<int64_t, AVX512> {
+ static const int N = 8;
+ static constexpr int64_t MAX = std::numeric_limits<int64_t>::Max();
public:
static INLINE void sort_01v_ascending(__m512i& d01) {
sort_02v_merge_descending(d01, d02);
sort_01v_merge_descending(d03);
}
- static INLINE void sort_04v_ascending(__m512i& d01, __m512i& d02, __m512i& d03, __m512i& d04) {
+ static NOINLINE void sort_04v_ascending(__m512i& d01, __m512i& d02, __m512i& d03, __m512i& d04) {
__m512i tmp;
sort_02v_ascending(d01, d02);
sort_02v_merge_ascending(d01, d02);
sort_02v_merge_ascending(d03, d04);
}
- static INLINE void sort_04v_descending(__m512i& d01, __m512i& d02, __m512i& d03, __m512i& d04) {
+ static NOINLINE void sort_04v_descending(__m512i& d01, __m512i& d02, __m512i& d03, __m512i& d04) {
__m512i tmp;
sort_02v_descending(d01, d02);
sort_02v_merge_descending(d01, d02);
sort_02v_merge_descending(d03, d04);
}
- static INLINE void sort_04v_merge_ascending(__m512i& d01, __m512i& d02, __m512i& d03, __m512i& d04) {
+ static NOINLINE void sort_04v_merge_ascending(__m512i& d01, __m512i& d02, __m512i& d03, __m512i& d04) {
__m512i tmp;
tmp = d01;
sort_02v_merge_ascending(d01, d02);
sort_02v_merge_ascending(d03, d04);
}
- static INLINE void sort_04v_merge_descending(__m512i& d01, __m512i& d02, __m512i& d03, __m512i& d04) {
+ static NOINLINE void sort_04v_merge_descending(__m512i& d01, __m512i& d02, __m512i& d03, __m512i& d04) {
__m512i tmp;
tmp = d01;
sort_04v_merge_descending(d01, d02, d03, d04);
sort_03v_merge_descending(d05, d06, d07);
}
- static INLINE void sort_08v_ascending(__m512i& d01, __m512i& d02, __m512i& d03, __m512i& d04, __m512i& d05, __m512i& d06, __m512i& d07, __m512i& d08) {
+ static NOINLINE void sort_08v_ascending(__m512i& d01, __m512i& d02, __m512i& d03, __m512i& d04, __m512i& d05, __m512i& d06, __m512i& d07, __m512i& d08) {
__m512i tmp;
sort_04v_ascending(d01, d02, d03, d04);
sort_04v_merge_ascending(d01, d02, d03, d04);
sort_04v_merge_ascending(d05, d06, d07, d08);
}
- static INLINE void sort_08v_descending(__m512i& d01, __m512i& d02, __m512i& d03, __m512i& d04, __m512i& d05, __m512i& d06, __m512i& d07, __m512i& d08) {
+ static NOINLINE void sort_08v_descending(__m512i& d01, __m512i& d02, __m512i& d03, __m512i& d04, __m512i& d05, __m512i& d06, __m512i& d07, __m512i& d08) {
__m512i tmp;
sort_04v_descending(d01, d02, d03, d04);
sort_04v_merge_descending(d01, d02, d03, d04);
sort_04v_merge_descending(d05, d06, d07, d08);
}
- static INLINE void sort_08v_merge_ascending(__m512i& d01, __m512i& d02, __m512i& d03, __m512i& d04, __m512i& d05, __m512i& d06, __m512i& d07, __m512i& d08) {
+ static NOINLINE void sort_08v_merge_ascending(__m512i& d01, __m512i& d02, __m512i& d03, __m512i& d04, __m512i& d05, __m512i& d06, __m512i& d07, __m512i& d08) {
__m512i tmp;
tmp = d01;
sort_04v_merge_ascending(d01, d02, d03, d04);
sort_04v_merge_ascending(d05, d06, d07, d08);
}
- static INLINE void sort_08v_merge_descending(__m512i& d01, __m512i& d02, __m512i& d03, __m512i& d04, __m512i& d05, __m512i& d06, __m512i& d07, __m512i& d08) {
+ static NOINLINE void sort_08v_merge_descending(__m512i& d01, __m512i& d02, __m512i& d03, __m512i& d04, __m512i& d05, __m512i& d06, __m512i& d07, __m512i& d08) {
__m512i tmp;
tmp = d01;
sort_08v_merge_descending(d01, d02, d03, d04, d05, d06, d07, d08);
sort_03v_merge_descending(d09, d10, d11);
}
- static INLINE void sort_12v_ascending(__m512i& d01, __m512i& d02, __m512i& d03, __m512i& d04, __m512i& d05, __m512i& d06, __m512i& d07, __m512i& d08, __m512i& d09, __m512i& d10, __m512i& d11, __m512i& d12) {
+ static NOINLINE void sort_12v_ascending(__m512i& d01, __m512i& d02, __m512i& d03, __m512i& d04, __m512i& d05, __m512i& d06, __m512i& d07, __m512i& d08, __m512i& d09, __m512i& d10, __m512i& d11, __m512i& d12) {
__m512i tmp;
sort_08v_ascending(d01, d02, d03, d04, d05, d06, d07, d08);
sort_08v_merge_ascending(d01, d02, d03, d04, d05, d06, d07, d08);
sort_04v_merge_ascending(d09, d10, d11, d12);
}
- static INLINE void sort_12v_descending(__m512i& d01, __m512i& d02, __m512i& d03, __m512i& d04, __m512i& d05, __m512i& d06, __m512i& d07, __m512i& d08, __m512i& d09, __m512i& d10, __m512i& d11, __m512i& d12) {
+ static NOINLINE void sort_12v_descending(__m512i& d01, __m512i& d02, __m512i& d03, __m512i& d04, __m512i& d05, __m512i& d06, __m512i& d07, __m512i& d08, __m512i& d09, __m512i& d10, __m512i& d11, __m512i& d12) {
__m512i tmp;
sort_08v_descending(d01, d02, d03, d04, d05, d06, d07, d08);
sort_08v_merge_descending(d01, d02, d03, d04, d05, d06, d07, d08);
sort_07v_merge_descending(d09, d10, d11, d12, d13, d14, d15);
}
- static INLINE void sort_16v_ascending(__m512i& d01, __m512i& d02, __m512i& d03, __m512i& d04, __m512i& d05, __m512i& d06, __m512i& d07, __m512i& d08, __m512i& d09, __m512i& d10, __m512i& d11, __m512i& d12, __m512i& d13, __m512i& d14, __m512i& d15, __m512i& d16) {
+ static NOINLINE void sort_16v_ascending(__m512i& d01, __m512i& d02, __m512i& d03, __m512i& d04, __m512i& d05, __m512i& d06, __m512i& d07, __m512i& d08, __m512i& d09, __m512i& d10, __m512i& d11, __m512i& d12, __m512i& d13, __m512i& d14, __m512i& d15, __m512i& d16) {
__m512i tmp;
sort_08v_ascending(d01, d02, d03, d04, d05, d06, d07, d08);
sort_08v_merge_ascending(d01, d02, d03, d04, d05, d06, d07, d08);
sort_08v_merge_ascending(d09, d10, d11, d12, d13, d14, d15, d16);
}
- static INLINE void sort_16v_descending(__m512i& d01, __m512i& d02, __m512i& d03, __m512i& d04, __m512i& d05, __m512i& d06, __m512i& d07, __m512i& d08, __m512i& d09, __m512i& d10, __m512i& d11, __m512i& d12, __m512i& d13, __m512i& d14, __m512i& d15, __m512i& d16) {
+ static NOINLINE void sort_16v_descending(__m512i& d01, __m512i& d02, __m512i& d03, __m512i& d04, __m512i& d05, __m512i& d06, __m512i& d07, __m512i& d08, __m512i& d09, __m512i& d10, __m512i& d11, __m512i& d12, __m512i& d13, __m512i& d14, __m512i& d15, __m512i& d16) {
__m512i tmp;
sort_08v_descending(d01, d02, d03, d04, d05, d06, d07, d08);
sort_08v_merge_descending(d09, d10, d11, d12, d13, d14, d15, d16);
}
- static NOINLINE void sort_01v(int64_t *ptr) {
- __m512i d01 = _mm512_loadu_si512((__m512i const *) ptr + 0);;
+ static NOINLINE void sort_01v_alt(int64_t *ptr, int remainder) {
+ const auto mask = 0xFF >> ((N - remainder) & (N-1));
+
+ __m512i d01 = _mm512_mask_loadu_epi64(_mm512_set1_epi64(MAX),
+ mask,
+ (int64_t const *) ((__m512i const *) ptr + 0));
sort_01v_ascending(d01);
- _mm512_storeu_si512((__m512i *) ptr + 0, d01);
-}
+ _mm512_mask_storeu_epi64((__m512i *) ptr + 0, mask, d01);
+ }
+
+ static NOINLINE void sort_02v_alt(int64_t *ptr, int remainder) {
+ const auto mask = 0xFF >> ((N - remainder) & (N-1));
- static NOINLINE void sort_02v(int64_t *ptr) {
__m512i d01 = _mm512_loadu_si512((__m512i const *) ptr + 0);;
- __m512i d02 = _mm512_loadu_si512((__m512i const *) ptr + 1);;
+ __m512i d02 = _mm512_mask_loadu_epi64(_mm512_set1_epi64(MAX),
+ mask,
+ (int64_t const *) ((__m512i const *) ptr + 1));
sort_02v_ascending(d01, d02);
_mm512_storeu_si512((__m512i *) ptr + 0, d01);
- _mm512_storeu_si512((__m512i *) ptr + 1, d02);
-}
+ _mm512_mask_storeu_epi64((__m512i *) ptr + 1, mask, d02);
+ }
+
+ static NOINLINE void sort_03v_alt(int64_t *ptr, int remainder) {
+ const auto mask = 0xFF >> ((N - remainder) & (N-1));
- static NOINLINE void sort_03v(int64_t *ptr) {
__m512i d01 = _mm512_loadu_si512((__m512i const *) ptr + 0);;
__m512i d02 = _mm512_loadu_si512((__m512i const *) ptr + 1);;
- __m512i d03 = _mm512_loadu_si512((__m512i const *) ptr + 2);;
+ __m512i d03 = _mm512_mask_loadu_epi64(_mm512_set1_epi64(MAX),
+ mask,
+ (int64_t const *) ((__m512i const *) ptr + 2));
sort_03v_ascending(d01, d02, d03);
_mm512_storeu_si512((__m512i *) ptr + 0, d01);
_mm512_storeu_si512((__m512i *) ptr + 1, d02);
- _mm512_storeu_si512((__m512i *) ptr + 2, d03);
-}
+ _mm512_mask_storeu_epi64((__m512i *) ptr + 2, mask, d03);
+ }
+
+ static NOINLINE void sort_04v_alt(int64_t *ptr, int remainder) {
+ const auto mask = 0xFF >> ((N - remainder) & (N-1));
- static NOINLINE void sort_04v(int64_t *ptr) {
__m512i d01 = _mm512_loadu_si512((__m512i const *) ptr + 0);;
__m512i d02 = _mm512_loadu_si512((__m512i const *) ptr + 1);;
__m512i d03 = _mm512_loadu_si512((__m512i const *) ptr + 2);;
- __m512i d04 = _mm512_loadu_si512((__m512i const *) ptr + 3);;
+ __m512i d04 = _mm512_mask_loadu_epi64(_mm512_set1_epi64(MAX),
+ mask,
+ (int64_t const *) ((__m512i const *) ptr + 3));
sort_04v_ascending(d01, d02, d03, d04);
_mm512_storeu_si512((__m512i *) ptr + 0, d01);
_mm512_storeu_si512((__m512i *) ptr + 1, d02);
_mm512_storeu_si512((__m512i *) ptr + 2, d03);
- _mm512_storeu_si512((__m512i *) ptr + 3, d04);
-}
+ _mm512_mask_storeu_epi64((__m512i *) ptr + 3, mask, d04);
+ }
+
+ static NOINLINE void sort_05v_alt(int64_t *ptr, int remainder) {
+ const auto mask = 0xFF >> ((N - remainder) & (N-1));
- static NOINLINE void sort_05v(int64_t *ptr) {
__m512i d01 = _mm512_loadu_si512((__m512i const *) ptr + 0);;
__m512i d02 = _mm512_loadu_si512((__m512i const *) ptr + 1);;
__m512i d03 = _mm512_loadu_si512((__m512i const *) ptr + 2);;
__m512i d04 = _mm512_loadu_si512((__m512i const *) ptr + 3);;
- __m512i d05 = _mm512_loadu_si512((__m512i const *) ptr + 4);;
+ __m512i d05 = _mm512_mask_loadu_epi64(_mm512_set1_epi64(MAX),
+ mask,
+ (int64_t const *) ((__m512i const *) ptr + 4));
sort_05v_ascending(d01, d02, d03, d04, d05);
_mm512_storeu_si512((__m512i *) ptr + 0, d01);
_mm512_storeu_si512((__m512i *) ptr + 1, d02);
_mm512_storeu_si512((__m512i *) ptr + 2, d03);
_mm512_storeu_si512((__m512i *) ptr + 3, d04);
- _mm512_storeu_si512((__m512i *) ptr + 4, d05);
-}
+ _mm512_mask_storeu_epi64((__m512i *) ptr + 4, mask, d05);
+ }
+
+ static NOINLINE void sort_06v_alt(int64_t *ptr, int remainder) {
+ const auto mask = 0xFF >> ((N - remainder) & (N-1));
- static NOINLINE void sort_06v(int64_t *ptr) {
__m512i d01 = _mm512_loadu_si512((__m512i const *) ptr + 0);;
__m512i d02 = _mm512_loadu_si512((__m512i const *) ptr + 1);;
__m512i d03 = _mm512_loadu_si512((__m512i const *) ptr + 2);;
__m512i d04 = _mm512_loadu_si512((__m512i const *) ptr + 3);;
__m512i d05 = _mm512_loadu_si512((__m512i const *) ptr + 4);;
- __m512i d06 = _mm512_loadu_si512((__m512i const *) ptr + 5);;
+ __m512i d06 = _mm512_mask_loadu_epi64(_mm512_set1_epi64(MAX),
+ mask,
+ (int64_t const *) ((__m512i const *) ptr + 5));
sort_06v_ascending(d01, d02, d03, d04, d05, d06);
_mm512_storeu_si512((__m512i *) ptr + 0, d01);
_mm512_storeu_si512((__m512i *) ptr + 1, d02);
_mm512_storeu_si512((__m512i *) ptr + 2, d03);
_mm512_storeu_si512((__m512i *) ptr + 3, d04);
_mm512_storeu_si512((__m512i *) ptr + 4, d05);
- _mm512_storeu_si512((__m512i *) ptr + 5, d06);
-}
+ _mm512_mask_storeu_epi64((__m512i *) ptr + 5, mask, d06);
+ }
+
+ static NOINLINE void sort_07v_alt(int64_t *ptr, int remainder) {
+ const auto mask = 0xFF >> ((N - remainder) & (N-1));
- static NOINLINE void sort_07v(int64_t *ptr) {
__m512i d01 = _mm512_loadu_si512((__m512i const *) ptr + 0);;
__m512i d02 = _mm512_loadu_si512((__m512i const *) ptr + 1);;
__m512i d03 = _mm512_loadu_si512((__m512i const *) ptr + 2);;
__m512i d04 = _mm512_loadu_si512((__m512i const *) ptr + 3);;
__m512i d05 = _mm512_loadu_si512((__m512i const *) ptr + 4);;
__m512i d06 = _mm512_loadu_si512((__m512i const *) ptr + 5);;
- __m512i d07 = _mm512_loadu_si512((__m512i const *) ptr + 6);;
+ __m512i d07 = _mm512_mask_loadu_epi64(_mm512_set1_epi64(MAX),
+ mask,
+ (int64_t const *) ((__m512i const *) ptr + 6));
sort_07v_ascending(d01, d02, d03, d04, d05, d06, d07);
_mm512_storeu_si512((__m512i *) ptr + 0, d01);
_mm512_storeu_si512((__m512i *) ptr + 1, d02);
_mm512_storeu_si512((__m512i *) ptr + 3, d04);
_mm512_storeu_si512((__m512i *) ptr + 4, d05);
_mm512_storeu_si512((__m512i *) ptr + 5, d06);
- _mm512_storeu_si512((__m512i *) ptr + 6, d07);
-}
+ _mm512_mask_storeu_epi64((__m512i *) ptr + 6, mask, d07);
+ }
+
+ static NOINLINE void sort_08v_alt(int64_t *ptr, int remainder) {
+ const auto mask = 0xFF >> ((N - remainder) & (N-1));
- static NOINLINE void sort_08v(int64_t *ptr) {
__m512i d01 = _mm512_loadu_si512((__m512i const *) ptr + 0);;
__m512i d02 = _mm512_loadu_si512((__m512i const *) ptr + 1);;
__m512i d03 = _mm512_loadu_si512((__m512i const *) ptr + 2);;
__m512i d05 = _mm512_loadu_si512((__m512i const *) ptr + 4);;
__m512i d06 = _mm512_loadu_si512((__m512i const *) ptr + 5);;
__m512i d07 = _mm512_loadu_si512((__m512i const *) ptr + 6);;
- __m512i d08 = _mm512_loadu_si512((__m512i const *) ptr + 7);;
+ __m512i d08 = _mm512_mask_loadu_epi64(_mm512_set1_epi64(MAX),
+ mask,
+ (int64_t const *) ((__m512i const *) ptr + 7));
sort_08v_ascending(d01, d02, d03, d04, d05, d06, d07, d08);
_mm512_storeu_si512((__m512i *) ptr + 0, d01);
_mm512_storeu_si512((__m512i *) ptr + 1, d02);
_mm512_storeu_si512((__m512i *) ptr + 4, d05);
_mm512_storeu_si512((__m512i *) ptr + 5, d06);
_mm512_storeu_si512((__m512i *) ptr + 6, d07);
- _mm512_storeu_si512((__m512i *) ptr + 7, d08);
-}
+ _mm512_mask_storeu_epi64((__m512i *) ptr + 7, mask, d08);
+ }
+
+ static NOINLINE void sort_09v_alt(int64_t *ptr, int remainder) {
+ const auto mask = 0xFF >> ((N - remainder) & (N-1));
- static NOINLINE void sort_09v(int64_t *ptr) {
__m512i d01 = _mm512_loadu_si512((__m512i const *) ptr + 0);;
__m512i d02 = _mm512_loadu_si512((__m512i const *) ptr + 1);;
__m512i d03 = _mm512_loadu_si512((__m512i const *) ptr + 2);;
__m512i d06 = _mm512_loadu_si512((__m512i const *) ptr + 5);;
__m512i d07 = _mm512_loadu_si512((__m512i const *) ptr + 6);;
__m512i d08 = _mm512_loadu_si512((__m512i const *) ptr + 7);;
- __m512i d09 = _mm512_loadu_si512((__m512i const *) ptr + 8);;
+ __m512i d09 = _mm512_mask_loadu_epi64(_mm512_set1_epi64(MAX),
+ mask,
+ (int64_t const *) ((__m512i const *) ptr + 8));
sort_09v_ascending(d01, d02, d03, d04, d05, d06, d07, d08, d09);
_mm512_storeu_si512((__m512i *) ptr + 0, d01);
_mm512_storeu_si512((__m512i *) ptr + 1, d02);
_mm512_storeu_si512((__m512i *) ptr + 5, d06);
_mm512_storeu_si512((__m512i *) ptr + 6, d07);
_mm512_storeu_si512((__m512i *) ptr + 7, d08);
- _mm512_storeu_si512((__m512i *) ptr + 8, d09);
-}
+ _mm512_mask_storeu_epi64((__m512i *) ptr + 8, mask, d09);
+ }
+
+ static NOINLINE void sort_10v_alt(int64_t *ptr, int remainder) {
+ const auto mask = 0xFF >> ((N - remainder) & (N-1));
- static NOINLINE void sort_10v(int64_t *ptr) {
__m512i d01 = _mm512_loadu_si512((__m512i const *) ptr + 0);;
__m512i d02 = _mm512_loadu_si512((__m512i const *) ptr + 1);;
__m512i d03 = _mm512_loadu_si512((__m512i const *) ptr + 2);;
__m512i d07 = _mm512_loadu_si512((__m512i const *) ptr + 6);;
__m512i d08 = _mm512_loadu_si512((__m512i const *) ptr + 7);;
__m512i d09 = _mm512_loadu_si512((__m512i const *) ptr + 8);;
- __m512i d10 = _mm512_loadu_si512((__m512i const *) ptr + 9);;
+ __m512i d10 = _mm512_mask_loadu_epi64(_mm512_set1_epi64(MAX),
+ mask,
+ (int64_t const *) ((__m512i const *) ptr + 9));
sort_10v_ascending(d01, d02, d03, d04, d05, d06, d07, d08, d09, d10);
_mm512_storeu_si512((__m512i *) ptr + 0, d01);
_mm512_storeu_si512((__m512i *) ptr + 1, d02);
_mm512_storeu_si512((__m512i *) ptr + 6, d07);
_mm512_storeu_si512((__m512i *) ptr + 7, d08);
_mm512_storeu_si512((__m512i *) ptr + 8, d09);
- _mm512_storeu_si512((__m512i *) ptr + 9, d10);
-}
+ _mm512_mask_storeu_epi64((__m512i *) ptr + 9, mask, d10);
+ }
+
+ static NOINLINE void sort_11v_alt(int64_t *ptr, int remainder) {
+ const auto mask = 0xFF >> ((N - remainder) & (N-1));
- static NOINLINE void sort_11v(int64_t *ptr) {
__m512i d01 = _mm512_loadu_si512((__m512i const *) ptr + 0);;
__m512i d02 = _mm512_loadu_si512((__m512i const *) ptr + 1);;
__m512i d03 = _mm512_loadu_si512((__m512i const *) ptr + 2);;
__m512i d08 = _mm512_loadu_si512((__m512i const *) ptr + 7);;
__m512i d09 = _mm512_loadu_si512((__m512i const *) ptr + 8);;
__m512i d10 = _mm512_loadu_si512((__m512i const *) ptr + 9);;
- __m512i d11 = _mm512_loadu_si512((__m512i const *) ptr + 10);;
+ __m512i d11 = _mm512_mask_loadu_epi64(_mm512_set1_epi64(MAX),
+ mask,
+ (int64_t const *) ((__m512i const *) ptr + 10));
sort_11v_ascending(d01, d02, d03, d04, d05, d06, d07, d08, d09, d10, d11);
_mm512_storeu_si512((__m512i *) ptr + 0, d01);
_mm512_storeu_si512((__m512i *) ptr + 1, d02);
_mm512_storeu_si512((__m512i *) ptr + 7, d08);
_mm512_storeu_si512((__m512i *) ptr + 8, d09);
_mm512_storeu_si512((__m512i *) ptr + 9, d10);
- _mm512_storeu_si512((__m512i *) ptr + 10, d11);
-}
+ _mm512_mask_storeu_epi64((__m512i *) ptr + 10, mask, d11);
+ }
+
+ static NOINLINE void sort_12v_alt(int64_t *ptr, int remainder) {
+ const auto mask = 0xFF >> ((N - remainder) & (N-1));
- static NOINLINE void sort_12v(int64_t *ptr) {
__m512i d01 = _mm512_loadu_si512((__m512i const *) ptr + 0);;
__m512i d02 = _mm512_loadu_si512((__m512i const *) ptr + 1);;
__m512i d03 = _mm512_loadu_si512((__m512i const *) ptr + 2);;
__m512i d09 = _mm512_loadu_si512((__m512i const *) ptr + 8);;
__m512i d10 = _mm512_loadu_si512((__m512i const *) ptr + 9);;
__m512i d11 = _mm512_loadu_si512((__m512i const *) ptr + 10);;
- __m512i d12 = _mm512_loadu_si512((__m512i const *) ptr + 11);;
+ __m512i d12 = _mm512_mask_loadu_epi64(_mm512_set1_epi64(MAX),
+ mask,
+ (int64_t const *) ((__m512i const *) ptr + 11));
sort_12v_ascending(d01, d02, d03, d04, d05, d06, d07, d08, d09, d10, d11, d12);
_mm512_storeu_si512((__m512i *) ptr + 0, d01);
_mm512_storeu_si512((__m512i *) ptr + 1, d02);
_mm512_storeu_si512((__m512i *) ptr + 8, d09);
_mm512_storeu_si512((__m512i *) ptr + 9, d10);
_mm512_storeu_si512((__m512i *) ptr + 10, d11);
- _mm512_storeu_si512((__m512i *) ptr + 11, d12);
-}
+ _mm512_mask_storeu_epi64((__m512i *) ptr + 11, mask, d12);
+ }
+
+ static NOINLINE void sort_13v_alt(int64_t *ptr, int remainder) {
+ const auto mask = 0xFF >> ((N - remainder) & (N-1));
- static NOINLINE void sort_13v(int64_t *ptr) {
__m512i d01 = _mm512_loadu_si512((__m512i const *) ptr + 0);;
__m512i d02 = _mm512_loadu_si512((__m512i const *) ptr + 1);;
__m512i d03 = _mm512_loadu_si512((__m512i const *) ptr + 2);;
__m512i d10 = _mm512_loadu_si512((__m512i const *) ptr + 9);;
__m512i d11 = _mm512_loadu_si512((__m512i const *) ptr + 10);;
__m512i d12 = _mm512_loadu_si512((__m512i const *) ptr + 11);;
- __m512i d13 = _mm512_loadu_si512((__m512i const *) ptr + 12);;
+ __m512i d13 = _mm512_mask_loadu_epi64(_mm512_set1_epi64(MAX),
+ mask,
+ (int64_t const *) ((__m512i const *) ptr + 12));
sort_13v_ascending(d01, d02, d03, d04, d05, d06, d07, d08, d09, d10, d11, d12, d13);
_mm512_storeu_si512((__m512i *) ptr + 0, d01);
_mm512_storeu_si512((__m512i *) ptr + 1, d02);
_mm512_storeu_si512((__m512i *) ptr + 9, d10);
_mm512_storeu_si512((__m512i *) ptr + 10, d11);
_mm512_storeu_si512((__m512i *) ptr + 11, d12);
- _mm512_storeu_si512((__m512i *) ptr + 12, d13);
-}
+ _mm512_mask_storeu_epi64((__m512i *) ptr + 12, mask, d13);
+ }
+
+ static NOINLINE void sort_14v_alt(int64_t *ptr, int remainder) {
+ const auto mask = 0xFF >> ((N - remainder) & (N-1));
- static NOINLINE void sort_14v(int64_t *ptr) {
__m512i d01 = _mm512_loadu_si512((__m512i const *) ptr + 0);;
__m512i d02 = _mm512_loadu_si512((__m512i const *) ptr + 1);;
__m512i d03 = _mm512_loadu_si512((__m512i const *) ptr + 2);;
__m512i d11 = _mm512_loadu_si512((__m512i const *) ptr + 10);;
__m512i d12 = _mm512_loadu_si512((__m512i const *) ptr + 11);;
__m512i d13 = _mm512_loadu_si512((__m512i const *) ptr + 12);;
- __m512i d14 = _mm512_loadu_si512((__m512i const *) ptr + 13);;
+ __m512i d14 = _mm512_mask_loadu_epi64(_mm512_set1_epi64(MAX),
+ mask,
+ (int64_t const *) ((__m512i const *) ptr + 13));
sort_14v_ascending(d01, d02, d03, d04, d05, d06, d07, d08, d09, d10, d11, d12, d13, d14);
_mm512_storeu_si512((__m512i *) ptr + 0, d01);
_mm512_storeu_si512((__m512i *) ptr + 1, d02);
_mm512_storeu_si512((__m512i *) ptr + 10, d11);
_mm512_storeu_si512((__m512i *) ptr + 11, d12);
_mm512_storeu_si512((__m512i *) ptr + 12, d13);
- _mm512_storeu_si512((__m512i *) ptr + 13, d14);
-}
+ _mm512_mask_storeu_epi64((__m512i *) ptr + 13, mask, d14);
+ }
+
+ static NOINLINE void sort_15v_alt(int64_t *ptr, int remainder) {
+ const auto mask = 0xFF >> ((N - remainder) & (N-1));
- static NOINLINE void sort_15v(int64_t *ptr) {
__m512i d01 = _mm512_loadu_si512((__m512i const *) ptr + 0);;
__m512i d02 = _mm512_loadu_si512((__m512i const *) ptr + 1);;
__m512i d03 = _mm512_loadu_si512((__m512i const *) ptr + 2);;
__m512i d12 = _mm512_loadu_si512((__m512i const *) ptr + 11);;
__m512i d13 = _mm512_loadu_si512((__m512i const *) ptr + 12);;
__m512i d14 = _mm512_loadu_si512((__m512i const *) ptr + 13);;
- __m512i d15 = _mm512_loadu_si512((__m512i const *) ptr + 14);;
+ __m512i d15 = _mm512_mask_loadu_epi64(_mm512_set1_epi64(MAX),
+ mask,
+ (int64_t const *) ((__m512i const *) ptr + 14));
sort_15v_ascending(d01, d02, d03, d04, d05, d06, d07, d08, d09, d10, d11, d12, d13, d14, d15);
_mm512_storeu_si512((__m512i *) ptr + 0, d01);
_mm512_storeu_si512((__m512i *) ptr + 1, d02);
_mm512_storeu_si512((__m512i *) ptr + 11, d12);
_mm512_storeu_si512((__m512i *) ptr + 12, d13);
_mm512_storeu_si512((__m512i *) ptr + 13, d14);
- _mm512_storeu_si512((__m512i *) ptr + 14, d15);
-}
+ _mm512_mask_storeu_epi64((__m512i *) ptr + 14, mask, d15);
+ }
+
+ static NOINLINE void sort_16v_alt(int64_t *ptr, int remainder) {
+ const auto mask = 0xFF >> ((N - remainder) & (N-1));
- static NOINLINE void sort_16v(int64_t *ptr) {
__m512i d01 = _mm512_loadu_si512((__m512i const *) ptr + 0);;
__m512i d02 = _mm512_loadu_si512((__m512i const *) ptr + 1);;
__m512i d03 = _mm512_loadu_si512((__m512i const *) ptr + 2);;
__m512i d13 = _mm512_loadu_si512((__m512i const *) ptr + 12);;
__m512i d14 = _mm512_loadu_si512((__m512i const *) ptr + 13);;
__m512i d15 = _mm512_loadu_si512((__m512i const *) ptr + 14);;
- __m512i d16 = _mm512_loadu_si512((__m512i const *) ptr + 15);;
+ __m512i d16 = _mm512_mask_loadu_epi64(_mm512_set1_epi64(MAX),
+ mask,
+ (int64_t const *) ((__m512i const *) ptr + 15));
sort_16v_ascending(d01, d02, d03, d04, d05, d06, d07, d08, d09, d10, d11, d12, d13, d14, d15, d16);
_mm512_storeu_si512((__m512i *) ptr + 0, d01);
_mm512_storeu_si512((__m512i *) ptr + 1, d02);
_mm512_storeu_si512((__m512i *) ptr + 12, d13);
_mm512_storeu_si512((__m512i *) ptr + 13, d14);
_mm512_storeu_si512((__m512i *) ptr + 14, d15);
- _mm512_storeu_si512((__m512i *) ptr + 15, d16);
-}
+ _mm512_mask_storeu_epi64((__m512i *) ptr + 15, mask, d16);
+ }
static void sort(int64_t *ptr, size_t length);
};
#ifndef BITONIC_SORT_H
#define BITONIC_SORT_H
-#include <stdint.h>
#include "../defs.h"
#include "../machine_traits.h"
namespace vxsort {
namespace smallsort {
+using namespace std;
+
+// * We might read the last 4 bytes into a 128-bit vector for 64-bit element masking
+// * We might read the last 8 bytes into a 128-bit vector for 32-bit element masking
+// This mostly applies to debug mode, since without optimizations, most compilers
+// actually execute the instruction stream _mm256_cvtepi8_epiNN + _mm_loadu_si128 as they are given.
+// In contract, release/optimizing compilers, turn that very specific instruction pair to
+// a more reasonable: vpmovsxbq ymm0, dword [rax*4 + mask_table_4], eliminating the 128-bit
+// load completely and effectively reading 4/8 (depending if the instruction is vpmovsxb[q,d]
+#if !defined(__has_feature)
+#define __has_feature(a) (0)
+#endif
+#if !__has_feature(address_sanitizer)
+const int M4_SIZE = 16;
+const int M8_SIZE = 64;
+#else
+const int M4_SIZE = 16 + 12;
+const int M8_SIZE = 64 + 8;
+#endif
+
+extern "C" const uint8_t mask_table_4[M4_SIZE];
+extern "C" const uint8_t mask_table_8[M8_SIZE];
+
template <typename T, vector_machine M>
struct bitonic {
public:
static void sort(T* ptr, size_t length);
+ static void sort_alt(T* ptr, size_t length);
};
} // namespace smallsort
} // namespace gcsort
return f"_mm256_loadu_ps(({t} const *) ((__m256 const *) {v} + {offset}))"
return f"_mm256_lddqu_si256((__m256i const *) {v} + {offset});"
+ def get_mask_load_intrinsic(self, v, offset, mask):
+ t = self.type
+
+ if self.vector_size() == 4:
+ int_suffix = "epi64"
+ max_value = f"_mm256_andnot_si256({mask}, _mm256_set1_epi64x(MAX))"
+ elif self.vector_size() == 8:
+ int_suffix = "epi32"
+ max_value = f"_mm256_andnot_si256({mask}, _mm256_set1_epi32(MAX))"
+
+ if t == "double":
+ max_value = f"_mm256_andnot_pd(i2d(mask), _mm256_set1_pd(MAX))"
+ load = f"_mm256_maskload_pd(({t} const *) ((__m256d const *) {v} + {offset}), {mask})"
+ return f"_mm256_or_pd({load}, {max_value})"
+ if t == "float":
+ max_value = f"_mm256_andnot_ps(i2s(mask), _mm256_set1_ps(MAX))"
+ load = f"_mm256_maskload_ps(({t} const *) ((__m256 const *) {v} + {offset}), {mask})"
+ return f"_mm256_or_ps({load}, {max_value})"
+
+
+ if t == "int64_t" or t == "uint64_t":
+ it = "long long"
+ else:
+ it = t[1:] if t[0] == 'u' else t
+
+ load = f"_mm256_maskload_{int_suffix}(({it} const *) ((__m256i const *) {v} + {offset}), {mask})"
+ return f"_mm256_or_si256({load}, {max_value})"
def get_store_intrinsic(self, ptr, offset, value):
t = self.type
return f"_mm256_storeu_ps(({t} *) ((__m256 *) {ptr} + {offset}), {value})"
return f"_mm256_storeu_si256((__m256i *) {ptr} + {offset}, {value})"
+ def get_mask_store_intrinsic(self, ptr, offset, value, mask):
+ t = self.type
+
+ if self.vector_size() == 4:
+ int_suffix = "epi64"
+ elif self.vector_size() == 8:
+ int_suffix = "epi32"
+
+ if t == "double":
+ return f"_mm256_maskstore_pd(({t} *) ((__m256d *) {ptr} + {offset}), {mask}, {value})"
+ if t == "float":
+ return f"_mm256_maskstore_ps(({t} *) ((__m256 *) {ptr} + {offset}), {mask}, {value})"
+
+ if t == "int64_t" or t == "uint64_t":
+ it = "long long"
+ else:
+ it = t[1:] if t[0] == 'u' else t;
+ return f"_mm256_maskstore_{int_suffix}(({it} *) ((__m256i *) {ptr} + {offset}), {mask}, {value})"
+
+
def autogenerated_blabber(self):
return f"""/////////////////////////////////////////////////////////////////////////////
////
#endif
#endif
+#include <limits>
#include <immintrin.h>
#include "bitonic_sort.h"
namespace vxsort {{
namespace smallsort {{
+
+extern "C" const uint8_t mask_table_4[16];
+extern "C" const uint8_t mask_table_8[64];
+
template<> struct bitonic<{t}, AVX2> {{
+ static const int N = {self.vector_size()};
+ static constexpr {t} MAX = std::numeric_limits<{t}>::max();
public:
"""
print(s, file=f)
suffix = "ascending" if ascending else "descending"
rev_suffix = "descending" if ascending else "ascending"
-
+
inl = "INLINE" if inline else "NOINLINE"
s = f""" static {inl} void sort_{width:02d}v_merge_{suffix}({g.generate_param_def_list(width)}) {{
print(" }", file=f)
- def generate_entry_points(self, f):
+ def generate_entry_points_old(self, f):
type = self.type
g = self
for m in range(1, g.max_bitonic_sort_vectors() + 1):
s = f"""
- static NOINLINE void sort_{m:02d}v({type} *ptr) {{"""
+ static NOINLINE void sort_{m:02d}v_old({type} *ptr) {{"""
print(s, file=f)
for l in range(0, m):
s = f" {g.get_store_intrinsic('ptr', l, f'd{l + 1:02d}')};"
print(s, file=f)
- print("}", file=f)
+ print(" }", file=f)
+
+ def generate_entry_points(self, f):
+ type = self.type
+ g = self
+ for m in range(1, g.max_bitonic_sort_vectors() + 1):
+ s = f"""
+ static NOINLINE void sort_{m:02d}v_alt({type} *ptr, int remainder) {{
+ const auto mask = _mm256_cvtepi8_epi{int(256 / self.vector_size())}(_mm_loadu_si128((__m128i*)(mask_table_{self.vector_size()} + remainder * N)));
+"""
+ print(s, file=f)
+
+ for l in range(0, m-1):
+ s = f" {g.vector_type()} d{l + 1:02d} = {g.get_load_intrinsic('ptr', l)};"
+ print(s, file=f)
+ s = f" {g.vector_type()} d{m:02d} = {g.get_mask_load_intrinsic('ptr', m - 1, 'mask')};"
+ print(s, file=f)
+
+ s = f" sort_{m:02d}v_ascending({g.generate_param_list(1, m)});"
+ print(s, file=f)
+
+ for l in range(0, m-1):
+ s = f" {g.get_store_intrinsic('ptr', l, f'd{l + 1:02d}')};"
+ print(s, file=f)
+ s = f" {g.get_mask_store_intrinsic('ptr', m - 1, f'd{m:02d}', 'mask')};"
+ print(s, file=f)
+
+ print(" }", file=f)
def generate_master_entry_point(self, f_header, f_src):
t = self.type
g = self
+ # s = f""" static void sort_old({t} *ptr, size_t length);"""
+ # print(s, file=f_header)
+
s = f""" static void sort({t} *ptr, size_t length);"""
print(s, file=f_header)
- s = f"""void vxsort::smallsort::bitonic<{t}, vector_machine::AVX2 >::sort({t} *ptr, size_t length) {{
- const int N = {g.vector_size()};
- switch(length / N) {{"""
+ # s = f"""void vxsort::smallsort::bitonic<{t}, vector_machine::AVX2 >::sort({t} *ptr, size_t length) {{
+ # switch(length / N) {{"""
+ # print(s, file=f_src)
+ #
+ # for m in range(1, self.max_bitonic_sort_vectors() + 1):
+ # s = f" case {m}: sort_{m:02d}v(ptr); break;"
+ # print(s, file=f_src)
+ # print(" }", file=f_src)
+ # print("}", file=f_src)
+
+ s = f"""void vxsort::smallsort::bitonic<{t}, vector_machine::AVX2 >::sort({t} *ptr, size_t length) {{
+ const auto fullvlength = length / N;
+ const int remainder = (int) (length - fullvlength * N);
+ const auto v = fullvlength + ((remainder > 0) ? 1 : 0);
+ switch(v) {{"""
print(s, file=f_src)
for m in range(1, self.max_bitonic_sort_vectors() + 1):
- s = f" case {m}: sort_{m:02d}v(ptr); break;"
+ s = f" case {m}: sort_{m:02d}v_alt(ptr, remainder); break;"
print(s, file=f_src)
print(" }", file=f_src)
print("}", file=f_src)
+
pass
return f"_mm512_loadu_ps(({t} const *) ((__m512 const *) {v} + {offset}))"
return f"_mm512_loadu_si512((__m512i const *) {v} + {offset});"
+ def get_mask_load_intrinsic(self, v, offset, mask):
+ t = self.type
+
+ if self.vector_size() == 8:
+ int_suffix = "epi64"
+ max_value = f"_mm512_set1_epi64(MAX)"
+ elif self.vector_size() == 16:
+ int_suffix = "epi32"
+ max_value = f"_mm512_set1_epi32(MAX)"
+
+ if t == "double":
+ return f"""_mm512_mask_loadu_pd(_mm512_set1_pd(MAX),
+ {mask},
+ ({t} const *) ((__m512d const *) {v} + {offset}))"""
+ elif t == "float":
+ return f"""_mm512_mask_loadu_ps(_mm512_set1_ps(MAX),
+ {mask},
+ ({t} const *) ((__m512 const *) {v} + {offset}))"""
+
+ return f"""_mm512_mask_loadu_{int_suffix}({max_value},
+ {mask},
+ ({t} const *) ((__m512i const *) {v} + {offset}))"""
+
def get_store_intrinsic(self, ptr, offset, value):
t = self.type
return f"_mm512_storeu_ps(({t} *) ((__m512 *) {ptr} + {offset}), {value})"
return f"_mm512_storeu_si512((__m512i *) {ptr} + {offset}, {value})"
+ def get_mask_store_intrinsic(self, ptr, offset, value, mask):
+ t = self.type
+
+ if self.vector_size() == 8:
+ int_suffix = "epi64"
+ elif self.vector_size() == 16:
+ int_suffix = "epi32"
+
+ if t == "double":
+ return f"_mm512_mask_storeu_pd(({t} *) ((__m512d *) {ptr} + {offset}), {mask}, {value})"
+ if t == "float":
+ return f"_mm512_mask_storeu_ps(({t} *) ((__m512 *) {ptr} + {offset}), {mask}, {value})"
+ return f"_mm512_mask_storeu_{int_suffix}((__m512i *) {ptr} + {offset}, {mask}, {value})"
+
def autogenerated_blabber(self):
return f"""/////////////////////////////////////////////////////////////////////////////
////
#endif
#endif
+#include <limits>
#include <immintrin.h>
#include "bitonic_sort.h"
namespace vxsort {{
namespace smallsort {{
template<> struct bitonic<{t}, AVX512> {{
+ static const int N = {self.vector_size()};
+ static constexpr {t} MAX = std::numeric_limits<{t}>::max();
public:
"""
print(s, file=f)
print(" }", file=f)
- def generate_entry_points(self, f):
+ def generate_entry_points_old(self, f):
type = self.type
g = self
for m in range(1, g.max_bitonic_sort_vectors() + 1):
s = f"""
- static NOINLINE void sort_{m:02d}v({type} *ptr) {{"""
+ static NOINLINE void sort_{m:02d}v_old({type} *ptr) {{"""
print(s, file=f)
for l in range(0, m):
s = f" {g.get_store_intrinsic('ptr', l, f'd{l + 1:02d}')};"
print(s, file=f)
- print("}", file=f)
+ print(" }", file=f)
+
+ def generate_entry_points(self, f):
+ type = self.type
+ g = self
+ for m in range(1, g.max_bitonic_sort_vectors() + 1):
+ s = f"""
+ static NOINLINE void sort_{m:02d}v_alt({type} *ptr, int remainder) {{
+ const auto mask = 0x{((1 << self.vector_size()) - 1):X} >> ((N - remainder) & (N-1));
+"""
+ print(s, file=f)
+
+ for l in range(0, m-1):
+ s = f" {g.vector_type()} d{l + 1:02d} = {g.get_load_intrinsic('ptr', l)};"
+ print(s, file=f)
+ s = f" {g.vector_type()} d{m:02d} = {g.get_mask_load_intrinsic('ptr', m - 1, 'mask')};"
+ print(s, file=f)
+
+ s = f" sort_{m:02d}v_ascending({g.generate_param_list(1, m)});"
+ print(s, file=f)
+
+ for l in range(0, m-1):
+ s = f" {g.get_store_intrinsic('ptr', l, f'd{l + 1:02d}')};"
+ print(s, file=f)
+ s = f" {g.get_mask_store_intrinsic('ptr', m - 1, f'd{m:02d}', 'mask')};"
+ print(s, file=f)
+
+ print(" }", file=f)
def generate_master_entry_point(self, f_header, f_src):
t = self.type
g = self
+ # s = f""" static void sort_old({t} *ptr, size_t length);"""
+ # print(s, file=f_header)
+
s = f""" static void sort({t} *ptr, size_t length);"""
print(s, file=f_header)
- s = f"""void vxsort::smallsort::bitonic<{t}, vector_machine::AVX512 >::sort({t} *ptr, size_t length) {{
- const int N = {g.vector_size()};
- switch(length / N) {{"""
+ # s = f"""void vxsort::smallsort::bitonic<{t}, vector_machine::AVX512 >::sort_old({t} *ptr, size_t length) {{
+ # switch(length / N) {{"""
+ # print(s, file=f_src)
+ #
+ # for m in range(1, self.max_bitonic_sort_vectors() + 1):
+ # s = f" case {m}: sort_{m:02d}v(ptr); break;"
+ # print(s, file=f_src)
+ # print(" }", file=f_src)
+ # print("}", file=f_src)
+
+
+ s = f"""void vxsort::smallsort::bitonic<{t}, vector_machine::AVX512 >::sort({t} *ptr, size_t length) {{
+ const auto fullvlength = length / N;
+ const int remainder = (int) (length - fullvlength * N);
+ const auto v = fullvlength + ((remainder > 0) ? 1 : 0);
+ switch(v) {{"""
print(s, file=f_src)
for m in range(1, self.max_bitonic_sort_vectors() + 1):
- s = f" case {m}: sort_{m:02d}v(ptr); break;"
+ s = f" case {m}: sort_{m:02d}v_alt(ptr, remainder); break;"
print(s, file=f_src)
print(" }", file=f_src)
+
print("}", file=f_src)
pass
# usage: bitonic_gen.py [-h] [--vector-isa VECTOR_ISA [VECTOR_ISA ...]]
# [--break-inline BREAK_INLINE] [--output-dir OUTPUT_DIR]
#
-# the files in src/coreclr/src/gc/vxsort/smallsort checked in can be generated with:
-# python bitonic_gen.py --output-dir c:\temp --vector-isa AVX2 AVX512
+# the files in src/coreclr/src/gc/vxsort/smallsort that are currently checked in can be generated with:
+# python bitonic_gen.py --output-dir c:\temp --vector-isa AVX2 AVX512 --break-inline 4
#
import argparse
import os
for width in range(2, g.max_bitonic_sort_vectors() + 1):
# Allow breaking the inline chain once in a while (configurable)
- if break_inline == 0 or width & break_inline != 0:
+ if break_inline == 0 or width % break_inline != 0:
inline = True
else:
inline = False
g.generate_compounded_merger(f_header, width, ascending=True, inline=inline)
g.generate_compounded_merger(f_header, width, ascending=False, inline=inline)
+ #g.generate_entry_points_old(f_header)
g.generate_entry_points(f_header)
g.generate_master_entry_point(f_header, f_src)
g.generate_epilogue(f_header)
#include <assert.h>
#include <immintrin.h>
-
#include "defs.h"
-//#include "isa_detection.h"
#include "alignment.h"
#include "machine_traits.h"
+#ifdef VXSORT_STATS
+#include "vxsort_stats.h"
+#endif //VXSORT_STATS
+#include "packer.h"
#include "smallsort/bitonic_sort.h"
-//#include <algorithm>
-//#include <cstring>
-//#include <cstdint>
-
namespace vxsort {
using vxsort::smallsort::bitonic;
-
-template <typename T, vector_machine M, int Unroll=1>
+/**
+ * sort primitives, quickly
+ * @tparam T The primitive type being sorted
+ * @tparam M The vector machine model/ISA (e.g. AVX2, AVX512 etc.)
+ * @tparam Unroll The unroll factor, controls to some extent, the code-bloat/speedup ration at the call site
+ * Defaults to 1
+ * @tparam Shift Optional; specify how many LSB bits are known to be zero in the original input. Can be used
+ * to further speed up sorting.
+ */
+template <typename T, vector_machine M, int Unroll=1, int Shift=0>
class vxsort {
static_assert(Unroll >= 1, "Unroll can be in the range 1..12");
static_assert(Unroll <= 12, "Unroll can be in the range 1..12");
private:
using MT = vxsort_machine_traits<T, M>;
typedef typename MT::TV TV;
+ typedef typename MT::TPACK TPACK;
typedef alignment_hint<sizeof(TV)> AH;
static const int ELEMENT_ALIGN = sizeof(T) - 1;
static const int PARTITION_TMP_SIZE_IN_ELEMENTS =
(2 * SLACK_PER_SIDE_IN_ELEMENTS + N + 4*N);
+ void reset(T* start, T* end) {
+ _depth = 0;
+ _startPtr = start;
+ _endPtr = end;
+ }
+
+ T* _startPtr = nullptr;
+ T* _endPtr = nullptr;
+
+ T _temp[PARTITION_TMP_SIZE_IN_ELEMENTS];
+ int _depth = 0;
+
static int floor_log2_plus_one(size_t n) {
auto result = 0;
while (n >= 1) {
swap(left, right);
}
- static void insertion_sort(T* lo, T* hi) {
- for (auto i = lo + 1; i <= hi; i++) {
- auto j = i;
- auto t = *i;
- while ((j > lo) && (t < *(j - 1))) {
- *j = *(j - 1);
- j--;
- }
- *j = t;
- }
- }
-
static void heap_sort(T* lo, T* hi) {
size_t n = hi - lo + 1;
for (size_t i = n / 2; i >= 1; i--) {
*(lo + i - 1) = d;
}
- void reset(T* start, T* end) {
- _depth = 0;
- _startPtr = start;
- _endPtr = end;
- }
-
- T* _startPtr = nullptr;
- T* _endPtr = nullptr;
-
- T _temp[PARTITION_TMP_SIZE_IN_ELEMENTS];
- int _depth = 0;
-
NOINLINE
T* align_left_scalar_uncommon(T* read_left, T pivot,
T*& tmp_left, T*& tmp_right) {
return readRight;
}
- void sort(T* left, T* right, AH realignHint,
- int depthLimit) {
+ void sort(T* left, T* right, T left_hint, T right_hint, AH realignHint,
+ int depth_limit) {
auto length = (size_t)(right - left + 1);
T* mid;
// Go to insertion sort below this threshold
if (length <= SMALL_SORT_THRESHOLD_ELEMENTS) {
-
- auto nextLength = (length & (N-1)) > 0 ? (length + N) & ~(N-1) : length;
-
- auto extraSpaceNeeded = nextLength - length;
- auto fakeLeft = left - extraSpaceNeeded;
- if (fakeLeft >= _startPtr) {
- bitonic<T, M>::sort(fakeLeft, nextLength);
- } else {
- insertion_sort(left, right);
- }
+#ifdef VXSORT_STATS
+ vxsort_stats<T>::bump_small_sorts();
+ vxsort_stats<T>::record_small_sort_size(length);
+#endif
+ bitonic<T, M>::sort(left, length);
return;
}
// will not do well:
// 1. Reverse sorted array
// 2. High degree of repeated values (dutch flag problem, one value)
- if (depthLimit == 0) {
+ if (depth_limit == 0) {
heap_sort(left, right);
_depth--;
return;
}
- depthLimit--;
+
+ depth_limit--;
+
+
+ if (MT::supports_packing()) {
+ if (MT::template can_pack<Shift>(right_hint - left_hint)) {
+ packer<T, TPACK, M, Shift, 2, SMALL_SORT_THRESHOLD_ELEMENTS>::pack(left, length, left_hint);
+ auto packed_sorter = vxsort<TPACK, M, Unroll>();
+ packed_sorter.sort((TPACK *) left, ((TPACK *) left) + length - 1);
+ packer<T, TPACK, M, Shift, 2, SMALL_SORT_THRESHOLD_ELEMENTS>::unpack((TPACK *) left, length, left_hint);
+ return;
+ }
+ }
// This is going to be a bit weird:
// Pre/Post alignment calculations happen here: we prepare hints to the
vectorized_partition<SafeInnerUnroll>(left, right, realignHint) :
vectorized_partition<Unroll>(left, right, realignHint);
-
-
_depth++;
- sort(left, sep - 2, realignHint.realign_right(), depthLimit);
- sort(sep, right, realignHint.realign_left(), depthLimit);
+ sort(left, sep - 2, left_hint, *sep, realignHint.realign_right(), depth_limit);
+ sort(sep, right, *(sep - 2), right_hint, realignHint.realign_left(), depth_limit);
_depth--;
}
const TV& P,
T*& left,
T*& right) {
+#ifdef VXSORT_STATS
+ vxsort_stats<T>::bump_vec_loads();
+ vxsort_stats<T>::bump_vec_stores(2);
+#endif
if (MT::supports_compress_writes()) {
partition_block_with_compress(dataVec, P, left, right);
} else {
const TV& P,
T*& left,
T*& right) {
+#ifdef VXSORT_STATS
+ vxsort_stats<T>::bump_perms();
+#endif
auto mask = MT::get_cmpgt_mask(dataVec, P);
dataVec = MT::partition_vector(dataVec, mask);
MT::store_vec(reinterpret_cast<TV*>(left), dataVec);
assert((reinterpret_cast<size_t>(left) & ELEMENT_ALIGN) == 0);
assert((reinterpret_cast<size_t>(right) & ELEMENT_ALIGN) == 0);
+#ifdef VXSORT_STATS
+ vxsort_stats<T>::bump_partitions((right - left) + 1);
+#endif
+
// Vectorized double-pumped (dual-sided) partitioning:
// We start with picking a pivot using the media-of-3 "method"
// Once we have sensible pivot stored as the last element of the array
*writeLeft++ = pivot;
assert(writeLeft > left);
- assert(writeLeft <= right);
+ assert(writeLeft <= right+1);
return writeLeft;
}
const auto preAlignedLeft = (TV*) (left + leftAlign);
const auto preAlignedRight = (TV*) (right + rightAlign - N);
+#ifdef VXSORT_STATS
+ vxsort_stats<T>::bump_vec_loads(2);
+ vxsort_stats<T>::bump_vec_stores(4);
+#endif
+
// Alignment with vectorization is tricky, so read carefully before changing code:
// 1. We load data, which we might need to align, if the alignment hints
// mean pre-alignment (or overlapping alignment)
tmpStartRight -= rightAlign & rai;
}
else {
+#ifdef VXSORT_STATS
+ vxsort_stats<T>::bump_perms(2);
+#endif
RT0 = MT::partition_vector(RT0, rtMask);
LT0 = MT::partition_vector(LT0, ltMask);
MT::store_vec((TV*) tmpRight, RT0);
}
public:
- NOINLINE void sort(T* left, T* right) {
+ /**
+ * Sort a given range
+ * @param left The left edge of the range, including
+ * @param right The right edge of the range, including
+ * @param left_hint Optional; A hint, Use to speed up the sorting operation, describing a single value that is known to be
+ * smaller-than, or equalt to all values contained within the provided array.
+ * @param right_hint Optional; A hint, Use to speed up the sorting operation, describing a single value that is known to be
+ * larger-than than all values contained within the provided array.
+ */
+ NOINLINE void sort(T* left, T* right,
+ T left_hint = std::numeric_limits<T>::Min(),
+ T right_hint = std::numeric_limits<T>::Max())
+ {
+// init_isa_detection();
+
+#ifdef VXSORT_STATS
+ vxsort_stats<T>::bump_sorts((right - left) + 1);
+#endif
reset(left, right);
auto depthLimit = 2 * floor_log2_plus_one(right + 1 - left);
- sort(left, right, AH(), depthLimit);
+ sort(left, right, left_hint, right_hint, AH(), depthLimit);
}
};
#ifdef __GNUC__
#ifdef __clang__
-#pragma clang attribute push (__attribute__((target("avx512f"))), apply_to = any(function))
+#pragma clang attribute push (__attribute__((target("avx512f,avx512dq"))), apply_to = any(function))
#else
#pragma GCC push_options
-#pragma GCC target("avx512f")
+#pragma GCC target("avx512f,avx512dq")
#endif
#endif
../gc/vxsort/smallsort/bitonic_sort.AVX2.int32_t.generated.cpp
../gc/vxsort/smallsort/bitonic_sort.AVX512.int64_t.generated.cpp
../gc/vxsort/smallsort/bitonic_sort.AVX512.int32_t.generated.cpp
+ ../gc/vxsort/smallsort/avx2_load_mask_tables.cpp
)
endif (CLR_CMAKE_TARGET_ARCH_AMD64 AND CLR_CMAKE_TARGET_WIN32)
projects / platform / upstream / dotnet / runtime.git / commitdiff