andrew
6fed133212
Move SIMD operations on potentially-indeterminate Node16::index bytes into file-level assembly, where loading and operating on indeterminate values is well-defined (unlike C++). Restructure scalar fallback loops to iterate [0, numChildren) instead of [0, kMaxNodes). Fix TrivialSpan construction from indeterminate pointers in check::Job::init and insertPointWritesOrSorted to only construct when end.len > 0. Add MSan toolchain to the debug CI build to catch these issues going forward.
74 lines
3.0 KiB
ArmAsm
74 lines
3.0 KiB
ArmAsm
// SIMD operations on potentially-indeterminate Node16::index[16] bytes.
|
|
// Written in assembly so msan doesn't track the loads. The caller is
|
|
// responsible for masking the returned bitfield to [0, numChildren) before
|
|
// using it.
|
|
//
|
|
// All functions return a 16-bit bitmask in %eax (bit i set = condition true
|
|
// at index i). The upper 16 bits of %eax are zero.
|
|
//
|
|
// System V AMD64 ABI:
|
|
// %rdi = const uint8_t *idx (16 bytes)
|
|
// %esi = uint8_t key (find_eq_16, find_ge_16)
|
|
// %sil = uint8_t begin (mask_in_range_16)
|
|
// %dl = uint8_t end (mask_in_range_16)
|
|
|
|
.text
|
|
|
|
// uint32_t find_eq_16(const uint8_t idx[16], uint8_t key)
|
|
// Returns bit i set if idx[i] == key
|
|
.globl find_eq_16
|
|
.type find_eq_16, @function
|
|
find_eq_16:
|
|
vmovd %esi, %xmm1 // broadcast key
|
|
vpbroadcastb %xmm1, %xmm1
|
|
vmovdqu (%rdi), %xmm0 // load 16 bytes (may contain indeterminate data)
|
|
vpcmpeqb %xmm0, %xmm1, %xmm0 // 0xff for each match
|
|
vpmovmskb %xmm0, %eax // 16-bit bitmask
|
|
movzwl %ax, %eax // zero-extend to 32 bits
|
|
ret
|
|
.size find_eq_16, .-find_eq_16
|
|
|
|
// uint32_t find_ge_16(const uint8_t idx[16], uint8_t child)
|
|
// Returns bit i set if idx[i] >= child
|
|
// x86 doesn't have a "compare unsigned >=" for bytes directly, so we use:
|
|
// min(key, idx[i]) == key iff idx[i] >= key
|
|
.globl find_ge_16
|
|
.type find_ge_16, @function
|
|
find_ge_16:
|
|
vmovd %esi, %xmm1
|
|
vpbroadcastb %xmm1, %xmm1 // key broadcast
|
|
vmovdqu (%rdi), %xmm0 // load 16 bytes
|
|
vpminub %xmm0, %xmm1, %xmm2 // min(key, idx[i])
|
|
vpcmpeqb %xmm2, %xmm1, %xmm0 // 0xff where min == key, i.e. idx[i] >= key
|
|
vpmovmskb %xmm0, %eax
|
|
movzwl %ax, %eax
|
|
ret
|
|
.size find_ge_16, .-find_ge_16
|
|
|
|
// uint32_t mask_in_range_16(const uint8_t idx[16], uint8_t begin, uint8_t end)
|
|
// Returns bit i set if begin <= idx[i] < end
|
|
// Logic: (idx[i] - begin) < (end - begin) [unsigned wrapping arithmetic]
|
|
// Equivalently: idx[i] - begin != max(idx[i] - begin, end - begin)
|
|
// i.e. idx[i] - begin is NOT equal to the saturated value.
|
|
// We compute: sub = idx - begin; result = (sub < (end-begin)) for each byte.
|
|
// Using: sub == max(sub, end-begin) means NOT in range.
|
|
// So: in_range = ~(movemask(cmpeq(sub, max(sub, range_size))))
|
|
.globl mask_in_range_16
|
|
.type mask_in_range_16, @function
|
|
mask_in_range_16:
|
|
vmovd %esi, %xmm1 // begin
|
|
vpbroadcastb %xmm1, %xmm1
|
|
vmovd %edx, %xmm2 // end
|
|
vpbroadcastb %xmm2, %xmm2
|
|
vmovdqu (%rdi), %xmm0 // load 16 bytes
|
|
vpsubb %xmm1, %xmm0, %xmm0 // idx - begin (wrapping)
|
|
vpsubb %xmm1, %xmm2, %xmm2 // end - begin (range size)
|
|
vpmaxub %xmm0, %xmm2, %xmm3 // max(idx-begin, range_size)
|
|
vpcmpeqb %xmm3, %xmm0, %xmm0 // 0xff where NOT in range
|
|
vpmovmskb %xmm0, %eax
|
|
not %eax // invert: 1 = in range
|
|
movzwl %ax, %eax
|
|
ret
|
|
.size mask_in_range_16, .-mask_in_range_16
|
|
|
|
.section .note.GNU-stack,"",@progbits
|