Skip to content

A
abseil-cpp
Commit c2e9ce1d by Derek Mauro Committed by Copybara-Service
Options

CRC: Get CPU detection and hardware acceleration working on MSVC x86(_64) 

Using /arch:AVX on MSVC now uses the accelerated implementation

PiperOrigin-RevId: 490550573
Change-Id: I924259845f38ee41d15f23f95ad085ad664642b5
parent 4c5eb49d
Showing with 111 additions and 74 deletions
absl/crc/internal/cpu_detect.cc
......@@ -24,30 +24,29 @@
#include <sys/auxv.h>
#endif
#if defined(_WIN32) || defined(_WIN64)
#include <intrin.h>
#endif
namespace absl {
ABSL_NAMESPACE_BEGIN
namespace crc_internal {
#if defined(__x86_64__)
// Inline cpuid instruction. %rbx is occasionally used to address stack
// variables in presence of dynamic allocas. Preserve the %rbx register via
// %rdi to work around a clang bug https://bugs.llvm.org/show_bug.cgi?id=17907
// (%rbx in an output constraint is not considered a clobbered register).
//
// a_inp and c_inp are the input parameters eax and ecx of the CPUID
// instruction.
// a, b, c, and d contain the contents of eax, ebx, ecx, and edx as returned by
// the CPUID instruction
#define ABSL_INTERNAL_GETCPUID(a, b, c, d, a_inp, c_inp) \
asm("mov %%rbx, %%rdi\n" \
"cpuid\n" \
"xchg %%rdi, %%rbx\n" \
: "=a"(a), "=D"(b), "=c"(c), "=d"(d) \
: "a"(a_inp), "2"(c_inp))
#if defined(__x86_64__) || defined(_M_X64)
namespace {
#if !defined(_WIN32) && !defined(_WIN64)
// MSVC defines this function for us.
// https://learn.microsoft.com/en-us/cpp/intrinsics/cpuid-cpuidex
static void __cpuid(int cpu_info[4], int info_type) {
__asm__ volatile("cpuid \n\t"
: "=a"(cpu_info[0]), "=b"(cpu_info[1]), "=c"(cpu_info[2]),
"=d"(cpu_info[3])
: "a"(info_type), "c"(0));
}
#endif // !defined(_WIN32) && !defined(_WIN64)
enum class Vendor {
kUnknown,
kIntel,
......@@ -55,14 +54,14 @@ enum class Vendor {
};
Vendor GetVendor() {
uint32_t eax, ebx, ecx, edx;
// Get the vendor string (issue CPUID with eax = 0).
int cpu_info[4];
__cpuid(cpu_info, 0);
// Get vendor string (issue CPUID with eax = 0)
ABSL_INTERNAL_GETCPUID(eax, ebx, ecx, edx, 0, 0);
std::string vendor;
vendor.append(reinterpret_cast<char*>(&ebx), 4);
vendor.append(reinterpret_cast<char*>(&edx), 4);
vendor.append(reinterpret_cast<char*>(&ecx), 4);
vendor.append(reinterpret_cast<char*>(&cpu_info[1]), 4);
vendor.append(reinterpret_cast<char*>(&cpu_info[3]), 4);
vendor.append(reinterpret_cast<char*>(&cpu_info[2]), 4);
if (vendor == "GenuineIntel") {
return Vendor::kIntel;
} else if (vendor == "AuthenticAmd") {
......@@ -73,13 +72,14 @@ Vendor GetVendor() {
}
CpuType GetIntelCpuType() {
uint32_t eax, ebx, ecx, edx;
// to get general information and extended features we send eax = 1 and
// To get general information and extended features we send eax = 1 and
// ecx = 0 to cpuid. The response is returned in eax, ebx, ecx and edx.
// (See Intel 64 and IA-32 Architectures Software Developer's Manual
// Volume 2A: Instruction Set Reference, A-M CPUID).
// https://www.intel.com/content/www/us/en/architecture-and-technology/64-ia-32-architectures-software-developer-vol-2a-manual.html
ABSL_INTERNAL_GETCPUID(eax, ebx, ecx, edx, 1, 0);
// https://learn.microsoft.com/en-us/cpp/intrinsics/cpuid-cpuidex
int cpu_info[4];
__cpuid(cpu_info, 1);
// Response in eax bits as follows:
// 0-3 (stepping id)
......@@ -89,12 +89,12 @@ CpuType GetIntelCpuType() {
// 16-19 (extended model)
// 20-27 (extended family)
int family = (eax >> 8) & 0x0f;
int model_num = (eax >> 4) & 0x0f;
int ext_family = (eax >> 20) & 0xff;
int ext_model_num = (eax >> 16) & 0x0f;
int family = (cpu_info[0] >> 8) & 0x0f;
int model_num = (cpu_info[0] >> 4) & 0x0f;
int ext_family = (cpu_info[0] >> 20) & 0xff;
int ext_model_num = (cpu_info[0] >> 16) & 0x0f;
int brand_id = ebx & 0xff;
int brand_id = cpu_info[1] & 0xff;
// Process the extended family and model info if necessary
if (family == 0x0f) {
......@@ -123,7 +123,7 @@ CpuType GetIntelCpuType() {
case 0x56: // BroadwellDE
return CpuType::kIntelBroadwell;
case 0x55: // Skylake Xeon
if ((eax & 0x0f) < 5) { // stepping < 5 is skylake
if ((cpu_info[0] & 0x0f) < 5) { // stepping < 5 is skylake
return CpuType::kIntelSkylakeXeon;
} else { // stepping >= 5 is cascadelake
return CpuType::kIntelCascadelakeXeon;
......@@ -142,12 +142,13 @@ CpuType GetIntelCpuType() {
}
CpuType GetAmdCpuType() {
uint32_t eax, ebx, ecx, edx;
// to get general information and extended features we send eax = 1 and
// To get general information and extended features we send eax = 1 and
// ecx = 0 to cpuid. The response is returned in eax, ebx, ecx and edx.
// (See Intel 64 and IA-32 Architectures Software Developer's Manual
// Volume 2A: Instruction Set Reference, A-M CPUID).
ABSL_INTERNAL_GETCPUID(eax, ebx, ecx, edx, 1, 0);
// https://learn.microsoft.com/en-us/cpp/intrinsics/cpuid-cpuidex
int cpu_info[4];
__cpuid(cpu_info, 1);
// Response in eax bits as follows:
// 0-3 (stepping id)
......@@ -157,10 +158,10 @@ CpuType GetAmdCpuType() {
// 16-19 (extended model)
// 20-27 (extended family)
int family = (eax >> 8) & 0x0f;
int model_num = (eax >> 4) & 0x0f;
int ext_family = (eax >> 20) & 0xff;
int ext_model_num = (eax >> 16) & 0x0f;
int family = (cpu_info[0] >> 8) & 0x0f;
int model_num = (cpu_info[0] >> 4) & 0x0f;
int ext_family = (cpu_info[0] >> 20) & 0xff;
int ext_model_num = (cpu_info[0] >> 16) & 0x0f;
if (family == 0x0f) {
family += ext_family;
......
absl/crc/internal/crc32_x86_arm_combined_simd.h
......@@ -25,12 +25,18 @@
// We define a translation layer for both x86 and ARM for the ease of use and
// most performance gains.
// We need CRC (part of sse4.2) and PCLMULQDQ instructions.
// We need CRC (part of SSE 4.2) and PCLMULQDQ instructions.
#if defined(__SSE4_2__) && defined(__PCLMUL__)
#include <x86intrin.h>
#define ABSL_CRC_INTERNAL_HAVE_X86_SIMD
#elif defined(_MSC_VER) && defined(__AVX__)
// MSVC AVX (/arch:AVX) implies SSE 4.2 and PCLMULQDQ.
#include <intrin.h>
#define ABSL_CRC_INTERNAL_HAVE_X86_SIMD
#elif defined(__aarch64__) && defined(__LITTLE_ENDIAN__) && \
defined(__ARM_FEATURE_CRC32) && defined(__ARM_NEON)
......
absl/crc/internal/crc_memcpy_x86_64.cc
......@@ -38,24 +38,27 @@
// using 3 CRCs over fixed-size blocks where the zero-extensions required for
// CRC32C::Concat can be precomputed.
#include <cstddef>
#include <cstdint>
#include "absl/crc/crc32c.h"
#include "absl/strings/string_view.h"
#ifdef __SSE4_2__
#include <immintrin.h>
#endif
#include <emmintrin.h>
#include <x86intrin.h>
#ifdef _MSC_VER
#include <intrin.h>
#endif
#include <cstddef>
#include <cstdint>
#include <type_traits>
#include "absl/base/dynamic_annotations.h"
#include "absl/base/internal/prefetch.h"
#include "absl/base/optimization.h"
#include "absl/crc/crc32c.h"
#include "absl/crc/internal/cpu_detect.h"
#include "absl/crc/internal/crc_memcpy.h"
#include "absl/strings/string_view.h"
#if defined(__SSE4_2__) || (defined(_MSC_VER) && defined(__AVX__))
namespace absl {
ABSL_NAMESPACE_BEGIN
......@@ -88,7 +91,9 @@ inline void LargeTailCopy(crc32c_t* crcs, char** dst, const char** src,
uint64_t int_data[kIntLoadsPerVec * int_regions];
while (copy_rounds > 0) {
#ifdef __GNUC__
#pragma unroll_completely
#endif
for (int i = 0; i < vec_regions; i++) {
int region = i;
......@@ -109,7 +114,9 @@ inline void LargeTailCopy(crc32c_t* crcs, char** dst, const char** src,
_mm_extract_epi64(data[i], 1)));
}
#ifdef __GNUC__
#pragma unroll_completely
#endif
for (int i = 0; i < int_regions; i++) {
int region = vec_regions + i;
......@@ -117,7 +124,9 @@ inline void LargeTailCopy(crc32c_t* crcs, char** dst, const char** src,
reinterpret_cast<const uint64_t*>(*src + region_size * region);
auto* udst = reinterpret_cast<uint64_t*>(*dst + region_size * region);
#ifdef __GNUC__
#pragma unroll_completely
#endif
for (int j = 0; j < kIntLoadsPerVec; j++) {
int data_index = i * kIntLoadsPerVec + j;
......@@ -238,7 +247,9 @@ crc32c_t AcceleratedCrcMemcpyEngine<vec_regions, int_regions>::Compute(
// Main loop.
while (copy_rounds > kBlocksPerCacheLine) {
// Prefetch kPrefetchAhead bytes ahead of each pointer.
#ifdef __GNUC__
#pragma unroll_completely
#endif
for (int i = 0; i < kRegions; i++) {
absl::base_internal::PrefetchT0(src_bytes + kPrefetchAhead +
region_size * i);
......@@ -247,10 +258,14 @@ crc32c_t AcceleratedCrcMemcpyEngine<vec_regions, int_regions>::Compute(
}
// Load and store data, computing CRC on the way.
#ifdef __GNUC__
#pragma unroll_completely
#endif
for (int i = 0; i < kBlocksPerCacheLine; i++) {
// Copy and CRC the data for the CRC regions.
#ifdef __GNUC__
#pragma unroll_completely
#endif
for (int j = 0; j < vec_regions; j++) {
// Cycle which regions get vector load/store and integer load/store, to
// engage prefetching logic around vector load/stores and save issue
......@@ -276,7 +291,9 @@ crc32c_t AcceleratedCrcMemcpyEngine<vec_regions, int_regions>::Compute(
}
// Preload the partial CRCs for the CLMUL subregions.
#ifdef __GNUC__
#pragma unroll_completely
#endif
for (int j = 0; j < int_regions; j++) {
// Cycle which regions get vector load/store and integer load/store, to
// engage prefetching logic around vector load/stores and save issue
......@@ -288,7 +305,9 @@ crc32c_t AcceleratedCrcMemcpyEngine<vec_regions, int_regions>::Compute(
auto* udst =
reinterpret_cast<uint64_t*>(dst_bytes + region_size * region);
#ifdef __GNUC__
#pragma unroll_completely
#endif
for (int k = 0; k < kIntLoadsPerVec; k++) {
int data_index = j * kIntLoadsPerVec + k;
......@@ -432,4 +451,4 @@ std::unique_ptr<CrcMemcpyEngine> CrcMemcpy::GetTestEngine(int vector,
ABSL_NAMESPACE_END
} // namespace absl
#endif // __SSE4_2__
#endif // defined(__SSE4_2__) || (defined(_MSC_VER) && defined(__AVX__))
absl/crc/internal/crc_x86_arm_combined.cc
......@@ -35,6 +35,9 @@
#define ABSL_INTERNAL_CAN_USE_SIMD_CRC32C
#elif defined(__SSE4_2__) && defined(__PCLMUL__)
#define ABSL_INTERNAL_CAN_USE_SIMD_CRC32C
#elif defined(_MSC_VER) && defined(__AVX__)
// MSVC AVX support (/arch:AVX) implies SSE 4.2 and PCLMUL support.
#define ABSL_INTERNAL_CAN_USE_SIMD_CRC32C
#endif
namespace absl {
......
absl/crc/internal/non_temporal_memcpy.h
......@@ -15,46 +15,56 @@
#ifndef ABSL_CRC_INTERNAL_NON_TEMPORAL_MEMCPY_H_
#define ABSL_CRC_INTERNAL_NON_TEMPORAL_MEMCPY_H_
#include <algorithm>
#include <cassert>
#include <cstring>
#include <iostream>
#include "absl/base/config.h"
#include "absl/base/optimization.h"
#ifdef _MSC_VER
#include <intrin.h>
#endif
#ifdef __SSE__
// Only include if we're running on a CPU that supports SSE ISA, needed for
// sfence
#include <immintrin.h> // IWYU pragma: keep
#include <xmmintrin.h>
#endif
#ifdef __SSE2__
// Only include if we're running on a CPU that supports SSE2 ISA, needed for
// movdqa, movdqu, movntdq
#include <emmintrin.h> // IWYU pragma: keep
#include <emmintrin.h>
#endif
#ifdef __SSE3__
#include <pmmintrin.h>
#endif
#ifdef __AVX__
#include <immintrin.h>
#endif
#ifdef __aarch64__
// Only include if we're running on a CPU that supports ARM NEON ISA, needed for
// sfence, movdqa, movdqu, movntdq
#include "absl/crc/internal/non_temporal_arm_intrinsics.h"
#endif
#include <algorithm>
#include <cassert>
#include <cstring>
#include <iostream>
#include "absl/base/config.h"
#include "absl/base/optimization.h"
namespace absl {
ABSL_NAMESPACE_BEGIN
namespace crc_internal {
// This non-temporal memcpy does regular load and non-temporal store memory
// copy. It is compatible to both 16-byte aligned and unaligned addresses. If
// data at the destination is not immediately accessed, using non-temporal
// memcpy can save 1 DRAM load of the destination cacheline.
constexpr int kCacheLineSize = ABSL_CACHELINE_SIZE;
constexpr size_t kCacheLineSize = ABSL_CACHELINE_SIZE;
// If the objects overlap, the behavior is undefined.
// MSVC does not have proper header support for some of these intrinsics,
// so it should go to fallback
inline void *non_temporal_store_memcpy(void *__restrict dst,
const void *__restrict src, size_t len) {
#if (defined(__SSE3__) || defined(__aarch64__)) && !defined(_MSC_VER)
#if defined(__SSE3__) || defined(__aarch64__) || \
(defined(_MSC_VER) && defined(__AVX__))
// This implementation requires SSE3.
// MSVC cannot target SSE3 directly, but when MSVC targets AVX,
// SSE3 support is implied.
uint8_t *d = reinterpret_cast<uint8_t *>(dst);
const uint8_t *s = reinterpret_cast<const uint8_t *>(src);
......@@ -104,17 +114,15 @@ inline void *non_temporal_store_memcpy(void *__restrict dst,
}
return dst;
#else
// Fallback to regular memcpy when SSE2/3 & aarch64 is not available.
// Fallback to regular memcpy.
return memcpy(dst, src, len);
#endif // __SSE3__ || __aarch64__
#endif // __SSE3__ || __aarch64__ || (_MSC_VER && __AVX__)
}
// MSVC does not have proper header support for some of these intrinsics,
// so it should go to fallback
inline void *non_temporal_store_memcpy_avx(void *__restrict dst,
const void *__restrict src,
size_t len) {
#if defined(__AVX__) && !defined(_MSC_VER)
#ifdef __AVX__
uint8_t *d = reinterpret_cast<uint8_t *>(dst);
const uint8_t *s = reinterpret_cast<const uint8_t *>(src);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment