27 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__MMX__) 31 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__SSE__) 35 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__SSE2__) 39 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__SSE3__) 43 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__SSSE3__) 47 #if !defined(_MSC_VER) || __has_feature(modules) || \ 48 (defined(__SSE4_2__) || defined(__SSE4_1__)) 52 #if !defined(_MSC_VER) || __has_feature(modules) || \ 53 (defined(__AES__) || defined(__PCLMUL__)) 57 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__CLFLUSHOPT__) 61 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__CLWB__) 65 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__AVX__) 69 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__AVX2__) 73 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__F16C__) 77 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__VPCLMULQDQ__) 81 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__BMI__) 85 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__BMI2__) 89 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__LZCNT__) 93 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__POPCNT__) 97 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__FMA__) 101 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__AVX512F__) 105 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__AVX512VL__) 109 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__AVX512BW__) 113 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__AVX512BITALG__) 117 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__AVX512CD__) 121 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__AVX512VPOPCNTDQ__) 125 #if !defined(_MSC_VER) || __has_feature(modules) || \ 126 (defined(__AVX512VL__) && defined(__AVX512VPOPCNTDQ__)) 130 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__AVX512VNNI__) 134 #if !defined(_MSC_VER) || __has_feature(modules) || \ 135 (defined(__AVX512VL__) && defined(__AVX512VNNI__)) 139 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__AVX512DQ__) 143 #if !defined(_MSC_VER) || __has_feature(modules) || \ 144 (defined(__AVX512VL__) && defined(__AVX512BITALG__)) 148 #if !defined(_MSC_VER) || __has_feature(modules) || \ 149 (defined(__AVX512VL__) && defined(__AVX512BW__)) 153 #if !defined(_MSC_VER) || __has_feature(modules) || \ 154 (defined(__AVX512VL__) && defined(__AVX512CD__)) 158 #if !defined(_MSC_VER) || __has_feature(modules) || \ 159 (defined(__AVX512VL__) && defined(__AVX512DQ__)) 163 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__AVX512ER__) 167 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__AVX512IFMA__) 171 #if !defined(_MSC_VER) || __has_feature(modules) || \ 172 (defined(__AVX512IFMA__) && defined(__AVX512VL__)) 176 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__AVX512VBMI__) 180 #if !defined(_MSC_VER) || __has_feature(modules) || \ 181 (defined(__AVX512VBMI__) && defined(__AVX512VL__)) 185 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__AVX512VBMI2__) 189 #if !defined(_MSC_VER) || __has_feature(modules) || \ 190 (defined(__AVX512VBMI2__) && defined(__AVX512VL__)) 194 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__AVX512PF__) 198 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__PKU__) 202 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__VAES__) 206 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__GFNI__) 210 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__RDPID__) 216 static __inline__
unsigned int __attribute__((__always_inline__, __nodebug__, __target__(
"rdpid")))
218 return __builtin_ia32_rdpid();
222 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__RDRND__) 223 static __inline__
int __attribute__((__always_inline__, __nodebug__, __target__(
"rdrnd")))
224 _rdrand16_step(
unsigned short *
__p)
226 return __builtin_ia32_rdrand16_step(__p);
229 static __inline__
int __attribute__((__always_inline__, __nodebug__, __target__(
"rdrnd")))
230 _rdrand32_step(
unsigned int *
__p)
232 return __builtin_ia32_rdrand32_step(__p);
236 static __inline__
int __attribute__((__always_inline__, __nodebug__, __target__(
"rdrnd")))
237 _rdrand64_step(
unsigned long long *__p)
239 return __builtin_ia32_rdrand64_step(__p);
246 _bit_scan_forward(
int __A) {
247 return __builtin_ctz(__A);
251 static __inline__
int __attribute__((__always_inline__, __nodebug__))
252 _bit_scan_reverse(
int __A) {
253 return 31 - __builtin_clz(__A);
256 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__FSGSBASE__) 258 static __inline__
unsigned int __attribute__((__always_inline__, __nodebug__, __target__(
"fsgsbase")))
259 _readfsbase_u32(
void)
261 return __builtin_ia32_rdfsbase32();
264 static __inline__
unsigned long long __attribute__((__always_inline__, __nodebug__, __target__(
"fsgsbase")))
265 _readfsbase_u64(
void)
267 return __builtin_ia32_rdfsbase64();
270 static __inline__
unsigned int __attribute__((__always_inline__, __nodebug__, __target__(
"fsgsbase")))
271 _readgsbase_u32(
void)
273 return __builtin_ia32_rdgsbase32();
276 static __inline__
unsigned long long __attribute__((__always_inline__, __nodebug__, __target__(
"fsgsbase")))
277 _readgsbase_u64(
void)
279 return __builtin_ia32_rdgsbase64();
282 static __inline__
void __attribute__((__always_inline__, __nodebug__, __target__(
"fsgsbase")))
283 _writefsbase_u32(
unsigned int __V)
285 __builtin_ia32_wrfsbase32(__V);
288 static __inline__
void __attribute__((__always_inline__, __nodebug__, __target__(
"fsgsbase")))
289 _writefsbase_u64(
unsigned long long __V)
291 __builtin_ia32_wrfsbase64(__V);
294 static __inline__
void __attribute__((__always_inline__, __nodebug__, __target__(
"fsgsbase")))
295 _writegsbase_u32(
unsigned int __V)
297 __builtin_ia32_wrgsbase32(__V);
300 static __inline__
void __attribute__((__always_inline__, __nodebug__, __target__(
"fsgsbase")))
301 _writegsbase_u64(
unsigned long long __V)
303 __builtin_ia32_wrgsbase64(__V);
309 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__MOVBE__) 317 static __inline__
short __attribute__((__always_inline__, __nodebug__, __target__(
"movbe")))
318 _loadbe_i16(
void const * __P) {
322 return __builtin_bswap16(((
struct __loadu_i16*)__P)->
__v);
325 static __inline__
void __attribute__((__always_inline__, __nodebug__, __target__(
"movbe")))
326 _storebe_i16(
void * __P,
short __D) {
327 struct __storeu_i16 {
330 ((
struct __storeu_i16*)__P)->__v = __builtin_bswap16(__D);
333 static __inline__
int __attribute__((__always_inline__, __nodebug__, __target__(
"movbe")))
334 _loadbe_i32(
void const * __P) {
338 return __builtin_bswap32(((
struct __loadu_i32*)__P)->
__v);
341 static __inline__
void __attribute__((__always_inline__, __nodebug__, __target__(
"movbe")))
342 _storebe_i32(
void * __P,
int __D) {
343 struct __storeu_i32 {
346 ((
struct __storeu_i32*)__P)->__v = __builtin_bswap32(__D);
350 static __inline__
long long __attribute__((__always_inline__, __nodebug__, __target__(
"movbe")))
351 _loadbe_i64(
void const * __P) {
355 return __builtin_bswap64(((
struct __loadu_i64*)__P)->
__v);
358 static __inline__
void __attribute__((__always_inline__, __nodebug__, __target__(
"movbe")))
359 _storebe_i64(
void * __P,
long long __D) {
360 struct __storeu_i64 {
363 ((
struct __storeu_i64*)__P)->__v = __builtin_bswap64(__D);
368 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__RTM__) 373 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__SHA__) 377 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__FXSR__) 381 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__XSAVE__) 385 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__XSAVEOPT__) 389 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__XSAVEC__) 393 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__XSAVES__) 397 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__SHSTK__) 405 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__RDSEED__) 409 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__WBNOINVD__) 413 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__CLDEMOTE__) 417 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__WAITPKG__) 421 #if !defined(_MSC_VER) || __has_feature(modules) || \ 422 defined(__MOVDIRI__) || defined(__MOVDIR64B__) 426 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__PCONFIG__) 430 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__SGX__) 434 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__PTWRITE__) 438 #if !defined(_MSC_VER) || __has_feature(modules) || defined(__INVPCID__) 444 #define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__)) 451 #if defined(__i386__) || defined(__x86_64__) 453 _InterlockedExchange_HLEAcquire(
long volatile *_Target,
long _Value) {
454 __asm__ __volatile__(
".byte 0xf2 ; lock ; xchg %0, %1" 455 :
"+r" (_Value),
"+m" (*_Target) ::
"memory");
459 _InterlockedExchange_HLERelease(
long volatile *_Target,
long _Value) {
460 __asm__ __volatile__(
".byte 0xf3 ; lock ; xchg %0, %1" 461 :
"+r" (_Value),
"+m" (*_Target) ::
"memory");
465 #if defined(__x86_64__) 467 _InterlockedExchange64_HLEAcquire(__int64
volatile *_Target, __int64 _Value) {
468 __asm__ __volatile__(
".byte 0xf2 ; lock ; xchg %0, %1" 469 :
"+r" (_Value),
"+m" (*_Target) ::
"memory");
473 _InterlockedExchange64_HLERelease(__int64
volatile *_Target, __int64 _Value) {
474 __asm__ __volatile__(
".byte 0xf3 ; lock ; xchg %0, %1" 475 :
"+r" (_Value),
"+m" (*_Target) ::
"memory");
482 #if defined(__i386__) || defined(__x86_64__) 484 _InterlockedCompareExchange_HLEAcquire(
long volatile *_Destination,
485 long _Exchange,
long _Comparand) {
486 __asm__ __volatile__(
".byte 0xf2 ; lock ; cmpxchg %2, %1" 487 :
"+a" (_Comparand),
"+m" (*_Destination)
488 :
"r" (_Exchange) :
"memory");
492 _InterlockedCompareExchange_HLERelease(
long volatile *_Destination,
493 long _Exchange,
long _Comparand) {
494 __asm__ __volatile__(
".byte 0xf3 ; lock ; cmpxchg %2, %1" 495 :
"+a" (_Comparand),
"+m" (*_Destination)
496 :
"r" (_Exchange) :
"memory");
500 #if defined(__x86_64__) 502 _InterlockedCompareExchange64_HLEAcquire(__int64
volatile *_Destination,
503 __int64 _Exchange, __int64 _Comparand) {
504 __asm__ __volatile__(
".byte 0xf2 ; lock ; cmpxchg %2, %1" 505 :
"+a" (_Comparand),
"+m" (*_Destination)
506 :
"r" (_Exchange) :
"memory");
510 _InterlockedCompareExchange64_HLERelease(__int64
volatile *_Destination,
511 __int64 _Exchange, __int64 _Comparand) {
512 __asm__ __volatile__(
".byte 0xf3 ; lock ; cmpxchg %2, %1" 513 :
"+a" (_Comparand),
"+m" (*_Destination)
514 :
"r" (_Exchange) :
"memory");
522 #undef __DEFAULT_FN_ATTRS
struct __storeu_i16 *__P __v
static __inline__ void short __D
static __inline unsigned char unsigned int unsigned int unsigned int * __p
#define __DEFAULT_FN_ATTRS
static __inline__ unsigned int __attribute__((__always_inline__, __nodebug__, __target__("rdpid"))) _rdpid_u32(void)
Returns the value of the IA32_TSC_AUX MSR (0xc0000103).