cpu_x86.go

     1  // Copyright 2018 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  //go:build 386 || amd64 || amd64p32
     6  
     7  package cpu
     8  
     9  import "runtime"
    10  
    11  const cacheLineSize = 64
    12  
    13  func initOptions() {
    14  	options = []option{
    15  		{Name: "adx", Feature: &X86.HasADX},
    16  		{Name: "aes", Feature: &X86.HasAES},
    17  		{Name: "avx", Feature: &X86.HasAVX},
    18  		{Name: "avx2", Feature: &X86.HasAVX2},
    19  		{Name: "avx512", Feature: &X86.HasAVX512},
    20  		{Name: "avx512f", Feature: &X86.HasAVX512F},
    21  		{Name: "avx512cd", Feature: &X86.HasAVX512CD},
    22  		{Name: "avx512er", Feature: &X86.HasAVX512ER},
    23  		{Name: "avx512pf", Feature: &X86.HasAVX512PF},
    24  		{Name: "avx512vl", Feature: &X86.HasAVX512VL},
    25  		{Name: "avx512bw", Feature: &X86.HasAVX512BW},
    26  		{Name: "avx512dq", Feature: &X86.HasAVX512DQ},
    27  		{Name: "avx512ifma", Feature: &X86.HasAVX512IFMA},
    28  		{Name: "avx512vbmi", Feature: &X86.HasAVX512VBMI},
    29  		{Name: "avx512vnniw", Feature: &X86.HasAVX5124VNNIW},
    30  		{Name: "avx5124fmaps", Feature: &X86.HasAVX5124FMAPS},
    31  		{Name: "avx512vpopcntdq", Feature: &X86.HasAVX512VPOPCNTDQ},
    32  		{Name: "avx512vpclmulqdq", Feature: &X86.HasAVX512VPCLMULQDQ},
    33  		{Name: "avx512vnni", Feature: &X86.HasAVX512VNNI},
    34  		{Name: "avx512gfni", Feature: &X86.HasAVX512GFNI},
    35  		{Name: "avx512vaes", Feature: &X86.HasAVX512VAES},
    36  		{Name: "avx512vbmi2", Feature: &X86.HasAVX512VBMI2},
    37  		{Name: "avx512bitalg", Feature: &X86.HasAVX512BITALG},
    38  		{Name: "avx512bf16", Feature: &X86.HasAVX512BF16},
    39  		{Name: "amxtile", Feature: &X86.HasAMXTile},
    40  		{Name: "amxint8", Feature: &X86.HasAMXInt8},
    41  		{Name: "amxbf16", Feature: &X86.HasAMXBF16},
    42  		{Name: "bmi1", Feature: &X86.HasBMI1},
    43  		{Name: "bmi2", Feature: &X86.HasBMI2},
    44  		{Name: "cx16", Feature: &X86.HasCX16},
    45  		{Name: "erms", Feature: &X86.HasERMS},
    46  		{Name: "fma", Feature: &X86.HasFMA},
    47  		{Name: "osxsave", Feature: &X86.HasOSXSAVE},
    48  		{Name: "pclmulqdq", Feature: &X86.HasPCLMULQDQ},
    49  		{Name: "popcnt", Feature: &X86.HasPOPCNT},
    50  		{Name: "rdrand", Feature: &X86.HasRDRAND},
    51  		{Name: "rdseed", Feature: &X86.HasRDSEED},
    52  		{Name: "sse3", Feature: &X86.HasSSE3},
    53  		{Name: "sse41", Feature: &X86.HasSSE41},
    54  		{Name: "sse42", Feature: &X86.HasSSE42},
    55  		{Name: "ssse3", Feature: &X86.HasSSSE3},
    56  		{Name: "avxifma", Feature: &X86.HasAVXIFMA},
    57  		{Name: "avxvnni", Feature: &X86.HasAVXVNNI},
    58  		{Name: "avxvnniint8", Feature: &X86.HasAVXVNNIInt8},
    59  
    60  		// These capabilities should always be enabled on amd64:
    61  		{Name: "sse2", Feature: &X86.HasSSE2, Required: runtime.GOARCH == "amd64"},
    62  	}
    63  }
    64  
    65  func archInit() {
    66  
    67  	Initialized = true
    68  
    69  	maxID, _, _, _ := cpuid(0, 0)
    70  
    71  	if maxID < 1 {
    72  		return
    73  	}
    74  
    75  	_, _, ecx1, edx1 := cpuid(1, 0)
    76  	X86.HasSSE2 = isSet(26, edx1)
    77  
    78  	X86.HasSSE3 = isSet(0, ecx1)
    79  	X86.HasPCLMULQDQ = isSet(1, ecx1)
    80  	X86.HasSSSE3 = isSet(9, ecx1)
    81  	X86.HasFMA = isSet(12, ecx1)
    82  	X86.HasCX16 = isSet(13, ecx1)
    83  	X86.HasSSE41 = isSet(19, ecx1)
    84  	X86.HasSSE42 = isSet(20, ecx1)
    85  	X86.HasPOPCNT = isSet(23, ecx1)
    86  	X86.HasAES = isSet(25, ecx1)
    87  	X86.HasOSXSAVE = isSet(27, ecx1)
    88  	X86.HasRDRAND = isSet(30, ecx1)
    89  
    90  	var osSupportsAVX, osSupportsAVX512 bool
    91  	// For XGETBV, OSXSAVE bit is required and sufficient.
    92  	if X86.HasOSXSAVE {
    93  		eax, _ := xgetbv()
    94  		// Check if XMM and YMM registers have OS support.
    95  		osSupportsAVX = isSet(1, eax) && isSet(2, eax)
    96  
    97  		if runtime.GOOS == "darwin" {
    98  			// Darwin requires special AVX512 checks, see cpu_darwin_x86.go
    99  			osSupportsAVX512 = osSupportsAVX && darwinSupportsAVX512()
   100  		} else {
   101  			// Check if OPMASK and ZMM registers have OS support.
   102  			osSupportsAVX512 = osSupportsAVX && isSet(5, eax) && isSet(6, eax) && isSet(7, eax)
   103  		}
   104  	}
   105  
   106  	X86.HasAVX = isSet(28, ecx1) && osSupportsAVX
   107  
   108  	if maxID < 7 {
   109  		return
   110  	}
   111  
   112  	eax7, ebx7, ecx7, edx7 := cpuid(7, 0)
   113  	X86.HasBMI1 = isSet(3, ebx7)
   114  	X86.HasAVX2 = isSet(5, ebx7) && osSupportsAVX
   115  	X86.HasBMI2 = isSet(8, ebx7)
   116  	X86.HasERMS = isSet(9, ebx7)
   117  	X86.HasRDSEED = isSet(18, ebx7)
   118  	X86.HasADX = isSet(19, ebx7)
   119  
   120  	X86.HasAVX512 = isSet(16, ebx7) && osSupportsAVX512 // Because avx-512 foundation is the core required extension
   121  	if X86.HasAVX512 {
   122  		X86.HasAVX512F = true
   123  		X86.HasAVX512CD = isSet(28, ebx7)
   124  		X86.HasAVX512ER = isSet(27, ebx7)
   125  		X86.HasAVX512PF = isSet(26, ebx7)
   126  		X86.HasAVX512VL = isSet(31, ebx7)
   127  		X86.HasAVX512BW = isSet(30, ebx7)
   128  		X86.HasAVX512DQ = isSet(17, ebx7)
   129  		X86.HasAVX512IFMA = isSet(21, ebx7)
   130  		X86.HasAVX512VBMI = isSet(1, ecx7)
   131  		X86.HasAVX5124VNNIW = isSet(2, edx7)
   132  		X86.HasAVX5124FMAPS = isSet(3, edx7)
   133  		X86.HasAVX512VPOPCNTDQ = isSet(14, ecx7)
   134  		X86.HasAVX512VPCLMULQDQ = isSet(10, ecx7)
   135  		X86.HasAVX512VNNI = isSet(11, ecx7)
   136  		X86.HasAVX512GFNI = isSet(8, ecx7)
   137  		X86.HasAVX512VAES = isSet(9, ecx7)
   138  		X86.HasAVX512VBMI2 = isSet(6, ecx7)
   139  		X86.HasAVX512BITALG = isSet(12, ecx7)
   140  	}
   141  
   142  	X86.HasAMXTile = isSet(24, edx7)
   143  	X86.HasAMXInt8 = isSet(25, edx7)
   144  	X86.HasAMXBF16 = isSet(22, edx7)
   145  
   146  	// These features depend on the second level of extended features.
   147  	if eax7 >= 1 {
   148  		eax71, _, _, edx71 := cpuid(7, 1)
   149  		if X86.HasAVX512 {
   150  			X86.HasAVX512BF16 = isSet(5, eax71)
   151  		}
   152  		if X86.HasAVX {
   153  			X86.HasAVXIFMA = isSet(23, eax71)
   154  			X86.HasAVXVNNI = isSet(4, eax71)
   155  			X86.HasAVXVNNIInt8 = isSet(4, edx71)
   156  		}
   157  	}
   158  }
   159  
   160  func isSet(bitpos uint, value uint32) bool {
   161  	return value&(1<<bitpos) != 0
   162  }
   163
View as plain text