cpufeatures.go

     1  // Copyright 2025 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package ssacompile
     6  
     7  import (
     8  	"fmt"
     9  	"internal/goarch"
    10  
    11  	"cmd/compile/internal/ssa"
    12  	"cmd/compile/internal/ssa/block"
    13  	"cmd/compile/internal/ssa/ssaop"
    14  	"cmd/compile/internal/types"
    15  	"cmd/internal/obj"
    16  )
    17  
    18  type localEffect struct {
    19  	start    ssa.CPUfeatures    // features present at beginning of block
    20  	internal ssa.CPUfeatures    // features implied by execution of block
    21  	end      [2]ssa.CPUfeatures // for BlockIf, features present on outgoing edges
    22  	visited  bool               // On the first iteration this will be false for backedges.
    23  }
    24  
    25  func (e localEffect) String() string {
    26  	return fmt.Sprintf("visited=%v, start=%v, internal=%v, end[0]=%v, end[1]=%v", e.visited, e.start, e.internal, e.end[0], e.end[1])
    27  }
    28  
    29  // ifEffect pattern matches for a BlockIf conditional on a load
    30  // of a field from internal/cpu.X86 and returns the corresponding
    31  // effect.
    32  func ifEffect(b *ssa.Block) (features ssa.CPUfeatures, taken int) {
    33  	// TODO generalize for other architectures.
    34  	if b.Kind != block.BlockIf {
    35  		return
    36  	}
    37  	c := b.Controls[0]
    38  
    39  	if c.Op == ssaop.OpNot {
    40  		taken = 1
    41  		c = c.Args[0]
    42  	}
    43  	if c.Op != ssaop.OpLoad {
    44  		return
    45  	}
    46  	offPtr := c.Args[0]
    47  	if offPtr.Op != ssaop.OpOffPtr {
    48  		return
    49  	}
    50  	addr := offPtr.Args[0]
    51  	if addr.Op != ssaop.OpAddr || addr.Args[0].Op != ssaop.OpSB {
    52  		return
    53  	}
    54  	sym := addr.Aux.(*obj.LSym)
    55  	if sym.Name != "internal/cpu.X86" && sym.Name != "internal/cpu.ARM64" {
    56  		return
    57  	}
    58  	o := offPtr.AuxInt
    59  	t := addr.Type
    60  	if !t.IsPtr() {
    61  		b.Func.Fatalf("The symbol %s is not a pointer, found %v instead", sym.Name, t)
    62  	}
    63  	t = t.Elem()
    64  	if !t.IsStruct() {
    65  		b.Func.Fatalf("The referent of symbol %s is not a struct, found %v instead", sym.Name, t)
    66  	}
    67  	match := ""
    68  	for _, f := range t.Fields() {
    69  		if o == f.Offset && f.Sym != nil {
    70  			match = f.Sym.Name
    71  			break
    72  		}
    73  	}
    74  
    75  	switch match {
    76  
    77  	case "HasAVX":
    78  		features = ssa.CPUavx
    79  	case "HasAVXVNNI":
    80  		features = ssa.CPUavx | ssa.CPUavxvnni
    81  	case "HasAVX2":
    82  		features = ssa.CPUavx2 | ssa.CPUavx
    83  
    84  		// Compiler currently treats these all alike.
    85  	case "HasAVX512", "HasAVX512F", "HasAVX512CD", "HasAVX512BW",
    86  		"HasAVX512DQ", "HasAVX512VL", "HasAVX512VPCLMULQDQ":
    87  		features = ssa.CPUavx512 | ssa.CPUavx2 | ssa.CPUavx
    88  
    89  	case "HasAVX512GFNI":
    90  		features = ssa.CPUavx512 | ssa.CPUgfni | ssa.CPUavx2 | ssa.CPUavx
    91  	case "HasAVX512VNNI":
    92  		features = ssa.CPUavx512 | ssa.CPUavx512vnni | ssa.CPUavx2 | ssa.CPUavx
    93  	case "HasAVX512VBMI":
    94  		features = ssa.CPUavx512 | ssa.CPUvbmi | ssa.CPUavx2 | ssa.CPUavx
    95  	case "HasAVX512VBMI2":
    96  		features = ssa.CPUavx512 | ssa.CPUvbmi2 | ssa.CPUavx2 | ssa.CPUavx
    97  	case "HasAVX512BITALG":
    98  		features = ssa.CPUavx512 | ssa.CPUbitalg | ssa.CPUavx2 | ssa.CPUavx
    99  	case "HasAVX512VPOPCNTDQ":
   100  		features = ssa.CPUavx512 | ssa.CPUvpopcntdq | ssa.CPUavx2 | ssa.CPUavx
   101  
   102  	case "HasBMI1":
   103  		features = ssa.CPUvbmi
   104  	case "HasBMI2":
   105  		features = ssa.CPUvbmi2
   106  
   107  	case "HasSVE2":
   108  		features = ssa.CPUsve2
   109  
   110  		// Features that are not currently interesting to the compiler.
   111  	// HasSVE is not among them: rules only ever upgrade an SVE lowering to an
   112  	// SVE2 encoding, so baseline SVE is never a rule condition.
   113  	case "HasAES", "HasADX", "HasERMS", "HasFSRM", "HasFMA", "HasGFNI", "HasOSXSAVE",
   114  		"HasPCLMULQDQ", "HasPOPCNT", "HasRDTSCP", "HasSHA",
   115  		"HasSSE3", "HasSSSE3", "HasSSE41", "HasSSE42":
   116  
   117  	}
   118  	if b.Func.Pass.Debug > 2 {
   119  		b.Func.Warnl(b.Pos, "%s, block b%v has features offset %d, match is %s, features is %v", b.Func.Name, b.ID, o, match, features)
   120  	}
   121  	return
   122  }
   123  
   124  // noCodeValue reports whether v is known to emit no machine code, so
   125  // that even a SIMD type does not imply the presence of any CPU feature.
   126  func noCodeValue(v *ssa.Value) bool {
   127  	switch v.Op {
   128  	case ssaop.OpZeroSIMD, ssaop.OpPhi, ssaop.OpCopy, ssaop.OpSelectN,
   129  		ssaop.OpArg, ssaop.OpArgIntReg, ssaop.OpArgFloatReg:
   130  		return true
   131  	}
   132  	return false
   133  }
   134  
   135  func cpufeatures(f *ssa.Func) {
   136  	arch := f.Config.Ctxt.Arch.Family
   137  	// TODO there are other SIMD architectures
   138  	if arch != goarch.AMD64 && arch != goarch.ARM64 {
   139  		return
   140  	}
   141  
   142  	po := f.Postorder()
   143  
   144  	effects := make([]localEffect, 1+f.NumBlocks(), 1+f.NumBlocks())
   145  
   146  	features := func(t *types.Type) ssa.CPUfeatures {
   147  		if t.IsSIMD() && arch == goarch.AMD64 {
   148  			// On arm64 a SIMD type implies no feature a rule conditions on:
   149  			// rules only upgrade SVE lowerings to SVE2 encodings, and using an
   150  			// SVE type does not imply SVE2.
   151  			switch t.Size() {
   152  			case 16, 32:
   153  				return ssa.CPUavx
   154  			case 64:
   155  				return ssa.CPUavx512 | ssa.CPUavx2 | ssa.CPUavx
   156  			}
   157  		}
   158  		return ssa.CPUNone
   159  	}
   160  
   161  	// visit blocks in reverse post order
   162  	// when b is visited, all of its predecessors (except for loop back edges)
   163  	// will have been visited
   164  	for i := len(po) - 1; i >= 0; i-- {
   165  		b := po[i]
   166  
   167  		var feat ssa.CPUfeatures
   168  
   169  		if b == f.Entry {
   170  			// Check the types of inputs and outputs, as well as annotations.
   171  			// Start with none and union all that is implied by all the types seen.
   172  			if f.Type != nil { // a problem for SSA tests
   173  				for _, field := range f.Type.RecvParamsResults() {
   174  					feat |= features(field.Type)
   175  				}
   176  			}
   177  
   178  		} else {
   179  			// Start with all and intersect over predecessors
   180  			feat = ssa.CPUAll
   181  			for _, p := range b.Preds {
   182  				pb := p.Block()
   183  				if !effects[pb.ID].visited {
   184  
   185  					continue
   186  				}
   187  				pi := p.Index()
   188  				if pb.Kind != block.BlockIf {
   189  					pi = 0
   190  				}
   191  
   192  				feat &= effects[pb.ID].end[pi]
   193  			}
   194  		}
   195  
   196  		e := localEffect{start: feat, visited: true}
   197  
   198  		// Separately capture the internal effects of this block
   199  		var internal ssa.CPUfeatures
   200  		for _, v := range b.Values {
   201  			// the rule applied here is, if the block contains any
   202  			// instruction that would fault if the feature (avx, avx512)
   203  			// were not present, then assume that the feature is present
   204  			// for all the instructions in the block, a fault is a fault.
   205  			if noCodeValue(v) {
   206  				// v emits no instruction, so its type implies
   207  				// nothing about the CPU. In particular a
   208  				// SIMD-typed zero is just a reference to the
   209  				// fixed all-zeros register and flows freely
   210  				// through code that must run on machines
   211  				// without AVX.
   212  				continue
   213  			}
   214  			t := v.Type
   215  			if t.IsResults() {
   216  				for i := 0; i < t.NumFields(); i++ {
   217  					feat |= features(t.FieldType(i))
   218  				}
   219  			} else {
   220  				internal |= features(v.Type)
   221  			}
   222  		}
   223  		e.internal = internal
   224  		feat |= internal
   225  
   226  		branchEffect, taken := ifEffect(b)
   227  		e.end = [2]ssa.CPUfeatures{feat, feat}
   228  		e.end[taken] |= branchEffect
   229  
   230  		effects[b.ID] = e
   231  		if f.Pass.Debug > 1 && feat != ssa.CPUNone {
   232  			f.Warnl(b.Pos, "%s, block b%v has features %v", b.Func.Name, b.ID, feat)
   233  		}
   234  
   235  		b.CPUfeatures = feat
   236  		f.MaxCPUFeatures |= feat // not necessary to refine this estimate below
   237  	}
   238  
   239  	// If the flow graph is irreducible, things can still change on backedges.
   240  	change := true
   241  	for change {
   242  		change = false
   243  		for i := len(po) - 1; i >= 0; i-- {
   244  			b := po[i]
   245  
   246  			if b == f.Entry {
   247  				continue // cannot change
   248  			}
   249  			feat := ssa.CPUAll
   250  			for _, p := range b.Preds {
   251  				pb := p.Block()
   252  				pi := p.Index()
   253  				if pb.Kind != block.BlockIf {
   254  					pi = 0
   255  				}
   256  				feat &= effects[pb.ID].end[pi]
   257  			}
   258  			e := effects[b.ID]
   259  			if feat == e.start {
   260  				continue
   261  			}
   262  			e.start = feat
   263  			effects[b.ID] = e
   264  			// uh-oh, something changed
   265  			if f.Pass.Debug > 1 {
   266  				f.Warnl(b.Pos, "%s, block b%v saw predecessor feature change", b.Func.Name, b.ID)
   267  			}
   268  
   269  			feat |= e.internal
   270  			if feat == e.end[0]&e.end[1] {
   271  				continue
   272  			}
   273  
   274  			branchEffect, taken := ifEffect(b)
   275  			e.end = [2]ssa.CPUfeatures{feat, feat}
   276  			e.end[taken] |= branchEffect
   277  
   278  			effects[b.ID] = e
   279  			b.CPUfeatures = feat
   280  			if f.Pass.Debug > 1 {
   281  				f.Warnl(b.Pos, "%s, block b%v has new features %v", b.Func.Name, b.ID, feat)
   282  			}
   283  			change = true
   284  		}
   285  	}
   286  	if f.Pass.Debug > 0 {
   287  		for _, b := range f.Blocks {
   288  			if b.CPUfeatures != ssa.CPUNone {
   289  				f.Warnl(b.Pos, "%s, block b%v has features %v", b.Func.Name, b.ID, b.CPUfeatures)
   290  			}
   291  
   292  		}
   293  	}
   294  }
   295
View as plain text