Source file src/cmd/compile/internal/ssagen/intrinsics.go

     1  // Copyright 2024 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package ssagen
     6  
     7  import (
     8  	"fmt"
     9  	"internal/abi"
    10  	"internal/buildcfg"
    11  
    12  	"cmd/compile/internal/base"
    13  	"cmd/compile/internal/ir"
    14  	"cmd/compile/internal/ssa"
    15  	"cmd/compile/internal/types"
    16  	"cmd/internal/sys"
    17  )
    18  
    19  var intrinsics intrinsicBuilders
    20  
    21  // An intrinsicBuilder converts a call node n into an ssa value that
    22  // implements that call as an intrinsic. args is a list of arguments to the func.
    23  type intrinsicBuilder func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value
    24  
    25  type intrinsicKey struct {
    26  	arch *sys.Arch
    27  	pkg  string
    28  	fn   string
    29  }
    30  
    31  // intrinsicBuildConfig specifies the config to use for intrinsic building.
    32  type intrinsicBuildConfig struct {
    33  	instrumenting bool
    34  
    35  	go386     string
    36  	goamd64   int
    37  	goarm     buildcfg.GoarmFeatures
    38  	goarm64   buildcfg.Goarm64Features
    39  	gomips    string
    40  	gomips64  string
    41  	goppc64   int
    42  	goriscv64 int
    43  }
    44  
    45  type intrinsicBuilders map[intrinsicKey]intrinsicBuilder
    46  
    47  // add adds the intrinsic builder b for pkg.fn for the given architecture.
    48  func (ib intrinsicBuilders) add(arch *sys.Arch, pkg, fn string, b intrinsicBuilder) {
    49  	if _, found := ib[intrinsicKey{arch, pkg, fn}]; found {
    50  		panic(fmt.Sprintf("intrinsic already exists for %v.%v on %v", pkg, fn, arch.Name))
    51  	}
    52  	ib[intrinsicKey{arch, pkg, fn}] = b
    53  }
    54  
    55  // addForArchs adds the intrinsic builder b for pkg.fn for the given architectures.
    56  func (ib intrinsicBuilders) addForArchs(pkg, fn string, b intrinsicBuilder, archs ...*sys.Arch) {
    57  	for _, arch := range archs {
    58  		ib.add(arch, pkg, fn, b)
    59  	}
    60  }
    61  
    62  // addForFamilies does the same as addForArchs but operates on architecture families.
    63  func (ib intrinsicBuilders) addForFamilies(pkg, fn string, b intrinsicBuilder, archFamilies ...sys.ArchFamily) {
    64  	for _, arch := range sys.Archs {
    65  		if arch.InFamily(archFamilies...) {
    66  			intrinsics.add(arch, pkg, fn, b)
    67  		}
    68  	}
    69  }
    70  
    71  // alias aliases pkg.fn to targetPkg.targetFn for all architectures in archs
    72  // for which targetPkg.targetFn already exists.
    73  func (ib intrinsicBuilders) alias(pkg, fn, targetPkg, targetFn string, archs ...*sys.Arch) {
    74  	// TODO(jsing): Consider making this work even if the alias is added
    75  	// before the intrinsic.
    76  	aliased := false
    77  	for _, arch := range archs {
    78  		if b := intrinsics.lookup(arch, targetPkg, targetFn); b != nil {
    79  			intrinsics.add(arch, pkg, fn, b)
    80  			aliased = true
    81  		}
    82  	}
    83  	if !aliased {
    84  		panic(fmt.Sprintf("attempted to alias undefined intrinsic: %s.%s", pkg, fn))
    85  	}
    86  }
    87  
    88  // lookup looks up the intrinsic for a pkg.fn on the specified architecture.
    89  func (ib intrinsicBuilders) lookup(arch *sys.Arch, pkg, fn string) intrinsicBuilder {
    90  	return intrinsics[intrinsicKey{arch, pkg, fn}]
    91  }
    92  
    93  func initIntrinsics(cfg *intrinsicBuildConfig) {
    94  	if cfg == nil {
    95  		cfg = &intrinsicBuildConfig{
    96  			instrumenting: base.Flag.Cfg.Instrumenting,
    97  			go386:         buildcfg.GO386,
    98  			goamd64:       buildcfg.GOAMD64,
    99  			goarm:         buildcfg.GOARM,
   100  			goarm64:       buildcfg.GOARM64,
   101  			gomips:        buildcfg.GOMIPS,
   102  			gomips64:      buildcfg.GOMIPS64,
   103  			goppc64:       buildcfg.GOPPC64,
   104  			goriscv64:     buildcfg.GORISCV64,
   105  		}
   106  	}
   107  	intrinsics = intrinsicBuilders{}
   108  
   109  	var p4 []*sys.Arch
   110  	var p8 []*sys.Arch
   111  	var lwatomics []*sys.Arch
   112  	for _, a := range sys.Archs {
   113  		if a.PtrSize == 4 {
   114  			p4 = append(p4, a)
   115  		} else {
   116  			p8 = append(p8, a)
   117  		}
   118  		if a.Family != sys.PPC64 {
   119  			lwatomics = append(lwatomics, a)
   120  		}
   121  	}
   122  	all := sys.Archs[:]
   123  
   124  	add := func(pkg, fn string, b intrinsicBuilder, archs ...*sys.Arch) {
   125  		intrinsics.addForArchs(pkg, fn, b, archs...)
   126  	}
   127  	addF := func(pkg, fn string, b intrinsicBuilder, archFamilies ...sys.ArchFamily) {
   128  		intrinsics.addForFamilies(pkg, fn, b, archFamilies...)
   129  	}
   130  	alias := func(pkg, fn, pkg2, fn2 string, archs ...*sys.Arch) {
   131  		intrinsics.alias(pkg, fn, pkg2, fn2, archs...)
   132  	}
   133  
   134  	/******** runtime ********/
   135  	if !cfg.instrumenting {
   136  		add("runtime", "slicebytetostringtmp",
   137  			func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   138  				// Compiler frontend optimizations emit OBYTES2STRTMP nodes
   139  				// for the backend instead of slicebytetostringtmp calls
   140  				// when not instrumenting.
   141  				return s.newValue2(ssa.OpStringMake, n.Type(), args[0], args[1])
   142  			},
   143  			all...)
   144  	}
   145  	addF("internal/runtime/math", "MulUintptr",
   146  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   147  			if s.config.PtrSize == 4 {
   148  				return s.newValue2(ssa.OpMul32uover, types.NewTuple(types.Types[types.TUINT], types.Types[types.TUINT]), args[0], args[1])
   149  			}
   150  			return s.newValue2(ssa.OpMul64uover, types.NewTuple(types.Types[types.TUINT], types.Types[types.TUINT]), args[0], args[1])
   151  		},
   152  		sys.AMD64, sys.I386, sys.Loong64, sys.MIPS64, sys.PPC64, sys.RISCV64, sys.ARM64)
   153  	add("runtime", "KeepAlive",
   154  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   155  			data := s.newValue1(ssa.OpIData, s.f.Config.Types.BytePtr, args[0])
   156  			s.vars[memVar] = s.newValue2(ssa.OpKeepAlive, types.TypeMem, data, s.mem())
   157  			return nil
   158  		},
   159  		all...)
   160  
   161  	addF("runtime", "publicationBarrier",
   162  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   163  			s.vars[memVar] = s.newValue1(ssa.OpPubBarrier, types.TypeMem, s.mem())
   164  			return nil
   165  		},
   166  		sys.ARM64, sys.Loong64, sys.PPC64, sys.RISCV64)
   167  
   168  	/******** internal/runtime/sys ********/
   169  	add("internal/runtime/sys", "GetCallerPC",
   170  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   171  			return s.newValue0(ssa.OpGetCallerPC, s.f.Config.Types.Uintptr)
   172  		},
   173  		all...)
   174  
   175  	add("internal/runtime/sys", "GetCallerSP",
   176  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   177  			return s.newValue1(ssa.OpGetCallerSP, s.f.Config.Types.Uintptr, s.mem())
   178  		},
   179  		all...)
   180  
   181  	add("internal/runtime/sys", "GetClosurePtr",
   182  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   183  			return s.newValue0(ssa.OpGetClosurePtr, s.f.Config.Types.Uintptr)
   184  		},
   185  		all...)
   186  
   187  	brev_arch := []sys.ArchFamily{sys.AMD64, sys.I386, sys.ARM64, sys.ARM, sys.Loong64, sys.S390X}
   188  	if cfg.goppc64 >= 10 {
   189  		// Use only on Power10 as the new byte reverse instructions that Power10 provide
   190  		// make it worthwhile as an intrinsic
   191  		brev_arch = append(brev_arch, sys.PPC64)
   192  	}
   193  	addF("internal/runtime/sys", "Bswap32",
   194  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   195  			return s.newValue1(ssa.OpBswap32, types.Types[types.TUINT32], args[0])
   196  		},
   197  		brev_arch...)
   198  	addF("internal/runtime/sys", "Bswap64",
   199  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   200  			return s.newValue1(ssa.OpBswap64, types.Types[types.TUINT64], args[0])
   201  		},
   202  		brev_arch...)
   203  
   204  	/****** Prefetch ******/
   205  	makePrefetchFunc := func(op ssa.Op) func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   206  		return func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   207  			s.vars[memVar] = s.newValue2(op, types.TypeMem, args[0], s.mem())
   208  			return nil
   209  		}
   210  	}
   211  
   212  	// Make Prefetch intrinsics for supported platforms
   213  	// On the unsupported platforms stub function will be eliminated
   214  	addF("internal/runtime/sys", "Prefetch", makePrefetchFunc(ssa.OpPrefetchCache),
   215  		sys.AMD64, sys.ARM64, sys.PPC64)
   216  	addF("internal/runtime/sys", "PrefetchStreamed", makePrefetchFunc(ssa.OpPrefetchCacheStreamed),
   217  		sys.AMD64, sys.ARM64, sys.PPC64)
   218  
   219  	/******** internal/runtime/atomic ********/
   220  	type atomicOpEmitter func(s *state, n *ir.CallExpr, args []*ssa.Value, op ssa.Op, typ types.Kind, needReturn bool)
   221  
   222  	addF("internal/runtime/atomic", "Load",
   223  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   224  			v := s.newValue2(ssa.OpAtomicLoad32, types.NewTuple(types.Types[types.TUINT32], types.TypeMem), args[0], s.mem())
   225  			s.vars[memVar] = s.newValue1(ssa.OpSelect1, types.TypeMem, v)
   226  			return s.newValue1(ssa.OpSelect0, types.Types[types.TUINT32], v)
   227  		},
   228  		sys.AMD64, sys.ARM64, sys.Loong64, sys.MIPS, sys.MIPS64, sys.PPC64, sys.RISCV64, sys.S390X)
   229  	addF("internal/runtime/atomic", "Load8",
   230  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   231  			v := s.newValue2(ssa.OpAtomicLoad8, types.NewTuple(types.Types[types.TUINT8], types.TypeMem), args[0], s.mem())
   232  			s.vars[memVar] = s.newValue1(ssa.OpSelect1, types.TypeMem, v)
   233  			return s.newValue1(ssa.OpSelect0, types.Types[types.TUINT8], v)
   234  		},
   235  		sys.AMD64, sys.ARM64, sys.Loong64, sys.MIPS, sys.MIPS64, sys.PPC64, sys.RISCV64, sys.S390X)
   236  	addF("internal/runtime/atomic", "Load64",
   237  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   238  			v := s.newValue2(ssa.OpAtomicLoad64, types.NewTuple(types.Types[types.TUINT64], types.TypeMem), args[0], s.mem())
   239  			s.vars[memVar] = s.newValue1(ssa.OpSelect1, types.TypeMem, v)
   240  			return s.newValue1(ssa.OpSelect0, types.Types[types.TUINT64], v)
   241  		},
   242  		sys.AMD64, sys.ARM64, sys.Loong64, sys.MIPS64, sys.PPC64, sys.RISCV64, sys.S390X)
   243  	addF("internal/runtime/atomic", "LoadAcq",
   244  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   245  			v := s.newValue2(ssa.OpAtomicLoadAcq32, types.NewTuple(types.Types[types.TUINT32], types.TypeMem), args[0], s.mem())
   246  			s.vars[memVar] = s.newValue1(ssa.OpSelect1, types.TypeMem, v)
   247  			return s.newValue1(ssa.OpSelect0, types.Types[types.TUINT32], v)
   248  		},
   249  		sys.PPC64)
   250  	addF("internal/runtime/atomic", "LoadAcq64",
   251  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   252  			v := s.newValue2(ssa.OpAtomicLoadAcq64, types.NewTuple(types.Types[types.TUINT64], types.TypeMem), args[0], s.mem())
   253  			s.vars[memVar] = s.newValue1(ssa.OpSelect1, types.TypeMem, v)
   254  			return s.newValue1(ssa.OpSelect0, types.Types[types.TUINT64], v)
   255  		},
   256  		sys.PPC64)
   257  	addF("internal/runtime/atomic", "Loadp",
   258  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   259  			v := s.newValue2(ssa.OpAtomicLoadPtr, types.NewTuple(s.f.Config.Types.BytePtr, types.TypeMem), args[0], s.mem())
   260  			s.vars[memVar] = s.newValue1(ssa.OpSelect1, types.TypeMem, v)
   261  			return s.newValue1(ssa.OpSelect0, s.f.Config.Types.BytePtr, v)
   262  		},
   263  		sys.AMD64, sys.ARM64, sys.Loong64, sys.MIPS, sys.MIPS64, sys.PPC64, sys.RISCV64, sys.S390X)
   264  
   265  	addF("internal/runtime/atomic", "Store",
   266  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   267  			s.vars[memVar] = s.newValue3(ssa.OpAtomicStore32, types.TypeMem, args[0], args[1], s.mem())
   268  			return nil
   269  		},
   270  		sys.AMD64, sys.ARM64, sys.MIPS, sys.MIPS64, sys.PPC64, sys.RISCV64, sys.S390X)
   271  	addF("internal/runtime/atomic", "Store8",
   272  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   273  			s.vars[memVar] = s.newValue3(ssa.OpAtomicStore8, types.TypeMem, args[0], args[1], s.mem())
   274  			return nil
   275  		},
   276  		sys.AMD64, sys.ARM64, sys.MIPS, sys.MIPS64, sys.PPC64, sys.RISCV64, sys.S390X)
   277  	addF("internal/runtime/atomic", "Store64",
   278  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   279  			s.vars[memVar] = s.newValue3(ssa.OpAtomicStore64, types.TypeMem, args[0], args[1], s.mem())
   280  			return nil
   281  		},
   282  		sys.AMD64, sys.ARM64, sys.MIPS64, sys.PPC64, sys.RISCV64, sys.S390X)
   283  	addF("internal/runtime/atomic", "StorepNoWB",
   284  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   285  			s.vars[memVar] = s.newValue3(ssa.OpAtomicStorePtrNoWB, types.TypeMem, args[0], args[1], s.mem())
   286  			return nil
   287  		},
   288  		sys.AMD64, sys.ARM64, sys.Loong64, sys.MIPS, sys.MIPS64, sys.RISCV64, sys.S390X)
   289  	addF("internal/runtime/atomic", "StoreRel",
   290  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   291  			s.vars[memVar] = s.newValue3(ssa.OpAtomicStoreRel32, types.TypeMem, args[0], args[1], s.mem())
   292  			return nil
   293  		},
   294  		sys.PPC64)
   295  	addF("internal/runtime/atomic", "StoreRel64",
   296  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   297  			s.vars[memVar] = s.newValue3(ssa.OpAtomicStoreRel64, types.TypeMem, args[0], args[1], s.mem())
   298  			return nil
   299  		},
   300  		sys.PPC64)
   301  
   302  	makeAtomicStoreGuardedIntrinsicLoong64 := func(op0, op1 ssa.Op, typ types.Kind, emit atomicOpEmitter) intrinsicBuilder {
   303  		return func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   304  			// Target Atomic feature is identified by dynamic detection
   305  			addr := s.entryNewValue1A(ssa.OpAddr, types.Types[types.TBOOL].PtrTo(), ir.Syms.Loong64HasLAM_BH, s.sb)
   306  			v := s.load(types.Types[types.TBOOL], addr)
   307  			b := s.endBlock()
   308  			b.Kind = ssa.BlockIf
   309  			b.SetControl(v)
   310  			bTrue := s.f.NewBlock(ssa.BlockPlain)
   311  			bFalse := s.f.NewBlock(ssa.BlockPlain)
   312  			bEnd := s.f.NewBlock(ssa.BlockPlain)
   313  			b.AddEdgeTo(bTrue)
   314  			b.AddEdgeTo(bFalse)
   315  			b.Likely = ssa.BranchLikely
   316  
   317  			// We have atomic instructions - use it directly.
   318  			s.startBlock(bTrue)
   319  			emit(s, n, args, op1, typ, false)
   320  			s.endBlock().AddEdgeTo(bEnd)
   321  
   322  			// Use original instruction sequence.
   323  			s.startBlock(bFalse)
   324  			emit(s, n, args, op0, typ, false)
   325  			s.endBlock().AddEdgeTo(bEnd)
   326  
   327  			// Merge results.
   328  			s.startBlock(bEnd)
   329  
   330  			return nil
   331  		}
   332  	}
   333  
   334  	atomicStoreEmitterLoong64 := func(s *state, n *ir.CallExpr, args []*ssa.Value, op ssa.Op, typ types.Kind, needReturn bool) {
   335  		v := s.newValue3(op, types.NewTuple(types.Types[typ], types.TypeMem), args[0], args[1], s.mem())
   336  		s.vars[memVar] = s.newValue1(ssa.OpSelect1, types.TypeMem, v)
   337  		if needReturn {
   338  			s.vars[n] = s.newValue1(ssa.OpSelect0, types.Types[typ], v)
   339  		}
   340  	}
   341  
   342  	addF("internal/runtime/atomic", "Store8",
   343  		makeAtomicStoreGuardedIntrinsicLoong64(ssa.OpAtomicStore8, ssa.OpAtomicStore8Variant, types.TUINT8, atomicStoreEmitterLoong64),
   344  		sys.Loong64)
   345  	addF("internal/runtime/atomic", "Store",
   346  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   347  			s.vars[memVar] = s.newValue3(ssa.OpAtomicStore32Variant, types.TypeMem, args[0], args[1], s.mem())
   348  			return nil
   349  		},
   350  		sys.Loong64)
   351  	addF("internal/runtime/atomic", "Store64",
   352  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   353  			s.vars[memVar] = s.newValue3(ssa.OpAtomicStore64Variant, types.TypeMem, args[0], args[1], s.mem())
   354  			return nil
   355  		},
   356  		sys.Loong64)
   357  
   358  	addF("internal/runtime/atomic", "Xchg8",
   359  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   360  			v := s.newValue3(ssa.OpAtomicExchange8, types.NewTuple(types.Types[types.TUINT8], types.TypeMem), args[0], args[1], s.mem())
   361  			s.vars[memVar] = s.newValue1(ssa.OpSelect1, types.TypeMem, v)
   362  			return s.newValue1(ssa.OpSelect0, types.Types[types.TUINT8], v)
   363  		},
   364  		sys.AMD64, sys.PPC64)
   365  	addF("internal/runtime/atomic", "Xchg",
   366  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   367  			v := s.newValue3(ssa.OpAtomicExchange32, types.NewTuple(types.Types[types.TUINT32], types.TypeMem), args[0], args[1], s.mem())
   368  			s.vars[memVar] = s.newValue1(ssa.OpSelect1, types.TypeMem, v)
   369  			return s.newValue1(ssa.OpSelect0, types.Types[types.TUINT32], v)
   370  		},
   371  		sys.AMD64, sys.Loong64, sys.MIPS, sys.MIPS64, sys.PPC64, sys.RISCV64, sys.S390X)
   372  	addF("internal/runtime/atomic", "Xchg64",
   373  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   374  			v := s.newValue3(ssa.OpAtomicExchange64, types.NewTuple(types.Types[types.TUINT64], types.TypeMem), args[0], args[1], s.mem())
   375  			s.vars[memVar] = s.newValue1(ssa.OpSelect1, types.TypeMem, v)
   376  			return s.newValue1(ssa.OpSelect0, types.Types[types.TUINT64], v)
   377  		},
   378  		sys.AMD64, sys.Loong64, sys.MIPS64, sys.PPC64, sys.RISCV64, sys.S390X)
   379  
   380  	makeAtomicGuardedIntrinsicARM64common := func(op0, op1 ssa.Op, typ types.Kind, emit atomicOpEmitter, needReturn bool) intrinsicBuilder {
   381  
   382  		return func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   383  			if cfg.goarm64.LSE {
   384  				emit(s, n, args, op1, typ, needReturn)
   385  			} else {
   386  				// Target Atomic feature is identified by dynamic detection
   387  				addr := s.entryNewValue1A(ssa.OpAddr, types.Types[types.TBOOL].PtrTo(), ir.Syms.ARM64HasATOMICS, s.sb)
   388  				v := s.load(types.Types[types.TBOOL], addr)
   389  				b := s.endBlock()
   390  				b.Kind = ssa.BlockIf
   391  				b.SetControl(v)
   392  				bTrue := s.f.NewBlock(ssa.BlockPlain)
   393  				bFalse := s.f.NewBlock(ssa.BlockPlain)
   394  				bEnd := s.f.NewBlock(ssa.BlockPlain)
   395  				b.AddEdgeTo(bTrue)
   396  				b.AddEdgeTo(bFalse)
   397  				b.Likely = ssa.BranchLikely
   398  
   399  				// We have atomic instructions - use it directly.
   400  				s.startBlock(bTrue)
   401  				emit(s, n, args, op1, typ, needReturn)
   402  				s.endBlock().AddEdgeTo(bEnd)
   403  
   404  				// Use original instruction sequence.
   405  				s.startBlock(bFalse)
   406  				emit(s, n, args, op0, typ, needReturn)
   407  				s.endBlock().AddEdgeTo(bEnd)
   408  
   409  				// Merge results.
   410  				s.startBlock(bEnd)
   411  			}
   412  			if needReturn {
   413  				return s.variable(n, types.Types[typ])
   414  			} else {
   415  				return nil
   416  			}
   417  		}
   418  	}
   419  	makeAtomicGuardedIntrinsicARM64 := func(op0, op1 ssa.Op, typ types.Kind, emit atomicOpEmitter) intrinsicBuilder {
   420  		return makeAtomicGuardedIntrinsicARM64common(op0, op1, typ, emit, true)
   421  	}
   422  	makeAtomicGuardedIntrinsicARM64old := func(op0, op1 ssa.Op, typ types.Kind, emit atomicOpEmitter) intrinsicBuilder {
   423  		return makeAtomicGuardedIntrinsicARM64common(op0, op1, typ, emit, false)
   424  	}
   425  
   426  	atomicEmitterARM64 := func(s *state, n *ir.CallExpr, args []*ssa.Value, op ssa.Op, typ types.Kind, needReturn bool) {
   427  		v := s.newValue3(op, types.NewTuple(types.Types[typ], types.TypeMem), args[0], args[1], s.mem())
   428  		s.vars[memVar] = s.newValue1(ssa.OpSelect1, types.TypeMem, v)
   429  		if needReturn {
   430  			s.vars[n] = s.newValue1(ssa.OpSelect0, types.Types[typ], v)
   431  		}
   432  	}
   433  	addF("internal/runtime/atomic", "Xchg8",
   434  		makeAtomicGuardedIntrinsicARM64(ssa.OpAtomicExchange8, ssa.OpAtomicExchange8Variant, types.TUINT8, atomicEmitterARM64),
   435  		sys.ARM64)
   436  	addF("internal/runtime/atomic", "Xchg",
   437  		makeAtomicGuardedIntrinsicARM64(ssa.OpAtomicExchange32, ssa.OpAtomicExchange32Variant, types.TUINT32, atomicEmitterARM64),
   438  		sys.ARM64)
   439  	addF("internal/runtime/atomic", "Xchg64",
   440  		makeAtomicGuardedIntrinsicARM64(ssa.OpAtomicExchange64, ssa.OpAtomicExchange64Variant, types.TUINT64, atomicEmitterARM64),
   441  		sys.ARM64)
   442  
   443  	makeAtomicXchg8GuardedIntrinsicLoong64 := func(op ssa.Op) func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   444  		return func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   445  			addr := s.entryNewValue1A(ssa.OpAddr, types.Types[types.TBOOL].PtrTo(), ir.Syms.Loong64HasLAM_BH, s.sb)
   446  			v := s.load(types.Types[types.TBOOL], addr)
   447  			b := s.endBlock()
   448  			b.Kind = ssa.BlockIf
   449  			b.SetControl(v)
   450  			bTrue := s.f.NewBlock(ssa.BlockPlain)
   451  			bFalse := s.f.NewBlock(ssa.BlockPlain)
   452  			bEnd := s.f.NewBlock(ssa.BlockPlain)
   453  			b.AddEdgeTo(bTrue)
   454  			b.AddEdgeTo(bFalse)
   455  			b.Likely = ssa.BranchLikely // most loong64 machines support the amswapdb.b
   456  
   457  			// We have the intrinsic - use it directly.
   458  			s.startBlock(bTrue)
   459  			s.vars[n] = s.newValue3(op, types.NewTuple(types.Types[types.TUINT8], types.TypeMem), args[0], args[1], s.mem())
   460  			s.vars[memVar] = s.newValue1(ssa.OpSelect1, types.TypeMem, s.vars[n])
   461  			s.vars[n] = s.newValue1(ssa.OpSelect0, types.Types[types.TUINT8], s.vars[n])
   462  			s.endBlock().AddEdgeTo(bEnd)
   463  
   464  			// Call the pure Go version.
   465  			s.startBlock(bFalse)
   466  			s.vars[n] = s.callResult(n, callNormal) // types.Types[TUINT8]
   467  			s.endBlock().AddEdgeTo(bEnd)
   468  
   469  			// Merge results.
   470  			s.startBlock(bEnd)
   471  			return s.variable(n, types.Types[types.TUINT8])
   472  		}
   473  	}
   474  	addF("internal/runtime/atomic", "Xchg8",
   475  		makeAtomicXchg8GuardedIntrinsicLoong64(ssa.OpAtomicExchange8Variant),
   476  		sys.Loong64)
   477  
   478  	addF("internal/runtime/atomic", "Xadd",
   479  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   480  			v := s.newValue3(ssa.OpAtomicAdd32, types.NewTuple(types.Types[types.TUINT32], types.TypeMem), args[0], args[1], s.mem())
   481  			s.vars[memVar] = s.newValue1(ssa.OpSelect1, types.TypeMem, v)
   482  			return s.newValue1(ssa.OpSelect0, types.Types[types.TUINT32], v)
   483  		},
   484  		sys.AMD64, sys.Loong64, sys.MIPS, sys.MIPS64, sys.PPC64, sys.RISCV64, sys.S390X)
   485  	addF("internal/runtime/atomic", "Xadd64",
   486  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   487  			v := s.newValue3(ssa.OpAtomicAdd64, types.NewTuple(types.Types[types.TUINT64], types.TypeMem), args[0], args[1], s.mem())
   488  			s.vars[memVar] = s.newValue1(ssa.OpSelect1, types.TypeMem, v)
   489  			return s.newValue1(ssa.OpSelect0, types.Types[types.TUINT64], v)
   490  		},
   491  		sys.AMD64, sys.Loong64, sys.MIPS64, sys.PPC64, sys.RISCV64, sys.S390X)
   492  
   493  	addF("internal/runtime/atomic", "Xadd",
   494  		makeAtomicGuardedIntrinsicARM64(ssa.OpAtomicAdd32, ssa.OpAtomicAdd32Variant, types.TUINT32, atomicEmitterARM64),
   495  		sys.ARM64)
   496  	addF("internal/runtime/atomic", "Xadd64",
   497  		makeAtomicGuardedIntrinsicARM64(ssa.OpAtomicAdd64, ssa.OpAtomicAdd64Variant, types.TUINT64, atomicEmitterARM64),
   498  		sys.ARM64)
   499  
   500  	addF("internal/runtime/atomic", "Cas",
   501  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   502  			v := s.newValue4(ssa.OpAtomicCompareAndSwap32, types.NewTuple(types.Types[types.TBOOL], types.TypeMem), args[0], args[1], args[2], s.mem())
   503  			s.vars[memVar] = s.newValue1(ssa.OpSelect1, types.TypeMem, v)
   504  			return s.newValue1(ssa.OpSelect0, types.Types[types.TBOOL], v)
   505  		},
   506  		sys.AMD64, sys.MIPS, sys.MIPS64, sys.PPC64, sys.RISCV64, sys.S390X)
   507  	addF("internal/runtime/atomic", "Cas64",
   508  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   509  			v := s.newValue4(ssa.OpAtomicCompareAndSwap64, types.NewTuple(types.Types[types.TBOOL], types.TypeMem), args[0], args[1], args[2], s.mem())
   510  			s.vars[memVar] = s.newValue1(ssa.OpSelect1, types.TypeMem, v)
   511  			return s.newValue1(ssa.OpSelect0, types.Types[types.TBOOL], v)
   512  		},
   513  		sys.AMD64, sys.MIPS64, sys.PPC64, sys.RISCV64, sys.S390X)
   514  	addF("internal/runtime/atomic", "CasRel",
   515  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   516  			v := s.newValue4(ssa.OpAtomicCompareAndSwap32, types.NewTuple(types.Types[types.TBOOL], types.TypeMem), args[0], args[1], args[2], s.mem())
   517  			s.vars[memVar] = s.newValue1(ssa.OpSelect1, types.TypeMem, v)
   518  			return s.newValue1(ssa.OpSelect0, types.Types[types.TBOOL], v)
   519  		},
   520  		sys.PPC64)
   521  
   522  	atomicCasEmitterARM64 := func(s *state, n *ir.CallExpr, args []*ssa.Value, op ssa.Op, typ types.Kind, needReturn bool) {
   523  		v := s.newValue4(op, types.NewTuple(types.Types[types.TBOOL], types.TypeMem), args[0], args[1], args[2], s.mem())
   524  		s.vars[memVar] = s.newValue1(ssa.OpSelect1, types.TypeMem, v)
   525  		if needReturn {
   526  			s.vars[n] = s.newValue1(ssa.OpSelect0, types.Types[typ], v)
   527  		}
   528  	}
   529  
   530  	addF("internal/runtime/atomic", "Cas",
   531  		makeAtomicGuardedIntrinsicARM64(ssa.OpAtomicCompareAndSwap32, ssa.OpAtomicCompareAndSwap32Variant, types.TBOOL, atomicCasEmitterARM64),
   532  		sys.ARM64)
   533  	addF("internal/runtime/atomic", "Cas64",
   534  		makeAtomicGuardedIntrinsicARM64(ssa.OpAtomicCompareAndSwap64, ssa.OpAtomicCompareAndSwap64Variant, types.TBOOL, atomicCasEmitterARM64),
   535  		sys.ARM64)
   536  
   537  	atomicCasEmitterLoong64 := func(s *state, n *ir.CallExpr, args []*ssa.Value, op ssa.Op, typ types.Kind, needReturn bool) {
   538  		v := s.newValue4(op, types.NewTuple(types.Types[types.TBOOL], types.TypeMem), args[0], args[1], args[2], s.mem())
   539  		s.vars[memVar] = s.newValue1(ssa.OpSelect1, types.TypeMem, v)
   540  		if needReturn {
   541  			s.vars[n] = s.newValue1(ssa.OpSelect0, types.Types[typ], v)
   542  		}
   543  	}
   544  
   545  	makeAtomicCasGuardedIntrinsicLoong64 := func(op0, op1 ssa.Op, emit atomicOpEmitter) intrinsicBuilder {
   546  		return func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   547  			// Target Atomic feature is identified by dynamic detection
   548  			addr := s.entryNewValue1A(ssa.OpAddr, types.Types[types.TBOOL].PtrTo(), ir.Syms.Loong64HasLAMCAS, s.sb)
   549  			v := s.load(types.Types[types.TBOOL], addr)
   550  			b := s.endBlock()
   551  			b.Kind = ssa.BlockIf
   552  			b.SetControl(v)
   553  			bTrue := s.f.NewBlock(ssa.BlockPlain)
   554  			bFalse := s.f.NewBlock(ssa.BlockPlain)
   555  			bEnd := s.f.NewBlock(ssa.BlockPlain)
   556  			b.AddEdgeTo(bTrue)
   557  			b.AddEdgeTo(bFalse)
   558  			b.Likely = ssa.BranchLikely
   559  
   560  			// We have atomic instructions - use it directly.
   561  			s.startBlock(bTrue)
   562  			emit(s, n, args, op1, types.TBOOL, true)
   563  			s.endBlock().AddEdgeTo(bEnd)
   564  
   565  			// Use original instruction sequence.
   566  			s.startBlock(bFalse)
   567  			emit(s, n, args, op0, types.TBOOL, true)
   568  			s.endBlock().AddEdgeTo(bEnd)
   569  
   570  			// Merge results.
   571  			s.startBlock(bEnd)
   572  
   573  			return s.variable(n, types.Types[types.TBOOL])
   574  		}
   575  	}
   576  
   577  	addF("internal/runtime/atomic", "Cas",
   578  		makeAtomicCasGuardedIntrinsicLoong64(ssa.OpAtomicCompareAndSwap32, ssa.OpAtomicCompareAndSwap32Variant, atomicCasEmitterLoong64),
   579  		sys.Loong64)
   580  	addF("internal/runtime/atomic", "Cas64",
   581  		makeAtomicCasGuardedIntrinsicLoong64(ssa.OpAtomicCompareAndSwap64, ssa.OpAtomicCompareAndSwap64Variant, atomicCasEmitterLoong64),
   582  		sys.Loong64)
   583  
   584  	// Old-style atomic logical operation API (all supported archs except arm64).
   585  	addF("internal/runtime/atomic", "And8",
   586  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   587  			s.vars[memVar] = s.newValue3(ssa.OpAtomicAnd8, types.TypeMem, args[0], args[1], s.mem())
   588  			return nil
   589  		},
   590  		sys.AMD64, sys.Loong64, sys.MIPS, sys.MIPS64, sys.PPC64, sys.RISCV64, sys.S390X)
   591  	addF("internal/runtime/atomic", "And",
   592  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   593  			s.vars[memVar] = s.newValue3(ssa.OpAtomicAnd32, types.TypeMem, args[0], args[1], s.mem())
   594  			return nil
   595  		},
   596  		sys.AMD64, sys.Loong64, sys.MIPS, sys.MIPS64, sys.PPC64, sys.RISCV64, sys.S390X)
   597  	addF("internal/runtime/atomic", "Or8",
   598  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   599  			s.vars[memVar] = s.newValue3(ssa.OpAtomicOr8, types.TypeMem, args[0], args[1], s.mem())
   600  			return nil
   601  		},
   602  		sys.AMD64, sys.Loong64, sys.MIPS, sys.MIPS64, sys.PPC64, sys.RISCV64, sys.S390X)
   603  	addF("internal/runtime/atomic", "Or",
   604  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   605  			s.vars[memVar] = s.newValue3(ssa.OpAtomicOr32, types.TypeMem, args[0], args[1], s.mem())
   606  			return nil
   607  		},
   608  		sys.AMD64, sys.Loong64, sys.MIPS, sys.MIPS64, sys.PPC64, sys.RISCV64, sys.S390X)
   609  
   610  	// arm64 always uses the new-style atomic logical operations, for both the
   611  	// old and new style API.
   612  	addF("internal/runtime/atomic", "And8",
   613  		makeAtomicGuardedIntrinsicARM64old(ssa.OpAtomicAnd8value, ssa.OpAtomicAnd8valueVariant, types.TUINT8, atomicEmitterARM64),
   614  		sys.ARM64)
   615  	addF("internal/runtime/atomic", "Or8",
   616  		makeAtomicGuardedIntrinsicARM64old(ssa.OpAtomicOr8value, ssa.OpAtomicOr8valueVariant, types.TUINT8, atomicEmitterARM64),
   617  		sys.ARM64)
   618  	addF("internal/runtime/atomic", "And64",
   619  		makeAtomicGuardedIntrinsicARM64(ssa.OpAtomicAnd64value, ssa.OpAtomicAnd64valueVariant, types.TUINT64, atomicEmitterARM64),
   620  		sys.ARM64)
   621  	addF("internal/runtime/atomic", "And32",
   622  		makeAtomicGuardedIntrinsicARM64(ssa.OpAtomicAnd32value, ssa.OpAtomicAnd32valueVariant, types.TUINT32, atomicEmitterARM64),
   623  		sys.ARM64)
   624  	addF("internal/runtime/atomic", "And",
   625  		makeAtomicGuardedIntrinsicARM64old(ssa.OpAtomicAnd32value, ssa.OpAtomicAnd32valueVariant, types.TUINT32, atomicEmitterARM64),
   626  		sys.ARM64)
   627  	addF("internal/runtime/atomic", "Or64",
   628  		makeAtomicGuardedIntrinsicARM64(ssa.OpAtomicOr64value, ssa.OpAtomicOr64valueVariant, types.TUINT64, atomicEmitterARM64),
   629  		sys.ARM64)
   630  	addF("internal/runtime/atomic", "Or32",
   631  		makeAtomicGuardedIntrinsicARM64(ssa.OpAtomicOr32value, ssa.OpAtomicOr32valueVariant, types.TUINT32, atomicEmitterARM64),
   632  		sys.ARM64)
   633  	addF("internal/runtime/atomic", "Or",
   634  		makeAtomicGuardedIntrinsicARM64old(ssa.OpAtomicOr32value, ssa.OpAtomicOr32valueVariant, types.TUINT32, atomicEmitterARM64),
   635  		sys.ARM64)
   636  
   637  	// New-style atomic logical operations, which return the old memory value.
   638  	addF("internal/runtime/atomic", "And64",
   639  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   640  			v := s.newValue3(ssa.OpAtomicAnd64value, types.NewTuple(types.Types[types.TUINT64], types.TypeMem), args[0], args[1], s.mem())
   641  			p0, p1 := s.split(v)
   642  			s.vars[memVar] = p1
   643  			return p0
   644  		},
   645  		sys.AMD64, sys.Loong64)
   646  	addF("internal/runtime/atomic", "And32",
   647  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   648  			v := s.newValue3(ssa.OpAtomicAnd32value, types.NewTuple(types.Types[types.TUINT32], types.TypeMem), args[0], args[1], s.mem())
   649  			p0, p1 := s.split(v)
   650  			s.vars[memVar] = p1
   651  			return p0
   652  		},
   653  		sys.AMD64, sys.Loong64)
   654  	addF("internal/runtime/atomic", "Or64",
   655  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   656  			v := s.newValue3(ssa.OpAtomicOr64value, types.NewTuple(types.Types[types.TUINT64], types.TypeMem), args[0], args[1], s.mem())
   657  			p0, p1 := s.split(v)
   658  			s.vars[memVar] = p1
   659  			return p0
   660  		},
   661  		sys.AMD64, sys.Loong64)
   662  	addF("internal/runtime/atomic", "Or32",
   663  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   664  			v := s.newValue3(ssa.OpAtomicOr32value, types.NewTuple(types.Types[types.TUINT32], types.TypeMem), args[0], args[1], s.mem())
   665  			p0, p1 := s.split(v)
   666  			s.vars[memVar] = p1
   667  			return p0
   668  		},
   669  		sys.AMD64, sys.Loong64)
   670  
   671  	// Aliases for atomic load operations
   672  	alias("internal/runtime/atomic", "Loadint32", "internal/runtime/atomic", "Load", all...)
   673  	alias("internal/runtime/atomic", "Loadint64", "internal/runtime/atomic", "Load64", all...)
   674  	alias("internal/runtime/atomic", "Loaduintptr", "internal/runtime/atomic", "Load", p4...)
   675  	alias("internal/runtime/atomic", "Loaduintptr", "internal/runtime/atomic", "Load64", p8...)
   676  	alias("internal/runtime/atomic", "Loaduint", "internal/runtime/atomic", "Load", p4...)
   677  	alias("internal/runtime/atomic", "Loaduint", "internal/runtime/atomic", "Load64", p8...)
   678  	alias("internal/runtime/atomic", "LoadAcq", "internal/runtime/atomic", "Load", lwatomics...)
   679  	alias("internal/runtime/atomic", "LoadAcq64", "internal/runtime/atomic", "Load64", lwatomics...)
   680  	alias("internal/runtime/atomic", "LoadAcquintptr", "internal/runtime/atomic", "LoadAcq", p4...)
   681  	alias("sync", "runtime_LoadAcquintptr", "internal/runtime/atomic", "LoadAcq", p4...) // linknamed
   682  	alias("internal/runtime/atomic", "LoadAcquintptr", "internal/runtime/atomic", "LoadAcq64", p8...)
   683  	alias("sync", "runtime_LoadAcquintptr", "internal/runtime/atomic", "LoadAcq64", p8...) // linknamed
   684  
   685  	// Aliases for atomic store operations
   686  	alias("internal/runtime/atomic", "Storeint32", "internal/runtime/atomic", "Store", all...)
   687  	alias("internal/runtime/atomic", "Storeint64", "internal/runtime/atomic", "Store64", all...)
   688  	alias("internal/runtime/atomic", "Storeuintptr", "internal/runtime/atomic", "Store", p4...)
   689  	alias("internal/runtime/atomic", "Storeuintptr", "internal/runtime/atomic", "Store64", p8...)
   690  	alias("internal/runtime/atomic", "StoreRel", "internal/runtime/atomic", "Store", lwatomics...)
   691  	alias("internal/runtime/atomic", "StoreRel64", "internal/runtime/atomic", "Store64", lwatomics...)
   692  	alias("internal/runtime/atomic", "StoreReluintptr", "internal/runtime/atomic", "StoreRel", p4...)
   693  	alias("sync", "runtime_StoreReluintptr", "internal/runtime/atomic", "StoreRel", p4...) // linknamed
   694  	alias("internal/runtime/atomic", "StoreReluintptr", "internal/runtime/atomic", "StoreRel64", p8...)
   695  	alias("sync", "runtime_StoreReluintptr", "internal/runtime/atomic", "StoreRel64", p8...) // linknamed
   696  
   697  	// Aliases for atomic swap operations
   698  	alias("internal/runtime/atomic", "Xchgint32", "internal/runtime/atomic", "Xchg", all...)
   699  	alias("internal/runtime/atomic", "Xchgint64", "internal/runtime/atomic", "Xchg64", all...)
   700  	alias("internal/runtime/atomic", "Xchguintptr", "internal/runtime/atomic", "Xchg", p4...)
   701  	alias("internal/runtime/atomic", "Xchguintptr", "internal/runtime/atomic", "Xchg64", p8...)
   702  
   703  	// Aliases for atomic add operations
   704  	alias("internal/runtime/atomic", "Xaddint32", "internal/runtime/atomic", "Xadd", all...)
   705  	alias("internal/runtime/atomic", "Xaddint64", "internal/runtime/atomic", "Xadd64", all...)
   706  	alias("internal/runtime/atomic", "Xadduintptr", "internal/runtime/atomic", "Xadd", p4...)
   707  	alias("internal/runtime/atomic", "Xadduintptr", "internal/runtime/atomic", "Xadd64", p8...)
   708  
   709  	// Aliases for atomic CAS operations
   710  	alias("internal/runtime/atomic", "Casint32", "internal/runtime/atomic", "Cas", all...)
   711  	alias("internal/runtime/atomic", "Casint64", "internal/runtime/atomic", "Cas64", all...)
   712  	alias("internal/runtime/atomic", "Casuintptr", "internal/runtime/atomic", "Cas", p4...)
   713  	alias("internal/runtime/atomic", "Casuintptr", "internal/runtime/atomic", "Cas64", p8...)
   714  	alias("internal/runtime/atomic", "Casp1", "internal/runtime/atomic", "Cas", p4...)
   715  	alias("internal/runtime/atomic", "Casp1", "internal/runtime/atomic", "Cas64", p8...)
   716  	alias("internal/runtime/atomic", "CasRel", "internal/runtime/atomic", "Cas", lwatomics...)
   717  
   718  	// Aliases for atomic And/Or operations
   719  	alias("internal/runtime/atomic", "Anduintptr", "internal/runtime/atomic", "And64", sys.ArchARM64, sys.ArchLoong64)
   720  	alias("internal/runtime/atomic", "Oruintptr", "internal/runtime/atomic", "Or64", sys.ArchARM64, sys.ArchLoong64)
   721  
   722  	/******** math ********/
   723  	addF("math", "sqrt",
   724  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   725  			return s.newValue1(ssa.OpSqrt, types.Types[types.TFLOAT64], args[0])
   726  		},
   727  		sys.I386, sys.AMD64, sys.ARM, sys.ARM64, sys.Loong64, sys.MIPS, sys.MIPS64, sys.PPC64, sys.RISCV64, sys.S390X, sys.Wasm)
   728  	addF("math", "Trunc",
   729  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   730  			return s.newValue1(ssa.OpTrunc, types.Types[types.TFLOAT64], args[0])
   731  		},
   732  		sys.ARM64, sys.PPC64, sys.S390X, sys.Wasm)
   733  	addF("math", "Ceil",
   734  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   735  			return s.newValue1(ssa.OpCeil, types.Types[types.TFLOAT64], args[0])
   736  		},
   737  		sys.ARM64, sys.PPC64, sys.S390X, sys.Wasm)
   738  	addF("math", "Floor",
   739  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   740  			return s.newValue1(ssa.OpFloor, types.Types[types.TFLOAT64], args[0])
   741  		},
   742  		sys.ARM64, sys.PPC64, sys.S390X, sys.Wasm)
   743  	addF("math", "Round",
   744  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   745  			return s.newValue1(ssa.OpRound, types.Types[types.TFLOAT64], args[0])
   746  		},
   747  		sys.ARM64, sys.PPC64, sys.S390X)
   748  	addF("math", "RoundToEven",
   749  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   750  			return s.newValue1(ssa.OpRoundToEven, types.Types[types.TFLOAT64], args[0])
   751  		},
   752  		sys.ARM64, sys.S390X, sys.Wasm)
   753  	addF("math", "Abs",
   754  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   755  			return s.newValue1(ssa.OpAbs, types.Types[types.TFLOAT64], args[0])
   756  		},
   757  		sys.ARM64, sys.ARM, sys.Loong64, sys.PPC64, sys.RISCV64, sys.Wasm, sys.MIPS, sys.MIPS64)
   758  	addF("math", "Copysign",
   759  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   760  			return s.newValue2(ssa.OpCopysign, types.Types[types.TFLOAT64], args[0], args[1])
   761  		},
   762  		sys.Loong64, sys.PPC64, sys.RISCV64, sys.Wasm)
   763  	addF("math", "FMA",
   764  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   765  			return s.newValue3(ssa.OpFMA, types.Types[types.TFLOAT64], args[0], args[1], args[2])
   766  		},
   767  		sys.ARM64, sys.Loong64, sys.PPC64, sys.RISCV64, sys.S390X)
   768  	addF("math", "FMA",
   769  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   770  			if cfg.goamd64 >= 3 {
   771  				return s.newValue3(ssa.OpFMA, types.Types[types.TFLOAT64], args[0], args[1], args[2])
   772  			}
   773  
   774  			v := s.entryNewValue0A(ssa.OpHasCPUFeature, types.Types[types.TBOOL], ir.Syms.X86HasFMA)
   775  			b := s.endBlock()
   776  			b.Kind = ssa.BlockIf
   777  			b.SetControl(v)
   778  			bTrue := s.f.NewBlock(ssa.BlockPlain)
   779  			bFalse := s.f.NewBlock(ssa.BlockPlain)
   780  			bEnd := s.f.NewBlock(ssa.BlockPlain)
   781  			b.AddEdgeTo(bTrue)
   782  			b.AddEdgeTo(bFalse)
   783  			b.Likely = ssa.BranchLikely // >= haswell cpus are common
   784  
   785  			// We have the intrinsic - use it directly.
   786  			s.startBlock(bTrue)
   787  			s.vars[n] = s.newValue3(ssa.OpFMA, types.Types[types.TFLOAT64], args[0], args[1], args[2])
   788  			s.endBlock().AddEdgeTo(bEnd)
   789  
   790  			// Call the pure Go version.
   791  			s.startBlock(bFalse)
   792  			s.vars[n] = s.callResult(n, callNormal) // types.Types[TFLOAT64]
   793  			s.endBlock().AddEdgeTo(bEnd)
   794  
   795  			// Merge results.
   796  			s.startBlock(bEnd)
   797  			return s.variable(n, types.Types[types.TFLOAT64])
   798  		},
   799  		sys.AMD64)
   800  	addF("math", "FMA",
   801  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   802  			addr := s.entryNewValue1A(ssa.OpAddr, types.Types[types.TBOOL].PtrTo(), ir.Syms.ARMHasVFPv4, s.sb)
   803  			v := s.load(types.Types[types.TBOOL], addr)
   804  			b := s.endBlock()
   805  			b.Kind = ssa.BlockIf
   806  			b.SetControl(v)
   807  			bTrue := s.f.NewBlock(ssa.BlockPlain)
   808  			bFalse := s.f.NewBlock(ssa.BlockPlain)
   809  			bEnd := s.f.NewBlock(ssa.BlockPlain)
   810  			b.AddEdgeTo(bTrue)
   811  			b.AddEdgeTo(bFalse)
   812  			b.Likely = ssa.BranchLikely
   813  
   814  			// We have the intrinsic - use it directly.
   815  			s.startBlock(bTrue)
   816  			s.vars[n] = s.newValue3(ssa.OpFMA, types.Types[types.TFLOAT64], args[0], args[1], args[2])
   817  			s.endBlock().AddEdgeTo(bEnd)
   818  
   819  			// Call the pure Go version.
   820  			s.startBlock(bFalse)
   821  			s.vars[n] = s.callResult(n, callNormal) // types.Types[TFLOAT64]
   822  			s.endBlock().AddEdgeTo(bEnd)
   823  
   824  			// Merge results.
   825  			s.startBlock(bEnd)
   826  			return s.variable(n, types.Types[types.TFLOAT64])
   827  		},
   828  		sys.ARM)
   829  
   830  	makeRoundAMD64 := func(op ssa.Op) func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   831  		return func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   832  			if cfg.goamd64 >= 2 {
   833  				return s.newValue1(op, types.Types[types.TFLOAT64], args[0])
   834  			}
   835  
   836  			v := s.entryNewValue0A(ssa.OpHasCPUFeature, types.Types[types.TBOOL], ir.Syms.X86HasSSE41)
   837  			b := s.endBlock()
   838  			b.Kind = ssa.BlockIf
   839  			b.SetControl(v)
   840  			bTrue := s.f.NewBlock(ssa.BlockPlain)
   841  			bFalse := s.f.NewBlock(ssa.BlockPlain)
   842  			bEnd := s.f.NewBlock(ssa.BlockPlain)
   843  			b.AddEdgeTo(bTrue)
   844  			b.AddEdgeTo(bFalse)
   845  			b.Likely = ssa.BranchLikely // most machines have sse4.1 nowadays
   846  
   847  			// We have the intrinsic - use it directly.
   848  			s.startBlock(bTrue)
   849  			s.vars[n] = s.newValue1(op, types.Types[types.TFLOAT64], args[0])
   850  			s.endBlock().AddEdgeTo(bEnd)
   851  
   852  			// Call the pure Go version.
   853  			s.startBlock(bFalse)
   854  			s.vars[n] = s.callResult(n, callNormal) // types.Types[TFLOAT64]
   855  			s.endBlock().AddEdgeTo(bEnd)
   856  
   857  			// Merge results.
   858  			s.startBlock(bEnd)
   859  			return s.variable(n, types.Types[types.TFLOAT64])
   860  		}
   861  	}
   862  	addF("math", "RoundToEven",
   863  		makeRoundAMD64(ssa.OpRoundToEven),
   864  		sys.AMD64)
   865  	addF("math", "Floor",
   866  		makeRoundAMD64(ssa.OpFloor),
   867  		sys.AMD64)
   868  	addF("math", "Ceil",
   869  		makeRoundAMD64(ssa.OpCeil),
   870  		sys.AMD64)
   871  	addF("math", "Trunc",
   872  		makeRoundAMD64(ssa.OpTrunc),
   873  		sys.AMD64)
   874  
   875  	/******** math/bits ********/
   876  	addF("math/bits", "TrailingZeros64",
   877  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   878  			return s.newValue1(ssa.OpCtz64, types.Types[types.TINT], args[0])
   879  		},
   880  		sys.AMD64, sys.ARM64, sys.ARM, sys.Loong64, sys.S390X, sys.MIPS, sys.PPC64, sys.Wasm)
   881  	addF("math/bits", "TrailingZeros64",
   882  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   883  			lo := s.newValue1(ssa.OpInt64Lo, types.Types[types.TUINT32], args[0])
   884  			hi := s.newValue1(ssa.OpInt64Hi, types.Types[types.TUINT32], args[0])
   885  			return s.newValue2(ssa.OpCtz64On32, types.Types[types.TINT], lo, hi)
   886  		},
   887  		sys.I386)
   888  	addF("math/bits", "TrailingZeros32",
   889  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   890  			return s.newValue1(ssa.OpCtz32, types.Types[types.TINT], args[0])
   891  		},
   892  		sys.AMD64, sys.I386, sys.ARM64, sys.ARM, sys.Loong64, sys.S390X, sys.MIPS, sys.PPC64, sys.Wasm)
   893  	addF("math/bits", "TrailingZeros16",
   894  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   895  			return s.newValue1(ssa.OpCtz16, types.Types[types.TINT], args[0])
   896  		},
   897  		sys.AMD64, sys.ARM, sys.ARM64, sys.I386, sys.MIPS, sys.Loong64, sys.PPC64, sys.S390X, sys.Wasm)
   898  	addF("math/bits", "TrailingZeros8",
   899  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   900  			return s.newValue1(ssa.OpCtz8, types.Types[types.TINT], args[0])
   901  		},
   902  		sys.AMD64, sys.ARM, sys.ARM64, sys.I386, sys.MIPS, sys.Loong64, sys.PPC64, sys.S390X, sys.Wasm)
   903  
   904  	if cfg.goriscv64 >= 22 {
   905  		addF("math/bits", "TrailingZeros64",
   906  			func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   907  				return s.newValue1(ssa.OpCtz64, types.Types[types.TINT], args[0])
   908  			},
   909  			sys.RISCV64)
   910  		addF("math/bits", "TrailingZeros32",
   911  			func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   912  				return s.newValue1(ssa.OpCtz32, types.Types[types.TINT], args[0])
   913  			},
   914  			sys.RISCV64)
   915  		addF("math/bits", "TrailingZeros16",
   916  			func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   917  				return s.newValue1(ssa.OpCtz16, types.Types[types.TINT], args[0])
   918  			},
   919  			sys.RISCV64)
   920  		addF("math/bits", "TrailingZeros8",
   921  			func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   922  				return s.newValue1(ssa.OpCtz8, types.Types[types.TINT], args[0])
   923  			},
   924  			sys.RISCV64)
   925  	}
   926  
   927  	alias("math/bits", "ReverseBytes64", "internal/runtime/sys", "Bswap64", all...)
   928  	alias("math/bits", "ReverseBytes32", "internal/runtime/sys", "Bswap32", all...)
   929  	addF("math/bits", "ReverseBytes16",
   930  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   931  			return s.newValue1(ssa.OpBswap16, types.Types[types.TUINT16], args[0])
   932  		},
   933  		sys.Loong64)
   934  	// ReverseBytes inlines correctly, no need to intrinsify it.
   935  	// Nothing special is needed for targets where ReverseBytes16 lowers to a rotate
   936  	// On Power10, 16-bit rotate is not available so use BRH instruction
   937  	if cfg.goppc64 >= 10 {
   938  		addF("math/bits", "ReverseBytes16",
   939  			func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   940  				return s.newValue1(ssa.OpBswap16, types.Types[types.TUINT], args[0])
   941  			},
   942  			sys.PPC64)
   943  	}
   944  
   945  	addF("math/bits", "Len64",
   946  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   947  			return s.newValue1(ssa.OpBitLen64, types.Types[types.TINT], args[0])
   948  		},
   949  		sys.AMD64, sys.ARM, sys.ARM64, sys.Loong64, sys.MIPS, sys.PPC64, sys.S390X, sys.Wasm)
   950  	addF("math/bits", "Len32",
   951  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   952  			return s.newValue1(ssa.OpBitLen32, types.Types[types.TINT], args[0])
   953  		},
   954  		sys.AMD64, sys.ARM, sys.ARM64, sys.Loong64, sys.MIPS, sys.PPC64, sys.S390X, sys.Wasm)
   955  	addF("math/bits", "Len16",
   956  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   957  			return s.newValue1(ssa.OpBitLen16, types.Types[types.TINT], args[0])
   958  		},
   959  		sys.AMD64, sys.ARM, sys.ARM64, sys.Loong64, sys.MIPS, sys.PPC64, sys.S390X, sys.Wasm)
   960  	addF("math/bits", "Len8",
   961  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   962  			return s.newValue1(ssa.OpBitLen8, types.Types[types.TINT], args[0])
   963  		},
   964  		sys.AMD64, sys.ARM, sys.ARM64, sys.Loong64, sys.MIPS, sys.PPC64, sys.S390X, sys.Wasm)
   965  
   966  	if cfg.goriscv64 >= 22 {
   967  		addF("math/bits", "Len64",
   968  			func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   969  				return s.newValue1(ssa.OpBitLen64, types.Types[types.TINT], args[0])
   970  			},
   971  			sys.RISCV64)
   972  		addF("math/bits", "Len32",
   973  			func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   974  				return s.newValue1(ssa.OpBitLen32, types.Types[types.TINT], args[0])
   975  			},
   976  			sys.RISCV64)
   977  		addF("math/bits", "Len16",
   978  			func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   979  				return s.newValue1(ssa.OpBitLen16, types.Types[types.TINT], args[0])
   980  			},
   981  			sys.RISCV64)
   982  		addF("math/bits", "Len8",
   983  			func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   984  				return s.newValue1(ssa.OpBitLen8, types.Types[types.TINT], args[0])
   985  			},
   986  			sys.RISCV64)
   987  	}
   988  
   989  	alias("math/bits", "Len", "math/bits", "Len64", p8...)
   990  	alias("math/bits", "Len", "math/bits", "Len32", p4...)
   991  
   992  	// LeadingZeros is handled because it trivially calls Len.
   993  	addF("math/bits", "Reverse64",
   994  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   995  			return s.newValue1(ssa.OpBitRev64, types.Types[types.TINT], args[0])
   996  		},
   997  		sys.ARM64, sys.Loong64)
   998  	addF("math/bits", "Reverse32",
   999  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  1000  			return s.newValue1(ssa.OpBitRev32, types.Types[types.TINT], args[0])
  1001  		},
  1002  		sys.ARM64, sys.Loong64)
  1003  	addF("math/bits", "Reverse16",
  1004  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  1005  			return s.newValue1(ssa.OpBitRev16, types.Types[types.TINT], args[0])
  1006  		},
  1007  		sys.ARM64, sys.Loong64)
  1008  	addF("math/bits", "Reverse8",
  1009  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  1010  			return s.newValue1(ssa.OpBitRev8, types.Types[types.TINT], args[0])
  1011  		},
  1012  		sys.ARM64, sys.Loong64)
  1013  	addF("math/bits", "Reverse",
  1014  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  1015  			return s.newValue1(ssa.OpBitRev64, types.Types[types.TINT], args[0])
  1016  		},
  1017  		sys.ARM64, sys.Loong64)
  1018  	addF("math/bits", "RotateLeft8",
  1019  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  1020  			return s.newValue2(ssa.OpRotateLeft8, types.Types[types.TUINT8], args[0], args[1])
  1021  		},
  1022  		sys.AMD64, sys.RISCV64)
  1023  	addF("math/bits", "RotateLeft16",
  1024  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  1025  			return s.newValue2(ssa.OpRotateLeft16, types.Types[types.TUINT16], args[0], args[1])
  1026  		},
  1027  		sys.AMD64, sys.RISCV64)
  1028  	addF("math/bits", "RotateLeft32",
  1029  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  1030  			return s.newValue2(ssa.OpRotateLeft32, types.Types[types.TUINT32], args[0], args[1])
  1031  		},
  1032  		sys.AMD64, sys.ARM, sys.ARM64, sys.Loong64, sys.PPC64, sys.RISCV64, sys.S390X, sys.Wasm)
  1033  	addF("math/bits", "RotateLeft64",
  1034  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  1035  			return s.newValue2(ssa.OpRotateLeft64, types.Types[types.TUINT64], args[0], args[1])
  1036  		},
  1037  		sys.AMD64, sys.ARM64, sys.Loong64, sys.PPC64, sys.RISCV64, sys.S390X, sys.Wasm)
  1038  	alias("math/bits", "RotateLeft", "math/bits", "RotateLeft64", p8...)
  1039  
  1040  	makeOnesCountAMD64 := func(op ssa.Op) func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  1041  		return func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  1042  			if cfg.goamd64 >= 2 {
  1043  				return s.newValue1(op, types.Types[types.TINT], args[0])
  1044  			}
  1045  
  1046  			v := s.entryNewValue0A(ssa.OpHasCPUFeature, types.Types[types.TBOOL], ir.Syms.X86HasPOPCNT)
  1047  			b := s.endBlock()
  1048  			b.Kind = ssa.BlockIf
  1049  			b.SetControl(v)
  1050  			bTrue := s.f.NewBlock(ssa.BlockPlain)
  1051  			bFalse := s.f.NewBlock(ssa.BlockPlain)
  1052  			bEnd := s.f.NewBlock(ssa.BlockPlain)
  1053  			b.AddEdgeTo(bTrue)
  1054  			b.AddEdgeTo(bFalse)
  1055  			b.Likely = ssa.BranchLikely // most machines have popcnt nowadays
  1056  
  1057  			// We have the intrinsic - use it directly.
  1058  			s.startBlock(bTrue)
  1059  			s.vars[n] = s.newValue1(op, types.Types[types.TINT], args[0])
  1060  			s.endBlock().AddEdgeTo(bEnd)
  1061  
  1062  			// Call the pure Go version.
  1063  			s.startBlock(bFalse)
  1064  			s.vars[n] = s.callResult(n, callNormal) // types.Types[TINT]
  1065  			s.endBlock().AddEdgeTo(bEnd)
  1066  
  1067  			// Merge results.
  1068  			s.startBlock(bEnd)
  1069  			return s.variable(n, types.Types[types.TINT])
  1070  		}
  1071  	}
  1072  
  1073  	makeOnesCountLoong64 := func(op ssa.Op) func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  1074  		return func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  1075  			addr := s.entryNewValue1A(ssa.OpAddr, types.Types[types.TBOOL].PtrTo(), ir.Syms.Loong64HasLSX, s.sb)
  1076  			v := s.load(types.Types[types.TBOOL], addr)
  1077  			b := s.endBlock()
  1078  			b.Kind = ssa.BlockIf
  1079  			b.SetControl(v)
  1080  			bTrue := s.f.NewBlock(ssa.BlockPlain)
  1081  			bFalse := s.f.NewBlock(ssa.BlockPlain)
  1082  			bEnd := s.f.NewBlock(ssa.BlockPlain)
  1083  			b.AddEdgeTo(bTrue)
  1084  			b.AddEdgeTo(bFalse)
  1085  			b.Likely = ssa.BranchLikely // most loong64 machines support the LSX
  1086  
  1087  			// We have the intrinsic - use it directly.
  1088  			s.startBlock(bTrue)
  1089  			s.vars[n] = s.newValue1(op, types.Types[types.TINT], args[0])
  1090  			s.endBlock().AddEdgeTo(bEnd)
  1091  
  1092  			// Call the pure Go version.
  1093  			s.startBlock(bFalse)
  1094  			s.vars[n] = s.callResult(n, callNormal) // types.Types[TINT]
  1095  			s.endBlock().AddEdgeTo(bEnd)
  1096  
  1097  			// Merge results.
  1098  			s.startBlock(bEnd)
  1099  			return s.variable(n, types.Types[types.TINT])
  1100  		}
  1101  	}
  1102  
  1103  	addF("math/bits", "OnesCount64",
  1104  		makeOnesCountAMD64(ssa.OpPopCount64),
  1105  		sys.AMD64)
  1106  	addF("math/bits", "OnesCount64",
  1107  		makeOnesCountLoong64(ssa.OpPopCount64),
  1108  		sys.Loong64)
  1109  	addF("math/bits", "OnesCount64",
  1110  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  1111  			return s.newValue1(ssa.OpPopCount64, types.Types[types.TINT], args[0])
  1112  		},
  1113  		sys.PPC64, sys.ARM64, sys.S390X, sys.Wasm)
  1114  	addF("math/bits", "OnesCount32",
  1115  		makeOnesCountAMD64(ssa.OpPopCount32),
  1116  		sys.AMD64)
  1117  	addF("math/bits", "OnesCount32",
  1118  		makeOnesCountLoong64(ssa.OpPopCount32),
  1119  		sys.Loong64)
  1120  	addF("math/bits", "OnesCount32",
  1121  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  1122  			return s.newValue1(ssa.OpPopCount32, types.Types[types.TINT], args[0])
  1123  		},
  1124  		sys.PPC64, sys.ARM64, sys.S390X, sys.Wasm)
  1125  	addF("math/bits", "OnesCount16",
  1126  		makeOnesCountAMD64(ssa.OpPopCount16),
  1127  		sys.AMD64)
  1128  	addF("math/bits", "OnesCount16",
  1129  		makeOnesCountLoong64(ssa.OpPopCount16),
  1130  		sys.Loong64)
  1131  	addF("math/bits", "OnesCount16",
  1132  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  1133  			return s.newValue1(ssa.OpPopCount16, types.Types[types.TINT], args[0])
  1134  		},
  1135  		sys.ARM64, sys.S390X, sys.PPC64, sys.Wasm)
  1136  	addF("math/bits", "OnesCount8",
  1137  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  1138  			return s.newValue1(ssa.OpPopCount8, types.Types[types.TINT], args[0])
  1139  		},
  1140  		sys.S390X, sys.PPC64, sys.Wasm)
  1141  	alias("math/bits", "OnesCount", "math/bits", "OnesCount64", p8...)
  1142  
  1143  	addF("math/bits", "Mul64",
  1144  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  1145  			return s.newValue2(ssa.OpMul64uhilo, types.NewTuple(types.Types[types.TUINT64], types.Types[types.TUINT64]), args[0], args[1])
  1146  		},
  1147  		sys.AMD64, sys.ARM64, sys.PPC64, sys.S390X, sys.MIPS64, sys.RISCV64, sys.Loong64)
  1148  	alias("math/bits", "Mul", "math/bits", "Mul64", p8...)
  1149  	alias("internal/runtime/math", "Mul64", "math/bits", "Mul64", p8...)
  1150  	addF("math/bits", "Add64",
  1151  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  1152  			return s.newValue3(ssa.OpAdd64carry, types.NewTuple(types.Types[types.TUINT64], types.Types[types.TUINT64]), args[0], args[1], args[2])
  1153  		},
  1154  		sys.AMD64, sys.ARM64, sys.PPC64, sys.S390X, sys.RISCV64, sys.Loong64, sys.MIPS64)
  1155  	alias("math/bits", "Add", "math/bits", "Add64", p8...)
  1156  	alias("internal/runtime/math", "Add64", "math/bits", "Add64", all...)
  1157  	addF("math/bits", "Sub64",
  1158  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  1159  			return s.newValue3(ssa.OpSub64borrow, types.NewTuple(types.Types[types.TUINT64], types.Types[types.TUINT64]), args[0], args[1], args[2])
  1160  		},
  1161  		sys.AMD64, sys.ARM64, sys.PPC64, sys.S390X, sys.RISCV64, sys.Loong64, sys.MIPS64)
  1162  	alias("math/bits", "Sub", "math/bits", "Sub64", p8...)
  1163  	addF("math/bits", "Div64",
  1164  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  1165  			// check for divide-by-zero/overflow and panic with appropriate message
  1166  			cmpZero := s.newValue2(s.ssaOp(ir.ONE, types.Types[types.TUINT64]), types.Types[types.TBOOL], args[2], s.zeroVal(types.Types[types.TUINT64]))
  1167  			s.check(cmpZero, ir.Syms.Panicdivide)
  1168  			cmpOverflow := s.newValue2(s.ssaOp(ir.OLT, types.Types[types.TUINT64]), types.Types[types.TBOOL], args[0], args[2])
  1169  			s.check(cmpOverflow, ir.Syms.Panicoverflow)
  1170  			return s.newValue3(ssa.OpDiv128u, types.NewTuple(types.Types[types.TUINT64], types.Types[types.TUINT64]), args[0], args[1], args[2])
  1171  		},
  1172  		sys.AMD64)
  1173  	alias("math/bits", "Div", "math/bits", "Div64", sys.ArchAMD64)
  1174  
  1175  	alias("internal/runtime/sys", "TrailingZeros8", "math/bits", "TrailingZeros8", all...)
  1176  	alias("internal/runtime/sys", "TrailingZeros32", "math/bits", "TrailingZeros32", all...)
  1177  	alias("internal/runtime/sys", "TrailingZeros64", "math/bits", "TrailingZeros64", all...)
  1178  	alias("internal/runtime/sys", "Len8", "math/bits", "Len8", all...)
  1179  	alias("internal/runtime/sys", "Len64", "math/bits", "Len64", all...)
  1180  	alias("internal/runtime/sys", "OnesCount64", "math/bits", "OnesCount64", all...)
  1181  
  1182  	/******** sync/atomic ********/
  1183  
  1184  	// Note: these are disabled by flag_race in findIntrinsic below.
  1185  	alias("sync/atomic", "LoadInt32", "internal/runtime/atomic", "Load", all...)
  1186  	alias("sync/atomic", "LoadInt64", "internal/runtime/atomic", "Load64", all...)
  1187  	alias("sync/atomic", "LoadPointer", "internal/runtime/atomic", "Loadp", all...)
  1188  	alias("sync/atomic", "LoadUint32", "internal/runtime/atomic", "Load", all...)
  1189  	alias("sync/atomic", "LoadUint64", "internal/runtime/atomic", "Load64", all...)
  1190  	alias("sync/atomic", "LoadUintptr", "internal/runtime/atomic", "Load", p4...)
  1191  	alias("sync/atomic", "LoadUintptr", "internal/runtime/atomic", "Load64", p8...)
  1192  
  1193  	alias("sync/atomic", "StoreInt32", "internal/runtime/atomic", "Store", all...)
  1194  	alias("sync/atomic", "StoreInt64", "internal/runtime/atomic", "Store64", all...)
  1195  	// Note: not StorePointer, that needs a write barrier.  Same below for {CompareAnd}Swap.
  1196  	alias("sync/atomic", "StoreUint32", "internal/runtime/atomic", "Store", all...)
  1197  	alias("sync/atomic", "StoreUint64", "internal/runtime/atomic", "Store64", all...)
  1198  	alias("sync/atomic", "StoreUintptr", "internal/runtime/atomic", "Store", p4...)
  1199  	alias("sync/atomic", "StoreUintptr", "internal/runtime/atomic", "Store64", p8...)
  1200  
  1201  	alias("sync/atomic", "SwapInt32", "internal/runtime/atomic", "Xchg", all...)
  1202  	alias("sync/atomic", "SwapInt64", "internal/runtime/atomic", "Xchg64", all...)
  1203  	alias("sync/atomic", "SwapUint32", "internal/runtime/atomic", "Xchg", all...)
  1204  	alias("sync/atomic", "SwapUint64", "internal/runtime/atomic", "Xchg64", all...)
  1205  	alias("sync/atomic", "SwapUintptr", "internal/runtime/atomic", "Xchg", p4...)
  1206  	alias("sync/atomic", "SwapUintptr", "internal/runtime/atomic", "Xchg64", p8...)
  1207  
  1208  	alias("sync/atomic", "CompareAndSwapInt32", "internal/runtime/atomic", "Cas", all...)
  1209  	alias("sync/atomic", "CompareAndSwapInt64", "internal/runtime/atomic", "Cas64", all...)
  1210  	alias("sync/atomic", "CompareAndSwapUint32", "internal/runtime/atomic", "Cas", all...)
  1211  	alias("sync/atomic", "CompareAndSwapUint64", "internal/runtime/atomic", "Cas64", all...)
  1212  	alias("sync/atomic", "CompareAndSwapUintptr", "internal/runtime/atomic", "Cas", p4...)
  1213  	alias("sync/atomic", "CompareAndSwapUintptr", "internal/runtime/atomic", "Cas64", p8...)
  1214  
  1215  	alias("sync/atomic", "AddInt32", "internal/runtime/atomic", "Xadd", all...)
  1216  	alias("sync/atomic", "AddInt64", "internal/runtime/atomic", "Xadd64", all...)
  1217  	alias("sync/atomic", "AddUint32", "internal/runtime/atomic", "Xadd", all...)
  1218  	alias("sync/atomic", "AddUint64", "internal/runtime/atomic", "Xadd64", all...)
  1219  	alias("sync/atomic", "AddUintptr", "internal/runtime/atomic", "Xadd", p4...)
  1220  	alias("sync/atomic", "AddUintptr", "internal/runtime/atomic", "Xadd64", p8...)
  1221  
  1222  	alias("sync/atomic", "AndInt32", "internal/runtime/atomic", "And32", sys.ArchARM64, sys.ArchAMD64, sys.ArchLoong64)
  1223  	alias("sync/atomic", "AndUint32", "internal/runtime/atomic", "And32", sys.ArchARM64, sys.ArchAMD64, sys.ArchLoong64)
  1224  	alias("sync/atomic", "AndInt64", "internal/runtime/atomic", "And64", sys.ArchARM64, sys.ArchAMD64, sys.ArchLoong64)
  1225  	alias("sync/atomic", "AndUint64", "internal/runtime/atomic", "And64", sys.ArchARM64, sys.ArchAMD64, sys.ArchLoong64)
  1226  	alias("sync/atomic", "AndUintptr", "internal/runtime/atomic", "And64", sys.ArchARM64, sys.ArchAMD64, sys.ArchLoong64)
  1227  	alias("sync/atomic", "OrInt32", "internal/runtime/atomic", "Or32", sys.ArchARM64, sys.ArchAMD64, sys.ArchLoong64)
  1228  	alias("sync/atomic", "OrUint32", "internal/runtime/atomic", "Or32", sys.ArchARM64, sys.ArchAMD64, sys.ArchLoong64)
  1229  	alias("sync/atomic", "OrInt64", "internal/runtime/atomic", "Or64", sys.ArchARM64, sys.ArchAMD64, sys.ArchLoong64)
  1230  	alias("sync/atomic", "OrUint64", "internal/runtime/atomic", "Or64", sys.ArchARM64, sys.ArchAMD64, sys.ArchLoong64)
  1231  	alias("sync/atomic", "OrUintptr", "internal/runtime/atomic", "Or64", sys.ArchARM64, sys.ArchAMD64, sys.ArchLoong64)
  1232  
  1233  	/******** math/big ********/
  1234  	alias("math/big", "mulWW", "math/bits", "Mul64", p8...)
  1235  
  1236  	/******** internal/runtime/maps ********/
  1237  
  1238  	// Important: The intrinsic implementations below return a packed
  1239  	// bitset, while the portable Go implementation uses an unpacked
  1240  	// representation (one bit set in each byte).
  1241  	//
  1242  	// Thus we must replace most bitset methods with implementations that
  1243  	// work with the packed representation.
  1244  	//
  1245  	// TODO(prattmic): The bitset implementations don't use SIMD, so they
  1246  	// could be handled with build tags (though that would break
  1247  	// -d=ssa/intrinsics/off=1).
  1248  
  1249  	// With a packed representation we no longer need to shift the result
  1250  	// of TrailingZeros64.
  1251  	alias("internal/runtime/maps", "bitsetFirst", "internal/runtime/sys", "TrailingZeros64", sys.ArchAMD64)
  1252  
  1253  	addF("internal/runtime/maps", "bitsetRemoveBelow",
  1254  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  1255  			b := args[0]
  1256  			i := args[1]
  1257  
  1258  			// Clear the lower i bits in b.
  1259  			//
  1260  			// out = b &^ ((1 << i) - 1)
  1261  
  1262  			one := s.constInt64(types.Types[types.TUINT64], 1)
  1263  
  1264  			mask := s.newValue2(ssa.OpLsh8x8, types.Types[types.TUINT64], one, i)
  1265  			mask = s.newValue2(ssa.OpSub64, types.Types[types.TUINT64], mask, one)
  1266  			mask = s.newValue1(ssa.OpCom64, types.Types[types.TUINT64], mask)
  1267  
  1268  			return s.newValue2(ssa.OpAnd64, types.Types[types.TUINT64], b, mask)
  1269  		},
  1270  		sys.AMD64)
  1271  
  1272  	addF("internal/runtime/maps", "bitsetLowestSet",
  1273  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  1274  			b := args[0]
  1275  
  1276  			// Test the lowest bit in b.
  1277  			//
  1278  			// out = (b & 1) == 1
  1279  
  1280  			one := s.constInt64(types.Types[types.TUINT64], 1)
  1281  			and := s.newValue2(ssa.OpAnd64, types.Types[types.TUINT64], b, one)
  1282  			return s.newValue2(ssa.OpEq64, types.Types[types.TBOOL], and, one)
  1283  		},
  1284  		sys.AMD64)
  1285  
  1286  	addF("internal/runtime/maps", "bitsetShiftOutLowest",
  1287  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  1288  			b := args[0]
  1289  
  1290  			// Right shift out the lowest bit in b.
  1291  			//
  1292  			// out = b >> 1
  1293  
  1294  			one := s.constInt64(types.Types[types.TUINT64], 1)
  1295  			return s.newValue2(ssa.OpRsh64Ux64, types.Types[types.TUINT64], b, one)
  1296  		},
  1297  		sys.AMD64)
  1298  
  1299  	addF("internal/runtime/maps", "ctrlGroupMatchH2",
  1300  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  1301  			g := args[0]
  1302  			h := args[1]
  1303  
  1304  			// Explicit copies to fp registers. See
  1305  			// https://go.dev/issue/70451.
  1306  			gfp := s.newValue1(ssa.OpAMD64MOVQi2f, types.TypeInt128, g)
  1307  			hfp := s.newValue1(ssa.OpAMD64MOVQi2f, types.TypeInt128, h)
  1308  
  1309  			// Broadcast h2 into each byte of a word.
  1310  			var broadcast *ssa.Value
  1311  			if buildcfg.GOAMD64 >= 4 {
  1312  				// VPBROADCASTB saves 1 instruction vs PSHUFB
  1313  				// because the input can come from a GP
  1314  				// register, while PSHUFB requires moving into
  1315  				// an FP register first.
  1316  				//
  1317  				// Nominally PSHUFB would require a second
  1318  				// additional instruction to load the control
  1319  				// mask into a FP register. But broadcast uses
  1320  				// a control mask of 0, and the register ABI
  1321  				// already defines X15 as a zero register.
  1322  				broadcast = s.newValue1(ssa.OpAMD64VPBROADCASTB, types.TypeInt128, h) // use gp copy of h
  1323  			} else if buildcfg.GOAMD64 >= 2 {
  1324  				// PSHUFB performs a byte broadcast when given
  1325  				// a control input of 0.
  1326  				broadcast = s.newValue1(ssa.OpAMD64PSHUFBbroadcast, types.TypeInt128, hfp)
  1327  			} else {
  1328  				// No direct byte broadcast. First we must
  1329  				// duplicate the lower byte and then do a
  1330  				// 16-bit broadcast.
  1331  
  1332  				// "Unpack" h2 with itself. This duplicates the
  1333  				// input, resulting in h2 in the lower two
  1334  				// bytes.
  1335  				unpack := s.newValue2(ssa.OpAMD64PUNPCKLBW, types.TypeInt128, hfp, hfp)
  1336  
  1337  				// Copy the lower 16-bits of unpack into every
  1338  				// 16-bit slot in the lower 64-bits of the
  1339  				// output register. Note that immediate 0
  1340  				// selects the low word as the source for every
  1341  				// destination slot.
  1342  				broadcast = s.newValue1I(ssa.OpAMD64PSHUFLW, types.TypeInt128, 0, unpack)
  1343  
  1344  				// No need to broadcast into the upper 64-bits,
  1345  				// as we don't use those.
  1346  			}
  1347  
  1348  			// Compare each byte of the control word with h2. Each
  1349  			// matching byte has every bit set.
  1350  			eq := s.newValue2(ssa.OpAMD64PCMPEQB, types.TypeInt128, broadcast, gfp)
  1351  
  1352  			// Construct a "byte mask": each output bit is equal to
  1353  			// the sign bit each input byte.
  1354  			//
  1355  			// This results in a packed output (bit N set means
  1356  			// byte N matched).
  1357  			//
  1358  			// NOTE: See comment above on bitsetFirst.
  1359  			out := s.newValue1(ssa.OpAMD64PMOVMSKB, types.Types[types.TUINT16], eq)
  1360  
  1361  			// g is only 64-bits so the upper 64-bits of the
  1362  			// 128-bit register will be zero. If h2 is also zero,
  1363  			// then we'll get matches on those bytes. Truncate the
  1364  			// upper bits to ignore such matches.
  1365  			ret := s.newValue1(ssa.OpZeroExt8to64, types.Types[types.TUINT64], out)
  1366  
  1367  			return ret
  1368  		},
  1369  		sys.AMD64)
  1370  
  1371  	addF("internal/runtime/maps", "ctrlGroupMatchEmpty",
  1372  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  1373  			// An empty slot is   1000 0000
  1374  			// A deleted slot is  1111 1110
  1375  			// A full slot is     0??? ????
  1376  
  1377  			g := args[0]
  1378  
  1379  			// Explicit copy to fp register. See
  1380  			// https://go.dev/issue/70451.
  1381  			gfp := s.newValue1(ssa.OpAMD64MOVQi2f, types.TypeInt128, g)
  1382  
  1383  			if buildcfg.GOAMD64 >= 2 {
  1384  				// "PSIGNB negates each data element of the
  1385  				// destination operand (the first operand) if
  1386  				// the signed integer value of the
  1387  				// corresponding data element in the source
  1388  				// operand (the second operand) is less than
  1389  				// zero. If the signed integer value of a data
  1390  				// element in the source operand is positive,
  1391  				// the corresponding data element in the
  1392  				// destination operand is unchanged. If a data
  1393  				// element in the source operand is zero, the
  1394  				// corresponding data element in the
  1395  				// destination operand is set to zero" - Intel SDM
  1396  				//
  1397  				// If we pass the group control word as both
  1398  				// arguments:
  1399  				// - Full slots are unchanged.
  1400  				// - Deleted slots are negated, becoming
  1401  				//   0000 0010.
  1402  				// - Empty slots are negated, becoming
  1403  				//   1000 0000 (unchanged!).
  1404  				//
  1405  				// The result is that only empty slots have the
  1406  				// sign bit set. We then use PMOVMSKB to
  1407  				// extract the sign bits.
  1408  				sign := s.newValue2(ssa.OpAMD64PSIGNB, types.TypeInt128, gfp, gfp)
  1409  
  1410  				// Construct a "byte mask": each output bit is
  1411  				// equal to the sign bit each input byte. The
  1412  				// sign bit is only set for empty or deleted
  1413  				// slots.
  1414  				//
  1415  				// This results in a packed output (bit N set
  1416  				// means byte N matched).
  1417  				//
  1418  				// NOTE: See comment above on bitsetFirst.
  1419  				ret := s.newValue1(ssa.OpAMD64PMOVMSKB, types.Types[types.TUINT16], sign)
  1420  
  1421  				// g is only 64-bits so the upper 64-bits of
  1422  				// the 128-bit register will be zero. PSIGNB
  1423  				// will keep all of these bytes zero, so no
  1424  				// need to truncate.
  1425  
  1426  				return ret
  1427  			}
  1428  
  1429  			// No PSIGNB, simply do byte equality with ctrlEmpty.
  1430  
  1431  			// Load ctrlEmpty into each byte of a control word.
  1432  			var ctrlsEmpty uint64 = abi.SwissMapCtrlEmpty
  1433  			e := s.constInt64(types.Types[types.TUINT64], int64(ctrlsEmpty))
  1434  			// Explicit copy to fp register. See
  1435  			// https://go.dev/issue/70451.
  1436  			efp := s.newValue1(ssa.OpAMD64MOVQi2f, types.TypeInt128, e)
  1437  
  1438  			// Compare each byte of the control word with ctrlEmpty. Each
  1439  			// matching byte has every bit set.
  1440  			eq := s.newValue2(ssa.OpAMD64PCMPEQB, types.TypeInt128, efp, gfp)
  1441  
  1442  			// Construct a "byte mask": each output bit is equal to
  1443  			// the sign bit each input byte.
  1444  			//
  1445  			// This results in a packed output (bit N set means
  1446  			// byte N matched).
  1447  			//
  1448  			// NOTE: See comment above on bitsetFirst.
  1449  			out := s.newValue1(ssa.OpAMD64PMOVMSKB, types.Types[types.TUINT16], eq)
  1450  
  1451  			// g is only 64-bits so the upper 64-bits of the
  1452  			// 128-bit register will be zero. The upper 64-bits of
  1453  			// efp are also zero, so we'll get matches on those
  1454  			// bytes. Truncate the upper bits to ignore such
  1455  			// matches.
  1456  			return s.newValue1(ssa.OpZeroExt8to64, types.Types[types.TUINT64], out)
  1457  		},
  1458  		sys.AMD64)
  1459  
  1460  	addF("internal/runtime/maps", "ctrlGroupMatchEmptyOrDeleted",
  1461  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  1462  			// An empty slot is   1000 0000
  1463  			// A deleted slot is  1111 1110
  1464  			// A full slot is     0??? ????
  1465  			//
  1466  			// A slot is empty or deleted iff bit 7 (sign bit) is
  1467  			// set.
  1468  
  1469  			g := args[0]
  1470  
  1471  			// Explicit copy to fp register. See
  1472  			// https://go.dev/issue/70451.
  1473  			gfp := s.newValue1(ssa.OpAMD64MOVQi2f, types.TypeInt128, g)
  1474  
  1475  			// Construct a "byte mask": each output bit is equal to
  1476  			// the sign bit each input byte. The sign bit is only
  1477  			// set for empty or deleted slots.
  1478  			//
  1479  			// This results in a packed output (bit N set means
  1480  			// byte N matched).
  1481  			//
  1482  			// NOTE: See comment above on bitsetFirst.
  1483  			ret := s.newValue1(ssa.OpAMD64PMOVMSKB, types.Types[types.TUINT16], gfp)
  1484  
  1485  			// g is only 64-bits so the upper 64-bits of the
  1486  			// 128-bit register will be zero. Zero will never match
  1487  			// ctrlEmpty or ctrlDeleted, so no need to truncate.
  1488  
  1489  			return ret
  1490  		},
  1491  		sys.AMD64)
  1492  
  1493  	addF("internal/runtime/maps", "ctrlGroupMatchFull",
  1494  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  1495  			// An empty slot is   1000 0000
  1496  			// A deleted slot is  1111 1110
  1497  			// A full slot is     0??? ????
  1498  			//
  1499  			// A slot is full iff bit 7 (sign bit) is unset.
  1500  
  1501  			g := args[0]
  1502  
  1503  			// Explicit copy to fp register. See
  1504  			// https://go.dev/issue/70451.
  1505  			gfp := s.newValue1(ssa.OpAMD64MOVQi2f, types.TypeInt128, g)
  1506  
  1507  			// Construct a "byte mask": each output bit is equal to
  1508  			// the sign bit each input byte. The sign bit is only
  1509  			// set for empty or deleted slots.
  1510  			//
  1511  			// This results in a packed output (bit N set means
  1512  			// byte N matched).
  1513  			//
  1514  			// NOTE: See comment above on bitsetFirst.
  1515  			mask := s.newValue1(ssa.OpAMD64PMOVMSKB, types.Types[types.TUINT16], gfp)
  1516  
  1517  			// Invert the mask to set the bits for the full slots.
  1518  			out := s.newValue1(ssa.OpCom16, types.Types[types.TUINT16], mask)
  1519  
  1520  			// g is only 64-bits so the upper 64-bits of the
  1521  			// 128-bit register will be zero, with bit 7 unset.
  1522  			// Truncate the upper bits to ignore these.
  1523  			return s.newValue1(ssa.OpZeroExt8to64, types.Types[types.TUINT64], out)
  1524  		},
  1525  		sys.AMD64)
  1526  }
  1527  
  1528  // findIntrinsic returns a function which builds the SSA equivalent of the
  1529  // function identified by the symbol sym.  If sym is not an intrinsic call, returns nil.
  1530  func findIntrinsic(sym *types.Sym) intrinsicBuilder {
  1531  	if sym == nil || sym.Pkg == nil {
  1532  		return nil
  1533  	}
  1534  	pkg := sym.Pkg.Path
  1535  	if sym.Pkg == ir.Pkgs.Runtime {
  1536  		pkg = "runtime"
  1537  	}
  1538  	if base.Flag.Race && pkg == "sync/atomic" {
  1539  		// The race detector needs to be able to intercept these calls.
  1540  		// We can't intrinsify them.
  1541  		return nil
  1542  	}
  1543  	// Skip intrinsifying math functions (which may contain hard-float
  1544  	// instructions) when soft-float
  1545  	if Arch.SoftFloat && pkg == "math" {
  1546  		return nil
  1547  	}
  1548  
  1549  	fn := sym.Name
  1550  	if ssa.IntrinsicsDisable {
  1551  		if pkg == "internal/runtime/sys" && (fn == "GetCallerPC" || fn == "GrtCallerSP" || fn == "GetClosurePtr") {
  1552  			// These runtime functions don't have definitions, must be intrinsics.
  1553  		} else {
  1554  			return nil
  1555  		}
  1556  	}
  1557  	return intrinsics.lookup(Arch.LinkArch.Arch, pkg, fn)
  1558  }
  1559  
  1560  func IsIntrinsicCall(n *ir.CallExpr) bool {
  1561  	if n == nil {
  1562  		return false
  1563  	}
  1564  	name, ok := n.Fun.(*ir.Name)
  1565  	if !ok {
  1566  		return false
  1567  	}
  1568  	return findIntrinsic(name.Sym()) != nil
  1569  }
  1570  

View as plain text