writebarrier.go

     1  // Copyright 2016 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package ssa
     6  
     7  import (
     8  	"cmd/compile/internal/reflectdata"
     9  	"cmd/compile/internal/types"
    10  	"cmd/internal/obj"
    11  	"cmd/internal/objabi"
    12  	"cmd/internal/src"
    13  	"fmt"
    14  	"internal/buildcfg"
    15  )
    16  
    17  // A ZeroRegion records parts of an object which are known to be zero.
    18  // A ZeroRegion only applies to a single memory state.
    19  // Each bit in mask is set if the corresponding pointer-sized word of
    20  // the base object is known to be zero.
    21  // In other words, if mask & (1<<i) != 0, then [base+i*ptrSize, base+(i+1)*ptrSize)
    22  // is known to be zero.
    23  type ZeroRegion struct {
    24  	base *Value
    25  	mask uint64
    26  }
    27  
    28  // mightBeHeapPointer reports whether v might point to the heap.
    29  // v must have pointer type.
    30  func mightBeHeapPointer(v *Value) bool {
    31  	if IsGlobalAddr(v) {
    32  		return false
    33  	}
    34  	return true
    35  }
    36  
    37  // mightContainHeapPointer reports whether the data currently at addresses
    38  // [ptr,ptr+size) might contain heap pointers. "currently" means at memory state mem.
    39  // zeroes contains ZeroRegion data to help make that decision (see computeZeroMap).
    40  func mightContainHeapPointer(ptr *Value, size int64, mem *Value, zeroes map[ID]ZeroRegion) bool {
    41  	if IsReadOnlyGlobalAddr(ptr) {
    42  		// The read-only globals section cannot contain any heap pointers.
    43  		return false
    44  	}
    45  
    46  	// See if we can prove that the queried memory is all zero.
    47  
    48  	// Find base pointer and offset. Hopefully, the base is the result of a new(T).
    49  	var off int64
    50  	for ptr.Op == OpOffPtr {
    51  		off += ptr.AuxInt
    52  		ptr = ptr.Args[0]
    53  	}
    54  
    55  	ptrSize := ptr.Block.Func.Config.PtrSize
    56  	if off%ptrSize != 0 {
    57  		return true // see issue 61187
    58  	}
    59  	if size%ptrSize != 0 {
    60  		ptr.Fatalf("unaligned pointer write")
    61  	}
    62  	if off < 0 || off+size > 64*ptrSize {
    63  		// memory range goes off end of tracked offsets
    64  		return true
    65  	}
    66  	z := zeroes[mem.ID]
    67  	if ptr != z.base {
    68  		// This isn't the object we know about at this memory state.
    69  		return true
    70  	}
    71  	// Mask of bits we're asking about
    72  	m := (uint64(1)<<(size/ptrSize) - 1) << (off / ptrSize)
    73  
    74  	if z.mask&m == m {
    75  		// All locations are known to be zero, so no heap pointers.
    76  		return false
    77  	}
    78  	return true
    79  }
    80  
    81  // needwb reports whether we need write barrier for store op v.
    82  // v must be Store/Move/Zero.
    83  // zeroes provides known zero information (keyed by ID of memory-type values).
    84  func needwb(v *Value, zeroes map[ID]ZeroRegion) bool {
    85  	t, ok := v.Aux.(*types.Type)
    86  	if !ok {
    87  		v.Fatalf("store aux is not a type: %s", v.LongString())
    88  	}
    89  	if !t.HasPointers() {
    90  		return false
    91  	}
    92  	dst := v.Args[0]
    93  	if IsStackAddr(dst) {
    94  		return false // writes into the stack don't need write barrier
    95  	}
    96  	// If we're writing to a place that might have heap pointers, we need
    97  	// the write barrier.
    98  	if mightContainHeapPointer(dst, t.Size(), v.MemoryArg(), zeroes) {
    99  		return true
   100  	}
   101  	// Lastly, check if the values we're writing might be heap pointers.
   102  	// If they aren't, we don't need a write barrier.
   103  	switch v.Op {
   104  	case OpStore:
   105  		if !mightBeHeapPointer(v.Args[1]) {
   106  			return false
   107  		}
   108  	case OpZero:
   109  		return false // nil is not a heap pointer
   110  	case OpMove:
   111  		if !mightContainHeapPointer(v.Args[1], t.Size(), v.Args[2], zeroes) {
   112  			return false
   113  		}
   114  	default:
   115  		v.Fatalf("store op unknown: %s", v.LongString())
   116  	}
   117  	return true
   118  }
   119  
   120  // needWBsrc reports whether GC needs to see v when it is the source of a store.
   121  func needWBsrc(v *Value) bool {
   122  	return !IsGlobalAddr(v)
   123  }
   124  
   125  // needWBdst reports whether GC needs to see what used to be in *ptr when ptr is
   126  // the target of a pointer store.
   127  func needWBdst(ptr, mem *Value, zeroes map[ID]ZeroRegion) bool {
   128  	// Detect storing to zeroed memory.
   129  	var off int64
   130  	for ptr.Op == OpOffPtr {
   131  		off += ptr.AuxInt
   132  		ptr = ptr.Args[0]
   133  	}
   134  	ptrSize := ptr.Block.Func.Config.PtrSize
   135  	if off%ptrSize != 0 {
   136  		return true // see issue 61187
   137  	}
   138  	if off < 0 || off >= 64*ptrSize {
   139  		// write goes off end of tracked offsets
   140  		return true
   141  	}
   142  	z := zeroes[mem.ID]
   143  	if ptr != z.base {
   144  		return true
   145  	}
   146  	// If destination is known to be zeroed, we don't need the write barrier
   147  	// to record the old value in *ptr.
   148  	return z.mask>>uint(off/ptrSize)&1 == 0
   149  }
   150  
   151  // writebarrier pass inserts write barriers for store ops (Store, Move, Zero)
   152  // when necessary (the condition above). It rewrites store ops to branches
   153  // and runtime calls, like
   154  //
   155  //	if writeBarrier.enabled {
   156  //		buf := gcWriteBarrier2()	// Not a regular Go call
   157  //		buf[0] = val
   158  //		buf[1] = *ptr
   159  //	}
   160  //	*ptr = val
   161  //
   162  // A sequence of WB stores for many pointer fields of a single type will
   163  // be emitted together, with a single branch.
   164  func writebarrier(f *Func) {
   165  	if !f.fe.UseWriteBarrier() {
   166  		return
   167  	}
   168  
   169  	// Number of write buffer entries we can request at once.
   170  	// Must match runtime/mwbbuf.go:wbMaxEntriesPerCall.
   171  	// It must also match the number of instances of runtime.gcWriteBarrier{X}.
   172  	const maxEntries = 8
   173  
   174  	var sb, sp, wbaddr, const0 *Value
   175  	var cgoCheckPtrWrite, cgoCheckMemmove *obj.LSym
   176  	var wbZero, wbMove *obj.LSym
   177  	var stores, after []*Value
   178  	var sset, sset2 *sparseSet
   179  	var storeNumber []int32
   180  
   181  	// Compute map from a value to the SelectN [1] value that uses it.
   182  	select1 := f.Cache.allocValueSlice(f.NumValues())
   183  	defer func() { f.Cache.freeValueSlice(select1) }()
   184  	for _, b := range f.Blocks {
   185  		for _, v := range b.Values {
   186  			if v.Op != OpSelectN {
   187  				continue
   188  			}
   189  			if v.AuxInt != 1 {
   190  				continue
   191  			}
   192  			select1[v.Args[0].ID] = v
   193  		}
   194  	}
   195  
   196  	zeroes := f.computeZeroMap(select1)
   197  	for _, b := range f.Blocks { // range loop is safe since the blocks we added contain no stores to expand
   198  		// first, identify all the stores that need to insert a write barrier.
   199  		// mark them with WB ops temporarily. record presence of WB ops.
   200  		nWBops := 0 // count of temporarily created WB ops remaining to be rewritten in the current block
   201  		for _, v := range b.Values {
   202  			switch v.Op {
   203  			case OpStore, OpMove, OpZero:
   204  				if needwb(v, zeroes) {
   205  					switch v.Op {
   206  					case OpStore:
   207  						v.Op = OpStoreWB
   208  					case OpMove:
   209  						v.Op = OpMoveWB
   210  					case OpZero:
   211  						v.Op = OpZeroWB
   212  					}
   213  					nWBops++
   214  				}
   215  			}
   216  		}
   217  		if nWBops == 0 {
   218  			continue
   219  		}
   220  
   221  		if wbaddr == nil {
   222  			// lazily initialize global values for write barrier test and calls
   223  			// find SB and SP values in entry block
   224  			initpos := f.Entry.Pos
   225  			sp, sb = f.spSb()
   226  			wbsym := f.fe.Syslook("writeBarrier")
   227  			wbaddr = f.Entry.NewValue1A(initpos, OpAddr, f.Config.Types.UInt32Ptr, wbsym, sb)
   228  			wbZero = f.fe.Syslook("wbZero")
   229  			wbMove = f.fe.Syslook("wbMove")
   230  			if buildcfg.Experiment.CgoCheck2 {
   231  				cgoCheckPtrWrite = f.fe.Syslook("cgoCheckPtrWrite")
   232  				cgoCheckMemmove = f.fe.Syslook("cgoCheckMemmove")
   233  			}
   234  			const0 = f.ConstInt32(f.Config.Types.UInt32, 0)
   235  
   236  			// allocate auxiliary data structures for computing store order
   237  			sset = f.newSparseSet(f.NumValues())
   238  			defer f.retSparseSet(sset)
   239  			sset2 = f.newSparseSet(f.NumValues())
   240  			defer f.retSparseSet(sset2)
   241  			storeNumber = f.Cache.allocInt32Slice(f.NumValues())
   242  			defer f.Cache.freeInt32Slice(storeNumber)
   243  		}
   244  
   245  		// order values in store order
   246  		b.Values = storeOrder(b.Values, sset, storeNumber)
   247  	again:
   248  		// find the start and end of the last contiguous WB store sequence.
   249  		// a branch will be inserted there. values after it will be moved
   250  		// to a new block.
   251  		var last *Value
   252  		var start, end int
   253  		var nonPtrStores int
   254  		values := b.Values
   255  		hasMove := false
   256  	FindSeq:
   257  		for i := len(values) - 1; i >= 0; i-- {
   258  			w := values[i]
   259  			switch w.Op {
   260  			case OpStoreWB, OpMoveWB, OpZeroWB:
   261  				start = i
   262  				if last == nil {
   263  					last = w
   264  					end = i + 1
   265  				}
   266  				nonPtrStores = 0
   267  				if w.Op == OpMoveWB {
   268  					hasMove = true
   269  				}
   270  			case OpVarDef, OpVarLive:
   271  				continue
   272  			case OpStore:
   273  				if last == nil {
   274  					continue
   275  				}
   276  				nonPtrStores++
   277  				if nonPtrStores > 2 {
   278  					break FindSeq
   279  				}
   280  				if hasMove {
   281  					// We need to ensure that this store happens
   282  					// before we issue a wbMove, as the wbMove might
   283  					// use the result of this store as its source.
   284  					// Even though this store is not write-barrier
   285  					// eligible, it might nevertheless be the store
   286  					// of a pointer to the stack, which is then the
   287  					// source of the move.
   288  					// See issue 71228.
   289  					break FindSeq
   290  				}
   291  			default:
   292  				if last == nil {
   293  					continue
   294  				}
   295  				break FindSeq
   296  			}
   297  		}
   298  		stores = append(stores[:0], b.Values[start:end]...) // copy to avoid aliasing
   299  		after = append(after[:0], b.Values[end:]...)
   300  		b.Values = b.Values[:start]
   301  
   302  		// find the memory before the WB stores
   303  		mem := stores[0].MemoryArg()
   304  		pos := stores[0].Pos
   305  
   306  		// If the source of a MoveWB is volatile (will be clobbered by a
   307  		// function call), we need to copy it to a temporary location, as
   308  		// marshaling the args of wbMove might clobber the value we're
   309  		// trying to move.
   310  		// Look for volatile source, copy it to temporary before we check
   311  		// the write barrier flag.
   312  		// It is unlikely to have more than one of them. Just do a linear
   313  		// search instead of using a map.
   314  		// See issue 15854.
   315  		type volatileCopy struct {
   316  			src *Value // address of original volatile value
   317  			tmp *Value // address of temporary we've copied the volatile value into
   318  		}
   319  		var volatiles []volatileCopy
   320  
   321  		if !(f.ABIDefault == f.ABI1 && len(f.Config.intParamRegs) >= 3) {
   322  			// We don't need to do this if the calls we're going to do take
   323  			// all their arguments in registers.
   324  			// 3 is the magic number because it covers wbZero, wbMove, cgoCheckMemmove.
   325  		copyLoop:
   326  			for _, w := range stores {
   327  				if w.Op == OpMoveWB {
   328  					val := w.Args[1]
   329  					if isVolatile(val) {
   330  						for _, c := range volatiles {
   331  							if val == c.src {
   332  								continue copyLoop // already copied
   333  							}
   334  						}
   335  
   336  						t := val.Type.Elem()
   337  						tmp := f.NewLocal(w.Pos, t)
   338  						mem = b.NewValue1A(w.Pos, OpVarDef, types.TypeMem, tmp, mem)
   339  						tmpaddr := b.NewValue2A(w.Pos, OpLocalAddr, t.PtrTo(), tmp, sp, mem)
   340  						siz := t.Size()
   341  						mem = b.NewValue3I(w.Pos, OpMove, types.TypeMem, siz, tmpaddr, val, mem)
   342  						mem.Aux = t
   343  						volatiles = append(volatiles, volatileCopy{val, tmpaddr})
   344  					}
   345  				}
   346  			}
   347  		}
   348  
   349  		// Build branch point.
   350  		bThen := f.NewBlock(BlockPlain)
   351  		bEnd := f.NewBlock(b.Kind)
   352  		bThen.Pos = pos
   353  		bEnd.Pos = b.Pos
   354  		b.Pos = pos
   355  
   356  		// Set up control flow for end block.
   357  		bEnd.CopyControls(b)
   358  		bEnd.Likely = b.Likely
   359  		for _, e := range b.Succs {
   360  			bEnd.Succs = append(bEnd.Succs, e)
   361  			e.b.Preds[e.i].b = bEnd
   362  		}
   363  
   364  		// set up control flow for write barrier test
   365  		// load word, test word, avoiding partial register write from load byte.
   366  		cfgtypes := &f.Config.Types
   367  		flag := b.NewValue2(pos, OpLoad, cfgtypes.UInt32, wbaddr, mem)
   368  		flag = b.NewValue2(pos, OpNeq32, cfgtypes.Bool, flag, const0)
   369  		b.Kind = BlockIf
   370  		b.SetControl(flag)
   371  		b.Likely = BranchUnlikely
   372  		b.Succs = b.Succs[:0]
   373  		b.AddEdgeTo(bThen)
   374  		b.AddEdgeTo(bEnd)
   375  		bThen.AddEdgeTo(bEnd)
   376  
   377  		// For each write barrier store, append write barrier code to bThen.
   378  		memThen := mem
   379  		var curCall *Value
   380  		var curPtr *Value
   381  		addEntry := func(pos src.XPos, v *Value) {
   382  			if curCall == nil || curCall.AuxInt == maxEntries {
   383  				t := types.NewTuple(types.Types[types.TUINTPTR].PtrTo(), types.TypeMem)
   384  				curCall = bThen.NewValue1(pos, OpWB, t, memThen)
   385  				curPtr = bThen.NewValue1(pos, OpSelect0, types.Types[types.TUINTPTR].PtrTo(), curCall)
   386  				memThen = bThen.NewValue1(pos, OpSelect1, types.TypeMem, curCall)
   387  			}
   388  			// Store value in write buffer
   389  			num := curCall.AuxInt
   390  			curCall.AuxInt = num + 1
   391  			wbuf := bThen.NewValue1I(pos, OpOffPtr, types.Types[types.TUINTPTR].PtrTo(), num*f.Config.PtrSize, curPtr)
   392  			memThen = bThen.NewValue3A(pos, OpStore, types.TypeMem, types.Types[types.TUINTPTR], wbuf, v, memThen)
   393  		}
   394  
   395  		// Note: we can issue the write barrier code in any order. In particular,
   396  		// it doesn't matter if they are in a different order *even if* they end
   397  		// up referring to overlapping memory regions. For instance if an OpStore
   398  		// stores to a location that is later read by an OpMove. In all cases
   399  		// any pointers we must get into the write barrier buffer still make it,
   400  		// possibly in a different order and possibly a different (but definitely
   401  		// more than 0) number of times.
   402  		// In light of that, we process all the OpStoreWBs first. This minimizes
   403  		// the amount of spill/restore code we need around the Zero/Move calls.
   404  
   405  		// srcs contains the value IDs of pointer values we've put in the write barrier buffer.
   406  		srcs := sset
   407  		srcs.clear()
   408  		// dsts contains the value IDs of locations which we've read a pointer out of
   409  		// and put the result in the write barrier buffer.
   410  		dsts := sset2
   411  		dsts.clear()
   412  
   413  		for _, w := range stores {
   414  			if w.Op != OpStoreWB {
   415  				continue
   416  			}
   417  			pos := w.Pos
   418  			ptr := w.Args[0]
   419  			val := w.Args[1]
   420  			if !srcs.contains(val.ID) && needWBsrc(val) {
   421  				srcs.add(val.ID)
   422  				addEntry(pos, val)
   423  			}
   424  			if !dsts.contains(ptr.ID) && needWBdst(ptr, w.Args[2], zeroes) {
   425  				dsts.add(ptr.ID)
   426  				// Load old value from store target.
   427  				// Note: This turns bad pointer writes into bad
   428  				// pointer reads, which could be confusing. We could avoid
   429  				// reading from obviously bad pointers, which would
   430  				// take care of the vast majority of these. We could
   431  				// patch this up in the signal handler, or use XCHG to
   432  				// combine the read and the write.
   433  				oldVal := bThen.NewValue2(pos, OpLoad, types.Types[types.TUINTPTR], ptr, memThen)
   434  				// Save old value to write buffer.
   435  				addEntry(pos, oldVal)
   436  			}
   437  			f.fe.Func().SetWBPos(pos)
   438  			nWBops--
   439  		}
   440  
   441  		for _, w := range stores {
   442  			pos := w.Pos
   443  			switch w.Op {
   444  			case OpZeroWB:
   445  				dst := w.Args[0]
   446  				typ := reflectdata.TypeLinksym(w.Aux.(*types.Type))
   447  				// zeroWB(&typ, dst)
   448  				taddr := b.NewValue1A(pos, OpAddr, b.Func.Config.Types.Uintptr, typ, sb)
   449  				memThen = wbcall(pos, bThen, wbZero, sp, memThen, taddr, dst)
   450  				f.fe.Func().SetWBPos(pos)
   451  				nWBops--
   452  			case OpMoveWB:
   453  				dst := w.Args[0]
   454  				src := w.Args[1]
   455  				if isVolatile(src) {
   456  					for _, c := range volatiles {
   457  						if src == c.src {
   458  							src = c.tmp
   459  							break
   460  						}
   461  					}
   462  				}
   463  				typ := reflectdata.TypeLinksym(w.Aux.(*types.Type))
   464  				// moveWB(&typ, dst, src)
   465  				taddr := b.NewValue1A(pos, OpAddr, b.Func.Config.Types.Uintptr, typ, sb)
   466  				memThen = wbcall(pos, bThen, wbMove, sp, memThen, taddr, dst, src)
   467  				f.fe.Func().SetWBPos(pos)
   468  				nWBops--
   469  			}
   470  		}
   471  
   472  		// merge memory
   473  		mem = bEnd.NewValue2(pos, OpPhi, types.TypeMem, mem, memThen)
   474  
   475  		// Do raw stores after merge point.
   476  		for _, w := range stores {
   477  			pos := w.Pos
   478  			switch w.Op {
   479  			case OpStoreWB:
   480  				ptr := w.Args[0]
   481  				val := w.Args[1]
   482  				if buildcfg.Experiment.CgoCheck2 {
   483  					// Issue cgo checking code.
   484  					mem = wbcall(pos, bEnd, cgoCheckPtrWrite, sp, mem, ptr, val)
   485  				}
   486  				mem = bEnd.NewValue3A(pos, OpStore, types.TypeMem, w.Aux, ptr, val, mem)
   487  			case OpZeroWB:
   488  				dst := w.Args[0]
   489  				mem = bEnd.NewValue2I(pos, OpZero, types.TypeMem, w.AuxInt, dst, mem)
   490  				mem.Aux = w.Aux
   491  			case OpMoveWB:
   492  				dst := w.Args[0]
   493  				src := w.Args[1]
   494  				if isVolatile(src) {
   495  					for _, c := range volatiles {
   496  						if src == c.src {
   497  							src = c.tmp
   498  							break
   499  						}
   500  					}
   501  				}
   502  				if buildcfg.Experiment.CgoCheck2 {
   503  					// Issue cgo checking code.
   504  					typ := reflectdata.TypeLinksym(w.Aux.(*types.Type))
   505  					taddr := b.NewValue1A(pos, OpAddr, b.Func.Config.Types.Uintptr, typ, sb)
   506  					mem = wbcall(pos, bEnd, cgoCheckMemmove, sp, mem, taddr, dst, src)
   507  				}
   508  				mem = bEnd.NewValue3I(pos, OpMove, types.TypeMem, w.AuxInt, dst, src, mem)
   509  				mem.Aux = w.Aux
   510  			case OpVarDef, OpVarLive:
   511  				mem = bEnd.NewValue1A(pos, w.Op, types.TypeMem, w.Aux, mem)
   512  			case OpStore:
   513  				ptr := w.Args[0]
   514  				val := w.Args[1]
   515  				mem = bEnd.NewValue3A(pos, OpStore, types.TypeMem, w.Aux, ptr, val, mem)
   516  			}
   517  		}
   518  
   519  		// The last store becomes the WBend marker. This marker is used by the liveness
   520  		// pass to determine what parts of the code are preemption-unsafe.
   521  		// All subsequent memory operations use this memory, so we have to sacrifice the
   522  		// previous last memory op to become this new value.
   523  		bEnd.Values = append(bEnd.Values, last)
   524  		last.Block = bEnd
   525  		last.reset(OpWBend)
   526  		last.Pos = last.Pos.WithNotStmt()
   527  		last.Type = types.TypeMem
   528  		last.AddArg(mem)
   529  
   530  		// Free all the old stores, except last which became the WBend marker.
   531  		for _, w := range stores {
   532  			if w != last {
   533  				w.resetArgs()
   534  			}
   535  		}
   536  		for _, w := range stores {
   537  			if w != last {
   538  				f.freeValue(w)
   539  			}
   540  		}
   541  
   542  		// put values after the store sequence into the end block
   543  		bEnd.Values = append(bEnd.Values, after...)
   544  		for _, w := range after {
   545  			w.Block = bEnd
   546  		}
   547  
   548  		// if we have more stores in this block, do this block again
   549  		if nWBops > 0 {
   550  			goto again
   551  		}
   552  	}
   553  }
   554  
   555  // computeZeroMap returns a map from an ID of a memory value to
   556  // a set of locations that are known to be zeroed at that memory value.
   557  func (f *Func) computeZeroMap(select1 []*Value) map[ID]ZeroRegion {
   558  
   559  	ptrSize := f.Config.PtrSize
   560  	// Keep track of which parts of memory are known to be zero.
   561  	// This helps with removing write barriers for various initialization patterns.
   562  	// This analysis is conservative. We only keep track, for each memory state, of
   563  	// which of the first 64 words of a single object are known to be zero.
   564  	zeroes := map[ID]ZeroRegion{}
   565  	// Find new objects.
   566  	for _, b := range f.Blocks {
   567  		for _, v := range b.Values {
   568  			if mem, ok := IsNewObject(v, select1); ok {
   569  				// While compiling package runtime itself, we might see user
   570  				// calls to newobject, which will have result type
   571  				// unsafe.Pointer instead. We can't easily infer how large the
   572  				// allocated memory is, so just skip it.
   573  				if types.LocalPkg.Path == "runtime" && v.Type.IsUnsafePtr() {
   574  					continue
   575  				}
   576  
   577  				nptr := v.Type.Elem().Size() / ptrSize
   578  				if nptr > 64 {
   579  					nptr = 64
   580  				}
   581  				zeroes[mem.ID] = ZeroRegion{base: v, mask: 1<<uint(nptr) - 1}
   582  			}
   583  		}
   584  	}
   585  	// Find stores to those new objects.
   586  	for {
   587  		changed := false
   588  		for _, b := range f.Blocks {
   589  			// Note: iterating forwards helps convergence, as values are
   590  			// typically (but not always!) in store order.
   591  			for _, v := range b.Values {
   592  				if v.Op != OpStore {
   593  					continue
   594  				}
   595  				z, ok := zeroes[v.MemoryArg().ID]
   596  				if !ok {
   597  					continue
   598  				}
   599  				ptr := v.Args[0]
   600  				var off int64
   601  				size := v.Aux.(*types.Type).Size()
   602  				for ptr.Op == OpOffPtr {
   603  					off += ptr.AuxInt
   604  					ptr = ptr.Args[0]
   605  				}
   606  				if ptr != z.base {
   607  					// Different base object - we don't know anything.
   608  					// We could even be writing to the base object we know
   609  					// about, but through an aliased but offset pointer.
   610  					// So we have to throw all the zero information we have away.
   611  					continue
   612  				}
   613  				// Round to cover any partially written pointer slots.
   614  				// Pointer writes should never be unaligned like this, but non-pointer
   615  				// writes to pointer-containing types will do this.
   616  				if d := off % ptrSize; d != 0 {
   617  					off -= d
   618  					size += d
   619  				}
   620  				if d := size % ptrSize; d != 0 {
   621  					size += ptrSize - d
   622  				}
   623  				// Clip to the 64 words that we track.
   624  				min := off
   625  				max := off + size
   626  				if min < 0 {
   627  					min = 0
   628  				}
   629  				if max > 64*ptrSize {
   630  					max = 64 * ptrSize
   631  				}
   632  				// Clear bits for parts that we are writing (and hence
   633  				// will no longer necessarily be zero).
   634  				for i := min; i < max; i += ptrSize {
   635  					bit := i / ptrSize
   636  					z.mask &^= 1 << uint(bit)
   637  				}
   638  				if z.mask == 0 {
   639  					// No more known zeros - don't bother keeping.
   640  					continue
   641  				}
   642  				// Save updated known zero contents for new store.
   643  				if zeroes[v.ID] != z {
   644  					zeroes[v.ID] = z
   645  					changed = true
   646  				}
   647  			}
   648  		}
   649  		if !changed {
   650  			break
   651  		}
   652  	}
   653  	if f.pass.debug > 0 {
   654  		fmt.Printf("func %s\n", f.Name)
   655  		for mem, z := range zeroes {
   656  			fmt.Printf("  memory=v%d ptr=%v zeromask=%b\n", mem, z.base, z.mask)
   657  		}
   658  	}
   659  	return zeroes
   660  }
   661  
   662  // wbcall emits write barrier runtime call in b, returns memory.
   663  func wbcall(pos src.XPos, b *Block, fn *obj.LSym, sp, mem *Value, args ...*Value) *Value {
   664  	config := b.Func.Config
   665  	typ := config.Types.Uintptr // type of all argument values
   666  	nargs := len(args)
   667  
   668  	// TODO (register args) this is a bit of a hack.
   669  	inRegs := b.Func.ABIDefault == b.Func.ABI1 && len(config.intParamRegs) >= 3
   670  
   671  	if !inRegs {
   672  		// Store arguments to the appropriate stack slot.
   673  		off := config.ctxt.Arch.FixedFrameSize
   674  		for _, arg := range args {
   675  			stkaddr := b.NewValue1I(pos, OpOffPtr, typ.PtrTo(), off, sp)
   676  			mem = b.NewValue3A(pos, OpStore, types.TypeMem, typ, stkaddr, arg, mem)
   677  			off += typ.Size()
   678  		}
   679  		args = args[:0]
   680  	}
   681  
   682  	args = append(args, mem)
   683  
   684  	// issue call
   685  	argTypes := make([]*types.Type, nargs, 3) // at most 3 args; allows stack allocation
   686  	for i := 0; i < nargs; i++ {
   687  		argTypes[i] = typ
   688  	}
   689  	call := b.NewValue0A(pos, OpStaticCall, types.TypeResultMem, StaticAuxCall(fn, b.Func.ABIDefault.ABIAnalyzeTypes(argTypes, nil)))
   690  	call.AddArgs(args...)
   691  	call.AuxInt = int64(nargs) * typ.Size()
   692  	return b.NewValue1I(pos, OpSelectN, types.TypeMem, 0, call)
   693  }
   694  
   695  // round to a multiple of r, r is a power of 2.
   696  func round(o int64, r int64) int64 {
   697  	return (o + r - 1) &^ (r - 1)
   698  }
   699  
   700  // IsStackAddr reports whether v is known to be an address of a stack slot.
   701  func IsStackAddr(v *Value) bool {
   702  	for v.Op == OpOffPtr || v.Op == OpAddPtr || v.Op == OpPtrIndex || v.Op == OpCopy {
   703  		v = v.Args[0]
   704  	}
   705  	switch v.Op {
   706  	case OpSP, OpLocalAddr, OpSelectNAddr, OpGetCallerSP:
   707  		return true
   708  	}
   709  	return false
   710  }
   711  
   712  // IsGlobalAddr reports whether v is known to be an address of a global (or nil).
   713  func IsGlobalAddr(v *Value) bool {
   714  	for v.Op == OpOffPtr || v.Op == OpAddPtr || v.Op == OpPtrIndex || v.Op == OpCopy {
   715  		v = v.Args[0]
   716  	}
   717  	if v.Op == OpAddr && v.Args[0].Op == OpSB {
   718  		return true // address of a global
   719  	}
   720  	if v.Op == OpConstNil {
   721  		return true
   722  	}
   723  	if v.Op == OpLoad && IsReadOnlyGlobalAddr(v.Args[0]) {
   724  		return true // loading from a read-only global - the resulting address can't be a heap address.
   725  	}
   726  	return false
   727  }
   728  
   729  // IsReadOnlyGlobalAddr reports whether v is known to be an address of a read-only global.
   730  func IsReadOnlyGlobalAddr(v *Value) bool {
   731  	if v.Op == OpConstNil {
   732  		// Nil pointers are read only. See issue 33438.
   733  		return true
   734  	}
   735  	if v.Op == OpAddr && v.Aux != nil && v.Aux.(*obj.LSym).Type == objabi.SRODATA {
   736  		return true
   737  	}
   738  	return false
   739  }
   740  
   741  // IsNewObject reports whether v is a pointer to a freshly allocated & zeroed object,
   742  // if so, also returns the memory state mem at which v is zero.
   743  func IsNewObject(v *Value, select1 []*Value) (mem *Value, ok bool) {
   744  	f := v.Block.Func
   745  	c := f.Config
   746  	if f.ABIDefault == f.ABI1 && len(c.intParamRegs) >= 1 {
   747  		if v.Op != OpSelectN || v.AuxInt != 0 {
   748  			return nil, false
   749  		}
   750  		mem = select1[v.Args[0].ID]
   751  		if mem == nil {
   752  			return nil, false
   753  		}
   754  	} else {
   755  		if v.Op != OpLoad {
   756  			return nil, false
   757  		}
   758  		mem = v.MemoryArg()
   759  		if mem.Op != OpSelectN {
   760  			return nil, false
   761  		}
   762  		if mem.Type != types.TypeMem {
   763  			return nil, false
   764  		} // assume it is the right selection if true
   765  	}
   766  	call := mem.Args[0]
   767  	if call.Op != OpStaticCall {
   768  		return nil, false
   769  	}
   770  	if !isSameCall(call.Aux, "runtime.newobject") {
   771  		return nil, false
   772  	}
   773  	if f.ABIDefault == f.ABI1 && len(c.intParamRegs) >= 1 {
   774  		if v.Args[0] == call {
   775  			return mem, true
   776  		}
   777  		return nil, false
   778  	}
   779  	if v.Args[0].Op != OpOffPtr {
   780  		return nil, false
   781  	}
   782  	if v.Args[0].Args[0].Op != OpSP {
   783  		return nil, false
   784  	}
   785  	if v.Args[0].AuxInt != c.ctxt.Arch.FixedFrameSize+c.RegSize { // offset of return value
   786  		return nil, false
   787  	}
   788  	return mem, true
   789  }
   790  
   791  // IsSanitizerSafeAddr reports whether v is known to be an address
   792  // that doesn't need instrumentation.
   793  func IsSanitizerSafeAddr(v *Value) bool {
   794  	for v.Op == OpOffPtr || v.Op == OpAddPtr || v.Op == OpPtrIndex || v.Op == OpCopy {
   795  		v = v.Args[0]
   796  	}
   797  	switch v.Op {
   798  	case OpSP, OpLocalAddr, OpSelectNAddr:
   799  		// Stack addresses are always safe.
   800  		return true
   801  	case OpITab, OpStringPtr, OpGetClosurePtr:
   802  		// Itabs, string data, and closure fields are
   803  		// read-only once initialized.
   804  		return true
   805  	case OpAddr:
   806  		vt := v.Aux.(*obj.LSym).Type
   807  		return vt == objabi.SRODATA || vt == objabi.SLIBFUZZER_8BIT_COUNTER || vt == objabi.SCOVERAGE_COUNTER || vt == objabi.SCOVERAGE_AUXVAR
   808  	}
   809  	return false
   810  }
   811  
   812  // isVolatile reports whether v is a pointer to argument region on stack which
   813  // will be clobbered by a function call.
   814  func isVolatile(v *Value) bool {
   815  	for v.Op == OpOffPtr || v.Op == OpAddPtr || v.Op == OpPtrIndex || v.Op == OpCopy || v.Op == OpSelectNAddr {
   816  		v = v.Args[0]
   817  	}
   818  	return v.Op == OpSP
   819  }
   820
View as plain text