cse.go

     1  // Copyright 2015 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package ssacompile
     6  
     7  import (
     8  	"cmp"
     9  	"fmt"
    10  	"slices"
    11  
    12  	"cmd/compile/internal/ssa"
    13  	"cmd/compile/internal/ssa/ssaop"
    14  	"cmd/compile/internal/types"
    15  	"cmd/internal/src"
    16  )
    17  
    18  // cse does common-subexpression elimination on the Function.
    19  // Values are just relinked, nothing is deleted. A subsequent deadcode
    20  // pass is required to actually remove duplicate expressions.
    21  func cse(f *ssa.Func) {
    22  	// Two values are equivalent if they satisfy the following definition:
    23  	// equivalent(v, w):
    24  	//   v.op == w.op
    25  	//   v.type == w.type
    26  	//   v.aux == w.aux
    27  	//   v.auxint == w.auxint
    28  	//   len(v.args) == len(w.args)
    29  	//   v.block == w.block if v.op == OpPhi
    30  	//   equivalent(v.args[i], w.args[i]) for i in 0..len(v.args)-1
    31  
    32  	// The algorithm searches for a partition of f's values into
    33  	// equivalence classes using the above definition.
    34  	// It starts with a coarse partition and iteratively refines it
    35  	// until it reaches a fixed point.
    36  
    37  	// Make initial coarse partitions by using a subset of the conditions above.
    38  	a := f.Cache.AllocValueSlice(f.NumValues())
    39  	defer func() { f.Cache.FreeValueSlice(a) }() // inside closure to use final value of a
    40  	a = a[:0]
    41  	o := f.Cache.AllocInt32Slice(f.NumValues()) // the ordering score for stores
    42  	defer func() { f.Cache.FreeInt32Slice(o) }()
    43  	if f.Auxmap == nil {
    44  		f.Auxmap = ssa.AuxMap{}
    45  	}
    46  	for _, b := range f.Blocks {
    47  		for _, v := range b.Values {
    48  			if v.Type.IsMemory() {
    49  				continue // memory values can never cse
    50  			}
    51  			if f.Auxmap[v.Aux] == 0 {
    52  				f.Auxmap[v.Aux] = int32(len(f.Auxmap)) + 1
    53  			}
    54  			a = append(a, v)
    55  		}
    56  	}
    57  	partition := partitionValues(a, f.Auxmap)
    58  
    59  	// map from value id back to eqclass id
    60  	valueEqClass := f.Cache.AllocIDSlice(f.NumValues())
    61  	defer f.Cache.FreeIDSlice(valueEqClass)
    62  	for _, b := range f.Blocks {
    63  		for _, v := range b.Values {
    64  			// Use negative equivalence class #s for unique values.
    65  			valueEqClass[v.ID] = -v.ID
    66  		}
    67  	}
    68  	var pNum ssa.ID = 1
    69  	for _, e := range partition {
    70  		if f.Pass.Debug > 1 && len(e) > 500 {
    71  			fmt.Printf("CSE.large partition (%d): ", len(e))
    72  			for j := 0; j < 3; j++ {
    73  				fmt.Printf("%s ", e[j].LongString())
    74  			}
    75  			fmt.Println()
    76  		}
    77  
    78  		for _, v := range e {
    79  			valueEqClass[v.ID] = pNum
    80  		}
    81  		if f.Pass.Debug > 2 && len(e) > 1 {
    82  			fmt.Printf("CSE.partition #%d:", pNum)
    83  			for _, v := range e {
    84  				fmt.Printf(" %s", v.String())
    85  			}
    86  			fmt.Printf("\n")
    87  		}
    88  		pNum++
    89  	}
    90  
    91  	// Keep a table to remap memory operand of any memory user which does not have a memory result (such as a regular load),
    92  	// to some dominating memory operation, skipping the memory defs that do not alias with it.
    93  	memTable := f.Cache.AllocInt32Slice(f.NumValues())
    94  	defer f.Cache.FreeInt32Slice(memTable)
    95  
    96  	// Split equivalence classes at points where they have
    97  	// non-equivalent arguments.  Repeat until we can't find any
    98  	// more splits.
    99  	var splitPoints []int
   100  	for {
   101  		changed := false
   102  
   103  		// partition can grow in the loop. By not using a range loop here,
   104  		// we process new additions as they arrive, avoiding O(n^2) behavior.
   105  		for i := 0; i < len(partition); i++ {
   106  			e := partition[i]
   107  
   108  			if ssaop.OpcodeTable[e[0].Op].Commutative {
   109  				// Order the first two args before comparison.
   110  				for _, v := range e {
   111  					if valueEqClass[v.Args[0].ID] > valueEqClass[v.Args[1].ID] {
   112  						v.Args[0], v.Args[1] = v.Args[1], v.Args[0]
   113  					}
   114  				}
   115  			}
   116  
   117  			// Sort by eq class of arguments.
   118  			slices.SortFunc(e, func(v, w *ssa.Value) int {
   119  				_, idxMem, _, _ := isMemUser(v)
   120  				for i, a := range v.Args {
   121  					var aId, bId ssa.ID
   122  					if i != idxMem {
   123  						b := w.Args[i]
   124  						aId = a.ID
   125  						bId = b.ID
   126  					} else {
   127  						// A memory user's mem argument may be remapped to allow matching
   128  						// identical load-like instructions across disjoint stores.
   129  						aId, _ = getEffectiveMemoryArg(memTable, v)
   130  						bId, _ = getEffectiveMemoryArg(memTable, w)
   131  					}
   132  					if valueEqClass[aId] < valueEqClass[bId] {
   133  						return -1
   134  					}
   135  					if valueEqClass[aId] > valueEqClass[bId] {
   136  						return +1
   137  					}
   138  				}
   139  				return 0
   140  			})
   141  
   142  			// Find split points.
   143  			splitPoints = append(splitPoints[:0], 0)
   144  			for j := 1; j < len(e); j++ {
   145  				v, w := e[j-1], e[j]
   146  				// Note: commutative args already correctly ordered by byArgClass.
   147  				eqArgs := true
   148  				_, idxMem, _, _ := isMemUser(v)
   149  				for k, a := range v.Args {
   150  					if v.Op == ssaop.OpLocalAddr && k == 1 {
   151  						continue
   152  					}
   153  					var aId, bId ssa.ID
   154  					if k != idxMem {
   155  						b := w.Args[k]
   156  						aId = a.ID
   157  						bId = b.ID
   158  					} else {
   159  						// A memory user's mem argument may be remapped to allow matching
   160  						// identical load-like instructions across disjoint stores.
   161  						aId, _ = getEffectiveMemoryArg(memTable, v)
   162  						bId, _ = getEffectiveMemoryArg(memTable, w)
   163  					}
   164  					if valueEqClass[aId] != valueEqClass[bId] {
   165  						eqArgs = false
   166  						break
   167  					}
   168  				}
   169  				if !eqArgs {
   170  					splitPoints = append(splitPoints, j)
   171  				}
   172  			}
   173  			if len(splitPoints) == 1 {
   174  				continue // no splits, leave equivalence class alone.
   175  			}
   176  
   177  			// Move another equivalence class down in place of e.
   178  			partition[i] = partition[len(partition)-1]
   179  			partition = partition[:len(partition)-1]
   180  			i--
   181  
   182  			// Add new equivalence classes for the parts of e we found.
   183  			splitPoints = append(splitPoints, len(e))
   184  			for j := 0; j < len(splitPoints)-1; j++ {
   185  				f := e[splitPoints[j]:splitPoints[j+1]]
   186  				if len(f) == 1 {
   187  					// Don't add singletons.
   188  					valueEqClass[f[0].ID] = -f[0].ID
   189  					continue
   190  				}
   191  				for _, v := range f {
   192  					valueEqClass[v.ID] = pNum
   193  				}
   194  				pNum++
   195  				partition = append(partition, f)
   196  			}
   197  			changed = true
   198  		}
   199  
   200  		if !changed {
   201  			break
   202  		}
   203  	}
   204  
   205  	sdom := f.Sdom()
   206  
   207  	// Compute substitutions we would like to do. We substitute v for w
   208  	// if v and w are in the same equivalence class and v dominates w.
   209  	rewrite := f.Cache.AllocValueSlice(f.NumValues())
   210  	defer f.Cache.FreeValueSlice(rewrite)
   211  	for _, e := range partition {
   212  		slices.SortFunc(e, func(v, w *ssa.Value) int {
   213  			if c := cmp.Compare(sdom.DomOrder(v.Block), sdom.DomOrder(w.Block)); c != 0 {
   214  				return c
   215  			}
   216  			if _, _, _, ok := isMemUser(v); ok {
   217  				// Additional ordering among the memory users within one block: prefer the earliest
   218  				// possible value among the set of equivalent values, that is the one with the lowest
   219  				// skip count (lowest number of memory defs skipped until their common def).
   220  				_, vSkips := getEffectiveMemoryArg(memTable, v)
   221  				_, wSkips := getEffectiveMemoryArg(memTable, w)
   222  				if c := cmp.Compare(vSkips, wSkips); c != 0 {
   223  					return c
   224  				}
   225  			}
   226  			if v.Op == ssaop.OpLocalAddr {
   227  				// compare the memory args for OpLocalAddrs in the same block
   228  				vm := v.Args[1]
   229  				wm := w.Args[1]
   230  				if vm == wm {
   231  					return 0
   232  				}
   233  				// if the two OpLocalAddrs are in the same block, and one's memory
   234  				// arg also in the same block, but the other one's memory arg not,
   235  				// the latter must be in an ancestor block
   236  				if vm.Block != v.Block {
   237  					return -1
   238  				}
   239  				if wm.Block != w.Block {
   240  					return +1
   241  				}
   242  				// use store order if the memory args are in the same block
   243  				vs := storeOrdering(vm, o)
   244  				ws := storeOrdering(wm, o)
   245  				if vs <= 0 {
   246  					f.Fatalf("unable to determine the order of %s", vm.LongString())
   247  				}
   248  				if ws <= 0 {
   249  					f.Fatalf("unable to determine the order of %s", wm.LongString())
   250  				}
   251  				return cmp.Compare(vs, ws)
   252  			}
   253  			vStmt := v.Pos.IsStmt() == src.PosIsStmt
   254  			wStmt := w.Pos.IsStmt() == src.PosIsStmt
   255  			if vStmt != wStmt {
   256  				if vStmt {
   257  					return -1
   258  				}
   259  				return +1
   260  			}
   261  			return 0
   262  		})
   263  
   264  		for i := 0; i < len(e)-1; i++ {
   265  			// e is sorted by domorder, so a maximal dominant element is first in the slice
   266  			v := e[i]
   267  			if v == nil {
   268  				continue
   269  			}
   270  
   271  			e[i] = nil
   272  			// Replace all elements of e which v dominates
   273  			for j := i + 1; j < len(e); j++ {
   274  				w := e[j]
   275  				if w == nil {
   276  					continue
   277  				}
   278  				if sdom.IsAncestorEq(v.Block, w.Block) {
   279  					rewrite[w.ID] = v
   280  					e[j] = nil
   281  				} else {
   282  					// e is sorted by domorder, so v.Block doesn't dominate any subsequent blocks in e
   283  					break
   284  				}
   285  			}
   286  		}
   287  	}
   288  
   289  	rewrites := int64(0)
   290  
   291  	// Apply substitutions
   292  	for _, b := range f.Blocks {
   293  		for _, v := range b.Values {
   294  			for i, w := range v.Args {
   295  				if x := rewrite[w.ID]; x != nil {
   296  					if w.Pos.IsStmt() == src.PosIsStmt && w.Op != ssaop.OpNilCheck {
   297  						// about to lose a statement marker, w
   298  						// w is an input to v; if they're in the same block
   299  						// and the same line, v is a good-enough new statement boundary.
   300  						if w.Block == v.Block && w.Pos.Line() == v.Pos.Line() {
   301  							v.Pos = v.Pos.WithIsStmt()
   302  							w.Pos = w.Pos.WithNotStmt()
   303  						} // TODO and if this fails?
   304  					}
   305  					v.SetArg(i, x)
   306  					rewrites++
   307  				}
   308  			}
   309  		}
   310  		for i, v := range b.ControlValues() {
   311  			if x := rewrite[v.ID]; x != nil {
   312  				if v.Op == ssaop.OpNilCheck {
   313  					// nilcheck pass will remove the nil checks and log
   314  					// them appropriately, so don't mess with them here.
   315  					continue
   316  				}
   317  				b.ReplaceControl(i, x)
   318  			}
   319  		}
   320  	}
   321  
   322  	if f.Pass.Stats > 0 {
   323  		f.LogStat("CSE REWRITES", rewrites)
   324  	}
   325  
   326  	// Annotate HTML dumps with each memory user's effective memory arg.
   327  	f.HTMLWriter.DebugInfo(func(v *ssa.Value) string {
   328  		_, idxMem, _, ok := isMemUser(v)
   329  		if !ok {
   330  			return ""
   331  		}
   332  		memID, skips := getEffectiveMemoryArg(memTable, v)
   333  		if memID == v.Args[idxMem].ID {
   334  			return ""
   335  		}
   336  		return fmt.Sprintf("effmem %s (skips %d)", ssa.ValueHTML(memID), skips)
   337  	})
   338  }
   339  
   340  // storeOrdering computes the order for stores by iterate over the store
   341  // chain, assigns a score to each store. The scores only make sense for
   342  // stores within the same block, and the first store by store order has
   343  // the lowest score. The cache was used to ensure only compute once.
   344  func storeOrdering(v *ssa.Value, cache []int32) int32 {
   345  	const minScore int32 = 1
   346  	score := minScore
   347  	w := v
   348  	for {
   349  		if s := cache[w.ID]; s >= minScore {
   350  			score += s
   351  			break
   352  		}
   353  		if w.Op == ssaop.OpPhi || w.Op == ssaop.OpInitMem {
   354  			break
   355  		}
   356  		a := w.MemoryArg()
   357  		if a.Block != w.Block {
   358  			break
   359  		}
   360  		w = a
   361  		score++
   362  	}
   363  	w = v
   364  	for cache[w.ID] == 0 {
   365  		cache[w.ID] = score
   366  		if score == minScore {
   367  			break
   368  		}
   369  		w = w.MemoryArg()
   370  		score--
   371  	}
   372  	return cache[v.ID]
   373  }
   374  
   375  // An eqclass approximates an equivalence class. During the
   376  // algorithm it may represent the union of several of the
   377  // final equivalence classes.
   378  type eqclass []*ssa.Value
   379  
   380  // partitionValues partitions the values into equivalence classes
   381  // based on having all the following features match:
   382  //   - opcode
   383  //   - type
   384  //   - auxint
   385  //   - aux
   386  //   - nargs
   387  //   - block # if a phi op
   388  //   - first two arg's opcodes and auxint
   389  //   - NOT first two arg's aux; that can break CSE.
   390  //
   391  // partitionValues returns a list of equivalence classes, each
   392  // being a sorted by ID list of *Values. The eqclass slices are
   393  // backed by the same storage as the input slice.
   394  // Equivalence classes of size 1 are ignored.
   395  func partitionValues(a []*ssa.Value, auxIDs ssa.AuxMap) []eqclass {
   396  	slices.SortFunc(a, func(v, w *ssa.Value) int {
   397  		switch cmpVal(v, w, auxIDs) {
   398  		case types.CMPlt:
   399  			return -1
   400  		case types.CMPgt:
   401  			return +1
   402  		default:
   403  			// Sort by value ID last to keep the sort result deterministic.
   404  			return cmp.Compare(v.ID, w.ID)
   405  		}
   406  	})
   407  
   408  	var partition []eqclass
   409  	for len(a) > 0 {
   410  		v := a[0]
   411  		j := 1
   412  		for ; j < len(a); j++ {
   413  			w := a[j]
   414  			if cmpVal(v, w, auxIDs) != types.CMPeq {
   415  				break
   416  			}
   417  		}
   418  		if j > 1 {
   419  			partition = append(partition, a[:j])
   420  		}
   421  		a = a[j:]
   422  	}
   423  
   424  	return partition
   425  }
   426  func lt2Cmp(isLt bool) types.Cmp {
   427  	if isLt {
   428  		return types.CMPlt
   429  	}
   430  	return types.CMPgt
   431  }
   432  
   433  func cmpVal(v, w *ssa.Value, auxIDs ssa.AuxMap) types.Cmp {
   434  	// Try to order these comparison by cost (cheaper first)
   435  	if v.Op != w.Op {
   436  		return lt2Cmp(v.Op < w.Op)
   437  	}
   438  	if v.AuxInt != w.AuxInt {
   439  		return lt2Cmp(v.AuxInt < w.AuxInt)
   440  	}
   441  	if len(v.Args) != len(w.Args) {
   442  		return lt2Cmp(len(v.Args) < len(w.Args))
   443  	}
   444  	if v.Op == ssaop.OpPhi && v.Block != w.Block {
   445  		return lt2Cmp(v.Block.ID < w.Block.ID)
   446  	}
   447  	if v.Type.IsMemory() {
   448  		// We will never be able to CSE two values
   449  		// that generate memory.
   450  		return lt2Cmp(v.ID < w.ID)
   451  	}
   452  	// OpSelect is a pseudo-op. We need to be more aggressive
   453  	// regarding CSE to keep multiple OpSelect's of the same
   454  	// argument from existing.
   455  	if v.Op != ssaop.OpSelect0 && v.Op != ssaop.OpSelect1 && v.Op != ssaop.OpSelectN {
   456  		if tc := v.Type.Compare(w.Type); tc != types.CMPeq {
   457  			return tc
   458  		}
   459  	}
   460  
   461  	if v.Aux != w.Aux {
   462  		if v.Aux == nil {
   463  			return types.CMPlt
   464  		}
   465  		if w.Aux == nil {
   466  			return types.CMPgt
   467  		}
   468  		return lt2Cmp(auxIDs[v.Aux] < auxIDs[w.Aux])
   469  	}
   470  
   471  	return types.CMPeq
   472  }
   473  
   474  // Query if the given instruction only uses "memory" argument and we may try to skip some memory "defs" if they do not alias with its address.
   475  // Return index of pointer argument, index of "memory" argument, the access width and true on such instructions, otherwise return (-1, -1, 0, false).
   476  func isMemUser(v *ssa.Value) (int, int, int64, bool) {
   477  	switch v.Op {
   478  	case ssaop.OpLoad:
   479  		return 0, 1, v.Type.Size(), true
   480  	case ssaop.OpNilCheck:
   481  		return 0, 1, 0, true
   482  	default:
   483  		return -1, -1, 0, false
   484  	}
   485  }
   486  
   487  // Query if the given "memory"-defining instruction's memory destination can be analyzed for aliasing with a memory "user" instructions.
   488  // Return index of pointer argument, index of "memory" argument, the access width and true on such instructions, otherwise return (-1, -1, 0, false).
   489  // If the access width is 0, the pointer index may be -1 (no pointer operand is needed).
   490  func isMemDef(v *ssa.Value) (int, int, int64, bool) {
   491  	switch v.Op {
   492  	case ssaop.OpStore:
   493  		return 0, 2, ssa.AuxToType(v.Aux).Size(), true
   494  	case ssaop.OpVarDef:
   495  		return -1, 0, 0, true
   496  	case ssaop.OpZero:
   497  		return 0, 1, v.AuxInt, true
   498  	default:
   499  		return -1, -1, 0, false
   500  	}
   501  }
   502  
   503  // Mem table keeps memTableSkipBits lower bits to store the number of skips of "memory" operand
   504  // and the rest to store the ID of the destination "memory"-producing instruction.
   505  const memTableSkipBits = 8
   506  
   507  // The maximum ID value we are able to store in the memTable, otherwise fall back to v.ID
   508  const maxId = ssa.ID(1<<(31-memTableSkipBits)) - 1
   509  
   510  // Return the first possibly-aliased store along the memory chain starting at v's memory argument and the number of not-aliased stores skipped.
   511  func getEffectiveMemoryArg(memTable []int32, v *ssa.Value) (ssa.ID, uint32) {
   512  	if code := uint32(memTable[v.ID]); code != 0 {
   513  		return ssa.ID(code >> memTableSkipBits), code & ((1 << memTableSkipBits) - 1)
   514  	}
   515  	if idxPtr, idxMem, width, ok := isMemUser(v); ok {
   516  		// TODO: We could early return some predefined value if width==0
   517  		memId := v.Args[idxMem].ID
   518  		if memId > maxId {
   519  			return memId, 0
   520  		}
   521  		mem, skips := skipDisjointMemDefs(v, idxPtr, idxMem, width)
   522  		if mem.ID <= maxId {
   523  			memId = mem.ID
   524  		} else {
   525  			skips = 0 // avoid the skip
   526  		}
   527  		memTable[v.ID] = int32(memId<<memTableSkipBits) | int32(skips)
   528  		return memId, skips
   529  	} else {
   530  		v.Block.Func.Fatalf("expected memory user instruction: %v", v.LongString())
   531  	}
   532  	return 0, 0
   533  }
   534  
   535  // Find a memory def that's not trivially disjoint with the user instruction, count the number
   536  // of "skips" along the path. Return the corresponding memory def's value and the number of skips.
   537  func skipDisjointMemDefs(user *ssa.Value, idxUserPtr, idxUserMem int, useWidth int64) (*ssa.Value, uint32) {
   538  	usePtr, mem := user.Args[idxUserPtr], user.Args[idxUserMem]
   539  	const maxSkips = (1 << memTableSkipBits) - 1
   540  	var skips uint32
   541  	for skips = 0; skips < maxSkips; skips++ {
   542  		if idxPtr, idxMem, width, ok := isMemDef(mem); ok {
   543  			if mem.Args[idxMem].Uses > 50 {
   544  				// Skipping a memory def with a lot of uses may potentially increase register pressure.
   545  				break
   546  			}
   547  			if width == 0 {
   548  				mem = mem.Args[idxMem]
   549  				continue
   550  			}
   551  			defPtr := mem.Args[idxPtr]
   552  			if ssa.Disjoint1(defPtr, width, usePtr, useWidth) {
   553  				mem = mem.Args[idxMem]
   554  				continue
   555  			}
   556  		}
   557  		break
   558  	}
   559  	return mem, skips
   560  }
   561
View as plain text