cc_test.go

     1  // Copyright 2017 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  // This test uses the Pdeathsig field of syscall.SysProcAttr, so it only works
     6  // on platforms that support that.
     7  
     8  //go:build linux || (freebsd && amd64)
     9  
    10  // sanitizers_test checks the use of Go with sanitizers like msan, asan, etc.
    11  // See https://github.com/google/sanitizers.
    12  package sanitizers_test
    13  
    14  import (
    15  	"bytes"
    16  	"encoding/json"
    17  	"errors"
    18  	"fmt"
    19  	"internal/testenv"
    20  	"os"
    21  	"os/exec"
    22  	"os/user"
    23  	"path/filepath"
    24  	"regexp"
    25  	"strconv"
    26  	"strings"
    27  	"sync"
    28  	"syscall"
    29  	"testing"
    30  	"time"
    31  	"unicode"
    32  )
    33  
    34  var overcommit struct {
    35  	sync.Once
    36  	value int
    37  	err   error
    38  }
    39  
    40  // requireOvercommit skips t if the kernel does not allow overcommit.
    41  func requireOvercommit(t *testing.T) {
    42  	t.Helper()
    43  
    44  	overcommit.Once.Do(func() {
    45  		var out []byte
    46  		out, overcommit.err = os.ReadFile("/proc/sys/vm/overcommit_memory")
    47  		if overcommit.err != nil {
    48  			return
    49  		}
    50  		overcommit.value, overcommit.err = strconv.Atoi(string(bytes.TrimSpace(out)))
    51  	})
    52  
    53  	if overcommit.err != nil {
    54  		t.Skipf("couldn't determine vm.overcommit_memory (%v); assuming no overcommit", overcommit.err)
    55  	}
    56  	if overcommit.value == 2 {
    57  		t.Skip("vm.overcommit_memory=2")
    58  	}
    59  }
    60  
    61  var env struct {
    62  	sync.Once
    63  	m   map[string]string
    64  	err error
    65  }
    66  
    67  // goEnv returns the output of $(go env) as a map.
    68  func goEnv(key string) (string, error) {
    69  	env.Once.Do(func() {
    70  		var out []byte
    71  		out, env.err = exec.Command("go", "env", "-json").Output()
    72  		if env.err != nil {
    73  			return
    74  		}
    75  
    76  		env.m = make(map[string]string)
    77  		env.err = json.Unmarshal(out, &env.m)
    78  	})
    79  	if env.err != nil {
    80  		return "", env.err
    81  	}
    82  
    83  	v, ok := env.m[key]
    84  	if !ok {
    85  		return "", fmt.Errorf("`go env`: no entry for %v", key)
    86  	}
    87  	return v, nil
    88  }
    89  
    90  // replaceEnv sets the key environment variable to value in cmd.
    91  func replaceEnv(cmd *exec.Cmd, key, value string) {
    92  	if cmd.Env == nil {
    93  		cmd.Env = cmd.Environ()
    94  	}
    95  	cmd.Env = append(cmd.Env, key+"="+value)
    96  }
    97  
    98  // appendExperimentEnv appends comma-separated experiments to GOEXPERIMENT.
    99  func appendExperimentEnv(cmd *exec.Cmd, experiments []string) {
   100  	if cmd.Env == nil {
   101  		cmd.Env = cmd.Environ()
   102  	}
   103  	exps := strings.Join(experiments, ",")
   104  	for _, evar := range cmd.Env {
   105  		c := strings.SplitN(evar, "=", 2)
   106  		if c[0] == "GOEXPERIMENT" {
   107  			exps = c[1] + "," + exps
   108  		}
   109  	}
   110  	cmd.Env = append(cmd.Env, "GOEXPERIMENT="+exps)
   111  }
   112  
   113  // mustRun executes t and fails cmd with a well-formatted message if it fails.
   114  func mustRun(t *testing.T, cmd *exec.Cmd) {
   115  	t.Helper()
   116  	out := new(strings.Builder)
   117  	cmd.Stdout = out
   118  	cmd.Stderr = out
   119  
   120  	err := cmd.Start()
   121  	if err != nil {
   122  		t.Fatalf("%v: %v", cmd, err)
   123  	}
   124  
   125  	if deadline, ok := t.Deadline(); ok {
   126  		timeout := time.Until(deadline)
   127  		timeout -= timeout / 10 // Leave 10% headroom for logging and cleanup.
   128  		timer := time.AfterFunc(timeout, func() {
   129  			cmd.Process.Signal(syscall.SIGQUIT)
   130  		})
   131  		defer timer.Stop()
   132  	}
   133  
   134  	if err := cmd.Wait(); err != nil {
   135  		t.Fatalf("%v exited with %v\n%s", cmd, err, out)
   136  	}
   137  }
   138  
   139  // cc returns a cmd that executes `$(go env CC) $(go env GOGCCFLAGS) $args`.
   140  func cc(args ...string) (*exec.Cmd, error) {
   141  	CC, err := goEnv("CC")
   142  	if err != nil {
   143  		return nil, err
   144  	}
   145  
   146  	GOGCCFLAGS, err := goEnv("GOGCCFLAGS")
   147  	if err != nil {
   148  		return nil, err
   149  	}
   150  
   151  	// Split GOGCCFLAGS, respecting quoting.
   152  	//
   153  	// TODO(bcmills): This code also appears in
   154  	// cmd/cgo/internal/testcarchive/carchive_test.go, and perhaps ought to go in
   155  	// src/cmd/dist/test.go as well. Figure out where to put it so that it can be
   156  	// shared.
   157  	var flags []string
   158  	quote := '\000'
   159  	start := 0
   160  	lastSpace := true
   161  	backslash := false
   162  	for i, c := range GOGCCFLAGS {
   163  		if quote == '\000' && unicode.IsSpace(c) {
   164  			if !lastSpace {
   165  				flags = append(flags, GOGCCFLAGS[start:i])
   166  				lastSpace = true
   167  			}
   168  		} else {
   169  			if lastSpace {
   170  				start = i
   171  				lastSpace = false
   172  			}
   173  			if quote == '\000' && !backslash && (c == '"' || c == '\'') {
   174  				quote = c
   175  				backslash = false
   176  			} else if !backslash && quote == c {
   177  				quote = '\000'
   178  			} else if (quote == '\000' || quote == '"') && !backslash && c == '\\' {
   179  				backslash = true
   180  			} else {
   181  				backslash = false
   182  			}
   183  		}
   184  	}
   185  	if !lastSpace {
   186  		flags = append(flags, GOGCCFLAGS[start:])
   187  	}
   188  
   189  	cmd := exec.Command(CC, flags...)
   190  	cmd.Args = append(cmd.Args, args...)
   191  	return cmd, nil
   192  }
   193  
   194  type version struct {
   195  	name         string
   196  	major, minor int
   197  }
   198  
   199  var compiler struct {
   200  	sync.Once
   201  	version
   202  	err error
   203  }
   204  
   205  // compilerVersion detects the version of $(go env CC).
   206  //
   207  // It returns a non-nil error if the compiler matches a known version schema but
   208  // the version could not be parsed, or if $(go env CC) could not be determined.
   209  func compilerVersion() (version, error) {
   210  	compiler.Once.Do(func() {
   211  		compiler.err = func() error {
   212  			compiler.name = "unknown"
   213  
   214  			cmd, err := cc("--version")
   215  			if err != nil {
   216  				return err
   217  			}
   218  			out, err := cmd.Output()
   219  			if err != nil {
   220  				// Compiler does not support "--version" flag: not Clang or GCC.
   221  				return nil
   222  			}
   223  
   224  			var match [][]byte
   225  			if bytes.HasPrefix(out, []byte("gcc")) {
   226  				compiler.name = "gcc"
   227  				cmd, err := cc("-dumpfullversion", "-dumpversion")
   228  				if err != nil {
   229  					return err
   230  				}
   231  				out, err := cmd.Output()
   232  				if err != nil {
   233  					// gcc, but does not support gcc's "-v" flag?!
   234  					return err
   235  				}
   236  				gccRE := regexp.MustCompile(`(\d+)\.(\d+)`)
   237  				match = gccRE.FindSubmatch(out)
   238  			} else {
   239  				clangRE := regexp.MustCompile(`clang version (\d+)\.(\d+)`)
   240  				if match = clangRE.FindSubmatch(out); len(match) > 0 {
   241  					compiler.name = "clang"
   242  				}
   243  			}
   244  
   245  			if len(match) < 3 {
   246  				return nil // "unknown"
   247  			}
   248  			if compiler.major, err = strconv.Atoi(string(match[1])); err != nil {
   249  				return err
   250  			}
   251  			if compiler.minor, err = strconv.Atoi(string(match[2])); err != nil {
   252  				return err
   253  			}
   254  			return nil
   255  		}()
   256  	})
   257  	return compiler.version, compiler.err
   258  }
   259  
   260  // compilerSupportsLocation reports whether the compiler should be
   261  // able to provide file/line information in backtraces.
   262  func compilerSupportsLocation() bool {
   263  	compiler, err := compilerVersion()
   264  	if err != nil {
   265  		return false
   266  	}
   267  	switch compiler.name {
   268  	case "gcc":
   269  		// TODO(72752): the asan runtime support library
   270  		// (libasan.so.6) shipped with GCC 10 has problems digesting
   271  		// version 5 DWARF produced by the Go toolchain. Disable
   272  		// location checking if gcc is not sufficiently up to date in
   273  		// this case.
   274  		return compiler.major > 10
   275  	case "clang":
   276  		// TODO(65606): The clang toolchain on the LUCI builders is not built against
   277  		// zlib, the ASAN runtime can't actually symbolize its own stack trace. Once
   278  		// this is resolved, one way or another, switch this back to 'true'. We still
   279  		// have coverage from the 'gcc' case above.
   280  		if inLUCIBuild() {
   281  			return false
   282  		}
   283  		return true
   284  	default:
   285  		return false
   286  	}
   287  }
   288  
   289  // inLUCIBuild returns true if we're currently executing in a LUCI build.
   290  func inLUCIBuild() bool {
   291  	u, err := user.Current()
   292  	if err != nil {
   293  		return false
   294  	}
   295  	return testenv.Builder() != "" && u.Username == "swarming"
   296  }
   297  
   298  // compilerRequiredTsanVersion reports whether the compiler is the version required by Tsan.
   299  // Only restrictions for ppc64le are known; otherwise return true.
   300  func compilerRequiredTsanVersion(goos, goarch string) bool {
   301  	compiler, err := compilerVersion()
   302  	if err != nil {
   303  		return false
   304  	}
   305  	if compiler.name == "gcc" && goarch == "ppc64le" {
   306  		return compiler.major >= 9
   307  	}
   308  	return true
   309  }
   310  
   311  // compilerRequiredAsanVersion reports whether the compiler is the version required by Asan.
   312  func compilerRequiredAsanVersion(goos, goarch string) bool {
   313  	compiler, err := compilerVersion()
   314  	if err != nil {
   315  		return false
   316  	}
   317  	switch compiler.name {
   318  	case "gcc":
   319  		if goarch == "loong64" {
   320  			return compiler.major >= 14
   321  		}
   322  		if goarch == "ppc64le" {
   323  			return compiler.major >= 9
   324  		}
   325  		return compiler.major >= 7
   326  	case "clang":
   327  		if goarch == "loong64" {
   328  			return compiler.major >= 16
   329  		}
   330  		return compiler.major >= 9
   331  	default:
   332  		return false
   333  	}
   334  }
   335  
   336  // compilerRequiredLsanVersion reports whether the compiler is the
   337  // version required by Lsan.
   338  func compilerRequiredLsanVersion(goos, goarch string) bool {
   339  	return compilerRequiredAsanVersion(goos, goarch)
   340  }
   341  
   342  type compilerCheck struct {
   343  	once sync.Once
   344  	err  error
   345  	skip bool // If true, skip with err instead of failing with it.
   346  }
   347  
   348  type config struct {
   349  	sanitizer string
   350  
   351  	cFlags, ldFlags, goFlags []string
   352  
   353  	sanitizerCheck, runtimeCheck compilerCheck
   354  }
   355  
   356  var configs struct {
   357  	sync.Mutex
   358  	m map[string]*config
   359  }
   360  
   361  // configure returns the configuration for the given sanitizer.
   362  func configure(sanitizer string) *config {
   363  	configs.Lock()
   364  	defer configs.Unlock()
   365  	if c, ok := configs.m[sanitizer]; ok {
   366  		return c
   367  	}
   368  
   369  	sanitizerOpt := sanitizer
   370  	// For the leak detector, we use "go build -asan",
   371  	// which implies the address sanitizer.
   372  	// We may want to adjust this someday.
   373  	if sanitizer == "leak" {
   374  		sanitizerOpt = "address"
   375  	}
   376  
   377  	c := &config{
   378  		sanitizer: sanitizer,
   379  		cFlags:    []string{"-fsanitize=" + sanitizerOpt},
   380  		ldFlags:   []string{"-fsanitize=" + sanitizerOpt},
   381  	}
   382  
   383  	if testing.Verbose() {
   384  		c.goFlags = append(c.goFlags, "-x")
   385  	}
   386  
   387  	switch sanitizer {
   388  	case "memory":
   389  		c.goFlags = append(c.goFlags, "-msan")
   390  
   391  	case "thread":
   392  		c.goFlags = append(c.goFlags, "--installsuffix=tsan")
   393  		compiler, _ := compilerVersion()
   394  		if compiler.name == "gcc" {
   395  			c.cFlags = append(c.cFlags, "-fPIC")
   396  			c.ldFlags = append(c.ldFlags, "-fPIC", "-static-libtsan")
   397  		}
   398  
   399  	case "address", "leak":
   400  		c.goFlags = append(c.goFlags, "-asan")
   401  		// Set the debug mode to print the C stack trace.
   402  		c.cFlags = append(c.cFlags, "-g")
   403  
   404  	case "fuzzer":
   405  		c.goFlags = append(c.goFlags, "-tags=libfuzzer", "-gcflags=-d=libfuzzer")
   406  
   407  	default:
   408  		panic(fmt.Sprintf("unrecognized sanitizer: %q", sanitizer))
   409  	}
   410  
   411  	if configs.m == nil {
   412  		configs.m = make(map[string]*config)
   413  	}
   414  	configs.m[sanitizer] = c
   415  	return c
   416  }
   417  
   418  // goCmd returns a Cmd that executes "go $subcommand $args" with appropriate
   419  // additional flags and environment.
   420  func (c *config) goCmd(subcommand string, args ...string) *exec.Cmd {
   421  	return c.goCmdWithExperiments(subcommand, args, nil)
   422  }
   423  
   424  // goCmdWithExperiments returns a Cmd that executes
   425  // "GOEXPERIMENT=$experiments go $subcommand $args" with appropriate
   426  // additional flags and CGO-related environment variables.
   427  func (c *config) goCmdWithExperiments(subcommand string, args []string, experiments []string) *exec.Cmd {
   428  	cmd := exec.Command("go", subcommand)
   429  	cmd.Args = append(cmd.Args, c.goFlags...)
   430  	cmd.Args = append(cmd.Args, args...)
   431  	replaceEnv(cmd, "CGO_CFLAGS", strings.Join(c.cFlags, " "))
   432  	replaceEnv(cmd, "CGO_LDFLAGS", strings.Join(c.ldFlags, " "))
   433  	appendExperimentEnv(cmd, experiments)
   434  	return cmd
   435  }
   436  
   437  // skipIfCSanitizerBroken skips t if the C compiler does not produce working
   438  // binaries as configured.
   439  func (c *config) skipIfCSanitizerBroken(t *testing.T) {
   440  	check := &c.sanitizerCheck
   441  	check.once.Do(func() {
   442  		check.skip, check.err = c.checkCSanitizer()
   443  	})
   444  	if check.err != nil {
   445  		t.Helper()
   446  		if check.skip {
   447  			t.Skip(check.err)
   448  		}
   449  		t.Fatal(check.err)
   450  	}
   451  }
   452  
   453  var cMain = []byte(`
   454  int main() {
   455  	return 0;
   456  }
   457  `)
   458  
   459  var cLibFuzzerInput = []byte(`
   460  #include <stddef.h>
   461  int LLVMFuzzerTestOneInput(char *data, size_t size) {
   462  	return 0;
   463  }
   464  `)
   465  
   466  func (c *config) checkCSanitizer() (skip bool, err error) {
   467  	dir, err := os.MkdirTemp("", c.sanitizer)
   468  	if err != nil {
   469  		return false, fmt.Errorf("failed to create temp directory: %v", err)
   470  	}
   471  	defer os.RemoveAll(dir)
   472  
   473  	src := filepath.Join(dir, "return0.c")
   474  	cInput := cMain
   475  	if c.sanitizer == "fuzzer" {
   476  		// libFuzzer generates the main function itself, and uses a different input.
   477  		cInput = cLibFuzzerInput
   478  	}
   479  	if err := os.WriteFile(src, cInput, 0600); err != nil {
   480  		return false, fmt.Errorf("failed to write C source file: %v", err)
   481  	}
   482  
   483  	dst := filepath.Join(dir, "return0")
   484  	cmd, err := cc(c.cFlags...)
   485  	if err != nil {
   486  		return false, err
   487  	}
   488  	cmd.Args = append(cmd.Args, c.ldFlags...)
   489  	cmd.Args = append(cmd.Args, "-o", dst, src)
   490  	out, err := cmd.CombinedOutput()
   491  	if err != nil {
   492  		if bytes.Contains(out, []byte("-fsanitize")) &&
   493  			(bytes.Contains(out, []byte("unrecognized")) ||
   494  				bytes.Contains(out, []byte("unsupported"))) {
   495  			return true, errors.New(string(out))
   496  		}
   497  		return true, fmt.Errorf("%#q failed: %v\n%s", cmd, err, out)
   498  	}
   499  
   500  	if c.sanitizer == "fuzzer" {
   501  		// For fuzzer, don't try running the test binary. It never finishes.
   502  		return false, nil
   503  	}
   504  
   505  	if out, err := exec.Command(dst).CombinedOutput(); err != nil {
   506  		if os.IsNotExist(err) {
   507  			return true, fmt.Errorf("%#q failed to produce executable: %v", cmd, err)
   508  		}
   509  		snippet, _, _ := bytes.Cut(out, []byte("\n"))
   510  		return true, fmt.Errorf("%#q generated broken executable: %v\n%s", cmd, err, snippet)
   511  	}
   512  
   513  	return false, nil
   514  }
   515  
   516  // skipIfRuntimeIncompatible skips t if the Go runtime is suspected not to work
   517  // with cgo as configured.
   518  func (c *config) skipIfRuntimeIncompatible(t *testing.T) {
   519  	check := &c.runtimeCheck
   520  	check.once.Do(func() {
   521  		check.skip, check.err = c.checkRuntime()
   522  	})
   523  	if check.err != nil {
   524  		t.Helper()
   525  		if check.skip {
   526  			t.Skip(check.err)
   527  		}
   528  		t.Fatal(check.err)
   529  	}
   530  }
   531  
   532  func (c *config) checkRuntime() (skip bool, err error) {
   533  	if c.sanitizer != "thread" {
   534  		return false, nil
   535  	}
   536  
   537  	// libcgo.h sets CGO_TSAN if it detects TSAN support in the C compiler.
   538  	// Dump the preprocessor defines to check that works.
   539  	// (Sometimes it doesn't: see https://golang.org/issue/15983.)
   540  	cmd, err := cc(c.cFlags...)
   541  	if err != nil {
   542  		return false, err
   543  	}
   544  	cmd.Args = append(cmd.Args, "-dM", "-E", "../../../../runtime/cgo/libcgo.h")
   545  	out, err := cmd.CombinedOutput()
   546  	if err != nil {
   547  		return false, fmt.Errorf("%#q exited with %v\n%s", cmd, err, out)
   548  	}
   549  	if !bytes.Contains(out, []byte("#define CGO_TSAN")) {
   550  		return true, fmt.Errorf("%#q did not define CGO_TSAN", cmd)
   551  	}
   552  	return false, nil
   553  }
   554  
   555  // srcPath returns the path to the given file relative to this test's source tree.
   556  func srcPath(path string) string {
   557  	return filepath.Join("testdata", path)
   558  }
   559  
   560  // A tempDir manages a temporary directory within a test.
   561  type tempDir struct {
   562  	base string
   563  }
   564  
   565  func (d *tempDir) RemoveAll(t *testing.T) {
   566  	t.Helper()
   567  	if d.base == "" {
   568  		return
   569  	}
   570  	if err := os.RemoveAll(d.base); err != nil {
   571  		t.Fatalf("Failed to remove temp dir: %v", err)
   572  	}
   573  }
   574  
   575  func (d *tempDir) Base() string {
   576  	return d.base
   577  }
   578  
   579  func (d *tempDir) Join(name string) string {
   580  	return filepath.Join(d.base, name)
   581  }
   582  
   583  func newTempDir(t *testing.T) *tempDir {
   584  	return &tempDir{base: t.TempDir()}
   585  }
   586  
   587  // hangProneCmd returns an exec.Cmd for a command that is likely to hang.
   588  //
   589  // If one of these tests hangs, the caller is likely to kill the test process
   590  // using SIGINT, which will be sent to all of the processes in the test's group.
   591  // Unfortunately, TSAN in particular is prone to dropping signals, so the SIGINT
   592  // may terminate the test binary but leave the subprocess running. hangProneCmd
   593  // configures subprocess to receive SIGKILL instead to ensure that it won't
   594  // leak.
   595  func hangProneCmd(name string, arg ...string) *exec.Cmd {
   596  	cmd := exec.Command(name, arg...)
   597  	cmd.SysProcAttr = &syscall.SysProcAttr{
   598  		Pdeathsig: syscall.SIGKILL,
   599  	}
   600  	return cmd
   601  }
   602
View as plain text