tests.go

     1  // Copyright 2015 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package tests
     6  
     7  import (
     8  	_ "embed"
     9  	"go/ast"
    10  	"go/token"
    11  	"go/types"
    12  	"regexp"
    13  	"strings"
    14  	"unicode"
    15  	"unicode/utf8"
    16  
    17  	"golang.org/x/tools/go/analysis"
    18  	"golang.org/x/tools/go/analysis/passes/internal/analysisutil"
    19  )
    20  
    21  //go:embed doc.go
    22  var doc string
    23  
    24  var Analyzer = &analysis.Analyzer{
    25  	Name: "tests",
    26  	Doc:  analysisutil.MustExtractDoc(doc, "tests"),
    27  	URL:  "https://pkg.go.dev/golang.org/x/tools/go/analysis/passes/tests",
    28  	Run:  run,
    29  }
    30  
    31  var acceptedFuzzTypes = []types.Type{
    32  	types.Typ[types.String],
    33  	types.Typ[types.Bool],
    34  	types.Typ[types.Float32],
    35  	types.Typ[types.Float64],
    36  	types.Typ[types.Int],
    37  	types.Typ[types.Int8],
    38  	types.Typ[types.Int16],
    39  	types.Typ[types.Int32],
    40  	types.Typ[types.Int64],
    41  	types.Typ[types.Uint],
    42  	types.Typ[types.Uint8],
    43  	types.Typ[types.Uint16],
    44  	types.Typ[types.Uint32],
    45  	types.Typ[types.Uint64],
    46  	types.NewSlice(types.Universe.Lookup("byte").Type()),
    47  }
    48  
    49  func run(pass *analysis.Pass) (interface{}, error) {
    50  	for _, f := range pass.Files {
    51  		if !strings.HasSuffix(pass.Fset.File(f.FileStart).Name(), "_test.go") {
    52  			continue
    53  		}
    54  		for _, decl := range f.Decls {
    55  			fn, ok := decl.(*ast.FuncDecl)
    56  			if !ok || fn.Recv != nil {
    57  				// Ignore non-functions or functions with receivers.
    58  				continue
    59  			}
    60  			switch {
    61  			case strings.HasPrefix(fn.Name.Name, "Example"):
    62  				checkExampleName(pass, fn)
    63  				checkExampleOutput(pass, fn, f.Comments)
    64  			case strings.HasPrefix(fn.Name.Name, "Test"):
    65  				checkTest(pass, fn, "Test")
    66  			case strings.HasPrefix(fn.Name.Name, "Benchmark"):
    67  				checkTest(pass, fn, "Benchmark")
    68  			case strings.HasPrefix(fn.Name.Name, "Fuzz"):
    69  				checkTest(pass, fn, "Fuzz")
    70  				checkFuzz(pass, fn)
    71  			}
    72  		}
    73  	}
    74  	return nil, nil
    75  }
    76  
    77  // checkFuzz checks the contents of a fuzz function.
    78  func checkFuzz(pass *analysis.Pass, fn *ast.FuncDecl) {
    79  	params := checkFuzzCall(pass, fn)
    80  	if params != nil {
    81  		checkAddCalls(pass, fn, params)
    82  	}
    83  }
    84  
    85  // checkFuzzCall checks the arguments of f.Fuzz() calls:
    86  //
    87  //  1. f.Fuzz() should call a function and it should be of type (*testing.F).Fuzz().
    88  //  2. The called function in f.Fuzz(func(){}) should not return result.
    89  //  3. First argument of func() should be of type *testing.T
    90  //  4. Second argument onwards should be of type []byte, string, bool, byte,
    91  //     rune, float32, float64, int, int8, int16, int32, int64, uint, uint8, uint16,
    92  //     uint32, uint64
    93  //  5. func() must not call any *F methods, e.g. (*F).Log, (*F).Error, (*F).Skip
    94  //     The only *F methods that are allowed in the (*F).Fuzz function are (*F).Failed and (*F).Name.
    95  //
    96  // Returns the list of parameters to the fuzz function, if they are valid fuzz parameters.
    97  func checkFuzzCall(pass *analysis.Pass, fn *ast.FuncDecl) (params *types.Tuple) {
    98  	ast.Inspect(fn, func(n ast.Node) bool {
    99  		call, ok := n.(*ast.CallExpr)
   100  		if ok {
   101  			if !isFuzzTargetDotFuzz(pass, call) {
   102  				return true
   103  			}
   104  
   105  			// Only one argument (func) must be passed to (*testing.F).Fuzz.
   106  			if len(call.Args) != 1 {
   107  				return true
   108  			}
   109  			expr := call.Args[0]
   110  			if pass.TypesInfo.Types[expr].Type == nil {
   111  				return true
   112  			}
   113  			t := pass.TypesInfo.Types[expr].Type.Underlying()
   114  			tSign, argOk := t.(*types.Signature)
   115  			// Argument should be a function
   116  			if !argOk {
   117  				pass.ReportRangef(expr, "argument to Fuzz must be a function")
   118  				return false
   119  			}
   120  			// ff Argument function should not return
   121  			if tSign.Results().Len() != 0 {
   122  				pass.ReportRangef(expr, "fuzz target must not return any value")
   123  			}
   124  			// ff Argument function should have 1 or more argument
   125  			if tSign.Params().Len() == 0 {
   126  				pass.ReportRangef(expr, "fuzz target must have 1 or more argument")
   127  				return false
   128  			}
   129  			ok := validateFuzzArgs(pass, tSign.Params(), expr)
   130  			if ok && params == nil {
   131  				params = tSign.Params()
   132  			}
   133  			// Inspect the function that was passed as an argument to make sure that
   134  			// there are no calls to *F methods, except for Name and Failed.
   135  			ast.Inspect(expr, func(n ast.Node) bool {
   136  				if call, ok := n.(*ast.CallExpr); ok {
   137  					if !isFuzzTargetDot(pass, call, "") {
   138  						return true
   139  					}
   140  					if !isFuzzTargetDot(pass, call, "Name") && !isFuzzTargetDot(pass, call, "Failed") {
   141  						pass.ReportRangef(call, "fuzz target must not call any *F methods")
   142  					}
   143  				}
   144  				return true
   145  			})
   146  			// We do not need to look at any calls to f.Fuzz inside of a Fuzz call,
   147  			// since they are not allowed.
   148  			return false
   149  		}
   150  		return true
   151  	})
   152  	return params
   153  }
   154  
   155  // checkAddCalls checks that the arguments of f.Add calls have the same number and type of arguments as
   156  // the signature of the function passed to (*testing.F).Fuzz
   157  func checkAddCalls(pass *analysis.Pass, fn *ast.FuncDecl, params *types.Tuple) {
   158  	ast.Inspect(fn, func(n ast.Node) bool {
   159  		call, ok := n.(*ast.CallExpr)
   160  		if ok {
   161  			if !isFuzzTargetDotAdd(pass, call) {
   162  				return true
   163  			}
   164  
   165  			// The first argument to function passed to (*testing.F).Fuzz is (*testing.T).
   166  			if len(call.Args) != params.Len()-1 {
   167  				pass.ReportRangef(call, "wrong number of values in call to (*testing.F).Add: %d, fuzz target expects %d", len(call.Args), params.Len()-1)
   168  				return true
   169  			}
   170  			var mismatched []int
   171  			for i, expr := range call.Args {
   172  				if pass.TypesInfo.Types[expr].Type == nil {
   173  					return true
   174  				}
   175  				t := pass.TypesInfo.Types[expr].Type
   176  				if !types.Identical(t, params.At(i+1).Type()) {
   177  					mismatched = append(mismatched, i)
   178  				}
   179  			}
   180  			// If just one of the types is mismatched report for that
   181  			// type only. Otherwise report for the whole call to (*testing.F).Add
   182  			if len(mismatched) == 1 {
   183  				i := mismatched[0]
   184  				expr := call.Args[i]
   185  				t := pass.TypesInfo.Types[expr].Type
   186  				pass.ReportRangef(expr, "mismatched type in call to (*testing.F).Add: %v, fuzz target expects %v", t, params.At(i+1).Type())
   187  			} else if len(mismatched) > 1 {
   188  				var gotArgs, wantArgs []types.Type
   189  				for i := 0; i < len(call.Args); i++ {
   190  					gotArgs, wantArgs = append(gotArgs, pass.TypesInfo.Types[call.Args[i]].Type), append(wantArgs, params.At(i+1).Type())
   191  				}
   192  				pass.ReportRangef(call, "mismatched types in call to (*testing.F).Add: %v, fuzz target expects %v", gotArgs, wantArgs)
   193  			}
   194  		}
   195  		return true
   196  	})
   197  }
   198  
   199  // isFuzzTargetDotFuzz reports whether call is (*testing.F).Fuzz().
   200  func isFuzzTargetDotFuzz(pass *analysis.Pass, call *ast.CallExpr) bool {
   201  	return isFuzzTargetDot(pass, call, "Fuzz")
   202  }
   203  
   204  // isFuzzTargetDotAdd reports whether call is (*testing.F).Add().
   205  func isFuzzTargetDotAdd(pass *analysis.Pass, call *ast.CallExpr) bool {
   206  	return isFuzzTargetDot(pass, call, "Add")
   207  }
   208  
   209  // isFuzzTargetDot reports whether call is (*testing.F).<name>().
   210  func isFuzzTargetDot(pass *analysis.Pass, call *ast.CallExpr, name string) bool {
   211  	if selExpr, ok := call.Fun.(*ast.SelectorExpr); ok {
   212  		if !isTestingType(pass.TypesInfo.Types[selExpr.X].Type, "F") {
   213  			return false
   214  		}
   215  		if name == "" || selExpr.Sel.Name == name {
   216  			return true
   217  		}
   218  	}
   219  	return false
   220  }
   221  
   222  // Validate the arguments of fuzz target.
   223  func validateFuzzArgs(pass *analysis.Pass, params *types.Tuple, expr ast.Expr) bool {
   224  	fLit, isFuncLit := expr.(*ast.FuncLit)
   225  	exprRange := expr
   226  	ok := true
   227  	if !isTestingType(params.At(0).Type(), "T") {
   228  		if isFuncLit {
   229  			exprRange = fLit.Type.Params.List[0].Type
   230  		}
   231  		pass.ReportRangef(exprRange, "the first parameter of a fuzz target must be *testing.T")
   232  		ok = false
   233  	}
   234  	for i := 1; i < params.Len(); i++ {
   235  		if !isAcceptedFuzzType(params.At(i).Type()) {
   236  			if isFuncLit {
   237  				curr := 0
   238  				for _, field := range fLit.Type.Params.List {
   239  					curr += len(field.Names)
   240  					if i < curr {
   241  						exprRange = field.Type
   242  						break
   243  					}
   244  				}
   245  			}
   246  			pass.ReportRangef(exprRange, "fuzzing arguments can only have the following types: %s", formatAcceptedFuzzType())
   247  			ok = false
   248  		}
   249  	}
   250  	return ok
   251  }
   252  
   253  func isTestingType(typ types.Type, testingType string) bool {
   254  	// No Unalias here: I doubt "go test" recognizes
   255  	// "type A = *testing.T; func Test(A) {}" as a test.
   256  	ptr, ok := typ.(*types.Pointer)
   257  	if !ok {
   258  		return false
   259  	}
   260  	return analysisutil.IsNamedType(ptr.Elem(), "testing", testingType)
   261  }
   262  
   263  // Validate that fuzz target function's arguments are of accepted types.
   264  func isAcceptedFuzzType(paramType types.Type) bool {
   265  	for _, typ := range acceptedFuzzTypes {
   266  		if types.Identical(typ, paramType) {
   267  			return true
   268  		}
   269  	}
   270  	return false
   271  }
   272  
   273  func formatAcceptedFuzzType() string {
   274  	var acceptedFuzzTypesStrings []string
   275  	for _, typ := range acceptedFuzzTypes {
   276  		acceptedFuzzTypesStrings = append(acceptedFuzzTypesStrings, typ.String())
   277  	}
   278  	acceptedFuzzTypesMsg := strings.Join(acceptedFuzzTypesStrings, ", ")
   279  	return acceptedFuzzTypesMsg
   280  }
   281  
   282  func isExampleSuffix(s string) bool {
   283  	r, size := utf8.DecodeRuneInString(s)
   284  	return size > 0 && unicode.IsLower(r)
   285  }
   286  
   287  func isTestSuffix(name string) bool {
   288  	if len(name) == 0 {
   289  		// "Test" is ok.
   290  		return true
   291  	}
   292  	r, _ := utf8.DecodeRuneInString(name)
   293  	return !unicode.IsLower(r)
   294  }
   295  
   296  func isTestParam(typ ast.Expr, wantType string) bool {
   297  	ptr, ok := typ.(*ast.StarExpr)
   298  	if !ok {
   299  		// Not a pointer.
   300  		return false
   301  	}
   302  	// No easy way of making sure it's a *testing.T or *testing.B:
   303  	// ensure the name of the type matches.
   304  	if name, ok := ptr.X.(*ast.Ident); ok {
   305  		return name.Name == wantType
   306  	}
   307  	if sel, ok := ptr.X.(*ast.SelectorExpr); ok {
   308  		return sel.Sel.Name == wantType
   309  	}
   310  	return false
   311  }
   312  
   313  func lookup(pkg *types.Package, name string) []types.Object {
   314  	if o := pkg.Scope().Lookup(name); o != nil {
   315  		return []types.Object{o}
   316  	}
   317  
   318  	var ret []types.Object
   319  	// Search through the imports to see if any of them define name.
   320  	// It's hard to tell in general which package is being tested, so
   321  	// for the purposes of the analysis, allow the object to appear
   322  	// in any of the imports. This guarantees there are no false positives
   323  	// because the example needs to use the object so it must be defined
   324  	// in the package or one if its imports. On the other hand, false
   325  	// negatives are possible, but should be rare.
   326  	for _, imp := range pkg.Imports() {
   327  		if obj := imp.Scope().Lookup(name); obj != nil {
   328  			ret = append(ret, obj)
   329  		}
   330  	}
   331  	return ret
   332  }
   333  
   334  // This pattern is taken from /go/src/go/doc/example.go
   335  var outputRe = regexp.MustCompile(`(?i)^[[:space:]]*(unordered )?output:`)
   336  
   337  type commentMetadata struct {
   338  	isOutput bool
   339  	pos      token.Pos
   340  }
   341  
   342  func checkExampleOutput(pass *analysis.Pass, fn *ast.FuncDecl, fileComments []*ast.CommentGroup) {
   343  	commentsInExample := []commentMetadata{}
   344  	numOutputs := 0
   345  
   346  	// Find the comment blocks that are in the example. These comments are
   347  	// guaranteed to be in order of appearance.
   348  	for _, cg := range fileComments {
   349  		if cg.Pos() < fn.Pos() {
   350  			continue
   351  		} else if cg.End() > fn.End() {
   352  			break
   353  		}
   354  
   355  		isOutput := outputRe.MatchString(cg.Text())
   356  		if isOutput {
   357  			numOutputs++
   358  		}
   359  
   360  		commentsInExample = append(commentsInExample, commentMetadata{
   361  			isOutput: isOutput,
   362  			pos:      cg.Pos(),
   363  		})
   364  	}
   365  
   366  	// Change message based on whether there are multiple output comment blocks.
   367  	msg := "output comment block must be the last comment block"
   368  	if numOutputs > 1 {
   369  		msg = "there can only be one output comment block per example"
   370  	}
   371  
   372  	for i, cg := range commentsInExample {
   373  		// Check for output comments that are not the last comment in the example.
   374  		isLast := (i == len(commentsInExample)-1)
   375  		if cg.isOutput && !isLast {
   376  			pass.Report(
   377  				analysis.Diagnostic{
   378  					Pos:     cg.pos,
   379  					Message: msg,
   380  				},
   381  			)
   382  		}
   383  	}
   384  }
   385  
   386  func checkExampleName(pass *analysis.Pass, fn *ast.FuncDecl) {
   387  	fnName := fn.Name.Name
   388  	if params := fn.Type.Params; len(params.List) != 0 {
   389  		pass.Reportf(fn.Pos(), "%s should be niladic", fnName)
   390  	}
   391  	if results := fn.Type.Results; results != nil && len(results.List) != 0 {
   392  		pass.Reportf(fn.Pos(), "%s should return nothing", fnName)
   393  	}
   394  	if tparams := fn.Type.TypeParams; tparams != nil && len(tparams.List) > 0 {
   395  		pass.Reportf(fn.Pos(), "%s should not have type params", fnName)
   396  	}
   397  
   398  	if fnName == "Example" {
   399  		// Nothing more to do.
   400  		return
   401  	}
   402  
   403  	var (
   404  		exName = strings.TrimPrefix(fnName, "Example")
   405  		elems  = strings.SplitN(exName, "_", 3)
   406  		ident  = elems[0]
   407  		objs   = lookup(pass.Pkg, ident)
   408  	)
   409  	if ident != "" && len(objs) == 0 {
   410  		// Check ExampleFoo and ExampleBadFoo.
   411  		pass.Reportf(fn.Pos(), "%s refers to unknown identifier: %s", fnName, ident)
   412  		// Abort since obj is absent and no subsequent checks can be performed.
   413  		return
   414  	}
   415  	if len(elems) < 2 {
   416  		// Nothing more to do.
   417  		return
   418  	}
   419  
   420  	if ident == "" {
   421  		// Check Example_suffix and Example_BadSuffix.
   422  		if residual := strings.TrimPrefix(exName, "_"); !isExampleSuffix(residual) {
   423  			pass.Reportf(fn.Pos(), "%s has malformed example suffix: %s", fnName, residual)
   424  		}
   425  		return
   426  	}
   427  
   428  	mmbr := elems[1]
   429  	if !isExampleSuffix(mmbr) {
   430  		// Check ExampleFoo_Method and ExampleFoo_BadMethod.
   431  		found := false
   432  		// Check if Foo.Method exists in this package or its imports.
   433  		for _, obj := range objs {
   434  			if obj, _, _ := types.LookupFieldOrMethod(obj.Type(), true, obj.Pkg(), mmbr); obj != nil {
   435  				found = true
   436  				break
   437  			}
   438  		}
   439  		if !found {
   440  			pass.Reportf(fn.Pos(), "%s refers to unknown field or method: %s.%s", fnName, ident, mmbr)
   441  		}
   442  	}
   443  	if len(elems) == 3 && !isExampleSuffix(elems[2]) {
   444  		// Check ExampleFoo_Method_suffix and ExampleFoo_Method_Badsuffix.
   445  		pass.Reportf(fn.Pos(), "%s has malformed example suffix: %s", fnName, elems[2])
   446  	}
   447  }
   448  
   449  type tokenRange struct {
   450  	p, e token.Pos
   451  }
   452  
   453  func (r tokenRange) Pos() token.Pos {
   454  	return r.p
   455  }
   456  
   457  func (r tokenRange) End() token.Pos {
   458  	return r.e
   459  }
   460  
   461  func checkTest(pass *analysis.Pass, fn *ast.FuncDecl, prefix string) {
   462  	// Want functions with 0 results and 1 parameter.
   463  	if fn.Type.Results != nil && len(fn.Type.Results.List) > 0 ||
   464  		fn.Type.Params == nil ||
   465  		len(fn.Type.Params.List) != 1 ||
   466  		len(fn.Type.Params.List[0].Names) > 1 {
   467  		return
   468  	}
   469  
   470  	// The param must look like a *testing.T or *testing.B.
   471  	if !isTestParam(fn.Type.Params.List[0].Type, prefix[:1]) {
   472  		return
   473  	}
   474  
   475  	if tparams := fn.Type.TypeParams; tparams != nil && len(tparams.List) > 0 {
   476  		// Note: cmd/go/internal/load also errors about TestXXX and BenchmarkXXX functions with type parameters.
   477  		// We have currently decided to also warn before compilation/package loading. This can help users in IDEs.
   478  		at := tokenRange{tparams.Opening, tparams.Closing}
   479  		pass.ReportRangef(at, "%s has type parameters: it will not be run by go test as a %sXXX function", fn.Name.Name, prefix)
   480  	}
   481  
   482  	if !isTestSuffix(fn.Name.Name[len(prefix):]) {
   483  		pass.ReportRangef(fn.Name, "%s has malformed name: first letter after '%s' must not be lowercase", fn.Name.Name, prefix)
   484  	}
   485  }
   486
View as plain text