Source file src/cmd/go/internal/modindex/read.go

     1  // Copyright 2022 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package modindex
     6  
     7  import (
     8  	"bytes"
     9  	"encoding/binary"
    10  	"errors"
    11  	"fmt"
    12  	"go/build"
    13  	"go/build/constraint"
    14  	"go/token"
    15  	"internal/godebug"
    16  	"internal/goroot"
    17  	"path"
    18  	"path/filepath"
    19  	"runtime"
    20  	"runtime/debug"
    21  	"sort"
    22  	"strings"
    23  	"sync"
    24  	"time"
    25  	"unsafe"
    26  
    27  	"cmd/go/internal/base"
    28  	"cmd/go/internal/cache"
    29  	"cmd/go/internal/cfg"
    30  	"cmd/go/internal/fsys"
    31  	"cmd/go/internal/imports"
    32  	"cmd/go/internal/str"
    33  	"cmd/internal/par"
    34  )
    35  
    36  // enabled is used to flag off the behavior of the module index on tip, for debugging.
    37  var enabled = godebug.New("#goindex").Value() != "0"
    38  
    39  // Module represents and encoded module index file. It is used to
    40  // do the equivalent of build.Import of packages in the module and answer other
    41  // questions based on the index file's data.
    42  type Module struct {
    43  	modroot string
    44  	d       *decoder
    45  	n       int // number of packages
    46  }
    47  
    48  // moduleHash returns an ActionID corresponding to the state of the module
    49  // located at filesystem path modroot.
    50  func moduleHash(modroot string, ismodcache bool) (cache.ActionID, error) {
    51  	// We expect modules stored within the module cache to be checksummed and
    52  	// immutable, and we expect released modules within GOROOT to change only
    53  	// infrequently (when the Go version changes).
    54  	if !ismodcache {
    55  		// The contents of this module may change over time. We don't want to pay
    56  		// the cost to detect changes and re-index whenever they occur, so just
    57  		// don't index it at all.
    58  		//
    59  		// Note that this is true even for modules in GOROOT/src: non-release builds
    60  		// of the Go toolchain may have arbitrary development changes on top of the
    61  		// commit reported by runtime.Version, or could be completely artificial due
    62  		// to lacking a `git` binary (like "devel gomote.XXXXX", as synthesized by
    63  		// "gomote push" as of 2022-06-15). (Release builds shouldn't have
    64  		// modifications, but we don't want to use a behavior for releases that we
    65  		// haven't tested during development.)
    66  		return cache.ActionID{}, ErrNotIndexed
    67  	}
    68  
    69  	h := cache.NewHash("moduleIndex")
    70  	// TODO(bcmills): Since modules in the index are checksummed, we could
    71  	// probably improve the cache hit rate by keying off of the module
    72  	// path@version (perhaps including the checksum?) instead of the module root
    73  	// directory.
    74  	fmt.Fprintf(h, "module index %s %s %v\n", runtime.Version(), indexVersion, modroot)
    75  	return h.Sum(), nil
    76  }
    77  
    78  const modTimeCutoff = 2 * time.Second
    79  
    80  // dirHash returns an ActionID corresponding to the state of the package
    81  // located at filesystem path pkgdir.
    82  func dirHash(modroot, pkgdir string) (cache.ActionID, error) {
    83  	h := cache.NewHash("moduleIndex")
    84  	fmt.Fprintf(h, "modroot %s\n", modroot)
    85  	fmt.Fprintf(h, "package %s %s %v\n", runtime.Version(), indexVersion, pkgdir)
    86  	dirs, err := fsys.ReadDir(pkgdir)
    87  	if err != nil {
    88  		// pkgdir might not be a directory. give up on hashing.
    89  		return cache.ActionID{}, ErrNotIndexed
    90  	}
    91  	cutoff := time.Now().Add(-modTimeCutoff)
    92  	for _, d := range dirs {
    93  		if d.IsDir() {
    94  			continue
    95  		}
    96  
    97  		if !d.Type().IsRegular() {
    98  			return cache.ActionID{}, ErrNotIndexed
    99  		}
   100  		// To avoid problems for very recent files where a new
   101  		// write might not change the mtime due to file system
   102  		// mtime precision, reject caching if a file was read that
   103  		// is less than modTimeCutoff old.
   104  		//
   105  		// This is the same strategy used for hashing test inputs.
   106  		// See hashOpen in cmd/go/internal/test/test.go for the
   107  		// corresponding code.
   108  		info, err := d.Info()
   109  		if err != nil {
   110  			return cache.ActionID{}, ErrNotIndexed
   111  		}
   112  		if info.ModTime().After(cutoff) {
   113  			return cache.ActionID{}, ErrNotIndexed
   114  		}
   115  
   116  		fmt.Fprintf(h, "file %v %v %v\n", info.Name(), info.ModTime(), info.Size())
   117  	}
   118  	return h.Sum(), nil
   119  }
   120  
   121  var ErrNotIndexed = errors.New("not in module index")
   122  
   123  var (
   124  	errDisabled           = fmt.Errorf("%w: module indexing disabled", ErrNotIndexed)
   125  	errNotFromModuleCache = fmt.Errorf("%w: not from module cache", ErrNotIndexed)
   126  	errFIPS140            = fmt.Errorf("%w: fips140 snapshots not indexed", ErrNotIndexed)
   127  )
   128  
   129  // GetPackage returns the IndexPackage for the directory at the given path.
   130  // It will return ErrNotIndexed if the directory should be read without
   131  // using the index, for instance because the index is disabled, or the package
   132  // is not in a module.
   133  func GetPackage(modroot, pkgdir string) (*IndexPackage, error) {
   134  	mi, err := GetModule(modroot)
   135  	if err == nil {
   136  		return mi.Package(relPath(pkgdir, modroot)), nil
   137  	}
   138  	if !errors.Is(err, errNotFromModuleCache) {
   139  		return nil, err
   140  	}
   141  	if cfg.BuildContext.Compiler == "gccgo" && str.HasPathPrefix(modroot, cfg.GOROOTsrc) {
   142  		return nil, err // gccgo has no sources for GOROOT packages.
   143  	}
   144  	// The pkgdir for fips140 has been replaced in the fsys overlay,
   145  	// but the module index does not see that. Do not try to use the module index.
   146  	if strings.Contains(filepath.ToSlash(pkgdir), "internal/fips140/v") {
   147  		return nil, errFIPS140
   148  	}
   149  	modroot = filepath.Clean(modroot)
   150  	pkgdir = filepath.Clean(pkgdir)
   151  	return openIndexPackage(modroot, pkgdir)
   152  }
   153  
   154  // GetModule returns the Module for the given modroot.
   155  // It will return ErrNotIndexed if the directory should be read without
   156  // using the index, for instance because the index is disabled, or the package
   157  // is not in a module.
   158  func GetModule(modroot string) (*Module, error) {
   159  	dir, _, _ := cache.DefaultDir()
   160  	if !enabled || dir == "off" {
   161  		return nil, errDisabled
   162  	}
   163  	if modroot == "" {
   164  		panic("modindex.GetPackage called with empty modroot")
   165  	}
   166  	if cfg.BuildMod == "vendor" {
   167  		// Even if the main module is in the module cache,
   168  		// its vendored dependencies are not loaded from their
   169  		// usual cached locations.
   170  		return nil, errNotFromModuleCache
   171  	}
   172  	modroot = filepath.Clean(modroot)
   173  	if str.HasFilePathPrefix(modroot, cfg.GOROOTsrc) || !str.HasFilePathPrefix(modroot, cfg.GOMODCACHE) {
   174  		return nil, errNotFromModuleCache
   175  	}
   176  	return openIndexModule(modroot, true)
   177  }
   178  
   179  var mcache par.ErrCache[string, *Module]
   180  
   181  // openIndexModule returns the module index for modPath.
   182  // It will return ErrNotIndexed if the module can not be read
   183  // using the index because it contains symlinks.
   184  func openIndexModule(modroot string, ismodcache bool) (*Module, error) {
   185  	return mcache.Do(modroot, func() (*Module, error) {
   186  		fsys.Trace("openIndexModule", modroot)
   187  		id, err := moduleHash(modroot, ismodcache)
   188  		if err != nil {
   189  			return nil, err
   190  		}
   191  		data, _, opened, err := cache.GetMmap(cache.Default(), id)
   192  		if err != nil {
   193  			// Couldn't read from modindex. Assume we couldn't read from
   194  			// the index because the module hasn't been indexed yet.
   195  			// But double check on Windows that we haven't opened the file yet,
   196  			// because once mmap opens the file, we can't close it, and
   197  			// Windows won't let us open an already opened file.
   198  			data, err = indexModule(modroot)
   199  			if err != nil {
   200  				return nil, err
   201  			}
   202  			if runtime.GOOS != "windows" || !opened {
   203  				if err = cache.PutBytes(cache.Default(), id, data); err != nil {
   204  					return nil, err
   205  				}
   206  			}
   207  		}
   208  		mi, err := fromBytes(modroot, data)
   209  		if err != nil {
   210  			return nil, err
   211  		}
   212  		return mi, nil
   213  	})
   214  }
   215  
   216  var pcache par.ErrCache[[2]string, *IndexPackage]
   217  
   218  func openIndexPackage(modroot, pkgdir string) (*IndexPackage, error) {
   219  	return pcache.Do([2]string{modroot, pkgdir}, func() (*IndexPackage, error) {
   220  		fsys.Trace("openIndexPackage", pkgdir)
   221  		id, err := dirHash(modroot, pkgdir)
   222  		if err != nil {
   223  			return nil, err
   224  		}
   225  		data, _, opened, err := cache.GetMmap(cache.Default(), id)
   226  		if err != nil {
   227  			// Couldn't read from index. Assume we couldn't read from
   228  			// the index because the package hasn't been indexed yet.
   229  			// But double check on Windows that we haven't opened the file yet,
   230  			// because once mmap opens the file, we can't close it, and
   231  			// Windows won't let us open an already opened file.
   232  			data = indexPackage(modroot, pkgdir)
   233  			if runtime.GOOS != "windows" || !opened {
   234  				if err = cache.PutBytes(cache.Default(), id, data); err != nil {
   235  					return nil, err
   236  				}
   237  			}
   238  		}
   239  		pkg, err := packageFromBytes(modroot, data)
   240  		if err != nil {
   241  			return nil, err
   242  		}
   243  		return pkg, nil
   244  	})
   245  }
   246  
   247  var errCorrupt = errors.New("corrupt index")
   248  
   249  // protect marks the start of a large section of code that accesses the index.
   250  // It should be used as:
   251  //
   252  //	defer unprotect(protect, &err)
   253  //
   254  // It should not be used for trivial accesses which would be
   255  // dwarfed by the overhead of the defer.
   256  func protect() bool {
   257  	return debug.SetPanicOnFault(true)
   258  }
   259  
   260  var isTest = false
   261  
   262  // unprotect marks the end of a large section of code that accesses the index.
   263  // It should be used as:
   264  //
   265  //	defer unprotect(protect, &err)
   266  //
   267  // end looks for panics due to errCorrupt or bad mmap accesses.
   268  // When it finds them, it adds explanatory text, consumes the panic, and sets *errp instead.
   269  // If errp is nil, end adds the explanatory text but then calls base.Fatalf.
   270  func unprotect(old bool, errp *error) {
   271  	// SetPanicOnFault's errors _may_ satisfy this interface. Even though it's not guaranteed
   272  	// that all its errors satisfy this interface, we'll only check for these errors so that
   273  	// we don't suppress panics that could have been produced from other sources.
   274  	type addrer interface {
   275  		Addr() uintptr
   276  	}
   277  
   278  	debug.SetPanicOnFault(old)
   279  
   280  	if e := recover(); e != nil {
   281  		if _, ok := e.(addrer); ok || e == errCorrupt {
   282  			// This panic was almost certainly caused by SetPanicOnFault or our panic(errCorrupt).
   283  			err := fmt.Errorf("error reading module index: %v", e)
   284  			if errp != nil {
   285  				*errp = err
   286  				return
   287  			}
   288  			if isTest {
   289  				panic(err)
   290  			}
   291  			base.Fatalf("%v", err)
   292  		}
   293  		// The panic was likely not caused by SetPanicOnFault.
   294  		panic(e)
   295  	}
   296  }
   297  
   298  // fromBytes returns a *Module given the encoded representation.
   299  func fromBytes(moddir string, data []byte) (m *Module, err error) {
   300  	if !enabled {
   301  		panic("use of index")
   302  	}
   303  
   304  	defer unprotect(protect(), &err)
   305  
   306  	if !bytes.HasPrefix(data, []byte(indexVersion+"\n")) {
   307  		return nil, errCorrupt
   308  	}
   309  
   310  	const hdr = len(indexVersion + "\n")
   311  	d := &decoder{data: data}
   312  	str := d.intAt(hdr)
   313  	if str < hdr+8 || len(d.data) < str {
   314  		return nil, errCorrupt
   315  	}
   316  	d.data, d.str = data[:str], d.data[str:]
   317  	// Check that string table looks valid.
   318  	// First string is empty string (length 0),
   319  	// and we leave a marker byte 0xFF at the end
   320  	// just to make sure that the file is not truncated.
   321  	if len(d.str) == 0 || d.str[0] != 0 || d.str[len(d.str)-1] != 0xFF {
   322  		return nil, errCorrupt
   323  	}
   324  
   325  	n := d.intAt(hdr + 4)
   326  	if n < 0 || n > (len(d.data)-8)/8 {
   327  		return nil, errCorrupt
   328  	}
   329  
   330  	m = &Module{
   331  		moddir,
   332  		d,
   333  		n,
   334  	}
   335  	return m, nil
   336  }
   337  
   338  // packageFromBytes returns a *IndexPackage given the encoded representation.
   339  func packageFromBytes(modroot string, data []byte) (p *IndexPackage, err error) {
   340  	m, err := fromBytes(modroot, data)
   341  	if err != nil {
   342  		return nil, err
   343  	}
   344  	if m.n != 1 {
   345  		return nil, fmt.Errorf("corrupt single-package index")
   346  	}
   347  	return m.pkg(0), nil
   348  }
   349  
   350  // pkgDir returns the dir string of the i'th package in the index.
   351  func (m *Module) pkgDir(i int) string {
   352  	if i < 0 || i >= m.n {
   353  		panic(errCorrupt)
   354  	}
   355  	return m.d.stringAt(12 + 8 + 8*i)
   356  }
   357  
   358  // pkgOff returns the offset of the data for the i'th package in the index.
   359  func (m *Module) pkgOff(i int) int {
   360  	if i < 0 || i >= m.n {
   361  		panic(errCorrupt)
   362  	}
   363  	return m.d.intAt(12 + 8 + 8*i + 4)
   364  }
   365  
   366  // Walk calls f for each package in the index, passing the path to that package relative to the module root.
   367  func (m *Module) Walk(f func(path string)) {
   368  	defer unprotect(protect(), nil)
   369  	for i := 0; i < m.n; i++ {
   370  		f(m.pkgDir(i))
   371  	}
   372  }
   373  
   374  // relPath returns the path relative to the module's root.
   375  func relPath(path, modroot string) string {
   376  	return str.TrimFilePathPrefix(filepath.Clean(path), filepath.Clean(modroot))
   377  }
   378  
   379  var installgorootAll = godebug.New("installgoroot").Value() == "all"
   380  
   381  // Import is the equivalent of build.Import given the information in Module.
   382  func (rp *IndexPackage) Import(bctxt build.Context, mode build.ImportMode) (p *build.Package, err error) {
   383  	defer unprotect(protect(), &err)
   384  
   385  	ctxt := (*Context)(&bctxt)
   386  
   387  	p = &build.Package{}
   388  
   389  	p.ImportPath = "."
   390  	p.Dir = filepath.Join(rp.modroot, rp.dir)
   391  
   392  	var pkgerr error
   393  	switch ctxt.Compiler {
   394  	case "gccgo", "gc":
   395  	default:
   396  		// Save error for end of function.
   397  		pkgerr = fmt.Errorf("import %q: unknown compiler %q", p.Dir, ctxt.Compiler)
   398  	}
   399  
   400  	if p.Dir == "" {
   401  		return p, fmt.Errorf("import %q: import of unknown directory", p.Dir)
   402  	}
   403  
   404  	// goroot and gopath
   405  	inTestdata := func(sub string) bool {
   406  		return strings.Contains(sub, "/testdata/") || strings.HasSuffix(sub, "/testdata") || str.HasPathPrefix(sub, "testdata")
   407  	}
   408  	var pkga string
   409  	if !inTestdata(rp.dir) {
   410  		// In build.go, p.Root should only be set in the non-local-import case, or in
   411  		// GOROOT or GOPATH. Since module mode only calls Import with path set to "."
   412  		// and the module index doesn't apply outside modules, the GOROOT case is
   413  		// the only case where p.Root needs to be set.
   414  		if ctxt.GOROOT != "" && str.HasFilePathPrefix(p.Dir, cfg.GOROOTsrc) && p.Dir != cfg.GOROOTsrc {
   415  			p.Root = ctxt.GOROOT
   416  			p.Goroot = true
   417  			modprefix := str.TrimFilePathPrefix(rp.modroot, cfg.GOROOTsrc)
   418  			p.ImportPath = rp.dir
   419  			if modprefix != "" {
   420  				p.ImportPath = filepath.Join(modprefix, p.ImportPath)
   421  			}
   422  
   423  			// Set GOROOT-specific fields (sometimes for modules in a GOPATH directory).
   424  			// The fields set below (SrcRoot, PkgRoot, BinDir, PkgTargetRoot, and PkgObj)
   425  			// are only set in build.Import if p.Root != "".
   426  			var pkgtargetroot string
   427  			suffix := ""
   428  			if ctxt.InstallSuffix != "" {
   429  				suffix = "_" + ctxt.InstallSuffix
   430  			}
   431  			switch ctxt.Compiler {
   432  			case "gccgo":
   433  				pkgtargetroot = "pkg/gccgo_" + ctxt.GOOS + "_" + ctxt.GOARCH + suffix
   434  				dir, elem := path.Split(p.ImportPath)
   435  				pkga = pkgtargetroot + "/" + dir + "lib" + elem + ".a"
   436  			case "gc":
   437  				pkgtargetroot = "pkg/" + ctxt.GOOS + "_" + ctxt.GOARCH + suffix
   438  				pkga = pkgtargetroot + "/" + p.ImportPath + ".a"
   439  			}
   440  			p.SrcRoot = ctxt.joinPath(p.Root, "src")
   441  			p.PkgRoot = ctxt.joinPath(p.Root, "pkg")
   442  			p.BinDir = ctxt.joinPath(p.Root, "bin")
   443  			if pkga != "" {
   444  				// Always set PkgTargetRoot. It might be used when building in shared
   445  				// mode.
   446  				p.PkgTargetRoot = ctxt.joinPath(p.Root, pkgtargetroot)
   447  
   448  				// Set the install target if applicable.
   449  				if !p.Goroot || (installgorootAll && p.ImportPath != "unsafe" && p.ImportPath != "builtin") {
   450  					p.PkgObj = ctxt.joinPath(p.Root, pkga)
   451  				}
   452  			}
   453  		}
   454  	}
   455  
   456  	if rp.error != nil {
   457  		if errors.Is(rp.error, errCannotFindPackage) && ctxt.Compiler == "gccgo" && p.Goroot {
   458  			return p, nil
   459  		}
   460  		return p, rp.error
   461  	}
   462  
   463  	if mode&build.FindOnly != 0 {
   464  		return p, pkgerr
   465  	}
   466  
   467  	// We need to do a second round of bad file processing.
   468  	var badGoError error
   469  	badGoFiles := make(map[string]bool)
   470  	badGoFile := func(name string, err error) {
   471  		if badGoError == nil {
   472  			badGoError = err
   473  		}
   474  		if !badGoFiles[name] {
   475  			p.InvalidGoFiles = append(p.InvalidGoFiles, name)
   476  			badGoFiles[name] = true
   477  		}
   478  	}
   479  
   480  	var Sfiles []string // files with ".S"(capital S)/.sx(capital s equivalent for case insensitive filesystems)
   481  	var firstFile string
   482  	embedPos := make(map[string][]token.Position)
   483  	testEmbedPos := make(map[string][]token.Position)
   484  	xTestEmbedPos := make(map[string][]token.Position)
   485  	importPos := make(map[string][]token.Position)
   486  	testImportPos := make(map[string][]token.Position)
   487  	xTestImportPos := make(map[string][]token.Position)
   488  	allTags := make(map[string]bool)
   489  	for _, tf := range rp.sourceFiles {
   490  		name := tf.name()
   491  		// Check errors for go files and call badGoFiles to put them in
   492  		// InvalidGoFiles if they do have an error.
   493  		if strings.HasSuffix(name, ".go") {
   494  			if error := tf.error(); error != "" {
   495  				badGoFile(name, errors.New(tf.error()))
   496  				continue
   497  			} else if parseError := tf.parseError(); parseError != "" {
   498  				badGoFile(name, parseErrorFromString(tf.parseError()))
   499  				// Fall through: we still want to list files with parse errors.
   500  			}
   501  		}
   502  
   503  		var shouldBuild = true
   504  		if !ctxt.goodOSArchFile(name, allTags) && !ctxt.UseAllFiles {
   505  			shouldBuild = false
   506  		} else if goBuildConstraint := tf.goBuildConstraint(); goBuildConstraint != "" {
   507  			x, err := constraint.Parse(goBuildConstraint)
   508  			if err != nil {
   509  				return p, fmt.Errorf("%s: parsing //go:build line: %v", name, err)
   510  			}
   511  			shouldBuild = ctxt.eval(x, allTags)
   512  		} else if plusBuildConstraints := tf.plusBuildConstraints(); len(plusBuildConstraints) > 0 {
   513  			for _, text := range plusBuildConstraints {
   514  				if x, err := constraint.Parse(text); err == nil {
   515  					if !ctxt.eval(x, allTags) {
   516  						shouldBuild = false
   517  					}
   518  				}
   519  			}
   520  		}
   521  
   522  		ext := nameExt(name)
   523  		if !shouldBuild || tf.ignoreFile() {
   524  			if ext == ".go" {
   525  				p.IgnoredGoFiles = append(p.IgnoredGoFiles, name)
   526  			} else if fileListForExt(p, ext) != nil {
   527  				p.IgnoredOtherFiles = append(p.IgnoredOtherFiles, name)
   528  			}
   529  			continue
   530  		}
   531  
   532  		// Going to save the file. For non-Go files, can stop here.
   533  		switch ext {
   534  		case ".go":
   535  			// keep going
   536  		case ".S", ".sx":
   537  			// special case for cgo, handled at end
   538  			Sfiles = append(Sfiles, name)
   539  			continue
   540  		default:
   541  			if list := fileListForExt(p, ext); list != nil {
   542  				*list = append(*list, name)
   543  			}
   544  			continue
   545  		}
   546  
   547  		pkg := tf.pkgName()
   548  		if pkg == "documentation" {
   549  			p.IgnoredGoFiles = append(p.IgnoredGoFiles, name)
   550  			continue
   551  		}
   552  		isTest := strings.HasSuffix(name, "_test.go")
   553  		isXTest := false
   554  		if isTest && strings.HasSuffix(tf.pkgName(), "_test") && p.Name != tf.pkgName() {
   555  			isXTest = true
   556  			pkg = pkg[:len(pkg)-len("_test")]
   557  		}
   558  
   559  		if !isTest && tf.binaryOnly() {
   560  			p.BinaryOnly = true
   561  		}
   562  
   563  		if p.Name == "" {
   564  			p.Name = pkg
   565  			firstFile = name
   566  		} else if pkg != p.Name {
   567  			// TODO(#45999): The choice of p.Name is arbitrary based on file iteration
   568  			// order. Instead of resolving p.Name arbitrarily, we should clear out the
   569  			// existing Name and mark the existing files as also invalid.
   570  			badGoFile(name, &MultiplePackageError{
   571  				Dir:      p.Dir,
   572  				Packages: []string{p.Name, pkg},
   573  				Files:    []string{firstFile, name},
   574  			})
   575  		}
   576  		// Grab the first package comment as docs, provided it is not from a test file.
   577  		if p.Doc == "" && !isTest && !isXTest {
   578  			if synopsis := tf.synopsis(); synopsis != "" {
   579  				p.Doc = synopsis
   580  			}
   581  		}
   582  
   583  		// Record Imports and information about cgo.
   584  		isCgo := false
   585  		imports := tf.imports()
   586  		for _, imp := range imports {
   587  			if imp.path == "C" {
   588  				if isTest {
   589  					badGoFile(name, fmt.Errorf("use of cgo in test %s not supported", name))
   590  					continue
   591  				}
   592  				isCgo = true
   593  			}
   594  		}
   595  		if directives := tf.cgoDirectives(); directives != "" {
   596  			if err := ctxt.saveCgo(name, p, directives); err != nil {
   597  				badGoFile(name, err)
   598  			}
   599  		}
   600  
   601  		var fileList *[]string
   602  		var importMap, embedMap map[string][]token.Position
   603  		var directives *[]build.Directive
   604  		switch {
   605  		case isCgo:
   606  			allTags["cgo"] = true
   607  			if ctxt.CgoEnabled {
   608  				fileList = &p.CgoFiles
   609  				importMap = importPos
   610  				embedMap = embedPos
   611  				directives = &p.Directives
   612  			} else {
   613  				// Ignore Imports and Embeds from cgo files if cgo is disabled.
   614  				fileList = &p.IgnoredGoFiles
   615  			}
   616  		case isXTest:
   617  			fileList = &p.XTestGoFiles
   618  			importMap = xTestImportPos
   619  			embedMap = xTestEmbedPos
   620  			directives = &p.XTestDirectives
   621  		case isTest:
   622  			fileList = &p.TestGoFiles
   623  			importMap = testImportPos
   624  			embedMap = testEmbedPos
   625  			directives = &p.TestDirectives
   626  		default:
   627  			fileList = &p.GoFiles
   628  			importMap = importPos
   629  			embedMap = embedPos
   630  			directives = &p.Directives
   631  		}
   632  		*fileList = append(*fileList, name)
   633  		if importMap != nil {
   634  			for _, imp := range imports {
   635  				importMap[imp.path] = append(importMap[imp.path], imp.position)
   636  			}
   637  		}
   638  		if embedMap != nil {
   639  			for _, e := range tf.embeds() {
   640  				embedMap[e.pattern] = append(embedMap[e.pattern], e.position)
   641  			}
   642  		}
   643  		if directives != nil {
   644  			*directives = append(*directives, tf.directives()...)
   645  		}
   646  	}
   647  
   648  	p.EmbedPatterns, p.EmbedPatternPos = cleanDecls(embedPos)
   649  	p.TestEmbedPatterns, p.TestEmbedPatternPos = cleanDecls(testEmbedPos)
   650  	p.XTestEmbedPatterns, p.XTestEmbedPatternPos = cleanDecls(xTestEmbedPos)
   651  
   652  	p.Imports, p.ImportPos = cleanDecls(importPos)
   653  	p.TestImports, p.TestImportPos = cleanDecls(testImportPos)
   654  	p.XTestImports, p.XTestImportPos = cleanDecls(xTestImportPos)
   655  
   656  	for tag := range allTags {
   657  		p.AllTags = append(p.AllTags, tag)
   658  	}
   659  	sort.Strings(p.AllTags)
   660  
   661  	if len(p.CgoFiles) > 0 {
   662  		p.SFiles = append(p.SFiles, Sfiles...)
   663  		sort.Strings(p.SFiles)
   664  	} else {
   665  		p.IgnoredOtherFiles = append(p.IgnoredOtherFiles, Sfiles...)
   666  		sort.Strings(p.IgnoredOtherFiles)
   667  	}
   668  
   669  	if badGoError != nil {
   670  		return p, badGoError
   671  	}
   672  	if len(p.GoFiles)+len(p.CgoFiles)+len(p.TestGoFiles)+len(p.XTestGoFiles) == 0 {
   673  		return p, &build.NoGoError{Dir: p.Dir}
   674  	}
   675  	return p, pkgerr
   676  }
   677  
   678  // IsStandardPackage reports whether path is a standard package
   679  // for the goroot and compiler using the module index if possible,
   680  // and otherwise falling back to internal/goroot.IsStandardPackage
   681  func IsStandardPackage(goroot_, compiler, path string) bool {
   682  	if !enabled || compiler != "gc" {
   683  		return goroot.IsStandardPackage(goroot_, compiler, path)
   684  	}
   685  
   686  	reldir := filepath.FromSlash(path) // relative dir path in module index for package
   687  	modroot := filepath.Join(goroot_, "src")
   688  	if str.HasFilePathPrefix(reldir, "cmd") {
   689  		reldir = str.TrimFilePathPrefix(reldir, "cmd")
   690  		modroot = filepath.Join(modroot, "cmd")
   691  	}
   692  	if pkg, err := GetPackage(modroot, filepath.Join(modroot, reldir)); err == nil {
   693  		hasGo, err := pkg.IsGoDir()
   694  		return err == nil && hasGo
   695  	} else if errors.Is(err, ErrNotIndexed) {
   696  		// Fall back because package isn't indexable. (Probably because
   697  		// a file was modified recently)
   698  		return goroot.IsStandardPackage(goroot_, compiler, path)
   699  	}
   700  	return false
   701  }
   702  
   703  // IsGoDir is the equivalent of fsys.IsGoDir using the information in the index.
   704  func (rp *IndexPackage) IsGoDir() (_ bool, err error) {
   705  	defer func() {
   706  		if e := recover(); e != nil {
   707  			err = fmt.Errorf("error reading module index: %v", e)
   708  		}
   709  	}()
   710  	for _, sf := range rp.sourceFiles {
   711  		if strings.HasSuffix(sf.name(), ".go") {
   712  			return true, nil
   713  		}
   714  	}
   715  	return false, nil
   716  }
   717  
   718  // ScanDir implements imports.ScanDir using the information in the index.
   719  func (rp *IndexPackage) ScanDir(tags map[string]bool) (sortedImports []string, sortedTestImports []string, err error) {
   720  	// TODO(matloob) dir should eventually be relative to indexed directory
   721  	// TODO(matloob): skip reading raw package and jump straight to data we need?
   722  
   723  	defer func() {
   724  		if e := recover(); e != nil {
   725  			err = fmt.Errorf("error reading module index: %v", e)
   726  		}
   727  	}()
   728  
   729  	imports_ := make(map[string]bool)
   730  	testImports := make(map[string]bool)
   731  	numFiles := 0
   732  
   733  Files:
   734  	for _, sf := range rp.sourceFiles {
   735  		name := sf.name()
   736  		if strings.HasPrefix(name, "_") || strings.HasPrefix(name, ".") || !strings.HasSuffix(name, ".go") || !imports.MatchFile(name, tags) {
   737  			continue
   738  		}
   739  
   740  		// The following section exists for backwards compatibility reasons:
   741  		// scanDir ignores files with import "C" when collecting the list
   742  		// of imports unless the "cgo" tag is provided. The following comment
   743  		// is copied from the original.
   744  		//
   745  		// import "C" is implicit requirement of cgo tag.
   746  		// When listing files on the command line (explicitFiles=true)
   747  		// we do not apply build tag filtering but we still do apply
   748  		// cgo filtering, so no explicitFiles check here.
   749  		// Why? Because we always have, and it's not worth breaking
   750  		// that behavior now.
   751  		imps := sf.imports() // TODO(matloob): directly read import paths to avoid the extra strings?
   752  		for _, imp := range imps {
   753  			if imp.path == "C" && !tags["cgo"] && !tags["*"] {
   754  				continue Files
   755  			}
   756  		}
   757  
   758  		if !shouldBuild(sf, tags) {
   759  			continue
   760  		}
   761  		numFiles++
   762  		m := imports_
   763  		if strings.HasSuffix(name, "_test.go") {
   764  			m = testImports
   765  		}
   766  		for _, p := range imps {
   767  			m[p.path] = true
   768  		}
   769  	}
   770  	if numFiles == 0 {
   771  		return nil, nil, imports.ErrNoGo
   772  	}
   773  	return keys(imports_), keys(testImports), nil
   774  }
   775  
   776  func keys(m map[string]bool) []string {
   777  	list := make([]string, 0, len(m))
   778  	for k := range m {
   779  		list = append(list, k)
   780  	}
   781  	sort.Strings(list)
   782  	return list
   783  }
   784  
   785  // implements imports.ShouldBuild in terms of an index sourcefile.
   786  func shouldBuild(sf *sourceFile, tags map[string]bool) bool {
   787  	if goBuildConstraint := sf.goBuildConstraint(); goBuildConstraint != "" {
   788  		x, err := constraint.Parse(goBuildConstraint)
   789  		if err != nil {
   790  			return false
   791  		}
   792  		return imports.Eval(x, tags, true)
   793  	}
   794  
   795  	plusBuildConstraints := sf.plusBuildConstraints()
   796  	for _, text := range plusBuildConstraints {
   797  		if x, err := constraint.Parse(text); err == nil {
   798  			if !imports.Eval(x, tags, true) {
   799  				return false
   800  			}
   801  		}
   802  	}
   803  
   804  	return true
   805  }
   806  
   807  // IndexPackage holds the information in the index
   808  // needed to load a package in a specific directory.
   809  type IndexPackage struct {
   810  	error error
   811  	dir   string // directory of the package relative to the modroot
   812  
   813  	modroot string
   814  
   815  	// Source files
   816  	sourceFiles []*sourceFile
   817  }
   818  
   819  var errCannotFindPackage = errors.New("cannot find package")
   820  
   821  // Package and returns finds the package with the given path (relative to the module root).
   822  // If the package does not exist, Package returns an IndexPackage that will return an
   823  // appropriate error from its methods.
   824  func (m *Module) Package(path string) *IndexPackage {
   825  	defer unprotect(protect(), nil)
   826  
   827  	i, ok := sort.Find(m.n, func(i int) int {
   828  		return strings.Compare(path, m.pkgDir(i))
   829  	})
   830  	if !ok {
   831  		return &IndexPackage{error: fmt.Errorf("%w %q in:\n\t%s", errCannotFindPackage, path, filepath.Join(m.modroot, path))}
   832  	}
   833  	return m.pkg(i)
   834  }
   835  
   836  // pkg returns the i'th IndexPackage in m.
   837  func (m *Module) pkg(i int) *IndexPackage {
   838  	r := m.d.readAt(m.pkgOff(i))
   839  	p := new(IndexPackage)
   840  	if errstr := r.string(); errstr != "" {
   841  		p.error = errors.New(errstr)
   842  	}
   843  	p.dir = r.string()
   844  	p.sourceFiles = make([]*sourceFile, r.int())
   845  	for i := range p.sourceFiles {
   846  		p.sourceFiles[i] = &sourceFile{
   847  			d:   m.d,
   848  			pos: r.int(),
   849  		}
   850  	}
   851  	p.modroot = m.modroot
   852  	return p
   853  }
   854  
   855  // sourceFile represents the information of a given source file in the module index.
   856  type sourceFile struct {
   857  	d               *decoder // encoding of this source file
   858  	pos             int      // start of sourceFile encoding in d
   859  	onceReadImports sync.Once
   860  	savedImports    []rawImport // saved imports so that they're only read once
   861  }
   862  
   863  // Offsets for fields in the sourceFile.
   864  const (
   865  	sourceFileError = 4 * iota
   866  	sourceFileParseError
   867  	sourceFileSynopsis
   868  	sourceFileName
   869  	sourceFilePkgName
   870  	sourceFileIgnoreFile
   871  	sourceFileBinaryOnly
   872  	sourceFileCgoDirectives
   873  	sourceFileGoBuildConstraint
   874  	sourceFileNumPlusBuildConstraints
   875  )
   876  
   877  func (sf *sourceFile) error() string {
   878  	return sf.d.stringAt(sf.pos + sourceFileError)
   879  }
   880  func (sf *sourceFile) parseError() string {
   881  	return sf.d.stringAt(sf.pos + sourceFileParseError)
   882  }
   883  func (sf *sourceFile) synopsis() string {
   884  	return sf.d.stringAt(sf.pos + sourceFileSynopsis)
   885  }
   886  func (sf *sourceFile) name() string {
   887  	return sf.d.stringAt(sf.pos + sourceFileName)
   888  }
   889  func (sf *sourceFile) pkgName() string {
   890  	return sf.d.stringAt(sf.pos + sourceFilePkgName)
   891  }
   892  func (sf *sourceFile) ignoreFile() bool {
   893  	return sf.d.boolAt(sf.pos + sourceFileIgnoreFile)
   894  }
   895  func (sf *sourceFile) binaryOnly() bool {
   896  	return sf.d.boolAt(sf.pos + sourceFileBinaryOnly)
   897  }
   898  func (sf *sourceFile) cgoDirectives() string {
   899  	return sf.d.stringAt(sf.pos + sourceFileCgoDirectives)
   900  }
   901  func (sf *sourceFile) goBuildConstraint() string {
   902  	return sf.d.stringAt(sf.pos + sourceFileGoBuildConstraint)
   903  }
   904  
   905  func (sf *sourceFile) plusBuildConstraints() []string {
   906  	pos := sf.pos + sourceFileNumPlusBuildConstraints
   907  	n := sf.d.intAt(pos)
   908  	pos += 4
   909  	ret := make([]string, n)
   910  	for i := 0; i < n; i++ {
   911  		ret[i] = sf.d.stringAt(pos)
   912  		pos += 4
   913  	}
   914  	return ret
   915  }
   916  
   917  func (sf *sourceFile) importsOffset() int {
   918  	pos := sf.pos + sourceFileNumPlusBuildConstraints
   919  	n := sf.d.intAt(pos)
   920  	// each build constraint is 1 uint32
   921  	return pos + 4 + n*4
   922  }
   923  
   924  func (sf *sourceFile) embedsOffset() int {
   925  	pos := sf.importsOffset()
   926  	n := sf.d.intAt(pos)
   927  	// each import is 5 uint32s (string + tokpos)
   928  	return pos + 4 + n*(4*5)
   929  }
   930  
   931  func (sf *sourceFile) directivesOffset() int {
   932  	pos := sf.embedsOffset()
   933  	n := sf.d.intAt(pos)
   934  	// each embed is 5 uint32s (string + tokpos)
   935  	return pos + 4 + n*(4*5)
   936  }
   937  
   938  func (sf *sourceFile) imports() []rawImport {
   939  	sf.onceReadImports.Do(func() {
   940  		importsOffset := sf.importsOffset()
   941  		r := sf.d.readAt(importsOffset)
   942  		numImports := r.int()
   943  		ret := make([]rawImport, numImports)
   944  		for i := 0; i < numImports; i++ {
   945  			ret[i] = rawImport{r.string(), r.tokpos()}
   946  		}
   947  		sf.savedImports = ret
   948  	})
   949  	return sf.savedImports
   950  }
   951  
   952  func (sf *sourceFile) embeds() []embed {
   953  	embedsOffset := sf.embedsOffset()
   954  	r := sf.d.readAt(embedsOffset)
   955  	numEmbeds := r.int()
   956  	ret := make([]embed, numEmbeds)
   957  	for i := range ret {
   958  		ret[i] = embed{r.string(), r.tokpos()}
   959  	}
   960  	return ret
   961  }
   962  
   963  func (sf *sourceFile) directives() []build.Directive {
   964  	directivesOffset := sf.directivesOffset()
   965  	r := sf.d.readAt(directivesOffset)
   966  	numDirectives := r.int()
   967  	ret := make([]build.Directive, numDirectives)
   968  	for i := range ret {
   969  		ret[i] = build.Directive{Text: r.string(), Pos: r.tokpos()}
   970  	}
   971  	return ret
   972  }
   973  
   974  func asString(b []byte) string {
   975  	return unsafe.String(unsafe.SliceData(b), len(b))
   976  }
   977  
   978  // A decoder helps decode the index format.
   979  type decoder struct {
   980  	data []byte // data after header
   981  	str  []byte // string table
   982  }
   983  
   984  // intAt returns the int at the given offset in d.data.
   985  func (d *decoder) intAt(off int) int {
   986  	if off < 0 || len(d.data)-off < 4 {
   987  		panic(errCorrupt)
   988  	}
   989  	i := binary.LittleEndian.Uint32(d.data[off : off+4])
   990  	if int32(i)>>31 != 0 {
   991  		panic(errCorrupt)
   992  	}
   993  	return int(i)
   994  }
   995  
   996  // boolAt returns the bool at the given offset in d.data.
   997  func (d *decoder) boolAt(off int) bool {
   998  	return d.intAt(off) != 0
   999  }
  1000  
  1001  // stringAt returns the string pointed at by the int at the given offset in d.data.
  1002  func (d *decoder) stringAt(off int) string {
  1003  	return d.stringTableAt(d.intAt(off))
  1004  }
  1005  
  1006  // stringTableAt returns the string at the given offset in the string table d.str.
  1007  func (d *decoder) stringTableAt(off int) string {
  1008  	if off < 0 || off >= len(d.str) {
  1009  		panic(errCorrupt)
  1010  	}
  1011  	s := d.str[off:]
  1012  	v, n := binary.Uvarint(s)
  1013  	if n <= 0 || v > uint64(len(s[n:])) {
  1014  		panic(errCorrupt)
  1015  	}
  1016  	return asString(s[n : n+int(v)])
  1017  }
  1018  
  1019  // A reader reads sequential fields from a section of the index format.
  1020  type reader struct {
  1021  	d   *decoder
  1022  	pos int
  1023  }
  1024  
  1025  // readAt returns a reader starting at the given position in d.
  1026  func (d *decoder) readAt(pos int) *reader {
  1027  	return &reader{d, pos}
  1028  }
  1029  
  1030  // int reads the next int.
  1031  func (r *reader) int() int {
  1032  	i := r.d.intAt(r.pos)
  1033  	r.pos += 4
  1034  	return i
  1035  }
  1036  
  1037  // string reads the next string.
  1038  func (r *reader) string() string {
  1039  	return r.d.stringTableAt(r.int())
  1040  }
  1041  
  1042  // bool reads the next bool.
  1043  func (r *reader) bool() bool {
  1044  	return r.int() != 0
  1045  }
  1046  
  1047  // tokpos reads the next token.Position.
  1048  func (r *reader) tokpos() token.Position {
  1049  	return token.Position{
  1050  		Filename: r.string(),
  1051  		Offset:   r.int(),
  1052  		Line:     r.int(),
  1053  		Column:   r.int(),
  1054  	}
  1055  }
  1056  

View as plain text