Source file src/cmd/go/internal/modindex/read.go

     1  // Copyright 2022 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package modindex
     6  
     7  import (
     8  	"bytes"
     9  	"encoding/binary"
    10  	"errors"
    11  	"fmt"
    12  	"go/build"
    13  	"go/build/constraint"
    14  	"go/token"
    15  	"internal/godebug"
    16  	"internal/goroot"
    17  	"path"
    18  	"path/filepath"
    19  	"runtime"
    20  	"runtime/debug"
    21  	"sort"
    22  	"strings"
    23  	"sync"
    24  	"time"
    25  	"unsafe"
    26  
    27  	"cmd/go/internal/base"
    28  	"cmd/go/internal/cache"
    29  	"cmd/go/internal/cfg"
    30  	"cmd/go/internal/fsys"
    31  	"cmd/go/internal/imports"
    32  	"cmd/go/internal/str"
    33  	"cmd/internal/par"
    34  )
    35  
    36  // enabled is used to flag off the behavior of the module index on tip, for debugging.
    37  var enabled = godebug.New("#goindex").Value() != "0"
    38  
    39  // Module represents and encoded module index file. It is used to
    40  // do the equivalent of build.Import of packages in the module and answer other
    41  // questions based on the index file's data.
    42  type Module struct {
    43  	modroot string
    44  	d       *decoder
    45  	n       int // number of packages
    46  }
    47  
    48  // moduleHash returns an ActionID corresponding to the state of the module
    49  // located at filesystem path modroot.
    50  func moduleHash(modroot string, ismodcache bool) (cache.ActionID, error) {
    51  	// We expect modules stored within the module cache to be checksummed and
    52  	// immutable, and we expect released modules within GOROOT to change only
    53  	// infrequently (when the Go version changes).
    54  	if !ismodcache {
    55  		// The contents of this module may change over time. We don't want to pay
    56  		// the cost to detect changes and re-index whenever they occur, so just
    57  		// don't index it at all.
    58  		//
    59  		// Note that this is true even for modules in GOROOT/src: non-release builds
    60  		// of the Go toolchain may have arbitrary development changes on top of the
    61  		// commit reported by runtime.Version, or could be completely artificial due
    62  		// to lacking a `git` binary (like "devel gomote.XXXXX", as synthesized by
    63  		// "gomote push" as of 2022-06-15). (Release builds shouldn't have
    64  		// modifications, but we don't want to use a behavior for releases that we
    65  		// haven't tested during development.)
    66  		return cache.ActionID{}, ErrNotIndexed
    67  	}
    68  
    69  	h := cache.NewHash("moduleIndex")
    70  	// TODO(bcmills): Since modules in the index are checksummed, we could
    71  	// probably improve the cache hit rate by keying off of the module
    72  	// path@version (perhaps including the checksum?) instead of the module root
    73  	// directory.
    74  	fmt.Fprintf(h, "module index %s %s %v\n", runtime.Version(), indexVersion, modroot)
    75  	return h.Sum(), nil
    76  }
    77  
    78  const modTimeCutoff = 2 * time.Second
    79  
    80  // dirHash returns an ActionID corresponding to the state of the package
    81  // located at filesystem path pkgdir.
    82  func dirHash(modroot, pkgdir string) (cache.ActionID, error) {
    83  	h := cache.NewHash("moduleIndex")
    84  	fmt.Fprintf(h, "modroot %s\n", modroot)
    85  	fmt.Fprintf(h, "package %s %s %v\n", runtime.Version(), indexVersion, pkgdir)
    86  	dirs, err := fsys.ReadDir(pkgdir)
    87  	if err != nil {
    88  		// pkgdir might not be a directory. give up on hashing.
    89  		return cache.ActionID{}, ErrNotIndexed
    90  	}
    91  	cutoff := time.Now().Add(-modTimeCutoff)
    92  	for _, d := range dirs {
    93  		if d.IsDir() {
    94  			continue
    95  		}
    96  
    97  		if !d.Type().IsRegular() {
    98  			return cache.ActionID{}, ErrNotIndexed
    99  		}
   100  		// To avoid problems for very recent files where a new
   101  		// write might not change the mtime due to file system
   102  		// mtime precision, reject caching if a file was read that
   103  		// is less than modTimeCutoff old.
   104  		//
   105  		// This is the same strategy used for hashing test inputs.
   106  		// See hashOpen in cmd/go/internal/test/test.go for the
   107  		// corresponding code.
   108  		info, err := d.Info()
   109  		if err != nil {
   110  			return cache.ActionID{}, ErrNotIndexed
   111  		}
   112  		if info.ModTime().After(cutoff) {
   113  			return cache.ActionID{}, ErrNotIndexed
   114  		}
   115  
   116  		fmt.Fprintf(h, "file %v %v %v\n", info.Name(), info.ModTime(), info.Size())
   117  	}
   118  	return h.Sum(), nil
   119  }
   120  
   121  var ErrNotIndexed = errors.New("not in module index")
   122  
   123  var (
   124  	errDisabled           = fmt.Errorf("%w: module indexing disabled", ErrNotIndexed)
   125  	errNotFromModuleCache = fmt.Errorf("%w: not from module cache", ErrNotIndexed)
   126  	errFIPS140            = fmt.Errorf("%w: fips140 snapshots not indexed", ErrNotIndexed)
   127  )
   128  
   129  // GetPackage returns the IndexPackage for the directory at the given path.
   130  // It will return ErrNotIndexed if the directory should be read without
   131  // using the index, for instance because the index is disabled, or the package
   132  // is not in a module.
   133  func GetPackage(modroot, pkgdir string) (*IndexPackage, error) {
   134  	mi, err := GetModule(modroot)
   135  	if err == nil {
   136  		return mi.Package(relPath(pkgdir, modroot)), nil
   137  	}
   138  	if !errors.Is(err, errNotFromModuleCache) {
   139  		return nil, err
   140  	}
   141  	if cfg.BuildContext.Compiler == "gccgo" && str.HasFilePathPrefix(modroot, cfg.GOROOTsrc) {
   142  		return nil, err // gccgo has no sources for GOROOT packages.
   143  	}
   144  	// The pkgdir for fips140 has been replaced in the fsys overlay,
   145  	// but the module index does not see that. Do not try to use the module index.
   146  	if strings.Contains(filepath.ToSlash(pkgdir), "internal/fips140/v") {
   147  		return nil, errFIPS140
   148  	}
   149  	modroot = filepath.Clean(modroot)
   150  	pkgdir = filepath.Clean(pkgdir)
   151  	return openIndexPackage(modroot, pkgdir)
   152  }
   153  
   154  // GetModule returns the Module for the given modroot.
   155  // It will return ErrNotIndexed if the directory should be read without
   156  // using the index, for instance because the index is disabled, or the package
   157  // is not in a module.
   158  func GetModule(modroot string) (*Module, error) {
   159  	dir, _, _ := cache.DefaultDir()
   160  	if !enabled || dir == "off" {
   161  		return nil, errDisabled
   162  	}
   163  	if modroot == "" {
   164  		panic("modindex.GetPackage called with empty modroot")
   165  	}
   166  	if cfg.BuildMod == "vendor" {
   167  		// Even if the main module is in the module cache,
   168  		// its vendored dependencies are not loaded from their
   169  		// usual cached locations.
   170  		return nil, errNotFromModuleCache
   171  	}
   172  	modroot = filepath.Clean(modroot)
   173  	if str.HasFilePathPrefix(modroot, cfg.GOROOTsrc) || !str.HasFilePathPrefix(modroot, cfg.GOMODCACHE) {
   174  		return nil, errNotFromModuleCache
   175  	}
   176  	return openIndexModule(modroot, true)
   177  }
   178  
   179  var mcache par.ErrCache[string, *Module]
   180  
   181  // openIndexModule returns the module index for modPath.
   182  // It will return ErrNotIndexed if the module can not be read
   183  // using the index because it contains symlinks.
   184  func openIndexModule(modroot string, ismodcache bool) (*Module, error) {
   185  	return mcache.Do(modroot, func() (*Module, error) {
   186  		fsys.Trace("openIndexModule", modroot)
   187  		id, err := moduleHash(modroot, ismodcache)
   188  		if err != nil {
   189  			return nil, err
   190  		}
   191  		data, _, opened, err := cache.GetMmap(cache.Default(), id)
   192  		if err != nil {
   193  			// Couldn't read from modindex. Assume we couldn't read from
   194  			// the index because the module hasn't been indexed yet.
   195  			// But double check on Windows that we haven't opened the file yet,
   196  			// because once mmap opens the file, we can't close it, and
   197  			// Windows won't let us open an already opened file.
   198  			data, err = indexModule(modroot)
   199  			if err != nil {
   200  				return nil, err
   201  			}
   202  			if runtime.GOOS != "windows" || !opened {
   203  				if err = cache.PutBytes(cache.Default(), id, data); err != nil {
   204  					return nil, err
   205  				}
   206  			}
   207  		}
   208  		mi, err := fromBytes(modroot, data)
   209  		if err != nil {
   210  			return nil, err
   211  		}
   212  		return mi, nil
   213  	})
   214  }
   215  
   216  var pcache par.ErrCache[[2]string, *IndexPackage]
   217  
   218  func openIndexPackage(modroot, pkgdir string) (*IndexPackage, error) {
   219  	return pcache.Do([2]string{modroot, pkgdir}, func() (*IndexPackage, error) {
   220  		fsys.Trace("openIndexPackage", pkgdir)
   221  		id, err := dirHash(modroot, pkgdir)
   222  		if err != nil {
   223  			return nil, err
   224  		}
   225  		data, _, opened, err := cache.GetMmap(cache.Default(), id)
   226  		if err != nil {
   227  			// Couldn't read from index. Assume we couldn't read from
   228  			// the index because the package hasn't been indexed yet.
   229  			// But double check on Windows that we haven't opened the file yet,
   230  			// because once mmap opens the file, we can't close it, and
   231  			// Windows won't let us open an already opened file.
   232  			data = indexPackage(modroot, pkgdir)
   233  			if runtime.GOOS != "windows" || !opened {
   234  				if err = cache.PutBytes(cache.Default(), id, data); err != nil {
   235  					return nil, err
   236  				}
   237  			}
   238  		}
   239  		pkg, err := packageFromBytes(modroot, data)
   240  		if err != nil {
   241  			return nil, err
   242  		}
   243  		return pkg, nil
   244  	})
   245  }
   246  
   247  var errCorrupt = errors.New("corrupt index")
   248  
   249  // protect marks the start of a large section of code that accesses the index.
   250  // It should be used as:
   251  //
   252  //	defer unprotect(protect, &err)
   253  //
   254  // It should not be used for trivial accesses which would be
   255  // dwarfed by the overhead of the defer.
   256  func protect() bool {
   257  	return debug.SetPanicOnFault(true)
   258  }
   259  
   260  var isTest = false
   261  
   262  // unprotect marks the end of a large section of code that accesses the index.
   263  // It should be used as:
   264  //
   265  //	defer unprotect(protect, &err)
   266  //
   267  // end looks for panics due to errCorrupt or bad mmap accesses.
   268  // When it finds them, it adds explanatory text, consumes the panic, and sets *errp instead.
   269  // If errp is nil, end adds the explanatory text but then calls base.Fatalf.
   270  func unprotect(old bool, errp *error) {
   271  	// SetPanicOnFault's errors _may_ satisfy this interface. Even though it's not guaranteed
   272  	// that all its errors satisfy this interface, we'll only check for these errors so that
   273  	// we don't suppress panics that could have been produced from other sources.
   274  	type addrer interface {
   275  		Addr() uintptr
   276  	}
   277  
   278  	debug.SetPanicOnFault(old)
   279  
   280  	if e := recover(); e != nil {
   281  		if _, ok := e.(addrer); ok || e == errCorrupt {
   282  			// This panic was almost certainly caused by SetPanicOnFault or our panic(errCorrupt).
   283  			err := fmt.Errorf("error reading module index: %v", e)
   284  			if errp != nil {
   285  				*errp = err
   286  				return
   287  			}
   288  			if isTest {
   289  				panic(err)
   290  			}
   291  			base.Fatalf("%v", err)
   292  		}
   293  		// The panic was likely not caused by SetPanicOnFault.
   294  		panic(e)
   295  	}
   296  }
   297  
   298  // fromBytes returns a *Module given the encoded representation.
   299  func fromBytes(moddir string, data []byte) (m *Module, err error) {
   300  	if !enabled {
   301  		panic("use of index")
   302  	}
   303  
   304  	defer unprotect(protect(), &err)
   305  
   306  	if !bytes.HasPrefix(data, []byte(indexVersion+"\n")) {
   307  		return nil, errCorrupt
   308  	}
   309  
   310  	const hdr = len(indexVersion + "\n")
   311  	d := &decoder{data: data}
   312  	str := d.intAt(hdr)
   313  	if str < hdr+8 || len(d.data) < str {
   314  		return nil, errCorrupt
   315  	}
   316  	d.data, d.str = data[:str], d.data[str:]
   317  	// Check that string table looks valid.
   318  	// First string is empty string (length 0),
   319  	// and we leave a marker byte 0xFF at the end
   320  	// just to make sure that the file is not truncated.
   321  	if len(d.str) == 0 || d.str[0] != 0 || d.str[len(d.str)-1] != 0xFF {
   322  		return nil, errCorrupt
   323  	}
   324  
   325  	n := d.intAt(hdr + 4)
   326  	if n < 0 || n > (len(d.data)-8)/8 {
   327  		return nil, errCorrupt
   328  	}
   329  
   330  	m = &Module{
   331  		moddir,
   332  		d,
   333  		n,
   334  	}
   335  	return m, nil
   336  }
   337  
   338  // packageFromBytes returns a *IndexPackage given the encoded representation.
   339  func packageFromBytes(modroot string, data []byte) (p *IndexPackage, err error) {
   340  	m, err := fromBytes(modroot, data)
   341  	if err != nil {
   342  		return nil, err
   343  	}
   344  	if m.n != 1 {
   345  		return nil, fmt.Errorf("corrupt single-package index")
   346  	}
   347  	return m.pkg(0), nil
   348  }
   349  
   350  // pkgDir returns the dir string of the i'th package in the index.
   351  func (m *Module) pkgDir(i int) string {
   352  	if i < 0 || i >= m.n {
   353  		panic(errCorrupt)
   354  	}
   355  	return m.d.stringAt(12 + 8 + 8*i)
   356  }
   357  
   358  // pkgOff returns the offset of the data for the i'th package in the index.
   359  func (m *Module) pkgOff(i int) int {
   360  	if i < 0 || i >= m.n {
   361  		panic(errCorrupt)
   362  	}
   363  	return m.d.intAt(12 + 8 + 8*i + 4)
   364  }
   365  
   366  // Walk calls f for each package in the index, passing the path to that package relative to the module root.
   367  func (m *Module) Walk(f func(path string)) {
   368  	defer unprotect(protect(), nil)
   369  	for i := 0; i < m.n; i++ {
   370  		f(m.pkgDir(i))
   371  	}
   372  }
   373  
   374  // relPath returns the path relative to the module's root.
   375  func relPath(path, modroot string) string {
   376  	return str.TrimFilePathPrefix(filepath.Clean(path), filepath.Clean(modroot))
   377  }
   378  
   379  var installgorootAll = godebug.New("installgoroot").Value() == "all"
   380  
   381  // Import is the equivalent of build.Import given the information in Module.
   382  func (rp *IndexPackage) Import(bctxt build.Context, mode build.ImportMode) (p *build.Package, err error) {
   383  	defer unprotect(protect(), &err)
   384  
   385  	ctxt := (*Context)(&bctxt)
   386  
   387  	p = &build.Package{}
   388  
   389  	p.ImportPath = "."
   390  	p.Dir = filepath.Join(rp.modroot, rp.dir)
   391  
   392  	var pkgerr error
   393  	switch ctxt.Compiler {
   394  	case "gccgo", "gc":
   395  	default:
   396  		// Save error for end of function.
   397  		pkgerr = fmt.Errorf("import %q: unknown compiler %q", p.Dir, ctxt.Compiler)
   398  	}
   399  
   400  	if p.Dir == "" {
   401  		return p, fmt.Errorf("import %q: import of unknown directory", p.Dir)
   402  	}
   403  
   404  	// goroot and gopath
   405  	inTestdata := func(sub string) bool {
   406  		sub = filepath.ToSlash(sub)
   407  		return strings.Contains(sub, "/testdata/") || strings.HasSuffix(sub, "/testdata") || str.HasPathPrefix(sub, "testdata")
   408  	}
   409  	var pkga string
   410  	if !inTestdata(rp.dir) {
   411  		// In build.go, p.Root should only be set in the non-local-import case, or in
   412  		// GOROOT or GOPATH. Since module mode only calls Import with path set to "."
   413  		// and the module index doesn't apply outside modules, the GOROOT case is
   414  		// the only case where p.Root needs to be set.
   415  		if ctxt.GOROOT != "" && str.HasFilePathPrefix(p.Dir, cfg.GOROOTsrc) && p.Dir != cfg.GOROOTsrc {
   416  			p.Root = ctxt.GOROOT
   417  			p.Goroot = true
   418  			modprefix := str.TrimFilePathPrefix(rp.modroot, cfg.GOROOTsrc)
   419  			p.ImportPath = rp.dir
   420  			if modprefix != "" {
   421  				p.ImportPath = filepath.Join(modprefix, p.ImportPath)
   422  			}
   423  
   424  			// Set GOROOT-specific fields (sometimes for modules in a GOPATH directory).
   425  			// The fields set below (SrcRoot, PkgRoot, BinDir, PkgTargetRoot, and PkgObj)
   426  			// are only set in build.Import if p.Root != "".
   427  			var pkgtargetroot string
   428  			suffix := ""
   429  			if ctxt.InstallSuffix != "" {
   430  				suffix = "_" + ctxt.InstallSuffix
   431  			}
   432  			switch ctxt.Compiler {
   433  			case "gccgo":
   434  				pkgtargetroot = "pkg/gccgo_" + ctxt.GOOS + "_" + ctxt.GOARCH + suffix
   435  				dir, elem := path.Split(p.ImportPath)
   436  				pkga = pkgtargetroot + "/" + dir + "lib" + elem + ".a"
   437  			case "gc":
   438  				pkgtargetroot = "pkg/" + ctxt.GOOS + "_" + ctxt.GOARCH + suffix
   439  				pkga = pkgtargetroot + "/" + p.ImportPath + ".a"
   440  			}
   441  			p.SrcRoot = ctxt.joinPath(p.Root, "src")
   442  			p.PkgRoot = ctxt.joinPath(p.Root, "pkg")
   443  			p.BinDir = ctxt.joinPath(p.Root, "bin")
   444  			if pkga != "" {
   445  				// Always set PkgTargetRoot. It might be used when building in shared
   446  				// mode.
   447  				p.PkgTargetRoot = ctxt.joinPath(p.Root, pkgtargetroot)
   448  
   449  				// Set the install target if applicable.
   450  				if !p.Goroot || (installgorootAll && p.ImportPath != "unsafe" && p.ImportPath != "builtin") {
   451  					p.PkgObj = ctxt.joinPath(p.Root, pkga)
   452  				}
   453  			}
   454  		}
   455  	}
   456  
   457  	if rp.error != nil {
   458  		if errors.Is(rp.error, errCannotFindPackage) && ctxt.Compiler == "gccgo" && p.Goroot {
   459  			return p, nil
   460  		}
   461  		return p, rp.error
   462  	}
   463  
   464  	if mode&build.FindOnly != 0 {
   465  		return p, pkgerr
   466  	}
   467  
   468  	// We need to do a second round of bad file processing.
   469  	var badGoError error
   470  	badGoFiles := make(map[string]bool)
   471  	badGoFile := func(name string, err error) {
   472  		if badGoError == nil {
   473  			badGoError = err
   474  		}
   475  		if !badGoFiles[name] {
   476  			p.InvalidGoFiles = append(p.InvalidGoFiles, name)
   477  			badGoFiles[name] = true
   478  		}
   479  	}
   480  
   481  	var Sfiles []string // files with ".S"(capital S)/.sx(capital s equivalent for case insensitive filesystems)
   482  	var firstFile string
   483  	embedPos := make(map[string][]token.Position)
   484  	testEmbedPos := make(map[string][]token.Position)
   485  	xTestEmbedPos := make(map[string][]token.Position)
   486  	importPos := make(map[string][]token.Position)
   487  	testImportPos := make(map[string][]token.Position)
   488  	xTestImportPos := make(map[string][]token.Position)
   489  	allTags := make(map[string]bool)
   490  	for _, tf := range rp.sourceFiles {
   491  		name := tf.name()
   492  		// Check errors for go files and call badGoFiles to put them in
   493  		// InvalidGoFiles if they do have an error.
   494  		if strings.HasSuffix(name, ".go") {
   495  			if error := tf.error(); error != "" {
   496  				badGoFile(name, errors.New(tf.error()))
   497  				continue
   498  			} else if parseError := tf.parseError(); parseError != "" {
   499  				badGoFile(name, parseErrorFromString(tf.parseError()))
   500  				// Fall through: we still want to list files with parse errors.
   501  			}
   502  		}
   503  
   504  		var shouldBuild = true
   505  		if !ctxt.goodOSArchFile(name, allTags) && !ctxt.UseAllFiles {
   506  			shouldBuild = false
   507  		} else if goBuildConstraint := tf.goBuildConstraint(); goBuildConstraint != "" {
   508  			x, err := constraint.Parse(goBuildConstraint)
   509  			if err != nil {
   510  				return p, fmt.Errorf("%s: parsing //go:build line: %v", name, err)
   511  			}
   512  			shouldBuild = ctxt.eval(x, allTags)
   513  		} else if plusBuildConstraints := tf.plusBuildConstraints(); len(plusBuildConstraints) > 0 {
   514  			for _, text := range plusBuildConstraints {
   515  				if x, err := constraint.Parse(text); err == nil {
   516  					if !ctxt.eval(x, allTags) {
   517  						shouldBuild = false
   518  					}
   519  				}
   520  			}
   521  		}
   522  
   523  		ext := nameExt(name)
   524  		if !shouldBuild || tf.ignoreFile() {
   525  			if ext == ".go" {
   526  				p.IgnoredGoFiles = append(p.IgnoredGoFiles, name)
   527  			} else if fileListForExt(p, ext) != nil {
   528  				p.IgnoredOtherFiles = append(p.IgnoredOtherFiles, name)
   529  			}
   530  			continue
   531  		}
   532  
   533  		// Going to save the file. For non-Go files, can stop here.
   534  		switch ext {
   535  		case ".go":
   536  			// keep going
   537  		case ".S", ".sx":
   538  			// special case for cgo, handled at end
   539  			Sfiles = append(Sfiles, name)
   540  			continue
   541  		default:
   542  			if list := fileListForExt(p, ext); list != nil {
   543  				*list = append(*list, name)
   544  			}
   545  			continue
   546  		}
   547  
   548  		pkg := tf.pkgName()
   549  		if pkg == "documentation" {
   550  			p.IgnoredGoFiles = append(p.IgnoredGoFiles, name)
   551  			continue
   552  		}
   553  		isTest := strings.HasSuffix(name, "_test.go")
   554  		isXTest := false
   555  		if isTest && strings.HasSuffix(tf.pkgName(), "_test") && p.Name != tf.pkgName() {
   556  			isXTest = true
   557  			pkg = pkg[:len(pkg)-len("_test")]
   558  		}
   559  
   560  		if !isTest && tf.binaryOnly() {
   561  			p.BinaryOnly = true
   562  		}
   563  
   564  		if p.Name == "" {
   565  			p.Name = pkg
   566  			firstFile = name
   567  		} else if pkg != p.Name {
   568  			// TODO(#45999): The choice of p.Name is arbitrary based on file iteration
   569  			// order. Instead of resolving p.Name arbitrarily, we should clear out the
   570  			// existing Name and mark the existing files as also invalid.
   571  			badGoFile(name, &MultiplePackageError{
   572  				Dir:      p.Dir,
   573  				Packages: []string{p.Name, pkg},
   574  				Files:    []string{firstFile, name},
   575  			})
   576  		}
   577  		// Grab the first package comment as docs, provided it is not from a test file.
   578  		if p.Doc == "" && !isTest && !isXTest {
   579  			if synopsis := tf.synopsis(); synopsis != "" {
   580  				p.Doc = synopsis
   581  			}
   582  		}
   583  
   584  		// Record Imports and information about cgo.
   585  		isCgo := false
   586  		imports := tf.imports()
   587  		for _, imp := range imports {
   588  			if imp.path == "C" {
   589  				if isTest {
   590  					badGoFile(name, fmt.Errorf("use of cgo in test %s not supported", name))
   591  					continue
   592  				}
   593  				isCgo = true
   594  			}
   595  		}
   596  		if directives := tf.cgoDirectives(); directives != "" {
   597  			if err := ctxt.saveCgo(name, p, directives); err != nil {
   598  				badGoFile(name, err)
   599  			}
   600  		}
   601  
   602  		var fileList *[]string
   603  		var importMap, embedMap map[string][]token.Position
   604  		var directives *[]build.Directive
   605  		switch {
   606  		case isCgo:
   607  			allTags["cgo"] = true
   608  			if ctxt.CgoEnabled {
   609  				fileList = &p.CgoFiles
   610  				importMap = importPos
   611  				embedMap = embedPos
   612  				directives = &p.Directives
   613  			} else {
   614  				// Ignore Imports and Embeds from cgo files if cgo is disabled.
   615  				fileList = &p.IgnoredGoFiles
   616  			}
   617  		case isXTest:
   618  			fileList = &p.XTestGoFiles
   619  			importMap = xTestImportPos
   620  			embedMap = xTestEmbedPos
   621  			directives = &p.XTestDirectives
   622  		case isTest:
   623  			fileList = &p.TestGoFiles
   624  			importMap = testImportPos
   625  			embedMap = testEmbedPos
   626  			directives = &p.TestDirectives
   627  		default:
   628  			fileList = &p.GoFiles
   629  			importMap = importPos
   630  			embedMap = embedPos
   631  			directives = &p.Directives
   632  		}
   633  		*fileList = append(*fileList, name)
   634  		if importMap != nil {
   635  			for _, imp := range imports {
   636  				importMap[imp.path] = append(importMap[imp.path], imp.position)
   637  			}
   638  		}
   639  		if embedMap != nil {
   640  			for _, e := range tf.embeds() {
   641  				embedMap[e.pattern] = append(embedMap[e.pattern], e.position)
   642  			}
   643  		}
   644  		if directives != nil {
   645  			*directives = append(*directives, tf.directives()...)
   646  		}
   647  	}
   648  
   649  	p.EmbedPatterns, p.EmbedPatternPos = cleanDecls(embedPos)
   650  	p.TestEmbedPatterns, p.TestEmbedPatternPos = cleanDecls(testEmbedPos)
   651  	p.XTestEmbedPatterns, p.XTestEmbedPatternPos = cleanDecls(xTestEmbedPos)
   652  
   653  	p.Imports, p.ImportPos = cleanDecls(importPos)
   654  	p.TestImports, p.TestImportPos = cleanDecls(testImportPos)
   655  	p.XTestImports, p.XTestImportPos = cleanDecls(xTestImportPos)
   656  
   657  	for tag := range allTags {
   658  		p.AllTags = append(p.AllTags, tag)
   659  	}
   660  	sort.Strings(p.AllTags)
   661  
   662  	if len(p.CgoFiles) > 0 {
   663  		p.SFiles = append(p.SFiles, Sfiles...)
   664  		sort.Strings(p.SFiles)
   665  	} else {
   666  		p.IgnoredOtherFiles = append(p.IgnoredOtherFiles, Sfiles...)
   667  		sort.Strings(p.IgnoredOtherFiles)
   668  	}
   669  
   670  	if badGoError != nil {
   671  		return p, badGoError
   672  	}
   673  	if len(p.GoFiles)+len(p.CgoFiles)+len(p.TestGoFiles)+len(p.XTestGoFiles) == 0 {
   674  		return p, &build.NoGoError{Dir: p.Dir}
   675  	}
   676  	return p, pkgerr
   677  }
   678  
   679  // IsStandardPackage reports whether path is a standard package
   680  // for the goroot and compiler using the module index if possible,
   681  // and otherwise falling back to internal/goroot.IsStandardPackage
   682  func IsStandardPackage(goroot_, compiler, path string) bool {
   683  	if !enabled || compiler != "gc" {
   684  		return goroot.IsStandardPackage(fsys.ReadDir, goroot_, compiler, path)
   685  	}
   686  
   687  	reldir := filepath.FromSlash(path) // relative dir path in module index for package
   688  	modroot := filepath.Join(goroot_, "src")
   689  	if str.HasFilePathPrefix(reldir, "cmd") {
   690  		reldir = str.TrimFilePathPrefix(reldir, "cmd")
   691  		modroot = filepath.Join(modroot, "cmd")
   692  	}
   693  	if pkg, err := GetPackage(modroot, filepath.Join(modroot, reldir)); err == nil {
   694  		hasGo, err := pkg.IsGoDir()
   695  		return err == nil && hasGo
   696  	} else if errors.Is(err, ErrNotIndexed) {
   697  		// Fall back because package isn't indexable. (Probably because
   698  		// a file was modified recently)
   699  		return goroot.IsStandardPackage(fsys.ReadDir, goroot_, compiler, path)
   700  	}
   701  	return false
   702  }
   703  
   704  // IsGoDir is the equivalent of fsys.IsGoDir using the information in the index.
   705  func (rp *IndexPackage) IsGoDir() (_ bool, err error) {
   706  	defer func() {
   707  		if e := recover(); e != nil {
   708  			err = fmt.Errorf("error reading module index: %v", e)
   709  		}
   710  	}()
   711  	for _, sf := range rp.sourceFiles {
   712  		if strings.HasSuffix(sf.name(), ".go") {
   713  			return true, nil
   714  		}
   715  	}
   716  	return false, nil
   717  }
   718  
   719  // ScanDir implements imports.ScanDir using the information in the index.
   720  func (rp *IndexPackage) ScanDir(tags map[string]bool) (sortedImports []string, sortedTestImports []string, err error) {
   721  	// TODO(matloob) dir should eventually be relative to indexed directory
   722  	// TODO(matloob): skip reading raw package and jump straight to data we need?
   723  
   724  	defer func() {
   725  		if e := recover(); e != nil {
   726  			err = fmt.Errorf("error reading module index: %v", e)
   727  		}
   728  	}()
   729  
   730  	imports_ := make(map[string]bool)
   731  	testImports := make(map[string]bool)
   732  	numFiles := 0
   733  
   734  Files:
   735  	for _, sf := range rp.sourceFiles {
   736  		name := sf.name()
   737  		if strings.HasPrefix(name, "_") || strings.HasPrefix(name, ".") || !strings.HasSuffix(name, ".go") || !imports.MatchFile(name, tags) {
   738  			continue
   739  		}
   740  
   741  		// The following section exists for backwards compatibility reasons:
   742  		// scanDir ignores files with import "C" when collecting the list
   743  		// of imports unless the "cgo" tag is provided. The following comment
   744  		// is copied from the original.
   745  		//
   746  		// import "C" is implicit requirement of cgo tag.
   747  		// When listing files on the command line (explicitFiles=true)
   748  		// we do not apply build tag filtering but we still do apply
   749  		// cgo filtering, so no explicitFiles check here.
   750  		// Why? Because we always have, and it's not worth breaking
   751  		// that behavior now.
   752  		imps := sf.imports() // TODO(matloob): directly read import paths to avoid the extra strings?
   753  		for _, imp := range imps {
   754  			if imp.path == "C" && !tags["cgo"] && !tags["*"] {
   755  				continue Files
   756  			}
   757  		}
   758  
   759  		if !shouldBuild(sf, tags) {
   760  			continue
   761  		}
   762  		numFiles++
   763  		m := imports_
   764  		if strings.HasSuffix(name, "_test.go") {
   765  			m = testImports
   766  		}
   767  		for _, p := range imps {
   768  			m[p.path] = true
   769  		}
   770  	}
   771  	if numFiles == 0 {
   772  		return nil, nil, imports.ErrNoGo
   773  	}
   774  	return keys(imports_), keys(testImports), nil
   775  }
   776  
   777  func keys(m map[string]bool) []string {
   778  	list := make([]string, 0, len(m))
   779  	for k := range m {
   780  		list = append(list, k)
   781  	}
   782  	sort.Strings(list)
   783  	return list
   784  }
   785  
   786  // implements imports.ShouldBuild in terms of an index sourcefile.
   787  func shouldBuild(sf *sourceFile, tags map[string]bool) bool {
   788  	if goBuildConstraint := sf.goBuildConstraint(); goBuildConstraint != "" {
   789  		x, err := constraint.Parse(goBuildConstraint)
   790  		if err != nil {
   791  			return false
   792  		}
   793  		return imports.Eval(x, tags, true)
   794  	}
   795  
   796  	plusBuildConstraints := sf.plusBuildConstraints()
   797  	for _, text := range plusBuildConstraints {
   798  		if x, err := constraint.Parse(text); err == nil {
   799  			if !imports.Eval(x, tags, true) {
   800  				return false
   801  			}
   802  		}
   803  	}
   804  
   805  	return true
   806  }
   807  
   808  // IndexPackage holds the information in the index
   809  // needed to load a package in a specific directory.
   810  type IndexPackage struct {
   811  	error error
   812  	dir   string // directory of the package relative to the modroot
   813  
   814  	modroot string
   815  
   816  	// Source files
   817  	sourceFiles []*sourceFile
   818  }
   819  
   820  var errCannotFindPackage = errors.New("cannot find package")
   821  
   822  // Package and returns finds the package with the given path (relative to the module root).
   823  // If the package does not exist, Package returns an IndexPackage that will return an
   824  // appropriate error from its methods.
   825  func (m *Module) Package(path string) *IndexPackage {
   826  	defer unprotect(protect(), nil)
   827  
   828  	i, ok := sort.Find(m.n, func(i int) int {
   829  		return strings.Compare(path, m.pkgDir(i))
   830  	})
   831  	if !ok {
   832  		return &IndexPackage{error: fmt.Errorf("%w %q in:\n\t%s", errCannotFindPackage, path, filepath.Join(m.modroot, path))}
   833  	}
   834  	return m.pkg(i)
   835  }
   836  
   837  // pkg returns the i'th IndexPackage in m.
   838  func (m *Module) pkg(i int) *IndexPackage {
   839  	r := m.d.readAt(m.pkgOff(i))
   840  	p := new(IndexPackage)
   841  	if errstr := r.string(); errstr != "" {
   842  		p.error = errors.New(errstr)
   843  	}
   844  	p.dir = r.string()
   845  	p.sourceFiles = make([]*sourceFile, r.int())
   846  	for i := range p.sourceFiles {
   847  		p.sourceFiles[i] = &sourceFile{
   848  			d:   m.d,
   849  			pos: r.int(),
   850  		}
   851  	}
   852  	p.modroot = m.modroot
   853  	return p
   854  }
   855  
   856  // sourceFile represents the information of a given source file in the module index.
   857  type sourceFile struct {
   858  	d               *decoder // encoding of this source file
   859  	pos             int      // start of sourceFile encoding in d
   860  	onceReadImports sync.Once
   861  	savedImports    []rawImport // saved imports so that they're only read once
   862  }
   863  
   864  // Offsets for fields in the sourceFile.
   865  const (
   866  	sourceFileError = 4 * iota
   867  	sourceFileParseError
   868  	sourceFileSynopsis
   869  	sourceFileName
   870  	sourceFilePkgName
   871  	sourceFileIgnoreFile
   872  	sourceFileBinaryOnly
   873  	sourceFileCgoDirectives
   874  	sourceFileGoBuildConstraint
   875  	sourceFileNumPlusBuildConstraints
   876  )
   877  
   878  func (sf *sourceFile) error() string {
   879  	return sf.d.stringAt(sf.pos + sourceFileError)
   880  }
   881  func (sf *sourceFile) parseError() string {
   882  	return sf.d.stringAt(sf.pos + sourceFileParseError)
   883  }
   884  func (sf *sourceFile) synopsis() string {
   885  	return sf.d.stringAt(sf.pos + sourceFileSynopsis)
   886  }
   887  func (sf *sourceFile) name() string {
   888  	return sf.d.stringAt(sf.pos + sourceFileName)
   889  }
   890  func (sf *sourceFile) pkgName() string {
   891  	return sf.d.stringAt(sf.pos + sourceFilePkgName)
   892  }
   893  func (sf *sourceFile) ignoreFile() bool {
   894  	return sf.d.boolAt(sf.pos + sourceFileIgnoreFile)
   895  }
   896  func (sf *sourceFile) binaryOnly() bool {
   897  	return sf.d.boolAt(sf.pos + sourceFileBinaryOnly)
   898  }
   899  func (sf *sourceFile) cgoDirectives() string {
   900  	return sf.d.stringAt(sf.pos + sourceFileCgoDirectives)
   901  }
   902  func (sf *sourceFile) goBuildConstraint() string {
   903  	return sf.d.stringAt(sf.pos + sourceFileGoBuildConstraint)
   904  }
   905  
   906  func (sf *sourceFile) plusBuildConstraints() []string {
   907  	pos := sf.pos + sourceFileNumPlusBuildConstraints
   908  	n := sf.d.intAt(pos)
   909  	pos += 4
   910  	ret := make([]string, n)
   911  	for i := 0; i < n; i++ {
   912  		ret[i] = sf.d.stringAt(pos)
   913  		pos += 4
   914  	}
   915  	return ret
   916  }
   917  
   918  func (sf *sourceFile) importsOffset() int {
   919  	pos := sf.pos + sourceFileNumPlusBuildConstraints
   920  	n := sf.d.intAt(pos)
   921  	// each build constraint is 1 uint32
   922  	return pos + 4 + n*4
   923  }
   924  
   925  func (sf *sourceFile) embedsOffset() int {
   926  	pos := sf.importsOffset()
   927  	n := sf.d.intAt(pos)
   928  	// each import is 5 uint32s (string + tokpos)
   929  	return pos + 4 + n*(4*5)
   930  }
   931  
   932  func (sf *sourceFile) directivesOffset() int {
   933  	pos := sf.embedsOffset()
   934  	n := sf.d.intAt(pos)
   935  	// each embed is 5 uint32s (string + tokpos)
   936  	return pos + 4 + n*(4*5)
   937  }
   938  
   939  func (sf *sourceFile) imports() []rawImport {
   940  	sf.onceReadImports.Do(func() {
   941  		importsOffset := sf.importsOffset()
   942  		r := sf.d.readAt(importsOffset)
   943  		numImports := r.int()
   944  		ret := make([]rawImport, numImports)
   945  		for i := 0; i < numImports; i++ {
   946  			ret[i] = rawImport{r.string(), r.tokpos()}
   947  		}
   948  		sf.savedImports = ret
   949  	})
   950  	return sf.savedImports
   951  }
   952  
   953  func (sf *sourceFile) embeds() []embed {
   954  	embedsOffset := sf.embedsOffset()
   955  	r := sf.d.readAt(embedsOffset)
   956  	numEmbeds := r.int()
   957  	ret := make([]embed, numEmbeds)
   958  	for i := range ret {
   959  		ret[i] = embed{r.string(), r.tokpos()}
   960  	}
   961  	return ret
   962  }
   963  
   964  func (sf *sourceFile) directives() []build.Directive {
   965  	directivesOffset := sf.directivesOffset()
   966  	r := sf.d.readAt(directivesOffset)
   967  	numDirectives := r.int()
   968  	ret := make([]build.Directive, numDirectives)
   969  	for i := range ret {
   970  		ret[i] = build.Directive{Text: r.string(), Pos: r.tokpos()}
   971  	}
   972  	return ret
   973  }
   974  
   975  func asString(b []byte) string {
   976  	return unsafe.String(unsafe.SliceData(b), len(b))
   977  }
   978  
   979  // A decoder helps decode the index format.
   980  type decoder struct {
   981  	data []byte // data after header
   982  	str  []byte // string table
   983  }
   984  
   985  // intAt returns the int at the given offset in d.data.
   986  func (d *decoder) intAt(off int) int {
   987  	if off < 0 || len(d.data)-off < 4 {
   988  		panic(errCorrupt)
   989  	}
   990  	i := binary.LittleEndian.Uint32(d.data[off : off+4])
   991  	if int32(i)>>31 != 0 {
   992  		panic(errCorrupt)
   993  	}
   994  	return int(i)
   995  }
   996  
   997  // boolAt returns the bool at the given offset in d.data.
   998  func (d *decoder) boolAt(off int) bool {
   999  	return d.intAt(off) != 0
  1000  }
  1001  
  1002  // stringAt returns the string pointed at by the int at the given offset in d.data.
  1003  func (d *decoder) stringAt(off int) string {
  1004  	return d.stringTableAt(d.intAt(off))
  1005  }
  1006  
  1007  // stringTableAt returns the string at the given offset in the string table d.str.
  1008  func (d *decoder) stringTableAt(off int) string {
  1009  	if off < 0 || off >= len(d.str) {
  1010  		panic(errCorrupt)
  1011  	}
  1012  	s := d.str[off:]
  1013  	v, n := binary.Uvarint(s)
  1014  	if n <= 0 || v > uint64(len(s[n:])) {
  1015  		panic(errCorrupt)
  1016  	}
  1017  	return asString(s[n : n+int(v)])
  1018  }
  1019  
  1020  // A reader reads sequential fields from a section of the index format.
  1021  type reader struct {
  1022  	d   *decoder
  1023  	pos int
  1024  }
  1025  
  1026  // readAt returns a reader starting at the given position in d.
  1027  func (d *decoder) readAt(pos int) *reader {
  1028  	return &reader{d, pos}
  1029  }
  1030  
  1031  // int reads the next int.
  1032  func (r *reader) int() int {
  1033  	i := r.d.intAt(r.pos)
  1034  	r.pos += 4
  1035  	return i
  1036  }
  1037  
  1038  // string reads the next string.
  1039  func (r *reader) string() string {
  1040  	return r.d.stringTableAt(r.int())
  1041  }
  1042  
  1043  // tokpos reads the next token.Position.
  1044  func (r *reader) tokpos() token.Position {
  1045  	return token.Position{
  1046  		Filename: r.string(),
  1047  		Offset:   r.int(),
  1048  		Line:     r.int(),
  1049  		Column:   r.int(),
  1050  	}
  1051  }
  1052  

View as plain text