proxy.go

     1  // Copyright 2018 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package modfetch
     6  
     7  import (
     8  	"context"
     9  	"encoding/json"
    10  	"errors"
    11  	"fmt"
    12  	"io"
    13  	"io/fs"
    14  	"net/url"
    15  	pathpkg "path"
    16  	"path/filepath"
    17  	"strings"
    18  	"sync"
    19  	"time"
    20  
    21  	"cmd/go/internal/base"
    22  	"cmd/go/internal/cfg"
    23  	"cmd/go/internal/modfetch/codehost"
    24  	"cmd/go/internal/web"
    25  
    26  	"golang.org/x/mod/module"
    27  	"golang.org/x/mod/semver"
    28  )
    29  
    30  var HelpGoproxy = &base.Command{
    31  	UsageLine: "goproxy",
    32  	Short:     "module proxy protocol",
    33  	Long: `
    34  A Go module proxy is any web server that can respond to GET requests for
    35  URLs of a specified form. The requests have no query parameters, so even
    36  a site serving from a fixed file system (including a file:/// URL)
    37  can be a module proxy.
    38  
    39  For details on the GOPROXY protocol, see
    40  https://golang.org/ref/mod#goproxy-protocol.
    41  `,
    42  }
    43  
    44  var proxyOnce struct {
    45  	sync.Once
    46  	list []proxySpec
    47  	err  error
    48  }
    49  
    50  type proxySpec struct {
    51  	// url is the proxy URL or one of "off", "direct", "noproxy".
    52  	url string
    53  
    54  	// fallBackOnError is true if a request should be attempted on the next proxy
    55  	// in the list after any error from this proxy. If fallBackOnError is false,
    56  	// the request will only be attempted on the next proxy if the error is
    57  	// equivalent to os.ErrNotFound, which is true for 404 and 410 responses.
    58  	fallBackOnError bool
    59  }
    60  
    61  func proxyList() ([]proxySpec, error) {
    62  	proxyOnce.Do(func() {
    63  		if cfg.GONOPROXY != "" && cfg.GOPROXY != "direct" {
    64  			proxyOnce.list = append(proxyOnce.list, proxySpec{url: "noproxy"})
    65  		}
    66  
    67  		goproxy := cfg.GOPROXY
    68  		for goproxy != "" {
    69  			var url string
    70  			fallBackOnError := false
    71  			if i := strings.IndexAny(goproxy, ",|"); i >= 0 {
    72  				url = goproxy[:i]
    73  				fallBackOnError = goproxy[i] == '|'
    74  				goproxy = goproxy[i+1:]
    75  			} else {
    76  				url = goproxy
    77  				goproxy = ""
    78  			}
    79  
    80  			url = strings.TrimSpace(url)
    81  			if url == "" {
    82  				continue
    83  			}
    84  			if url == "off" {
    85  				// "off" always fails hard, so can stop walking list.
    86  				proxyOnce.list = append(proxyOnce.list, proxySpec{url: "off"})
    87  				break
    88  			}
    89  			if url == "direct" {
    90  				proxyOnce.list = append(proxyOnce.list, proxySpec{url: "direct"})
    91  				// For now, "direct" is the end of the line. We may decide to add some
    92  				// sort of fallback behavior for them in the future, so ignore
    93  				// subsequent entries for forward-compatibility.
    94  				break
    95  			}
    96  
    97  			// Single-word tokens are reserved for built-in behaviors, and anything
    98  			// containing the string ":/" or matching an absolute file path must be a
    99  			// complete URL. For all other paths, implicitly add "https://".
   100  			if strings.ContainsAny(url, ".:/") && !strings.Contains(url, ":/") && !filepath.IsAbs(url) && !pathpkg.IsAbs(url) {
   101  				url = "https://" + url
   102  			}
   103  
   104  			// Check that newProxyRepo accepts the URL.
   105  			// It won't do anything with the path.
   106  			if _, err := newProxyRepo(url, "golang.org/x/text"); err != nil {
   107  				proxyOnce.err = err
   108  				return
   109  			}
   110  
   111  			proxyOnce.list = append(proxyOnce.list, proxySpec{
   112  				url:             url,
   113  				fallBackOnError: fallBackOnError,
   114  			})
   115  		}
   116  
   117  		if len(proxyOnce.list) == 0 ||
   118  			len(proxyOnce.list) == 1 && proxyOnce.list[0].url == "noproxy" {
   119  			// There were no proxies, other than the implicit "noproxy" added when
   120  			// GONOPROXY is set. This can happen if GOPROXY is a non-empty string
   121  			// like "," or " ".
   122  			proxyOnce.err = fmt.Errorf("GOPROXY list is not the empty string, but contains no entries")
   123  		}
   124  	})
   125  
   126  	return proxyOnce.list, proxyOnce.err
   127  }
   128  
   129  // TryProxies iterates f over each configured proxy (including "noproxy" and
   130  // "direct" if applicable) until f returns no error or until f returns an
   131  // error that is not equivalent to fs.ErrNotExist on a proxy configured
   132  // not to fall back on errors.
   133  //
   134  // TryProxies then returns that final error.
   135  //
   136  // If GOPROXY is set to "off", TryProxies invokes f once with the argument
   137  // "off".
   138  func TryProxies(f func(proxy string) error) error {
   139  	proxies, err := proxyList()
   140  	if err != nil {
   141  		return err
   142  	}
   143  	if len(proxies) == 0 {
   144  		panic("GOPROXY list is empty")
   145  	}
   146  
   147  	// We try to report the most helpful error to the user. "direct" and "noproxy"
   148  	// errors are best, followed by proxy errors other than ErrNotExist, followed
   149  	// by ErrNotExist.
   150  	//
   151  	// Note that errProxyOff, errNoproxy, and errUseProxy are equivalent to
   152  	// ErrNotExist. errUseProxy should only be returned if "noproxy" is the only
   153  	// proxy. errNoproxy should never be returned, since there should always be a
   154  	// more useful error from "noproxy" first.
   155  	const (
   156  		notExistRank = iota
   157  		proxyRank
   158  		directRank
   159  	)
   160  	var bestErr error
   161  	bestErrRank := notExistRank
   162  	for _, proxy := range proxies {
   163  		err := f(proxy.url)
   164  		if err == nil {
   165  			return nil
   166  		}
   167  		isNotExistErr := errors.Is(err, fs.ErrNotExist)
   168  
   169  		if proxy.url == "direct" || (proxy.url == "noproxy" && err != errUseProxy) {
   170  			bestErr = err
   171  			bestErrRank = directRank
   172  		} else if bestErrRank <= proxyRank && !isNotExistErr {
   173  			bestErr = err
   174  			bestErrRank = proxyRank
   175  		} else if bestErrRank == notExistRank {
   176  			bestErr = err
   177  		}
   178  
   179  		if !proxy.fallBackOnError && !isNotExistErr {
   180  			break
   181  		}
   182  	}
   183  	return bestErr
   184  }
   185  
   186  type proxyRepo struct {
   187  	url          *url.URL // The combined module proxy URL joined with the module path.
   188  	path         string   // The module path (unescaped).
   189  	redactedBase string   // The base module proxy URL in [url.URL.Redacted] form.
   190  
   191  	listLatestOnce sync.Once
   192  	listLatest     *RevInfo
   193  	listLatestErr  error
   194  }
   195  
   196  func newProxyRepo(baseURL, path string) (Repo, error) {
   197  	// Parse the base proxy URL.
   198  	base, err := url.Parse(baseURL)
   199  	if err != nil {
   200  		return nil, err
   201  	}
   202  	redactedBase := base.Redacted()
   203  	switch base.Scheme {
   204  	case "http", "https":
   205  		// ok
   206  	case "file":
   207  		if *base != (url.URL{Scheme: base.Scheme, Path: base.Path, RawPath: base.RawPath}) {
   208  			return nil, fmt.Errorf("invalid file:// proxy URL with non-path elements: %s", redactedBase)
   209  		}
   210  	case "":
   211  		return nil, fmt.Errorf("invalid proxy URL missing scheme: %s", redactedBase)
   212  	default:
   213  		return nil, fmt.Errorf("invalid proxy URL scheme (must be https, http, file): %s", redactedBase)
   214  	}
   215  
   216  	// Append the module path to the URL.
   217  	url := base
   218  	enc, err := module.EscapePath(path)
   219  	if err != nil {
   220  		return nil, err
   221  	}
   222  	url.Path = strings.TrimSuffix(base.Path, "/") + "/" + enc
   223  	url.RawPath = strings.TrimSuffix(base.RawPath, "/") + "/" + pathEscape(enc)
   224  
   225  	return &proxyRepo{url, path, redactedBase, sync.Once{}, nil, nil}, nil
   226  }
   227  
   228  func (p *proxyRepo) ModulePath() string {
   229  	return p.path
   230  }
   231  
   232  var errProxyReuse = fmt.Errorf("proxy does not support CheckReuse")
   233  
   234  func (p *proxyRepo) CheckReuse(ctx context.Context, old *codehost.Origin) error {
   235  	return errProxyReuse
   236  }
   237  
   238  // versionError returns err wrapped in a ModuleError for p.path.
   239  func (p *proxyRepo) versionError(version string, err error) error {
   240  	if version != "" && version != module.CanonicalVersion(version) {
   241  		return &module.ModuleError{
   242  			Path: p.path,
   243  			Err: &module.InvalidVersionError{
   244  				Version: version,
   245  				Pseudo:  module.IsPseudoVersion(version),
   246  				Err:     err,
   247  			},
   248  		}
   249  	}
   250  
   251  	return &module.ModuleError{
   252  		Path:    p.path,
   253  		Version: version,
   254  		Err:     err,
   255  	}
   256  }
   257  
   258  func (p *proxyRepo) getBytes(ctx context.Context, path string) ([]byte, error) {
   259  	body, redactedURL, err := p.getBody(ctx, path)
   260  	if err != nil {
   261  		return nil, err
   262  	}
   263  	defer body.Close()
   264  
   265  	b, err := io.ReadAll(body)
   266  	if err != nil {
   267  		// net/http doesn't add context to Body read errors, so add it here.
   268  		// (See https://go.dev/issue/52727.)
   269  		return b, &url.Error{Op: "read", URL: redactedURL, Err: err}
   270  	}
   271  	return b, nil
   272  }
   273  
   274  func (p *proxyRepo) getBody(ctx context.Context, path string) (r io.ReadCloser, redactedURL string, err error) {
   275  	fullPath := pathpkg.Join(p.url.Path, path)
   276  
   277  	target := *p.url
   278  	target.Path = fullPath
   279  	target.RawPath = pathpkg.Join(target.RawPath, pathEscape(path))
   280  
   281  	resp, err := web.Get(web.DefaultSecurity, &target)
   282  	if err != nil {
   283  		return nil, "", err
   284  	}
   285  	if err := resp.Err(); err != nil {
   286  		resp.Body.Close()
   287  		return nil, "", err
   288  	}
   289  	return resp.Body, resp.URL, nil
   290  }
   291  
   292  func (p *proxyRepo) Versions(ctx context.Context, prefix string) (*Versions, error) {
   293  	data, err := p.getBytes(ctx, "@v/list")
   294  	if err != nil {
   295  		p.listLatestOnce.Do(func() {
   296  			p.listLatest, p.listLatestErr = nil, p.versionError("", err)
   297  		})
   298  		return nil, p.versionError("", err)
   299  	}
   300  	var list []string
   301  	allLine := strings.Split(string(data), "\n")
   302  	for _, line := range allLine {
   303  		f := strings.Fields(line)
   304  		if len(f) >= 1 && semver.IsValid(f[0]) && strings.HasPrefix(f[0], prefix) && !module.IsPseudoVersion(f[0]) {
   305  			list = append(list, f[0])
   306  		}
   307  	}
   308  	p.listLatestOnce.Do(func() {
   309  		p.listLatest, p.listLatestErr = p.latestFromList(ctx, allLine)
   310  	})
   311  	semver.Sort(list)
   312  	return &Versions{List: list}, nil
   313  }
   314  
   315  func (p *proxyRepo) latest(ctx context.Context) (*RevInfo, error) {
   316  	p.listLatestOnce.Do(func() {
   317  		data, err := p.getBytes(ctx, "@v/list")
   318  		if err != nil {
   319  			p.listLatestErr = p.versionError("", err)
   320  			return
   321  		}
   322  		list := strings.Split(string(data), "\n")
   323  		p.listLatest, p.listLatestErr = p.latestFromList(ctx, list)
   324  	})
   325  	return p.listLatest, p.listLatestErr
   326  }
   327  
   328  func (p *proxyRepo) latestFromList(ctx context.Context, allLine []string) (*RevInfo, error) {
   329  	var (
   330  		bestTime    time.Time
   331  		bestVersion string
   332  	)
   333  	for _, line := range allLine {
   334  		f := strings.Fields(line)
   335  		if len(f) >= 1 && semver.IsValid(f[0]) {
   336  			// If the proxy includes timestamps, prefer the timestamp it reports.
   337  			// Otherwise, derive the timestamp from the pseudo-version.
   338  			var (
   339  				ft time.Time
   340  			)
   341  			if len(f) >= 2 {
   342  				ft, _ = time.Parse(time.RFC3339, f[1])
   343  			} else if module.IsPseudoVersion(f[0]) {
   344  				ft, _ = module.PseudoVersionTime(f[0])
   345  			} else {
   346  				// Repo.Latest promises that this method is only called where there are
   347  				// no tagged versions. Ignore any tagged versions that were added in the
   348  				// meantime.
   349  				continue
   350  			}
   351  			if bestTime.Before(ft) {
   352  				bestTime = ft
   353  				bestVersion = f[0]
   354  			}
   355  		}
   356  	}
   357  	if bestVersion == "" {
   358  		return nil, p.versionError("", codehost.ErrNoCommits)
   359  	}
   360  
   361  	// Call Stat to get all the other fields, including Origin information.
   362  	return p.Stat(ctx, bestVersion)
   363  }
   364  
   365  func (p *proxyRepo) Stat(ctx context.Context, rev string) (*RevInfo, error) {
   366  	encRev, err := module.EscapeVersion(rev)
   367  	if err != nil {
   368  		return nil, p.versionError(rev, err)
   369  	}
   370  	data, err := p.getBytes(ctx, "@v/"+encRev+".info")
   371  	if err != nil {
   372  		return nil, p.versionError(rev, err)
   373  	}
   374  	info := new(RevInfo)
   375  	if err := json.Unmarshal(data, info); err != nil {
   376  		return nil, p.versionError(rev, fmt.Errorf("invalid response from proxy %q: %w", p.redactedBase, err))
   377  	}
   378  	if info.Version != rev && rev == module.CanonicalVersion(rev) && module.Check(p.path, rev) == nil {
   379  		// If we request a correct, appropriate version for the module path, the
   380  		// proxy must return either exactly that version or an error — not some
   381  		// arbitrary other version.
   382  		return nil, p.versionError(rev, fmt.Errorf("proxy returned info for version %s instead of requested version", info.Version))
   383  	}
   384  	return info, nil
   385  }
   386  
   387  func (p *proxyRepo) Latest(ctx context.Context) (*RevInfo, error) {
   388  	data, err := p.getBytes(ctx, "@latest")
   389  	if err != nil {
   390  		if !errors.Is(err, fs.ErrNotExist) {
   391  			return nil, p.versionError("", err)
   392  		}
   393  		return p.latest(ctx)
   394  	}
   395  	info := new(RevInfo)
   396  	if err := json.Unmarshal(data, info); err != nil {
   397  		return nil, p.versionError("", fmt.Errorf("invalid response from proxy %q: %w", p.redactedBase, err))
   398  	}
   399  	return info, nil
   400  }
   401  
   402  func (p *proxyRepo) GoMod(ctx context.Context, version string) ([]byte, error) {
   403  	if version != module.CanonicalVersion(version) {
   404  		return nil, p.versionError(version, fmt.Errorf("internal error: version passed to GoMod is not canonical"))
   405  	}
   406  
   407  	encVer, err := module.EscapeVersion(version)
   408  	if err != nil {
   409  		return nil, p.versionError(version, err)
   410  	}
   411  	data, err := p.getBytes(ctx, "@v/"+encVer+".mod")
   412  	if err != nil {
   413  		return nil, p.versionError(version, err)
   414  	}
   415  	return data, nil
   416  }
   417  
   418  func (p *proxyRepo) Zip(ctx context.Context, dst io.Writer, version string) error {
   419  	if version != module.CanonicalVersion(version) {
   420  		return p.versionError(version, fmt.Errorf("internal error: version passed to Zip is not canonical"))
   421  	}
   422  
   423  	encVer, err := module.EscapeVersion(version)
   424  	if err != nil {
   425  		return p.versionError(version, err)
   426  	}
   427  	path := "@v/" + encVer + ".zip"
   428  	body, redactedURL, err := p.getBody(ctx, path)
   429  	if err != nil {
   430  		return p.versionError(version, err)
   431  	}
   432  	defer body.Close()
   433  
   434  	lr := &io.LimitedReader{R: body, N: codehost.MaxZipFile + 1}
   435  	if _, err := io.Copy(dst, lr); err != nil {
   436  		// net/http doesn't add context to Body read errors, so add it here.
   437  		// (See https://go.dev/issue/52727.)
   438  		err = &url.Error{Op: "read", URL: redactedURL, Err: err}
   439  		return p.versionError(version, err)
   440  	}
   441  	if lr.N <= 0 {
   442  		return p.versionError(version, fmt.Errorf("downloaded zip file too large"))
   443  	}
   444  	return nil
   445  }
   446  
   447  // pathEscape escapes s so it can be used in a path.
   448  // That is, it escapes things like ? and # (which really shouldn't appear anyway).
   449  // It does not escape / to %2F: our REST API is designed so that / can be left as is.
   450  func pathEscape(s string) string {
   451  	return strings.ReplaceAll(url.PathEscape(s), "%2F", "/")
   452  }
   453
View as plain text