stream.go

     1  // Copyright 2010 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  //go:build !goexperiment.jsonv2
     6  
     7  package json
     8  
     9  import (
    10  	"bytes"
    11  	"errors"
    12  	"io"
    13  )
    14  
    15  // A Decoder reads and decodes JSON values from an input stream.
    16  type Decoder struct {
    17  	r       io.Reader
    18  	buf     []byte
    19  	d       decodeState
    20  	scanp   int   // start of unread data in buf
    21  	scanned int64 // amount of data already scanned
    22  	scan    scanner
    23  	err     error
    24  
    25  	tokenState int
    26  	tokenStack []int
    27  }
    28  
    29  // NewDecoder returns a new decoder that reads from r.
    30  //
    31  // The decoder introduces its own buffering and may
    32  // read data from r beyond the JSON values requested.
    33  func NewDecoder(r io.Reader) *Decoder {
    34  	return &Decoder{r: r}
    35  }
    36  
    37  // UseNumber causes the Decoder to unmarshal a number into an
    38  // interface value as a [Number] instead of as a float64.
    39  func (dec *Decoder) UseNumber() { dec.d.useNumber = true }
    40  
    41  // DisallowUnknownFields causes the Decoder to return an error when the destination
    42  // is a struct and the input contains object keys which do not match any
    43  // non-ignored, exported fields in the destination.
    44  func (dec *Decoder) DisallowUnknownFields() { dec.d.disallowUnknownFields = true }
    45  
    46  // Decode reads the next JSON-encoded value from its
    47  // input and stores it in the value pointed to by v.
    48  //
    49  // See the documentation for [Unmarshal] for details about
    50  // the conversion of JSON into a Go value.
    51  func (dec *Decoder) Decode(v any) error {
    52  	if dec.err != nil {
    53  		return dec.err
    54  	}
    55  
    56  	if err := dec.tokenPrepareForDecode(); err != nil {
    57  		return err
    58  	}
    59  
    60  	if !dec.tokenValueAllowed() {
    61  		return &SyntaxError{msg: "not at beginning of value", Offset: dec.InputOffset()}
    62  	}
    63  
    64  	// Read whole value into buffer.
    65  	n, err := dec.readValue()
    66  	if err != nil {
    67  		return err
    68  	}
    69  	dec.d.init(dec.buf[dec.scanp : dec.scanp+n])
    70  	dec.scanp += n
    71  
    72  	// Don't save err from unmarshal into dec.err:
    73  	// the connection is still usable since we read a complete JSON
    74  	// object from it before the error happened.
    75  	err = dec.d.unmarshal(v)
    76  
    77  	// fixup token streaming state
    78  	dec.tokenValueEnd()
    79  
    80  	return err
    81  }
    82  
    83  // Buffered returns a reader of the data remaining in the unread buffer,
    84  // which may contain zero or more bytes.
    85  // This is the data already consumed from the input [io.Reader],
    86  // but not yet read by a [Decoder.Decode] or [Decoder.Token] call.
    87  // It may contain bytes that do not form valid JSON as it has not yet
    88  // been validated according to the JSON grammar.
    89  // The exact amount of buffered data is an implementation detail
    90  // of the Decoder and may change over time.
    91  //
    92  // It is the caller's responsibility to concatenate this buffer with
    93  // the remainder of the input Reader to obtain the full sequence
    94  // of bytes after the last decoded JSON value.
    95  //
    96  // The reader is valid until the next call to [Decoder.Decode] or [Decoder.Token].
    97  func (dec *Decoder) Buffered() io.Reader {
    98  	return bytes.NewReader(dec.buf[dec.scanp:])
    99  }
   100  
   101  // readValue reads a JSON value into dec.buf.
   102  // It returns the length of the encoding.
   103  func (dec *Decoder) readValue() (int, error) {
   104  	dec.scan.reset()
   105  
   106  	scanp := dec.scanp
   107  	var err error
   108  Input:
   109  	// help the compiler see that scanp is never negative, so it can remove
   110  	// some bounds checks below.
   111  	for scanp >= 0 {
   112  
   113  		// Look in the buffer for a new value.
   114  		for ; scanp < len(dec.buf); scanp++ {
   115  			c := dec.buf[scanp]
   116  			dec.scan.bytes++
   117  			switch dec.scan.step(&dec.scan, c) {
   118  			case scanEnd:
   119  				// scanEnd is delayed one byte so we decrement
   120  				// the scanner bytes count by 1 to ensure that
   121  				// this value is correct in the next call of Decode.
   122  				dec.scan.bytes--
   123  				break Input
   124  			case scanEndObject, scanEndArray:
   125  				// scanEnd is delayed one byte.
   126  				// We might block trying to get that byte from src,
   127  				// so instead invent a space byte.
   128  				if stateEndValue(&dec.scan, ' ') == scanEnd {
   129  					scanp++
   130  					break Input
   131  				}
   132  			case scanError:
   133  				dec.err = dec.scan.err
   134  				return 0, dec.scan.err
   135  			}
   136  		}
   137  
   138  		// Did the last read have an error?
   139  		// Delayed until now to allow buffer scan.
   140  		if err != nil {
   141  			if err == io.EOF {
   142  				if dec.scan.step(&dec.scan, ' ') == scanEnd {
   143  					break Input
   144  				}
   145  				if nonSpace(dec.buf) {
   146  					err = io.ErrUnexpectedEOF
   147  				}
   148  			}
   149  			dec.err = err
   150  			return 0, err
   151  		}
   152  
   153  		n := scanp - dec.scanp
   154  		err = dec.refill()
   155  		scanp = dec.scanp + n
   156  	}
   157  	return scanp - dec.scanp, nil
   158  }
   159  
   160  func (dec *Decoder) refill() error {
   161  	// Make room to read more into the buffer.
   162  	// First slide down data already consumed.
   163  	if dec.scanp > 0 {
   164  		dec.scanned += int64(dec.scanp)
   165  		n := copy(dec.buf, dec.buf[dec.scanp:])
   166  		dec.buf = dec.buf[:n]
   167  		dec.scanp = 0
   168  	}
   169  
   170  	// Grow buffer if not large enough.
   171  	const minRead = 512
   172  	if cap(dec.buf)-len(dec.buf) < minRead {
   173  		newBuf := make([]byte, len(dec.buf), 2*cap(dec.buf)+minRead)
   174  		copy(newBuf, dec.buf)
   175  		dec.buf = newBuf
   176  	}
   177  
   178  	// Read. Delay error for next iteration (after scan).
   179  	n, err := dec.r.Read(dec.buf[len(dec.buf):cap(dec.buf)])
   180  	dec.buf = dec.buf[0 : len(dec.buf)+n]
   181  
   182  	return err
   183  }
   184  
   185  func nonSpace(b []byte) bool {
   186  	for _, c := range b {
   187  		if !isSpace(c) {
   188  			return true
   189  		}
   190  	}
   191  	return false
   192  }
   193  
   194  // An Encoder writes JSON values to an output stream.
   195  type Encoder struct {
   196  	w          io.Writer
   197  	err        error
   198  	escapeHTML bool
   199  
   200  	indentBuf    []byte
   201  	indentPrefix string
   202  	indentValue  string
   203  }
   204  
   205  // NewEncoder returns a new encoder that writes to w.
   206  func NewEncoder(w io.Writer) *Encoder {
   207  	return &Encoder{w: w, escapeHTML: true}
   208  }
   209  
   210  // Encode writes the JSON encoding of v to the stream,
   211  // with insignificant space characters elided,
   212  // followed by a newline character.
   213  //
   214  // See the documentation for [Marshal] for details about the
   215  // conversion of Go values to JSON.
   216  func (enc *Encoder) Encode(v any) error {
   217  	if enc.err != nil {
   218  		return enc.err
   219  	}
   220  
   221  	e := newEncodeState()
   222  	defer encodeStatePool.Put(e)
   223  
   224  	err := e.marshal(v, encOpts{escapeHTML: enc.escapeHTML})
   225  	if err != nil {
   226  		return err
   227  	}
   228  
   229  	// Terminate each value with a newline.
   230  	// This makes the output look a little nicer
   231  	// when debugging, and some kind of space
   232  	// is required if the encoded value was a number,
   233  	// so that the reader knows there aren't more
   234  	// digits coming.
   235  	e.WriteByte('\n')
   236  
   237  	b := e.Bytes()
   238  	if enc.indentPrefix != "" || enc.indentValue != "" {
   239  		enc.indentBuf, err = appendIndent(enc.indentBuf[:0], b, enc.indentPrefix, enc.indentValue)
   240  		if err != nil {
   241  			return err
   242  		}
   243  		b = enc.indentBuf
   244  	}
   245  	if _, err = enc.w.Write(b); err != nil {
   246  		enc.err = err
   247  	}
   248  	return err
   249  }
   250  
   251  // SetIndent instructs the encoder to format each subsequent encoded
   252  // value as if indented by the package-level function Indent(dst, src, prefix, indent).
   253  // Calling SetIndent("", "") disables indentation.
   254  func (enc *Encoder) SetIndent(prefix, indent string) {
   255  	enc.indentPrefix = prefix
   256  	enc.indentValue = indent
   257  }
   258  
   259  // SetEscapeHTML specifies whether problematic HTML characters
   260  // should be escaped inside JSON quoted strings.
   261  // The default behavior is to escape &, <, and > to \u0026, \u003c, and \u003e
   262  // to avoid certain safety problems that can arise when embedding JSON in HTML.
   263  //
   264  // In non-HTML settings where the escaping interferes with the readability
   265  // of the output, SetEscapeHTML(false) disables this behavior.
   266  func (enc *Encoder) SetEscapeHTML(on bool) {
   267  	enc.escapeHTML = on
   268  }
   269  
   270  // RawMessage is a raw encoded JSON value.
   271  // It implements [Marshaler] and [Unmarshaler] and can
   272  // be used to delay JSON decoding or precompute a JSON encoding.
   273  type RawMessage []byte
   274  
   275  // MarshalJSON returns m as the JSON encoding of m.
   276  func (m RawMessage) MarshalJSON() ([]byte, error) {
   277  	if m == nil {
   278  		return []byte("null"), nil
   279  	}
   280  	return m, nil
   281  }
   282  
   283  // UnmarshalJSON sets *m to a copy of data.
   284  func (m *RawMessage) UnmarshalJSON(data []byte) error {
   285  	if m == nil {
   286  		return errors.New("json.RawMessage: UnmarshalJSON on nil pointer")
   287  	}
   288  	*m = append((*m)[0:0], data...)
   289  	return nil
   290  }
   291  
   292  var _ Marshaler = (*RawMessage)(nil)
   293  var _ Unmarshaler = (*RawMessage)(nil)
   294  
   295  // A Token holds a value of one of these types:
   296  //
   297  //   - [Delim], for the four JSON delimiters [ ] { }
   298  //   - bool, for JSON booleans
   299  //   - float64, for JSON numbers
   300  //   - [Number], for JSON numbers
   301  //   - string, for JSON string literals
   302  //   - nil, for JSON null
   303  type Token any
   304  
   305  const (
   306  	tokenTopValue = iota
   307  	tokenArrayStart
   308  	tokenArrayValue
   309  	tokenArrayComma
   310  	tokenObjectStart
   311  	tokenObjectKey
   312  	tokenObjectColon
   313  	tokenObjectValue
   314  	tokenObjectComma
   315  )
   316  
   317  // advance tokenstate from a separator state to a value state
   318  func (dec *Decoder) tokenPrepareForDecode() error {
   319  	// Note: Not calling peek before switch, to avoid
   320  	// putting peek into the standard Decode path.
   321  	// peek is only called when using the Token API.
   322  	switch dec.tokenState {
   323  	case tokenArrayComma:
   324  		c, err := dec.peek()
   325  		if err != nil {
   326  			return err
   327  		}
   328  		if c != ',' {
   329  			return &SyntaxError{"expected comma after array element", dec.InputOffset()}
   330  		}
   331  		dec.scanp++
   332  		dec.tokenState = tokenArrayValue
   333  	case tokenObjectColon:
   334  		c, err := dec.peek()
   335  		if err != nil {
   336  			return err
   337  		}
   338  		if c != ':' {
   339  			return &SyntaxError{"expected colon after object key", dec.InputOffset()}
   340  		}
   341  		dec.scanp++
   342  		dec.tokenState = tokenObjectValue
   343  	}
   344  	return nil
   345  }
   346  
   347  func (dec *Decoder) tokenValueAllowed() bool {
   348  	switch dec.tokenState {
   349  	case tokenTopValue, tokenArrayStart, tokenArrayValue, tokenObjectValue:
   350  		return true
   351  	}
   352  	return false
   353  }
   354  
   355  func (dec *Decoder) tokenValueEnd() {
   356  	switch dec.tokenState {
   357  	case tokenArrayStart, tokenArrayValue:
   358  		dec.tokenState = tokenArrayComma
   359  	case tokenObjectValue:
   360  		dec.tokenState = tokenObjectComma
   361  	}
   362  }
   363  
   364  // A Delim is a JSON array or object delimiter, one of [ ] { or }.
   365  type Delim rune
   366  
   367  func (d Delim) String() string {
   368  	return string(d)
   369  }
   370  
   371  // Token returns the next JSON token in the input stream.
   372  // At the end of the input stream, Token returns nil, [io.EOF].
   373  //
   374  // Token guarantees that the delimiters [ ] { } it returns are
   375  // properly nested and matched: if Token encounters an unexpected
   376  // delimiter in the input, it will return an error.
   377  //
   378  // The input stream consists of basic JSON values—bool, string,
   379  // number, and null—along with delimiters [ ] { } of type [Delim]
   380  // to mark the start and end of arrays and objects.
   381  // Commas and colons are elided.
   382  func (dec *Decoder) Token() (Token, error) {
   383  	for {
   384  		c, err := dec.peek()
   385  		if err != nil {
   386  			return nil, err
   387  		}
   388  		switch c {
   389  		case '[':
   390  			if !dec.tokenValueAllowed() {
   391  				return dec.tokenError(c)
   392  			}
   393  			dec.scanp++
   394  			dec.tokenStack = append(dec.tokenStack, dec.tokenState)
   395  			dec.tokenState = tokenArrayStart
   396  			return Delim('['), nil
   397  
   398  		case ']':
   399  			if dec.tokenState != tokenArrayStart && dec.tokenState != tokenArrayComma {
   400  				return dec.tokenError(c)
   401  			}
   402  			dec.scanp++
   403  			dec.tokenState = dec.tokenStack[len(dec.tokenStack)-1]
   404  			dec.tokenStack = dec.tokenStack[:len(dec.tokenStack)-1]
   405  			dec.tokenValueEnd()
   406  			return Delim(']'), nil
   407  
   408  		case '{':
   409  			if !dec.tokenValueAllowed() {
   410  				return dec.tokenError(c)
   411  			}
   412  			dec.scanp++
   413  			dec.tokenStack = append(dec.tokenStack, dec.tokenState)
   414  			dec.tokenState = tokenObjectStart
   415  			return Delim('{'), nil
   416  
   417  		case '}':
   418  			if dec.tokenState != tokenObjectStart && dec.tokenState != tokenObjectComma {
   419  				return dec.tokenError(c)
   420  			}
   421  			dec.scanp++
   422  			dec.tokenState = dec.tokenStack[len(dec.tokenStack)-1]
   423  			dec.tokenStack = dec.tokenStack[:len(dec.tokenStack)-1]
   424  			dec.tokenValueEnd()
   425  			return Delim('}'), nil
   426  
   427  		case ':':
   428  			if dec.tokenState != tokenObjectColon {
   429  				return dec.tokenError(c)
   430  			}
   431  			dec.scanp++
   432  			dec.tokenState = tokenObjectValue
   433  			continue
   434  
   435  		case ',':
   436  			if dec.tokenState == tokenArrayComma {
   437  				dec.scanp++
   438  				dec.tokenState = tokenArrayValue
   439  				continue
   440  			}
   441  			if dec.tokenState == tokenObjectComma {
   442  				dec.scanp++
   443  				dec.tokenState = tokenObjectKey
   444  				continue
   445  			}
   446  			return dec.tokenError(c)
   447  
   448  		case '"':
   449  			if dec.tokenState == tokenObjectStart || dec.tokenState == tokenObjectKey {
   450  				var x string
   451  				old := dec.tokenState
   452  				dec.tokenState = tokenTopValue
   453  				err := dec.Decode(&x)
   454  				dec.tokenState = old
   455  				if err != nil {
   456  					return nil, err
   457  				}
   458  				dec.tokenState = tokenObjectColon
   459  				return x, nil
   460  			}
   461  			fallthrough
   462  
   463  		default:
   464  			if !dec.tokenValueAllowed() {
   465  				return dec.tokenError(c)
   466  			}
   467  			var x any
   468  			if err := dec.Decode(&x); err != nil {
   469  				return nil, err
   470  			}
   471  			return x, nil
   472  		}
   473  	}
   474  }
   475  
   476  func (dec *Decoder) tokenError(c byte) (Token, error) {
   477  	var context string
   478  	switch dec.tokenState {
   479  	case tokenTopValue:
   480  		context = " looking for beginning of value"
   481  	case tokenArrayStart, tokenArrayValue, tokenObjectValue:
   482  		context = " looking for beginning of value"
   483  	case tokenArrayComma:
   484  		context = " after array element"
   485  	case tokenObjectKey:
   486  		context = " looking for beginning of object key string"
   487  	case tokenObjectColon:
   488  		context = " after object key"
   489  	case tokenObjectComma:
   490  		context = " after object key:value pair"
   491  	}
   492  	return nil, &SyntaxError{"invalid character " + quoteChar(c) + context, dec.InputOffset()}
   493  }
   494  
   495  // More reports whether there is another element in the
   496  // current array or object being parsed.
   497  func (dec *Decoder) More() bool {
   498  	c, err := dec.peek()
   499  	return err == nil && c != ']' && c != '}'
   500  }
   501  
   502  func (dec *Decoder) peek() (byte, error) {
   503  	var err error
   504  	for {
   505  		for i := dec.scanp; i < len(dec.buf); i++ {
   506  			c := dec.buf[i]
   507  			if isSpace(c) {
   508  				continue
   509  			}
   510  			dec.scanp = i
   511  			return c, nil
   512  		}
   513  		// buffer has been scanned, now report any error
   514  		if err != nil {
   515  			return 0, err
   516  		}
   517  		err = dec.refill()
   518  	}
   519  }
   520  
   521  // InputOffset returns the input stream byte offset of the current decoder position.
   522  // The offset gives the location of the end of the most recently returned token
   523  // and the beginning of the next token.
   524  func (dec *Decoder) InputOffset() int64 {
   525  	return dec.scanned + int64(dec.scanp)
   526  }
   527
View as plain text