Source file
src/encoding/json/v2_stream_test.go
1
2
3
4
5
6
7 package json
8
9 import (
10 "bytes"
11 "errors"
12 "fmt"
13 "io"
14 "log"
15 "net"
16 "net/http"
17 "net/http/httptest"
18 "reflect"
19 "runtime/debug"
20 "strings"
21 "testing"
22
23 "encoding/json/internal/jsontest"
24 )
25
26 type CaseName = jsontest.CaseName
27 type CasePos = jsontest.CasePos
28
29 var Name = jsontest.Name
30
31
32
33 var streamTest = []any{
34 0.1,
35 "hello",
36 nil,
37 true,
38 false,
39 []any{"a", "b", "c"},
40 map[string]any{"K": "Kelvin", "ß": "long s"},
41 3.14,
42 }
43
44 var streamEncoded = `0.1
45 "hello"
46 null
47 true
48 false
49 ["a","b","c"]
50 {"ß":"long s","K":"Kelvin"}
51 3.14
52 `
53
54 func TestEncoder(t *testing.T) {
55 for i := 0; i <= len(streamTest); i++ {
56 var buf strings.Builder
57 enc := NewEncoder(&buf)
58
59 enc.SetIndent(">", ".")
60 enc.SetIndent("", "")
61 for j, v := range streamTest[0:i] {
62 if err := enc.Encode(v); err != nil {
63 t.Fatalf("#%d.%d Encode error: %v", i, j, err)
64 }
65 }
66 if got, want := buf.String(), nlines(streamEncoded, i); got != want {
67 t.Errorf("encoding %d items: mismatch:", i)
68 diff(t, []byte(got), []byte(want))
69 break
70 }
71 }
72 }
73
74 func TestEncoderErrorAndReuseEncodeState(t *testing.T) {
75
76 percent := debug.SetGCPercent(-1)
77 defer debug.SetGCPercent(percent)
78
79
80 type Dummy struct {
81 Name string
82 Next *Dummy
83 }
84 dummy := Dummy{Name: "Dummy"}
85 dummy.Next = &dummy
86
87 var buf bytes.Buffer
88 enc := NewEncoder(&buf)
89 if err := enc.Encode(dummy); err == nil {
90 t.Errorf("Encode(dummy) error: got nil, want non-nil")
91 }
92
93 type Data struct {
94 A string
95 I int
96 }
97 want := Data{A: "a", I: 1}
98 if err := enc.Encode(want); err != nil {
99 t.Errorf("Marshal error: %v", err)
100 }
101
102 var got Data
103 if err := Unmarshal(buf.Bytes(), &got); err != nil {
104 t.Errorf("Unmarshal error: %v", err)
105 }
106 if got != want {
107 t.Errorf("Marshal/Unmarshal roundtrip:\n\tgot: %v\n\twant: %v", got, want)
108 }
109 }
110
111 var streamEncodedIndent = `0.1
112 "hello"
113 null
114 true
115 false
116 [
117 >."a",
118 >."b",
119 >."c"
120 >]
121 {
122 >."ß": "long s",
123 >."K": "Kelvin"
124 >}
125 3.14
126 `
127
128 func TestEncoderIndent(t *testing.T) {
129 var buf strings.Builder
130 enc := NewEncoder(&buf)
131 enc.SetIndent(">", ".")
132 for _, v := range streamTest {
133 enc.Encode(v)
134 }
135 if got, want := buf.String(), streamEncodedIndent; got != want {
136 t.Errorf("Encode mismatch:\ngot:\n%s\n\nwant:\n%s", got, want)
137 diff(t, []byte(got), []byte(want))
138 }
139 }
140
141 type strMarshaler string
142
143 func (s strMarshaler) MarshalJSON() ([]byte, error) {
144 return []byte(s), nil
145 }
146
147 type strPtrMarshaler string
148
149 func (s *strPtrMarshaler) MarshalJSON() ([]byte, error) {
150 return []byte(*s), nil
151 }
152
153 func TestEncoderSetEscapeHTML(t *testing.T) {
154 var c C
155 var ct CText
156 var tagStruct struct {
157 Valid int `json:"<>&#! "`
158 Invalid int `json:"\\"`
159 }
160
161
162
163
164 marshalerStruct := &struct {
165 NonPtr strMarshaler
166 Ptr strPtrMarshaler
167 }{`"<str>"`, `"<str>"`}
168
169
170 stringOption := struct {
171 Bar string `json:"bar,string"`
172 }{`<html>foobar</html>`}
173
174 tests := []struct {
175 CaseName
176 v any
177 wantEscape string
178 want string
179 }{
180 {Name("c"), c, `"\u003c\u0026\u003e"`, `"<&>"`},
181 {Name("ct"), ct, `"\"\u003c\u0026\u003e\""`, `"\"<&>\""`},
182 {Name(`"<&>"`), "<&>", `"\u003c\u0026\u003e"`, `"<&>"`},
183 {
184 Name("tagStruct"), tagStruct,
185 `{"\u003c\u003e\u0026#! ":0,"Invalid":0}`,
186 `{"<>&#! ":0,"Invalid":0}`,
187 },
188 {
189 Name(`"<str>"`), marshalerStruct,
190 `{"NonPtr":"\u003cstr\u003e","Ptr":"\u003cstr\u003e"}`,
191 `{"NonPtr":"<str>","Ptr":"<str>"}`,
192 },
193 {
194 Name("stringOption"), stringOption,
195 `{"bar":"\"\\u003chtml\\u003efoobar\\u003c/html\\u003e\""}`,
196 `{"bar":"\"<html>foobar</html>\""}`,
197 },
198 }
199 for _, tt := range tests {
200 t.Run(tt.Name, func(t *testing.T) {
201 var buf strings.Builder
202 enc := NewEncoder(&buf)
203 if err := enc.Encode(tt.v); err != nil {
204 t.Fatalf("%s: Encode(%s) error: %s", tt.Where, tt.Name, err)
205 }
206 if got := strings.TrimSpace(buf.String()); got != tt.wantEscape {
207 t.Errorf("%s: Encode(%s):\n\tgot: %s\n\twant: %s", tt.Where, tt.Name, got, tt.wantEscape)
208 }
209 buf.Reset()
210 enc.SetEscapeHTML(false)
211 if err := enc.Encode(tt.v); err != nil {
212 t.Fatalf("%s: SetEscapeHTML(false) Encode(%s) error: %s", tt.Where, tt.Name, err)
213 }
214 if got := strings.TrimSpace(buf.String()); got != tt.want {
215 t.Errorf("%s: SetEscapeHTML(false) Encode(%s):\n\tgot: %s\n\twant: %s",
216 tt.Where, tt.Name, got, tt.want)
217 }
218 })
219 }
220 }
221
222 func TestDecoder(t *testing.T) {
223 for i := 0; i <= len(streamTest); i++ {
224
225
226
227
228
229 var buf bytes.Buffer
230 for _, c := range nlines(streamEncoded, i) {
231 if c != '\n' {
232 buf.WriteRune(c)
233 }
234 }
235 out := make([]any, i)
236 dec := NewDecoder(&buf)
237 for j := range out {
238 if err := dec.Decode(&out[j]); err != nil {
239 t.Fatalf("decode #%d/%d error: %v", j, i, err)
240 }
241 }
242 if !reflect.DeepEqual(out, streamTest[0:i]) {
243 t.Errorf("decoding %d items: mismatch:", i)
244 for j := range out {
245 if !reflect.DeepEqual(out[j], streamTest[j]) {
246 t.Errorf("#%d:\n\tgot: %v\n\twant: %v", j, out[j], streamTest[j])
247 }
248 }
249 break
250 }
251 }
252 }
253
254 func TestDecoderBuffered(t *testing.T) {
255 r := strings.NewReader(`{"Name": "Gopher"} extra `)
256 var m struct {
257 Name string
258 }
259 d := NewDecoder(r)
260 err := d.Decode(&m)
261 if err != nil {
262 t.Fatal(err)
263 }
264 if m.Name != "Gopher" {
265 t.Errorf("Name = %s, want Gopher", m.Name)
266 }
267 rest, err := io.ReadAll(d.Buffered())
268 if err != nil {
269 t.Fatal(err)
270 }
271 if got, want := string(rest), " extra "; got != want {
272 t.Errorf("Remaining = %s, want %s", got, want)
273 }
274 }
275
276 func nlines(s string, n int) string {
277 if n <= 0 {
278 return ""
279 }
280 for i, c := range s {
281 if c == '\n' {
282 if n--; n == 0 {
283 return s[0 : i+1]
284 }
285 }
286 }
287 return s
288 }
289
290 func TestRawMessage(t *testing.T) {
291 var data struct {
292 X float64
293 Id RawMessage
294 Y float32
295 }
296 const raw = `["\u0056",null]`
297 const want = `{"X":0.1,"Id":["\u0056",null],"Y":0.2}`
298 err := Unmarshal([]byte(want), &data)
299 if err != nil {
300 t.Fatalf("Unmarshal error: %v", err)
301 }
302 if string([]byte(data.Id)) != raw {
303 t.Fatalf("Unmarshal:\n\tgot: %s\n\twant: %s", []byte(data.Id), raw)
304 }
305 got, err := Marshal(&data)
306 if err != nil {
307 t.Fatalf("Marshal error: %v", err)
308 }
309 if string(got) != want {
310 t.Fatalf("Marshal:\n\tgot: %s\n\twant: %s", got, want)
311 }
312 }
313
314 func TestNullRawMessage(t *testing.T) {
315 var data struct {
316 X float64
317 Id RawMessage
318 IdPtr *RawMessage
319 Y float32
320 }
321 const want = `{"X":0.1,"Id":null,"IdPtr":null,"Y":0.2}`
322 err := Unmarshal([]byte(want), &data)
323 if err != nil {
324 t.Fatalf("Unmarshal error: %v", err)
325 }
326 if want, got := "null", string(data.Id); want != got {
327 t.Fatalf("Unmarshal:\n\tgot: %s\n\twant: %s", got, want)
328 }
329 if data.IdPtr != nil {
330 t.Fatalf("pointer mismatch: got non-nil, want nil")
331 }
332 got, err := Marshal(&data)
333 if err != nil {
334 t.Fatalf("Marshal error: %v", err)
335 }
336 if string(got) != want {
337 t.Fatalf("Marshal:\n\tgot: %s\n\twant: %s", got, want)
338 }
339 }
340
341 func TestBlocking(t *testing.T) {
342 tests := []struct {
343 CaseName
344 in string
345 }{
346 {Name(""), `{"x": 1}`},
347 {Name(""), `[1, 2, 3]`},
348 }
349 for _, tt := range tests {
350 t.Run(tt.Name, func(t *testing.T) {
351 r, w := net.Pipe()
352 go w.Write([]byte(tt.in))
353 var val any
354
355
356
357 if err := NewDecoder(r).Decode(&val); err != nil {
358 t.Errorf("%s: NewDecoder(%s).Decode error: %v", tt.Where, tt.in, err)
359 }
360 r.Close()
361 w.Close()
362 })
363 }
364 }
365
366 type decodeThis struct {
367 v any
368 }
369
370 func TestDecodeInStream(t *testing.T) {
371 tests := []struct {
372 CaseName
373 json string
374 expTokens []any
375 }{
376
377 {CaseName: Name(""), json: `10`, expTokens: []any{float64(10)}},
378 {CaseName: Name(""), json: ` [10] `, expTokens: []any{
379 Delim('['), float64(10), Delim(']')}},
380 {CaseName: Name(""), json: ` [false,10,"b"] `, expTokens: []any{
381 Delim('['), false, float64(10), "b", Delim(']')}},
382 {CaseName: Name(""), json: `{ "a": 1 }`, expTokens: []any{
383 Delim('{'), "a", float64(1), Delim('}')}},
384 {CaseName: Name(""), json: `{"a": 1, "b":"3"}`, expTokens: []any{
385 Delim('{'), "a", float64(1), "b", "3", Delim('}')}},
386 {CaseName: Name(""), json: ` [{"a": 1},{"a": 2}] `, expTokens: []any{
387 Delim('['),
388 Delim('{'), "a", float64(1), Delim('}'),
389 Delim('{'), "a", float64(2), Delim('}'),
390 Delim(']')}},
391 {CaseName: Name(""), json: `{"obj": {"a": 1}}`, expTokens: []any{
392 Delim('{'), "obj", Delim('{'), "a", float64(1), Delim('}'),
393 Delim('}')}},
394 {CaseName: Name(""), json: `{"obj": [{"a": 1}]}`, expTokens: []any{
395 Delim('{'), "obj", Delim('['),
396 Delim('{'), "a", float64(1), Delim('}'),
397 Delim(']'), Delim('}')}},
398
399
400 {CaseName: Name(""), json: `{ "a": 1 }`, expTokens: []any{
401 Delim('{'), "a",
402 decodeThis{float64(1)},
403 Delim('}')}},
404 {CaseName: Name(""), json: ` [ { "a" : 1 } ] `, expTokens: []any{
405 Delim('['),
406 decodeThis{map[string]any{"a": float64(1)}},
407 Delim(']')}},
408 {CaseName: Name(""), json: ` [{"a": 1},{"a": 2}] `, expTokens: []any{
409 Delim('['),
410 decodeThis{map[string]any{"a": float64(1)}},
411 decodeThis{map[string]any{"a": float64(2)}},
412 Delim(']')}},
413 {CaseName: Name(""), json: `{ "obj" : [ { "a" : 1 } ] }`, expTokens: []any{
414 Delim('{'), "obj", Delim('['),
415 decodeThis{map[string]any{"a": float64(1)}},
416 Delim(']'), Delim('}')}},
417
418 {CaseName: Name(""), json: `{"obj": {"a": 1}}`, expTokens: []any{
419 Delim('{'), "obj",
420 decodeThis{map[string]any{"a": float64(1)}},
421 Delim('}')}},
422 {CaseName: Name(""), json: `{"obj": [{"a": 1}]}`, expTokens: []any{
423 Delim('{'), "obj",
424 decodeThis{[]any{
425 map[string]any{"a": float64(1)},
426 }},
427 Delim('}')}},
428 {CaseName: Name(""), json: ` [{"a": 1} {"a": 2}] `, expTokens: []any{
429 Delim('['),
430 decodeThis{map[string]any{"a": float64(1)}},
431 decodeThis{&SyntaxError{"invalid character '{' after array element", len64(` [{"a": 1} {`)}},
432 }},
433 {CaseName: Name(""), json: `{ "` + strings.Repeat("a", 513) + `" 1 }`, expTokens: []any{
434 Delim('{'), strings.Repeat("a", 513),
435 decodeThis{&SyntaxError{"invalid character '1' after object key", len64(`{ "`) + 513 + len64(`" 1`)}},
436 }},
437 {CaseName: Name(""), json: `{ "\a" }`, expTokens: []any{
438 Delim('{'),
439 &SyntaxError{"invalid escape sequence `\\a` in string", len64(`{ "\a`)},
440 }},
441 {CaseName: Name(""), json: ` \a`, expTokens: []any{
442 &SyntaxError{"invalid character '\\\\' looking for beginning of value", len64(` \`)},
443 }},
444 {CaseName: Name(""), json: `,`, expTokens: []any{
445 &SyntaxError{"invalid character ',' looking for beginning of value", len64(`,`)},
446 }},
447 }
448 for _, tt := range tests {
449 t.Run(tt.Name, func(t *testing.T) {
450 dec := NewDecoder(strings.NewReader(tt.json))
451 for i, want := range tt.expTokens {
452 var got any
453 var err error
454
455 wantMore := true
456 switch want {
457 case Delim(']'), Delim('}'):
458 wantMore = false
459 }
460 if got := dec.More(); got != wantMore {
461 t.Fatalf("%s:\n\tinput: %s\n\tdec.More() = %v, want %v (next token: %T(%v)) rem:%q", tt.Where, tt.json, got, wantMore, want, want, tt.json[dec.InputOffset():])
462 }
463
464 if dt, ok := want.(decodeThis); ok {
465 want = dt.v
466 err = dec.Decode(&got)
467 } else {
468 got, err = dec.Token()
469 }
470 if errWant, ok := want.(error); ok {
471 if err == nil || !reflect.DeepEqual(err, errWant) {
472 t.Fatalf("%s:\n\tinput: %s\n\tgot error: %#v\n\twant error: %#v", tt.Where, tt.json, err, errWant)
473 }
474 break
475 } else if err != nil {
476 t.Fatalf("%s:\n\tinput: %s\n\tgot error: %v\n\twant error: nil", tt.Where, tt.json, err)
477 }
478 if !reflect.DeepEqual(got, want) {
479 t.Fatalf("%s: token %d:\n\tinput: %s\n\tgot: %T(%v)\n\twant: %T(%v)", tt.Where, i, tt.json, got, got, want, want)
480 }
481 }
482 })
483 }
484 }
485
486
487 func TestHTTPDecoding(t *testing.T) {
488 const raw = `{ "foo": "bar" }`
489
490 ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
491 w.Write([]byte(raw))
492 }))
493 defer ts.Close()
494 res, err := http.Get(ts.URL)
495 if err != nil {
496 log.Fatalf("http.Get error: %v", err)
497 }
498 defer res.Body.Close()
499
500 foo := struct {
501 Foo string
502 }{}
503
504 d := NewDecoder(res.Body)
505 err = d.Decode(&foo)
506 if err != nil {
507 t.Fatalf("Decode error: %v", err)
508 }
509 if foo.Foo != "bar" {
510 t.Errorf(`Decode: got %q, want "bar"`, foo.Foo)
511 }
512
513
514 err = d.Decode(&foo)
515 if err != io.EOF {
516 t.Errorf("Decode error:\n\tgot: %v\n\twant: io.EOF", err)
517 }
518 }
519
520
521 type readerFunc func([]byte) (int, error)
522
523 func (f readerFunc) Read(b []byte) (int, error) {
524 return f(b)
525 }
526
527 func TestTokenError(t *testing.T) {
528 tests := []struct {
529 in string
530 inErr error
531 err error
532 }{
533 {in: ``, err: io.EOF},
534 {in: `{`, err: io.EOF},
535 {in: `{"`, err: io.ErrUnexpectedEOF},
536 {in: `{"k"`, err: io.EOF},
537 {in: `{"k":`, err: io.EOF},
538 {in: `{"k",`, err: &SyntaxError{"invalid character ',' after object key", len64(`{"k",`)}},
539 {in: `{"k"}`, err: &SyntaxError{"invalid character '}' after object key", len64(`{"k"}`)}},
540 {in: ` [0`, err: io.EOF},
541 {in: `[0.`, err: io.ErrUnexpectedEOF},
542 {in: `[0. `, err: &SyntaxError{"invalid character ' ' in numeric literal", len64(`[0. `)}},
543 {in: `[0,`, err: io.EOF},
544 {in: `[0:`, err: &SyntaxError{"invalid character ':' after array element", len64(`[0:`)}},
545 {in: `n`, err: io.ErrUnexpectedEOF},
546 {in: `nul`, err: io.ErrUnexpectedEOF},
547 {in: `fal `, err: &SyntaxError{"invalid character ' ' in literal false (expecting 's')", len64(`fal `)}},
548 {in: `false`, err: io.EOF},
549 {in: ` 1e1000`, err: &UnmarshalTypeError{Value: "number 1e1000", Type: reflect.TypeFor[float64](), Offset: len64(` 1e1000`)}},
550 {in: `{"foo":1}{"bar":2}`, err: io.EOF},
551 {in: `{"foo":1}{"bar":2}`, inErr: io.ErrUnexpectedEOF, err: io.ErrUnexpectedEOF},
552 {in: `{"foo":1}{"bar":2}`, inErr: fmt.Errorf("wrap: %w", io.ErrUnexpectedEOF), err: fmt.Errorf("wrap: %w", io.ErrUnexpectedEOF)},
553 }
554 for _, tt := range tests {
555 r := strings.NewReader(tt.in)
556 d := NewDecoder(readerFunc(func(b []byte) (int, error) {
557 n, err := r.Read(b)
558 if err == io.EOF && tt.inErr != nil {
559 return n, tt.inErr
560 }
561 return n, err
562 }))
563 for i := 0; true; i++ {
564 if _, err := d.Token(); err != nil {
565 if !reflect.DeepEqual(err, tt.err) {
566 t.Errorf("`%s`: %d.Token error = %#v, want %#v", tt.in, i, err, tt.err)
567 }
568 break
569 }
570 }
571 }
572 }
573
574 func TestDecoderInputOffset(t *testing.T) {
575 const input = ` [
576 [ ] , [ "one" ] , [ "one" , "two" ] ,
577 { } , { "alpha" : "bravo" } , { "alpha" : "bravo" , "fizz" : "buzz" }
578 ] `
579 wantOffsets := []int64{
580 0, 1, 2, 5, 6, 7, 8, 9, 12, 13, 18, 19, 20, 21, 24, 25, 30, 31,
581 38, 39, 40, 41, 46, 47, 48, 49, 52, 53, 60, 61, 70, 71, 72, 73,
582 76, 77, 84, 85, 94, 95, 103, 104, 112, 113, 114, 116, 117, 117,
583 117, 117,
584 }
585 wantMores := []bool{
586 true, true, false, true, true, false, true, true, true, false,
587 true, false, true, true, true, false, true, true, true, true,
588 true, false, false, false, false,
589 }
590
591 d := NewDecoder(strings.NewReader(input))
592 checkOffset := func() {
593 t.Helper()
594 got := d.InputOffset()
595 if len(wantOffsets) == 0 {
596 t.Fatalf("InputOffset = %d, want nil", got)
597 }
598 want := wantOffsets[0]
599 if got != want {
600 t.Fatalf("InputOffset = %d, want %d", got, want)
601 }
602 wantOffsets = wantOffsets[1:]
603 }
604 checkMore := func() {
605 t.Helper()
606 got := d.More()
607 if len(wantMores) == 0 {
608 t.Fatalf("More = %v, want nil", got)
609 }
610 want := wantMores[0]
611 if got != want {
612 t.Fatalf("More = %v, want %v", got, want)
613 }
614 wantMores = wantMores[1:]
615 }
616 checkOffset()
617 checkMore()
618 checkOffset()
619 for {
620 if _, err := d.Token(); err == io.EOF {
621 break
622 } else if err != nil {
623 t.Fatalf("Token error: %v", err)
624 }
625 checkOffset()
626 checkMore()
627 checkOffset()
628 }
629 checkOffset()
630 checkMore()
631 checkOffset()
632
633 if len(wantOffsets)+len(wantMores) > 0 {
634 t.Fatal("unconsumed testdata")
635 }
636
637 t.Run("ArrayEOF", func(t *testing.T) {
638 d := NewDecoder(strings.NewReader(` [ "fizz" , `))
639 for {
640 if _, err := d.Token(); err == io.EOF {
641 break
642 } else if err != nil {
643 t.Fatalf("Token error: %v", err)
644 }
645 }
646 got := d.InputOffset()
647 want := len64(` [ "fizz" ,`)
648 if got != want {
649 t.Errorf("InputOffset = %v, want %v", got, want)
650 }
651 })
652
653 t.Run("ObjectEOF", func(t *testing.T) {
654 d := NewDecoder(strings.NewReader(` { "fizz" : `))
655 for {
656 if _, err := d.Token(); err == io.EOF {
657 break
658 } else if err != nil {
659 t.Fatalf("Token error: %v", err)
660 }
661 }
662 got := d.InputOffset()
663 want := len64(` { "fizz" :`)
664 if got != want {
665 t.Errorf("InputOffset = %v, want %v", got, want)
666 }
667 })
668 }
669
670 func TestDecoderMaxBytesError(t *testing.T) {
671
672
673
674 oversized := strings.Repeat("x", 1<<20+1)
675 body := `{"name":"` + oversized + `"}`
676
677 req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(body))
678 rec := httptest.NewRecorder()
679 req.Body = http.MaxBytesReader(rec, req.Body, 1<<20)
680
681 var v map[string]any
682 err := NewDecoder(req.Body).Decode(&v)
683 if err == nil {
684 t.Fatal("expected error, got nil")
685 }
686
687 var maxBytesErr *http.MaxBytesError
688 if !errors.As(err, &maxBytesErr) {
689 t.Errorf("errors.As(err, *http.MaxBytesError) = false, want true\nerror type: %T\nerror: %v", err, err)
690 }
691 }
692
View as plain text