1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
|
package conf
import (
"fmt"
"io"
"io/ioutil"
"os"
"unicode"
"unicode/utf8"
)
type tokentype int
const (
Name tokentype = iota
Literal
ListDelimiter
ObjectDelimiter
EmptyList
EmptyObject
Nothing
IfStatement
)
type token struct {
Type tokentype
Offset int
LineNr int
Lit string
}
type scanner struct {
data []byte
offset int
curline int
peeked *token
}
func newScanner(r io.Reader) *scanner {
// TODO: don't be lazy
data, err := ioutil.ReadAll(r)
if err != nil {
fmt.Fprintf(os.Stderr, "could not read data from Reader: %v\n", err)
os.Exit(1)
}
sc := scanner{
data: data,
curline: 1,
}
return &sc
}
func getTokenType(s []byte) tokentype {
switch s[0] {
case '"', '\'':
return Literal
case '[', ']':
return ListDelimiter
case '{', '}':
return ObjectDelimiter
}
return Name
}
func (s *scanner) Scan() (token, error) {
processed := 0
tokenstarted := false
oldline := s.curline
if s.peeked != nil {
ret := *s.peeked
s.peeked = nil
return ret, nil
}
for {
r, rlen := utf8.DecodeRune(s.data[s.offset+processed:])
if r == utf8.RuneError {
if rlen == 1 {
return token{}, fmt.Errorf("found invalid UTF8 at offset %d, before: %s", s.offset, string(s.data[s.offset]))
} else if rlen == 0 {
return token{}, io.EOF
}
}
processed += rlen
if unicode.IsSpace(r) {
if r == '\n' {
s.curline++
}
if tokenstarted {
break
}
s.offset += rlen
processed = 0
continue
}
tokenstarted = true
}
tokbytes := s.data[s.offset : s.offset+processed-1]
tokent := getTokenType(tokbytes)
s.offset += processed
ret := token{
Type: tokent,
Offset: s.offset,
LineNr: oldline,
Lit: string(tokbytes),
}
return ret, nil
}
func (s *scanner) Peek() (token, error) {
if s.peeked != nil {
return *s.peeked, nil
}
tok, err := s.Scan()
if err != nil {
return token{}, err
}
s.peeked = &tok
return tok, nil
}
|
