1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
|
package conf
import (
"fmt"
"io"
"io/ioutil"
"os"
"unicode"
"unicode/utf8"
)
type tokentype int
const (
Name tokentype = iota
Literal
DeliOpen
DeliClose
Nothing
IfStatement
)
var tokentypestrings []string = []string{Name: "Name", Literal: "Literal",
DeliOpen: "Opening delimiter", DeliClose: "Closing delimiter",
Nothing: "Nothing", IfStatement: "If statement"}
func (tt tokentype) String() string {
return tokentypestrings[tt]
}
type token struct {
Type tokentype
Offset int
LineNr int
Lit string
}
type scanner struct {
data []byte
offset int
curline int
}
func newScanner(r io.Reader) *scanner {
// TODO: don't be lazy
data, err := ioutil.ReadAll(r)
if err != nil {
fmt.Fprintf(os.Stderr, "could not read data from Reader: %v\n", err)
os.Exit(1)
}
sc := scanner{
data: data,
curline: 1,
}
return &sc
}
func getTokenType(s []byte) tokentype {
return Nothing
}
func (s *scanner) Scan() (token, error) {
processed := 0
tokenstarted := false
oldline := s.curline
for {
r, rlen := utf8.DecodeRune(s.data[s.offset+processed:])
if r == utf8.RuneError {
if rlen == 1 {
return token{}, fmt.Errorf("found invalid UTF8 at offset %d, before: %s", s.offset, s.data[s.offset])
} else if rlen == 0 {
return token{}, io.EOF
}
}
processed += rlen
if unicode.IsSpace(r) {
if r == '\n' {
s.curline++
}
if tokenstarted {
break
}
s.offset += rlen
processed = 0
continue
}
tokenstarted = true
}
tokbytes := s.data[s.offset : s.offset+processed-1]
tokent := getTokenType(tokbytes)
s.offset += processed
ret := token{
Type: tokent,
Offset: s.offset,
LineNr: oldline,
Lit: string(tokbytes),
}
return ret, nil
}
func (s *scanner) peek() (tokentype, string, error) {
return Nothing, "", nil
}
|