package conf import ( "fmt" "io" "io/ioutil" "os" "unicode" "unicode/utf8" ) type tokentype int const ( Name tokentype = iota Literal DeliOpen DeliClose Nothing IfStatement ) var tokentypestrings []string = []string{Name: "Name", Literal: "Literal", DeliOpen: "Opening delimiter", DeliClose: "Closing delimiter", Nothing: "Nothing", IfStatement: "If statement"} func (tt tokentype) String() string { return tokentypestrings[tt] } type token struct { Type tokentype Offset int LineNr int Lit string } type scanner struct { data []byte offset int curline int } func newScanner(r io.Reader) *scanner { // TODO: don't be lazy data, err := ioutil.ReadAll(r) if err != nil { fmt.Fprintf(os.Stderr, "could not read data from Reader: %v\n", err) os.Exit(1) } sc := scanner{ data: data, curline: 1, } return &sc } func getTokenType(s []byte) tokentype { return Nothing } func (s *scanner) Scan() (token, error) { processed := 0 tokenstarted := false oldline := s.curline for { r, rlen := utf8.DecodeRune(s.data[s.offset+processed:]) if r == utf8.RuneError { if rlen == 1 { return token{}, fmt.Errorf("found invalid UTF8 at offset %d, before: %s", s.offset, s.data[s.offset]) } else if rlen == 0 { return token{}, io.EOF } } processed += rlen if unicode.IsSpace(r) { if r == '\n' { s.curline++ } if tokenstarted { break } s.offset += rlen processed = 0 continue } tokenstarted = true } tokbytes := s.data[s.offset : s.offset+processed-1] tokent := getTokenType(tokbytes) s.offset += processed ret := token{ Type: tokent, Offset: s.offset, LineNr: oldline, Lit: string(tokbytes), } return ret, nil } func (s *scanner) peek() (tokentype, string, error) { return Nothing, "", nil }