From 25013841d2c28eca33a8a1f94a33babad36d80aa Mon Sep 17 00:00:00 2001 From: Silvan Jegen Date: Mon, 26 Dec 2016 14:09:32 +0100 Subject: Fix "go vet" issue --- conf/scanner.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/scanner.go b/conf/scanner.go index 9bd7a4b..fd9868a 100644 --- a/conf/scanner.go +++ b/conf/scanner.go @@ -61,7 +61,7 @@ func (s *scanner) Scan() (token, error) { r, rlen := utf8.DecodeRune(s.data[s.offset+processed:]) if r == utf8.RuneError { if rlen == 1 { - return token{}, fmt.Errorf("found invalid UTF8 at offset %d, before: %s", s.offset, s.data[s.offset]) + return token{}, fmt.Errorf("found invalid UTF8 at offset %d, before: %s", s.offset, string(s.data[s.offset])) } else if rlen == 0 { return token{}, io.EOF } -- cgit v1.2.3 From d700b7019e8fc380ee5ca454f3aa3edd36f30faf Mon Sep 17 00:00:00 2001 From: Silvan Jegen Date: Tue, 27 Dec 2016 14:38:28 +0100 Subject: Parse more of our current config format --- conf/parser.go | 92 ++++++++++++++++++++++++++++++++++++++++++++++++++------- conf/scanner.go | 17 +++++++++-- stasher.cfg | 1 + 3 files changed, 97 insertions(+), 13 deletions(-) diff --git a/conf/parser.go b/conf/parser.go index 75f610c..26784f4 100644 --- a/conf/parser.go +++ b/conf/parser.go @@ -1,8 +1,12 @@ // grammar of the config file // +// Notes: We parse exactly three modules ("input", "filter", "output") +// in our configuration. Each module consists of two name-object pairs +// that are nested. We don't want to allow limitless recursion here. +// // config = module module module -// module = name object -// name = 'input' | 'filter' | 'etc.' +// module = name object name object +// name = 'input' | 'filter' | 'output' | etc. // object = '{' keyvalue | keyvalue | ... '}' // keyvalue = statement '=>' value // statement = name | if @@ -17,8 +21,8 @@ import ( "os" ) -// Having a Config to Manager function could be nice? -// Or we could just return a Manager from here. +// Having a Config to Manager function could be nice? Or we could just +// return a Manager from here which may be even better... type Config struct { } @@ -44,20 +48,88 @@ func NewConfig(r io.Reader) *Config { } func (p *parser) startparsing() { + p.module("input") + + p.module("filter") + +} + +func (p *parser) advanceOneToken(place string) { var err error p.last = p.cur p.cur, err = p.s.Scan() - for err == nil { - fmt.Fprintf(os.Stderr, "tokentype: %d, token: %q offset: %d, line: %d\n", p.cur.Type, p.cur.Lit, p.cur.Offset, p.cur.LineNr) - p.last = p.cur - p.cur, err = p.s.Scan() + if err != nil { + fmt.Fprintf(os.Stderr, "error when parsing %s. Scanner returned an error %v.\n", place, err) + os.Exit(1) } - fmt.Fprintf(os.Stderr, "Error: tokentype: %d, token: %q, err: %v\n", p.cur.Type, p.cur.Lit, err) } -func (p *parser) module(name string) { +func (p *parser) module(firstname string) { + p.advanceOneToken("module") + + fmt.Fprintf(os.Stderr, "tokentype: %d, token: %q\n", p.cur.Type, p.cur.Lit) + + if p.cur.Lit != firstname { + fmt.Fprintf(os.Stderr, "error when parsing module. We were expecting name %q but got %q at line %d offset %d.\n", firstname, p.cur.Lit, p.cur.LineNr, p.cur.Offset) + } + + p.advanceOneToken("module") + if p.cur.Type != ObjectDelimiter { + fmt.Fprintf(os.Stderr, "error when parsing module. We were expecting an opening bracket but got %q at line %d offset %d.\n", p.cur.Lit, p.cur.LineNr, p.cur.Offset) + } + + p.advanceOneToken("module") + if p.cur.Type != Name { + fmt.Fprintf(os.Stderr, "error when parsing module. We were expecting another name but got %q at line %d offset %d.\n", p.cur.Lit, p.cur.LineNr, p.cur.Offset) + } + p.object() + + p.advanceOneToken("module") + if p.cur.Lit[0] != '}' { + fmt.Fprintf(os.Stderr, "error when parsing module. Was expecting a closing bracket but got %s at line %d offset %d.\n", p.cur.Lit, p.cur.LineNr, p.cur.Offset) + } + } func (p *parser) object() { + p.advanceOneToken("object") + + if p.cur.Lit[0] != '{' { + fmt.Fprintf(os.Stderr, "error when parsing object. Was expecting an opening bracket but got %s at line %d offset %d.\n", p.cur.Lit, p.cur.LineNr, p.cur.Offset) + } + // hack to deal with case where there is no white space between + // the delimiters. + if p.cur.Lit == "{}" { + return + } + + for { + more := p.keyvalue() + if !more { + break + } + } + +} + +func (p *parser) keyvalue() bool { + p.advanceOneToken("keyvalue") + + fmt.Fprintf(os.Stderr, "keyvalue: tokentype: %d, token: %q\n", p.cur.Type, p.cur.Lit) + if p.cur.Type != Literal { + fmt.Fprintf(os.Stderr, "error when parsing keyvalue. Was expecting a name but got %q at line %d offset %d.\n", p.cur.Lit, p.cur.LineNr, p.cur.Offset) + } + + p.advanceOneToken("keyvalue") + if p.cur.Lit != "=>" { + fmt.Fprintf(os.Stderr, "error when parsing keyvalue. Was expecting a '=>' but got %q at line %d offset %d.\n", p.cur.Lit, p.cur.LineNr, p.cur.Offset) + } + + p.advanceOneToken("keyvalue") + if p.cur.Type != Literal || p.cur.Type != Literal { + fmt.Fprintf(os.Stderr, "error when parsing keyvalue. Was expecting a literal or a name but got %q at line %d offset %d.\n", p.cur.Lit, p.cur.LineNr, p.cur.Offset) + } + + return false } diff --git a/conf/scanner.go b/conf/scanner.go index fd9868a..44301fe 100644 --- a/conf/scanner.go +++ b/conf/scanner.go @@ -14,8 +14,10 @@ type tokentype int const ( Name tokentype = iota Literal - DeliOpen - DeliClose + ListDelimiter + ObjectDelimiter + EmptyList + EmptyObject Nothing IfStatement ) @@ -50,7 +52,16 @@ func newScanner(r io.Reader) *scanner { } func getTokenType(s []byte) tokentype { - return Nothing + switch s[0] { + case '"', '\'': + return Literal + case '[': + return ListDelimiter + case '{': + return ObjectDelimiter + } + + return Name } func (s *scanner) Scan() (token, error) { diff --git a/stasher.cfg b/stasher.cfg index e0df958..d0fe9c1 100644 --- a/stasher.cfg +++ b/stasher.cfg @@ -5,6 +5,7 @@ input { filter { http { "F" => "localhost:9200" + "G" => "localhost:9201" } } -- cgit v1.2.3 From e2fdd3b1b714b9fa896218c68fdfe57d906730ba Mon Sep 17 00:00:00 2001 From: Silvan Jegen Date: Wed, 28 Dec 2016 19:48:53 +0100 Subject: Implement Peek() --- conf/scanner.go | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/conf/scanner.go b/conf/scanner.go index 44301fe..74e66e8 100644 --- a/conf/scanner.go +++ b/conf/scanner.go @@ -33,6 +33,8 @@ type scanner struct { data []byte offset int curline int + + peeked *token } func newScanner(r io.Reader) *scanner { @@ -55,9 +57,9 @@ func getTokenType(s []byte) tokentype { switch s[0] { case '"', '\'': return Literal - case '[': + case '[', ']': return ListDelimiter - case '{': + case '{', '}': return ObjectDelimiter } @@ -68,6 +70,11 @@ func (s *scanner) Scan() (token, error) { processed := 0 tokenstarted := false oldline := s.curline + if s.peeked != nil { + ret := *s.peeked + s.peeked = nil + return ret, nil + } for { r, rlen := utf8.DecodeRune(s.data[s.offset+processed:]) if r == utf8.RuneError { @@ -108,6 +115,15 @@ func (s *scanner) Scan() (token, error) { return ret, nil } -func (s *scanner) peek() (tokentype, string, error) { - return Nothing, "", nil +func (s *scanner) Peek() (token, error) { + if s.peeked != nil { + return *s.peeked, nil + } + + tok, err := s.Scan() + if err != nil { + return token{}, err + } + s.peeked = &tok + return tok, nil } -- cgit v1.2.3 From ca96c7fd5c43b99873a67eb239176bd7a9e762f0 Mon Sep 17 00:00:00 2001 From: Silvan Jegen Date: Wed, 28 Dec 2016 20:07:57 +0100 Subject: Finish parsing of keyvalues --- conf/parser.go | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/conf/parser.go b/conf/parser.go index 26784f4..cc85054 100644 --- a/conf/parser.go +++ b/conf/parser.go @@ -127,9 +127,21 @@ func (p *parser) keyvalue() bool { } p.advanceOneToken("keyvalue") - if p.cur.Type != Literal || p.cur.Type != Literal { + if p.cur.Type != Literal && p.cur.Type != Name { fmt.Fprintf(os.Stderr, "error when parsing keyvalue. Was expecting a literal or a name but got %q at line %d offset %d.\n", p.cur.Lit, p.cur.LineNr, p.cur.Offset) } - return false + tok, err := p.s.Peek() + if err != nil { + fmt.Fprintf(os.Stderr, "error when parsing keyvalue. Got an error when checking if there are more keyvalues: %v.\n", err) + return false + } + + fmt.Fprintf(os.Stderr, "peeked %q (type %d) at line %d offset %d.\n", tok.Lit, tok.Type, tok.LineNr, tok.Offset) + if tok.Type == ObjectDelimiter { + p.advanceOneToken("keyvaluelast") + return false + } + + return true } -- cgit v1.2.3