using System; using System.Collections.Generic; using System.Linq; using System.Text; using System.Threading.Tasks; using System.IO; using System.Globalization; namespace CompilerVVM { class TokenScanner { int idconst = 1; private IList tokens; private IList list; public IList Tokens { get { return tokens; } } private void Scan(System.IO.TextReader input, Dictionary dict) { while (input.Peek() != -1) { char ch = (char)input.Peek(); if (char.IsWhiteSpace(ch)) { input.Read(); } else if (char.IsLetter(ch) || ch == '_') { StringBuilder accum = new StringBuilder(); while (char.IsLetter(ch) || ch == '_' || char.IsNumber(ch)) { accum.Append(ch); input.Read(); if (input.Peek() == -1) { break; } else { ch = (char)input.Peek(); } } this.tokens.Add(accum.ToString()); } else if (ch == '"') { StringBuilder accum = new StringBuilder(); input.Read(); if (input.Peek() == -1) { throw new ScannerException("Unterminated string"); } while ((ch = (char)input.Peek()) != '"') { accum.Append(ch); input.Read(); if (input.Peek() == -1) { throw new ScannerException("Unterminated string"); } } input.Read(); dict.Add(accum.ToString(), string.Format("text_const_{0}", idconst++)); this.tokens.Add(accum); } else if (char.IsDigit(ch)) { StringBuilder accum = new StringBuilder(); while (char.IsDigit(ch) || ch == '.') { accum.Append(ch); input.Read(); if (input.Peek() == -1) { break; } else { ch = (char)input.Peek(); } } this.tokens.Add(new Number(accum.ToString())); } else if (ch == '(') { this.tokens.Add(OP.OpenParam); input.Read(); if (input.Peek() == -1) { throw new ScannerException("Unterminated parameter section"); } while ((ch = (char)input.Peek()) != ')') { if (char.IsLetter(ch) || ch == '_') { StringBuilder accum = new StringBuilder(); while (char.IsLetter(ch) || ch == '_') { accum.Append(ch); input.Read(); if (input.Peek() == -1) { break; } else { ch = (char)input.Peek(); } } this.tokens.Add(accum.ToString()); } else if (char.IsDigit(ch)) { StringBuilder accum = new StringBuilder(); while (char.IsDigit(ch) || ch == '.') { accum.Append(ch); input.Read(); if (input.Peek() == -1) { break; } else { ch = (char)input.Peek(); } } this.tokens.Add(new Number(accum.ToString())); } else if (ch == '"') { StringBuilder accum = new StringBuilder(); input.Read(); if (input.Peek() == -1) { throw new ScannerException("Unterminated string"); } while ((ch = (char)input.Peek()) != '"') { accum.Append(ch); input.Read(); if (input.Peek() == -1) { throw new ScannerException("Unterminated string"); } } input.Read(); dict.Add(accum.ToString(), string.Format("text_const_{0}", idconst++)); this.tokens.Add(accum); } else if (char.IsWhiteSpace(ch) || ch == ',') input.Read(); else switch (ch) { case '=': input.Read(); if ((char)input.Peek() == '=') { input.Read(); this.tokens.Add(OP.Equal); } else this.tokens.Add(OP.Assigment); break; case '<': input.Read(); if ((char)input.Peek() == '=') { input.Read(); this.tokens.Add(OP.LessEqual); } else this.tokens.Add(OP.Less); break; case '>': input.Read(); if ((char)input.Peek() == '=') { input.Read(); this.tokens.Add(OP.GreaterEqual); } else this.tokens.Add(OP.Greater); break; case '!': input.Read(); if ((char)input.Peek() == '=') { input.Read(); this.tokens.Add(OP.NotEqual); } else throw new Exception("!"); break; } } this.tokens.Add(OP.CloseParam); input.Read(); } else switch (ch) { case ';': input.Read(); this.tokens.Add(OP.Semicolon); break; case '{': input.Read(); this.tokens.Add(OP.OpenBlock); break; case '}': input.Read(); this.tokens.Add(OP.CloseBlock); break; case '+': input.Read(); this.tokens.Add(OP.Add); break; case '-': input.Read(); this.tokens.Add(OP.Sub); break; case '*': input.Read(); this.tokens.Add(OP.Mul); break; case '/': input.Read(); this.tokens.Add(OP.Div); break; case '%': input.Read(); this.tokens.Add(OP.Mod); break; case '=': input.Read(); if ((char)input.Peek() == '=') { input.Read(); this.tokens.Add(OP.Equal); } else this.tokens.Add(OP.Assigment); break; default: throw new ScannerException("Scanner encountered unrecognized character '" + ch + "'"); } } } public TokenScanner(TextReader input, Dictionary TextConstant) { tokens = new List(); this.Scan(input, TextConstant); } } enum OP { Semicolon, OpenBlock, CloseBlock, OpenParam, CloseParam, Assigment, Equal, NotEqual, Less, Greater, LessEqual, GreaterEqual, Add, Sub, Mul, Div, Mod } }