Changeset 241
- Timestamp:
- 08/13/06 22:08:47 (2 years ago)
- Files:
-
- trunk/enki/BaseParser.d (modified) (1 diff)
- trunk/enki/library/d/Token.d (modified) (5 diffs)
- trunk/enki/library/d/Tokenizer.bnf (modified) (6 diffs)
- trunk/enki/library/d/TokenizerBase.d (modified) (1 diff)
Legend:
- Unmodified
- Added
- Removed
- Modified
- Copied
- Moved
trunk/enki/BaseParser.d
r239 r241 54 54 } 55 55 56 //NOTE: oops. :( 57 alias initalize initialize; 58 56 59 public void initalize(String input){ 57 60 data = input; trunk/enki/library/d/Token.d
r239 r241 31 31 private import enki.library.d.Operator; 32 32 33 //TODO: get rid of std.string and std.utf 34 private import std.string; // used for conversion 35 private import std.utf; // used for conversion 33 36 alias ulong IntegerValue; //TODO: turn into a more advanced type 34 37 … … 47 50 _this.exponentSign = exponentSign; 48 51 return _this; 52 } 53 54 public String render(){ 55 char[] sign = exponentSign ? "+" : "-"; 56 char[] img = imaginary ? "i" : ""; 57 58 return toUTF32(std.string.toString(whole) ~ "." ~ std.string.toString(fraction) ~ sign ~ "e" ~ img); 49 59 } 50 60 } … … 252 262 Special, 253 263 LineDirective, 254 } 264 } 255 265 256 266 union ValueUnion{ … … 293 303 public static Token comment(String id){ 294 304 Token _this; 295 _this.type = Operator;305 _this.type = Comment; 296 306 _this.value.strValue = id; 297 307 return _this; … … 343 353 return _this; 344 354 } 345 355 346 356 String render(){ 347 if(type == Token.Identifier){ 357 switch(type){ 358 case Identifier: 348 359 return value.strValue; 360 case Keyword: 361 return keywordXref[value.token]; 362 case Operator: 363 return operatorXref[value.token]; 364 case Comment: 365 case StringLiteral: 366 case CharLiteral: 367 return value.strValue; 368 return value.strValue; 369 case FloatingPoint: 370 return value.floatingValue.render(); 371 case Integer: 372 return toUTF32(std.string.toString(value.integerValue)); 373 case Special: 374 //TODO: fix me 375 return value.strValue; 376 case LineDirective: 377 default: 378 return ""; 349 379 } 350 else if(type == Token.Keyword){ 351 return keywordXref[value.token]; 352 } 353 else if(type == Token.Operator){ 354 return operatorXref[value.token]; 355 } 356 return ""; 357 } 358 } 380 } 381 } trunk/enki/library/d/Tokenizer.bnf
r240 r241 42 42 .classname(Tokenizer); 43 43 44 .define("String","any","true","any"); 44 45 .define(bool,eoi,false,"End of Input"); 45 46 .define("String","letter","true","letter"); … … 68 69 69 70 Space 70 = Token tok 71 ::= #20 | #09 | #0B | #0C | EndOfLine | Comment:tok; 72 71 ::= #20 | #09 | #0B | #0C | EndOfLine; 72 73 73 Comment 74 74 = Token Token.comment(String comment) 75 ::= ("\x2F\x2A" { Character}:comment "\x2A\x2F") |76 ("\x2F\x2F" { Character}:comment EndOfLine) |75 ::= ("\x2F\x2A" {any}:comment "\x2A\x2F") | 76 ("\x2F\x2F" {any}:comment EndOfLine) | 77 77 ("\x2F\x2B" SlashPlusContent:comment); 78 78 79 79 SlashPlusContent 80 80 = String comment 81 ::= { Character}:comment ("\x2B\x2F" | ("\x2F\x2B" SlashPlusContent:~comment));81 ::= {any}:comment ("\x2B\x2F" | ("\x2F\x2B" SlashPlusContent:~comment)); 82 82 83 83 ##### Tokens ##### … … 86 86 = Token[] tokens 87 87 ::= { 88 Space:~tokens | 88 Space | 89 Comment:~tokens | 89 90 StringLiteral:~tokens | 91 SpecialToken:~tokens | 92 SpecialTokenSequence:~tokens | 93 Identifier:~tokens | 90 94 CharacterLiteral:~tokens | 91 IntegerLiteral:~tokens | 95 Keyword:~tokens | 96 Operator:~tokens | 92 97 FloatLiteral:~tokens | 93 Keyword:~tokens | 94 Operator:~tokens | 95 SpecialToken:~tokens | 96 SpecialTokenSequence:~tokens 98 IntegerLiteral:~tokens 97 99 } RealEndOfFile; 98 100 … … 141 143 142 144 DoubleQuotedCharacter 143 ::= Character | EscapeSequence | EndOfLine;145 ::= EscapeSequence | EndOfLine | any; 144 146 145 147 EscapeSequence … … 149 151 "\"":value | 150 152 "?":value | 151 "\ \" EndOfFile |152 "\ \":value |153 "\x5C" EndOfFile | 154 "\x5C":value | 153 155 "a" #07:value | 154 156 "b" #08:value | … … 260 262 261 263 NonZeroDigit 262 ::= "1" | "2" | "3" | "4" | "5" | "6" | "7" | "8" | "9"; 264 = String value 265 ::= ("1" | "2" | "3" | "4" | "5" | "6" | "7" | "8" | "9"):value; 263 266 264 267 DecimalDigit 265 ::= "0" | NonZeroDigit; 268 = String value 269 ::= ("0" | NonZeroDigit):value; 266 270 267 271 BinaryDigit 268 ::= "0" | "1"; 272 = String value 273 ::= ("0" | "1"):value; 269 274 270 275 OctalDigit 271 ::= "0" | "1" | "2" | "3" | "4" | "5" | "6" | "7"; 276 = String value 277 ::= ("0" | "1" | "2" | "3" | "4" | "5" | "6" | "7"):value; 272 278 273 279 HexDigit 274 ::= "0" | "1" | "2" | "3" | "4" | "5" | "6" | "7" | "8" | "9" | 280 = String value 281 ::= ("0" | "1" | "2" | "3" | "4" | "5" | "6" | "7" | "8" | "9" | 275 282 "a" | "b" | "c" | "d" | "e" | "f" | 276 "A" | "B" | "C" | "D" | "E" | "F" ;283 "A" | "B" | "C" | "D" | "E" | "F"):value; 277 284 278 285 trunk/enki/library/d/TokenizerBase.d
r239 r241 86 86 return result; 87 87 } 88 import std.stdio; 88 89 89 90 IntegerValue convertToDecimal(String value){ 90 91 IntegerValue result = 0; 91 92 92 93 foreach(ch; value){ 93 94 result = (result * 10) + (ch - '0');