Navigation

← Previous Changeset
Next Changeset →

Changeset 232

Timestamp:

08/04/06 16:35:53 (2 years ago)

Author:

pragma

Message:

Updated D tokenizer (WIP)

Files:

trunk/enki/library/d/Token.d (modified) (18 diffs)
trunk/enki/library/d/Tokenizer.bnf (modified) (11 diffs)
trunk/enki/library/d/TokenizerBase.d (modified) (1 diff)
trunk/enki/types.d (modified) (1 diff)

Legend:

: Unmodified
: Added
: Removed
: Modified
: Copied
: Moved

trunk/enki/library/d/Token.d

r230	r232
27	27	*/
28	28
	29	private import enki.types;
29	30	private import ddoc.Token;
	31
	32	alias ulong Integer; //TODO: turn into a more advanced type
	33
	34	struct FloatingPoint{
	35	Integer whole;
	36	Integer fraction;
	37	Integer exponent;
	38	bool exponentSign;
	39	bool imaginary;
	40
	41	static FloatingPoint opCall(Integer whole,Integer fraction,Integer exponent,bool exponentSign){
	42	FloatingPoint _this;
	43	_this.whole = whole;
	44	_this.fraction = fraction;
	45	_this.exponent = exponent;
	46	_this.exponentSign = exponentSign;
	47	return _this;
	48	}
	49	}
	50
	51	struct LineDirective{
	52	Integer line;
	53	String filespec;
	54
	55	static LineDirective opCall(Integer line,String filespec){
	56	LineDirective _this;
	57	_this.line = line;
	58	_this.filespec = filespec;
	59	returh _this;
	60	}
	61	}
30	62
31	63	enum TokenType{
…	…
33	65	Keyword,
34	66	Operator,
35		String,
	67	Comment,
	68	StringLiteral,
	69	CharLiteral,
	70	Integer,
	71	FloatingPoint,
	72	Special,
	73	LineDirective,
	74	}
	75
	76	enum TokenSubtype{
	77	None = 0,
36	78	Char,
37		Int,
38		Uint,
39		Long,
40		Ulong,
41		Float
	79	DChar,
	80	WChar,
	81	LongInteger,
	82	UnsignedInteger,
	83	LongUnsignedInteger,
	84	Float,
	85	Real,
42	86	}
43	87
…	…
51	95	Assert,
52	96	Auto,
53
54	97	Body,
55	98	Bool,
56	99	Break,
57	100	Byte,
58
59	101	Case,
60	102	Cast,
…	…
68	110	Continue,
69	111	Creal,
70
71	112	Dchar,
72	113	Debug,
…	…
77	118	Do,
78	119	Double,
79
80	120	Else,
81	121	Enum,
82	122	Export,
83	123	Extern,
84
85	124	False,
86	125	Final,
…	…
90	129	Foreach,
91	130	Function,
92
93	131	Goto,
94
95	132	Idouble,
96	133	If,
…	…
103	140	Invariant,
104	141	Ireal,
105		I,
106
	142	Is,
107	143	Long,
108
109	144	Mixin,
110	145	Module,
111
112	146	New,
113	147	Null,
114
115	148	Out,
116	149	Override,
117
118	150	Package,
119	151	Pragma,
…	…
121	153	Protected,
122	154	Public,
123
124	155	Real,
125	156	Return,
126
127	157	Scope,
128	158	Short,
…	…
132	162	Switch,
133	163	Synchronized,
134
135	164	Template,
136	165	Thi,
…	…
141	170	Typeid,
142	171	Typeof,
143
144	172	Ubyte,
145	173	Ucent,
…	…
149	177	Unittest,
150	178	Ushort,
151
152	179	Version,
153	180	Void,
154	181	Volatile,
155
156	182	Wchar,
157	183	While,
…	…
161	187	DivAssign,
162	188	Div,
163
164	189	Elipsis,
165	190	Slice,
166	191	Dot,
167
168	192	AndAssign,
169	193	AndAnd,
170	194	And,
171
172	195	OrAssign,
173	196	OrOr,
174	197	Or,
175
176	198	MinusMinus,
177	199	MinusAssign,
178	200	Minus,
179
180	201	PlusPlus,
181	202	PlusAssign,
182	203	Plus,
183
184	204	LessEquals,
185	205	LessLess,
…	…
204	224	NotCat,
205	225	Not,
206
207	226	OpenParen,
208	227	CloseParen,
…	…
211	230	OpenCurl,
212	231	CloseCurl,
213
214	232	Question,
215	233	Comma,
…	…
217	235	Colon,
218	236	Dollar,
219
220	237	EqualsEqualsEquals,
221	238	EqualsEquals,
222	239	Equals,
223
224	240	StarEquals,
225	241	Star,
226
227	242	ModEquals,
228	243	Mod,
229
230	244	InverseEquals,
231	245	Inverse,
232
233	246	CatEquals,
234	247	CatCat,
…	…
238	251	struct Token{
239	252	TokenType type;
240
241		union value{
242		char[] strValue;
243		ulong ulongValue;
244		long Value;
245		float floatValue;
246		double doubleValue;
247		}
248
249		public static Token createIdentifier(char[] value){
250		Token _this;
251		_this.type = TokenType.identifier;
	253	TokenSubtype subtype;
	254
	255	union ValueUnion{
	256	Tok token;
	257	String strValue;
	258	Integer integerValue;
	259	FloatingPoint floatingValue;
	260	LineDirective lineDirective;
	261	}
	262
	263	ValueUnion value;
	264
	265	public static Token identifier(String value){
	266	Token _this;
	267	_this.type = TokenType.Identifier;
252	268	_this.value.strValue = value;
253	269	return _this;
254	270	}
255	271
256		public static Token createString(char[] value){
257		Token _this;
258		_this.type = TokenType.String;
259		_this.value.strValue = value;
260		return _this;
261		}
262
263		public static Token createKeyword(Tok id){
	272	public static Token keyword(Tok id){
264	273	Token _this;
265	274	_this.type = TokenType.Keyword;
…	…
268	277	}
269	278
270		public static Token ~~createO~~perator(Tok id){
	279	public static Token operator(Tok id){
271	280	Token _this;
272	281	_this.type = TokenType.Operator;
…	…
274	283	return _this;
275	284	}
276		}
	285
	286	public static Token comment(String id){
	287	Token _this;
	288	_this.type = TokenType.Operator;
	289	_this.value.strValue = id;
	290	return _this;
	291	}
	292
	293	public static Token stringLiteral(String value,TokenSubtype subtype){
	294	Token _this;
	295	_this.type = TokenType.StringLiteral;
	296	_this.subtype = subtype;
	297	_this.value.strValue = value;
	298	return _this;
	299	}
	300
	301	public static Token charLiteral(String value){
	302	Token _this;
	303	_this.type = TokenType.CharLiteral;
	304	_this.value.strValue = value;
	305	return _this;
	306	}
	307
	308	public static Token integerLiteral(Integer value,TokenSubtype subtype){
	309	Token _this;
	310	_this.type = TokenType.Integer;
	311	_this.subtype = subtype;
	312	_this.value.integerValue = value;
	313	return _this;
	314	}
	315
	316	public static Token floatingLiteral(FloatingPoint value,TokenSubtype subtype,bool isImaginary){
	317	Token _this;
	318	_this.type = TokenType.FloatingPoint;
	319	_this.subtype = subtype;
	320	_this.value.floatingValue = value;
	321	_this.value.floatingValue.imaginary = isImaginary;
	322	return _this;
	323	}
	324
	325	public static Token special(String value){
	326	Token _this;
	327	_this.type = TokenType.Special;
	328	_this.value.strValue = value;
	329	return _this;
	330	}
	331
	332	public static Token lineDirective(Integer line,String filespec){
	333	Token _this;
	334	_this.type = TokenType.LineDirective;
	335	_this.value.lineDirective = LineDirective(line,filespec);
	336	return _this;
	337	}
	338	}

trunk/enki/library/d/Tokenizer.bnf

r230	r232
27	27	#Lexer Phase - Tokenization of the source file
28	28
29		.module(ddoc.Tokenizer);
30		.import(ddoc.TokenizerBase);
31		.baseclass(ddoc.TokenizerBase);
32		.classname(ddoc.Tokenizer);
33
	29	#TODO: UTF directive
	30	#.utf(32);
	31	.parsetype("String");
	32
	33	.module(d.Tokenizer);
	34	.import(d.TokenizerBase);
	35	.import(d.Token);
	36	.baseclass(TokenizerBase);
	37	.classname(Tokenizer);
34	38
35	39	.define(bool,eoi,false,"End of Input");
…	…
39	43	.define("String","any","true","any");
40	44
41
42	45	Syntax
	46	= Token[] tokens;
43	47	::= {
44		Space \| Token \| Identifier \| StringLiteral \|
45		CharacterLiteral \| IntegerLiteral \| FloatLiteral \| Keyword \|
46		SpecialToken \| SpecialTokenSequence
	48	Space:~tokens \|
	49	Token:~tokens \|
	50	Identifier:~tokens \|
	51	StringLiteral:~tokens \|
	52	CharacterLiteral:~tokens \|
	53	IntegerLiteral:~tokens \|
	54	FloatLiteral:~tokens \|
	55	Keyword:~tokens \|
	56	SpecialToken:~tokens \|
	57	SpecialTokenSequence:~tokens
47	58	} RealEndOfFile;
48
49
50	59
51	60	##### Space, Comments and EOF #####
…	…
64	73
65	74	Space
66		::= #20 \| #09 \| #0B \| #0C \| EndOfLine \| Comment;
	75	= Token tok;
	76	::= #20 \| #09 \| #0B \| #0C \| EndOfLine \| Comment:tok;
67	77
68	78	Comment
69		= ~~bool addC~~omment(comment)
	79	= Token Token.comment(comment)
70	80	::= ("/" {Character}:comment "/") \|
71	81	("//" {Character}:comment EndOfLine) \|
…	…
90	100
91	101	OtherToken
92		= ~~bool addOperator(tok)~~
93		::= ("/=" \|
94		"/" \|
95		"..." \|
96		".." \|
97		"." \|
98		"&=" \|
99		"&&" \|
100		"&" \|
101		"\|~~\|" \|~~
102		"\|" \|
103		"\|=" \|
104		"--" \|
105		"-=" \|
106		"-" \|
107		"+=" \|
108		"++" \|
109		"+" \|
110		"<=" \|
111		"<<" \|
112		"<<=" \|
113		"<>" \|
114		"<>=" \|
115		"<" \|
116		">>>" \|
117		">>>=" \|
118		">>=" \|
119		">>" \|
120		">=" \|
121		">" \|
122		"!==" \|
123		"!=" \|
124		"!<>" \|
125		"!<>=" \|
126		"!<" \|
127		"!<=" \|
128		"!>" \|
129		"!>=" \|
130		"!~" \|
131		"!" \|
132		"(" \|
133		")" \|
134		"[" \|
135		"]" \|
136		"{" \|
137		"}" \|
138		"?" \|
139		"," \|
140		";" \|
141		":" \|
142		"$" \|
143		"===" \|
144		"==" \|
145		"=" \|
146		"*=" \|
147		"*" \|
148		"%=" \|
149		"%" \|
150		"^=" \|
151		"^" \|
152		"~~" \|
153		"~" \|
154		"~="
	102	= Token Token.operator(Tok tok);
	103	::= ("/=" @DivAssign:tok, \|
	104	"/" @Div:tok, \|
	105	"..." @Elipsis:tok, \|
	106	".." @Slice:tok, \|
	107	"." @Dot:tok, \|
	108	"&=" @AndAssign:tok, \|
	109	"&&" @AndAnd:tok, \|
	110	"&" @And:tok, \|
	111	"\|=" @OrAssign:tok, \|
	112	"\|\|" @OrOr:tok, \|
	113	"\|" @Or:tok, \|
	114	"--" @MinusMinus:tok, \|
	115	"-=" @MinusAssign:tok, \|
	116	"-" @Minus:tok, \|
	117	"++" @PlusPlus:tok, \|
	118	"+=" @PlusAssign:tok, \|
	119	"+" @Plus:tok, \|
	120	"<=" @LessEquals:tok, \|
	121	"<<" @LessLess:tok, \|
	122	"<<=" @LessLessEquals:tok, \|
	123	"<>" @LessGreater:tok, \|
	124	"<>=" @LessGreaterEquals:tok, \|
	125	"<" @Less:tok, \|
	126	">>>" @GreaterGreaterGreater:tok, \|
	127	">>>=" @GreaterGreaterGreaterEquals:tok,\|
	128	">>=" @GreaterGreaterEquals:tok, \|
	129	">>" @GreaterGreater:tok, \|
	130	">=" @GreaterEquals:tok, \|
	131	">" @Greater:tok, \|
	132	"!==" @NotEqualsEquals:tok, \|
	133	"!=" @NotEquals:tok, \|
	134	"!<>" @NotGreaterLess:tok, \|
	135	"!<>=" @NotGreaterLessEquals:tok, \|
	136	"!<" @NotLess:tok, \|
	137	"!<=" @NotLessEquals:tok, \|
	138	"!>" @NotGreater:tok, \|
	139	"!>=" @NotGreaterEquals:tok, \|
	140	"!~" @NotCat:tok, \|
	141	"!" @Not:tok, \|
	142	"(" @OpenParen:tok, \|
	143	")" @CloseParen:tok, \|
	144	"[" @OpenBracket:tok, \|
	145	"]" @CloseBracket:tok, \|
	146	"{" @OpenCurl:tok, \|
	147	"}" @CloseCurl:tok, \|
	148	"?" @Question:tok, \|
	149	"," @Comma:tok, \|
	150	";" @Semi:tok, \|
	151	":" @Colon:tok, \|
	152	"$" @Dollar:tok, \|
	153	"===" @EqualsEqualsEquals:tok, \|
	154	"==" @EqualsEquals:tok, \|
	155	"=" @Equals:tok, \|
	156	"*=" @StarEquals:tok, \|
	157	"*" @Star:tok, \|
	158	"%=" @ModEquals:tok, \|
	159	"%" @Mod:tok, \|
	160	"^=" @InverseEquals:tok, \|
	161	"^" @Inverse:tok, \|
	162	"~=" @CatEquals:tok, \|
	163	"~~" @CatCat:tok, \|
	164	"~" @Cat:tok
155	165	):tok;
156	166
…	…
160	170
161	171	Identifier
162		= ~~bool addIdentifier(tok)~~
163		::= (IdentifierStart {IdentifierChar}):~~tok~~;
	172	= Token Token.identifier(name);
	173	::= (IdentifierStart {IdentifierChar}):name;
164	174
165	175	IdentifierStart
…	…
177	187
178	188	StringLiteral
179		::= WysiwygString \|
180		AlternateWysiwygString \|
181		DoubleQuotedString \|
182		EscapeSequence \|
183		HexString;
	189	= Token tok
	190	::= WysiwygString:tok \|
	191	AlternateWysiwygString:tok \|
	192	DoubleQuotedString:tok \|
	193	EscapeSequence:tok \|
	194	HexString:tok;
184	195
185	196	WysiwygString
186		= ~~bool addString~~(str,width)
	197	= Token Token.stringLiteral(str,width)
187	198	::= "r" {WysiwygCharacter}:str "\"" [Postfix:width];
188	199
189	200	AlternateWysiwygString
190		= ~~bool addString~~(str,width)
	201	= Token Token.stringLiteral(str,width)
191	202	::= "`" {WysiwygCharacter}:str "`" [Postfix:width];
192	203
…	…
195	206
196	207	DoubleQuotedString
197		= ~~bool addString~~(str,width)
	208	= Token Token.stringLiteral(str,width)
198	209	::= "\"" {DoubleQuotedCharacter}:str "\"" [Postfix:width];
199	210
…	…
202	213
203	214	EscapeSequence
	215	= String value
204	216	::= "\\" (
205		"'" \|
206		"\"" \|
207		"?" \|
	217	"'":value \|
	218	"\"":value \|
	219	"?":value \|
208	220	"\\" EndOfFile \|
209		"\\\\" \|
210		"a" \|
211		"b" \|
212		"f" \|
213		"n" \|
214		"r" \|
215		"t" \|
216		"v" \|
217		"x" HexDigit HexDigit \|
	221	"\\":value \|
	222	"a" #07:value \|
	223	"b" #08:value \|
	224	"t" #09:value \|
	225	"n" #0A:value \|
	226	"v" #0B:value \|
	227	"f" #0C:value \|
	228	"r" #0D:value \|
	229	"x" HexChar2:value \|
	230	"u" HexChar4:value \|
	231	"U" HexChar8:value \|
	232	"&" NamedCharacterEntity:value
	233	OctChar:value
	234	);
	235
	236	HexChar2
	237	= String hexToChar(digits)
	238	::= (HexDigit HexDigit):digits;
	239
	240	HexChar4
	241	= Char hexToChar(digits)
	242	::= (HexDigit HexDigit HexDigit HexDigit):digits;
	243
	244	HexChar8
	245	= Char hexToChar(digits)
	246	::= (HexDigit HexDigit HexDigit HexDigit HexDigit HexDigit HexDigit HexDigit):digits;
	247
	248	OctChar
	249	= Char octToChar(digits)
	250	::= (
218	251	OctalDigit OctalDigit OctalDigit \|
219	252	OctalDigit OctalDigit \|
220		OctalDigit \|
221		"u" HexDigit HexDigit HexDigit HexDigit \|
222		"U" HexDigit HexDigit HexDigit HexDigit HexDigit HexDigit HexDigit HexDigit \|
223		"&" NamedCharacterEntity
224		);
	253	OctalDigit
	254	):digits
225	255
226	256	HexString
227		= ~~bool addHexString~~(str,width)
228		::= "x\"" {HexStringChar~~}:str~~ "\"" [Postfix:width];
	257	= Token Token.stringLiteral(str,width)
	258	::= "x\"" {HexStringChar:~str} "\"" [Postfix:width];
229	259
230	260	HexStringChar
231		::= HexDigit \| WhiteSpace \| EndOfLine;
	261	= Char hexToChar(digits)
	262	::= WhiteSpace HexDigit:~value WhiteSpace HexDigit:~value;
232	263
233	264	Postfix
234		::= "c" \| "w" \| "d";
	265	= TokenSubtype subtype;
	266	::= "c" @TokenSubtype.Char:subtype \|
	267	"w" @TokenSubtype.WChar:subtype \|
	268	"d" @TokenSubtype.DChar:subtype;
235	269
236	270
…	…
238	272
239	273	CharacterLiteral
240		::= "'" SingleQuotedCharacter "'";
	274	= Token Token.charLiteral(value)
	275	::= "'" SingleQuotedCharacter:value "'";
241	276
242	277	SingleQuotedCharacter
243		::= Character \| EscapeSequence;
	278	= String value
	279	::= Character:value \| EscapeSequence:value;
244	280
245	281
…	…
248	284
249	285	IntegerLiteral
250		= ~~bool addInteger(value,~~type)
251		::= Integer:value [IntegerSuffix:type];
	286	= Token Token.integerLiteral(value,subtype)
	287	::= Integer:value [IntegerSuffix:subtype];
252	288
253	289	Integer
254		= ~~uint~~ value
	290	= Integer value
255	291	::= Decimal:value \| Binary:value \| Octal:value \| Hexadecimal:value;
256	292
257	293	IntegerSuffix
258		::= "L" \| "u" \| "U" \| "Lu" \| "LU" \| "uL" \| "UL";
	294	= TokenSubtype subtype;
	295	::= "L" @TokenSubtype.LongInteger:subtype \|
	296	"u" @TokenSubtype.UnsignedInteger:subtype \|
	297	"U" @TokenSubtype.UnsignedInteger:subtype \|
	298	"Lu" @TokenSubtype.LongUnsignedInteger:subtype \|
	299	"LU" @TokenSubtype.LongUnsignedInteger:subtype \|
	300	"uL" @TokenSubtype.LongUnsignedInteger:subtype \|
	301	"UL" @TokenSubtype.LongUnsignedInteger:subtype;
259	302
260	303	Decimal
261		= uint convertToDecimal(value)
262		::= ("0" \| (NonZeroDigit {DecimalDigit})):value;
	304	= Integer convertToDecimal(value)
	305	::= ("0":~value \| (NonZeroDigit:~value {"_"} {DecimalDigit:~value {"_"}}));
	306
	307	DecimalDigits
	308	= Integer convertToDecimal(value)
	309	::= {"_"} {DecimalDigit:~value {"_"}};
263	310
264	311	Binary
265		= ~~uint~~ convertToBinary(value)
266		::= ("0b"\|"0B") {~~BinaryDigit}:value~~;
	312	= Integer convertToBinary(value)
	313	::= ("0b"\|"0B") {"_"} {BinaryDigit:~value {"_"}};
267	314
268	315	Octal
269		= uint convertToOctal(value)
270		::= "0" {OctalDigit}:value;
	316	= Integer convertToOctal(value)
	317	::= "0" {"_"} {OctalDigit:~value {"_"}};
	318
	319	HexPrefix
	320	::= ("0x"\|"0X");
271	321
272	322	Hexadecimal
273		= uint convertToHex(value)
274		::= ("0x"\|"0X") {HexDigit}:value;
275
	323	= Integer convertToHex(value)
	324	::= HexPrefix {"_"} {HexDigit:~value {"_"}};
	325
	326	HexDigits
	327	= Integer convertToHex(value)
	328	::= {"_"} {HexDigit:~value {"_"}};
	329
276	330	NonZeroDigit
277	331	::= "1" \| "2" \| "3" \| "4" \| "5" \| "6" \| "7" \| "8" \| "9";
278	332
279	333	DecimalDigit
280		::= "0" \| NonZeroDigit ~~\| "_"~~;
	334	::= "0" \| NonZeroDigit;
281	335
282	336	BinaryDigit
283		::= "0" \| "1" ~~\| "_"~~;
	337	::= "0" \| "1";
284	338
285	339	OctalDigit
286		::= "0" \| "1" \| "2" \| "3" \| "4" \| "5" \| "6" \| "7" ~~\| "_"~~;
	340	::= "0" \| "1" \| "2" \| "3" \| "4" \| "5" \| "6" \| "7";
287	341
288	342	HexDigit
289	343	::= "0" \| "1" \| "2" \| "3" \| "4" \| "5" \| "6" \| "7" \| "8" \| "9" \|
290	344	"a" \| "b" \| "c" \| "d" \| "e" \| "f" \|
291		"A" \| "B" \| "C" \| "D" \| "E" \| "F" \|
292		"_";
	345	"A" \| "B" \| "C" \| "D" \| "E" \| "F";
	346
293	347
294	348	##### Floating Literals #####
295
296	349	#TODO: break down the various rules into more useful parts
297	350
298	351	FloatLiteral
299	352	= bool addFloat(value,type,bool isImaginary)
300		::= Float:value [FloatSuffix:type] [ImaginarySuffix:isImaginary];
	353	= Token Token.floatingLiteral(value,subtype,isImaginary)
	354	::= Float:value [FloatSuffix:subtype] [ImaginarySuffix:isImaginary];
301	355
302	356	Float
303		= value
	357	= FloatingPoint value
304	358	::= DecimalFloat:value \| HexFloat:value;
305	359
306	360	DecimalFloat
307		::= Decimal "." [Decimal] [DecimalExponent] \|
308		"." Decimal [DecimalExponent] \|
309		Decimal DecimalExponent;
310
311		DecimalExponent
312		::= "e" Decimal \|
313		"E" Decimal \|
314		"e+" Decimal \|
315		"E+" Decimal \|
316		"e-" Decimal \|
317		"E-" Decimal;
	361	= FloatingPoint FloatingPoint(whole,fraction,exponent,exponentSign)
	362	::= Decimal:whole "." [DecimalDigits:fraction [DecimalExponent!(exponent,exponentSign)]] \|
	363	"." DecimalDigits:fraction [DecimalExponent:exponent] \|
	364	Decimal:whole DecimalExponent!(exponent,exponentSign);
	365
	366	DecimalExponent(Integer exponent,bool exponentSign)
	367	::= "e" DecimalDigits:exponent @true:exponentSign \|
	368	"E" DecimalDigits:exponent @true:exponentSign \|
	369	"e+" DecimalDigits:exponent @true:exponentSign \|
	370	"E+" DecimalDigits:exponent @true:exponentSign \|
	371	"e-" DecimalDigits:exponent @false:exponentSign \|
	372	"E-" DecimalDigits:exponent @false:exponentSign;
318	373
319	374	HexFloat
320		::= HexPrefix HexDigits "."
321		HexPrefix HexDigits "." HexDigits \|
322		HexPrefix HexDigits "." HexDigits HexExponent \|
323		HexPrefix "." HexDigits \|
324		HexPrefix "." HexDigits HexExponent \|
325		HexPrefix HexDigits HexExponent;
326
327		HexDigits
328		::= HexDigit {HexDigit};
329
330		HexPrefix
331		::= "0x" \| "0X";
332
333		HexExponent
334		::= "p" Decimal \|
335		"P" Decimal \|
336		"p+" Decimal \|
337		"P+" Decimal \|
338		"p-" Decimal \|
339		"P-" Decimal;
	375	= FloatingPoint FloatingPoint(whole,fraction,exponent,exponentSign)
	376	::= Hexadecimal:whole "." [HexDigits:fraction [HexExponent!(exponent,exponentSign)]] \|
	377	HexPrefix "." HexDigits:fraction [HexExponent!(exponent,exponentSign)] \|
	378	Hexadecimal:whole HexExponent!(exponent,exponentSign);
	379
	380	HexExponent(Integer exponent,bool exponentSign)
	381	::= "p" DecimalDigits:exponent @true:exponentSign \|
	382	"P" DecimalDigits:exponent @true:exponentSign \|
	383	"p+" DecimalDigits:exponent @true:exponentSign \|
	384	"P+" DecimalDigits:exponent @true:exponentSign \|
	385	"p-" DecimalDigits:exponent @false:exponentSign \|
	386	"P-" DecimalDigits:exponent @false:exponentSign;
340	387
341	388	FloatSuffix
342		::= "f" \| "F" \| "L";
	389	= TokenSubtype subtype
	390	::= "f" @TokenSubtype.Float:subtype \|
	391	"F" @TokenSubtype.Float:subtype \|
	392	"L" @TokenSubtype.Real:subtype;
343	393
344	394	ImaginarySuffix
345		::= "i";
	395	= bool value;
	396	::= "i" @true:value \| @false:value;
346	397
347	398	##### Keywords #####
…	…
349	400
350	401	Keyword
351		= bool addKeyword(tok)
352		::= ("abstract" \|
353		"alias" \|
354		"align" \|
355		"asm" \|
356		"assert" \|
357		"auto" \|
358
359		"body" \|
360		"bool" \|
361		"break" \|
362		"byte" \|
363
364		"case" \|
365		"cast" \|
366		"catch" \|
367		"cdouble" \|
368		"cent" \|
369		"cfloat" \|
370		"char" \|
371		"class" \|
372		"const" \|
373		"continue" \|
374		"creal" \|
375
376		"dchar" \|
377		"debug" \|
378		"default" \|
379		"delegate" \|
380		"delete" \|
381		"deprecated" \|
382		"do" \|
383		"double" \|
384
385		"else" \|
386		"enum" \|
387		"export" \|
388		"extern" \|
389
390		"false" \|
391		"final" \|
392		"finally" \|
393		"float" \|
394		"for" \|
395		"foreach" \|
396		"function" \|
397
398		"goto" \|
399
400		"idouble" \|
401		"if" \|
402		"ifloat" \|
403		"import" \|
404		"in" \|
405		"inout" \|
406		"int" \|
407		"interface" \|
408		"invariant" \|
409		"ireal" \|
410		"is" \|
411
412		"long" \|
413
414		"mixin" \|
415		"module" \|
416
417		"new" \|
418		"null" \|
419
420		"out" \|
421		"override" \|
422
423		"package" \|
424		"pragma" \|
425		"private" \|
426		"protected" \|
427		"public" \|
428
429		"real" \|
430		"return" \|
431
432		"scope" \|
433		"short" \|
434		"static" \|
435		"struct" \|
436		"super" \|
437		"switch" \|
438		"synchronized" \|
439
440		"template" \|
441		"this" \|
442		"throw" \|
443		"true" \|
444		"try" \|
445		"typedef" \|
446		"typeid" \|
447		"typeof" \|
448
449		"ubyte" \|
450		"ucent" \|
451		"uint" \|
452		"ulong" \|
453		"union" \|
454		"unittest" \|
455		"ushort" \|
456
457		"version" \|
458		"void" \|
459		"volatile" \|
460
461		"wchar" \|
462		"while" \|
463		"with"
	402	= Token Token.keyword(tok)
	403	::= ("abstract" @Abstract:tok \|
	404	"alias" @Alia:tok \|
	405	"align" @Align:tok \|
	406	"asm" @Asm:tok \|
	407	"assert" @Assert:tok \|
	408	"auto" @Auto:tok \|
	409	"body" @Body:tok \|
	410	"bool" @Bool:tok \|
	411	"break" @Break:tok \|
	412	"byte" @Byte:tok \|
	413	"case" @Case:tok \|
	414	"cast" @Cast:tok \|
	415	"catch" @Catch:tok \|
	416	"cdouble" @Cdouble:tok \|
	417	"cent" @Cent:tok \|
	418	"cfloat" @Cfloat:tok \|
	419	"char" @Char:tok \|
	420	"class" @Cla:tok \|
	421	"const" @Const:tok \|
	422	"continue" @Continue:tok \|
	423	"creal" @Creal:tok \|
	424	"dchar" @Dchar:tok \|
	425	"debug" @Debug:tok \|
	426	"default" @Default:tok \|
	427	"delegate" @Delegate:tok \|
	428	"delete" @Delete:tok \|
	429	"deprecated" @Deprecated:tok \|
	430	"do" @Do:tok \|
	431	"double" @Double:tok \|
	432	"else" @Else:tok \|
	433	"enum" @Enum:tok \|
	434	"export" @Export:tok \|
	435	"extern" @Extern:tok \|
	436	"false" @False:tok \|
	437	"final" @Final:tok \|
	438	"finally" @Finally:tok \|
	439	"float" @Float:tok \|
	440	"for" @For:tok \|
	441	"foreach" @Foreach:tok \|
	442	"function" @Function:tok \|
	443	"goto" @Goto:tok \|
	444	"idouble" @Idouble:tok \|
	445	"if" @If:tok \|
	446	"ifloat" @Ifloat:tok \|
	447	"import" @Import:tok \|
	448	"in" @In:tok \|
	449	"inout" @Inout:tok \|
	450	"int" @Int:tok \|
	451	"interface" @Interface:tok \|
	452	"invariant" @Invariant:tok \|
	453	"ireal" @Ireal:tok \|
	454	"is" @Is:tok \|
	455	"long" @Long:tok \|
	456	"mixin" @Mixin:tok \|
	457	"module" @Module:tok \|
	458	"new" @New:tok \|
	459	"null" @Null:tok \|
	460	"out" @Out:tok \|
	461	"override" @Override:tok \|
	462	"package" @Package:tok \|
	463	"pragma" @Pragma:tok \|
	464	"private" @Private:tok \|
	465	"protected" @Protected:tok \|
	466	"public" @Public:tok \|
	467	"real" @Real:tok \|
	468	"return" @Return:tok \|
	469	"scope" @Scope:tok \|
	470	"short" @Short:tok \|
	471	"static" @Static:tok \|
	472	"struct" @Struct:tok \|
	473	"super" @Super:tok \|
	474	"switch" @Switch:tok \|
	475	"synchronized" @Synchronized:tok \|
	476	"template" @Template:tok \|
	477	"this" @Thi:tok \|
	478	"throw" @Throw:tok \|
	479	"true" @True:tok \|
	480	"try" @Try:tok \|
	481	"typedef" @Typedef:tok \|
	482	"typeid" @Typeid:tok \|
	483	"typeof" @Typeof:tok \|
	484	"ubyte" @Ubyte:tok \|
	485	"ucent" @Ucent:tok \|
	486	"uint" @UInt:tok \|
	487	"ulong" @Ulong:tok \|
	488	"union" @Union:tok \|
	489	"unittest" @Unittest:tok \|
	490	"ushort" @Ushort:tok \|
	491	"version" @Version:tok \|
	492	"void" @Void:tok &nbs

ddl

Navigation

Changeset 232

Legend:

trunk/enki/library/d/Token.d

trunk/enki/library/d/Tokenizer.bnf