Navigation

← Previous Changeset
Next Changeset →

Changeset 233

Timestamp:

08/07/06 12:28:56 (2 years ago)

Author:

pragma

Message:

Updates for D lexer.

Files:

trunk/enki/enki_bn.d (modified) (1 diff)
trunk/enki/library/d/Token.d (modified) (1 diff)
trunk/enki/library/d/Tokenizer.bnf (modified) (13 diffs)
trunk/enki/library/d/TokenizerBase.d (modified) (1 diff)

Legend:

: Unmodified
: Added
: Removed
: Modified
: Copied
: Moved

trunk/enki/enki_bn.d

r231	r233
2	2	// This file is automatically maintained by the BUILD utility,
3	3	// Please refrain from manually editing it.
4		long auto_build_number = 154;
	4	long auto_build_number = 156;

trunk/enki/library/d/Token.d

r232 r233

28 28
29 29 private import enki.types;
30 ~~private import ddoc.Token;~~
31 30
32 31 alias ulong Integer; //TODO: turn into a more advanced type

trunk/enki/library/d/Tokenizer.bnf

r232	r233
1		{{{/+
	1	.code{{{/+
2	2	Copyright (c) 2006 Eric Anderton
3	3
…	…
38	38
39	39	.define(bool,eoi,false,"End of Input");
40		~~.define(String,any,false);~~
41	40	.define("String","letter","true","Letter");
42	41	.define("String","digit","true","Digit");
43		.define("String","any","true","any");
	42
	43	.include("enki/library/d/CharEntity.bnf");
	44
	45	.define("String","NamedCharacterEntity","true","NamedCharacterEntity");
	46
	47	.include("enki/library/d/Character.bnf");
	48
	49	.define("String","Character","true","Character");
44	50
45	51	Syntax
46		= Token[] tokens;
	52	= Token[] tokens
47	53	::= {
48	54	Space:~tokens \|
49	55	Token:~tokens \|
50		~~Identifier:~tokens \|~~
51	56	StringLiteral:~tokens \|
52	57	CharacterLiteral:~tokens \|
…	…
73	78
74	79	Space
75		= Token tok;
	80	= Token tok
76	81	::= #20 \| #09 \| #0B \| #0C \| EndOfLine \| Comment:tok;
77	82
…	…
87	92
88	93
89
90	94	##### Tokens #####
91	95
92	96	Token
93		::= Identifier \|
94		StringLiteral \|
95		CharacterLiteral \|
96		IntegerLiteral \|
97		FloatLiteral \|
98		Keyword \|
99		OtherToken;
	97	= Token tok
	98	::= Identifier:tok \|
	99	StringLiteral:tok \|
	100	CharacterLiteral:tok \|
	101	IntegerLiteral:tok \|
	102	FloatLiteral:tok \|
	103	Keyword:tok \|
	104	OtherToken:tok;
100	105
101	106	OtherToken
102		= Token Token.operator(Tok tok);
103		::= ~~("/=" @DivAssign:tok,~~ \|
104		"/" @~~Div:tok,~~ \|
105		"..." @~~Elipsis:tok,~~ \|
106		".." @~~Slice:tok,~~ \|
107		"." @~~Dot:tok,~~ \|
108		"&=" @~~AndAssign:tok,~~ \|
109		"&&" @~~AndAnd:tok,~~ \|
110		"&" @~~And:tok,~~ \|
111		"\|=" @~~OrAssign:tok,~~ \|
112		"\|\|" @~~OrOr:tok,~~ \|
113		"\|" @~~Or:tok,~~ \|
114		"--" @~~MinusMinus:tok,~~ \|
115		"-=" @~~MinusAssign:tok,~~ \|
116		"-" @~~Minus:tok,~~ \|
117		"++" @~~PlusPlus:tok,~~ \|
118		"+=" @~~PlusAssign:tok,~~ \|
119		"+" @~~Plus:tok,~~ \|
120		"<=" @~~LessEquals:tok,~~ \|
121		"<<" @~~LessLess:tok,~~ \|
122		"<<=" @~~LessLessEquals:tok,~~ \|
123		"<>" @~~LessGreater:tok,~~ \|
124		"<>=" @~~LessGreaterEquals:tok,~~ \|
125		"<" @~~Less:tok,~~ \|
126		">>>" @~~GreaterGreaterGreater:tok,~~ \|
127		">>>=" @~~GreaterGreaterGreaterEquals:tok,~~\|
128		">>=" @~~GreaterGreaterEquals:tok,~~ \|
129		">>" @~~GreaterGreater:tok,~~ \|
130		">=" @~~GreaterEquals:tok,~~ \|
131		">" @~~Greater:tok,~~ \|
132		"!==" @~~NotEqualsEquals:tok,~~ \|
133		"!=" @~~NotEquals:tok,~~ \|
134		"!<>" @~~NotGreaterLess:tok,~~ \|
135		"!<>=" @~~NotGreaterLessEquals:tok,~~ \|
136		"!<" @~~NotLess:tok,~~ \|
137		"!<=" @~~NotLessEquals:tok,~~ \|
138		"!>" @~~NotGreater:tok,~~ \|
139		"!>=" @~~NotGreaterEquals:tok,~~ \|
140		"!~" @~~NotCat:tok,~~ \|
141		"!" @~~Not:tok,~~ \|
142		"(" @~~OpenParen:tok,~~ \|
143		")" @~~CloseParen:tok,~~ \|
144		"[" @~~OpenBracket:tok,~~ \|
145		"]" @~~CloseBracket:tok,~~ \|
146		"{" @~~OpenCurl:tok,~~ \|
147		"}" @~~CloseCurl:tok,~~ \|
148		"?" @~~Question:tok,~~ \|
149		"," @~~Comma:tok,~~ \|
150		";" @~~Semi:tok,~~ \|
151		":" @~~Colon:tok,~~ \|
152		"$" @~~Dollar:tok,~~ \|
153		"===" @~~EqualsEqualsEquals:tok,~~ \|
154		"==" @~~EqualsEquals:tok,~~ \|
155		"=" @~~Equals:tok,~~ \|
156		"*=" @~~StarEquals:tok,~~ \|
157		"*" @~~Star:tok,~~ \|
158		"%=" @~~ModEquals:tok,~~ \|
159		"%" @~~Mod:tok,~~ \|
160		"^=" @~~InverseEquals:tok,~~ \|
161		"^" @~~Inverse:tok,~~ \|
162		"~=" @~~CatEquals:tok,~~ \|
163		"~~" @~~CatCat:tok,~~ \|
164		"~" @Cat:tok
165		~~):tok~~;
	107	= Token Token.operator(Tok tok)
	108	::= "/=" @Tok.DivAssign:tok \|
	109	"/" @Tok.Div:tok \|
	110	"..." @Tok.Elipsis:tok \|
	111	".." @Tok.Slice:tok \|
	112	"." @Tok.Dot:tok \|
	113	"&=" @Tok.AndAssign:tok \|
	114	"&&" @Tok.AndAnd:tok \|
	115	"&" @Tok.And:tok \|
	116	"\|=" @Tok.OrAssign:tok \|
	117	"\|\|" @Tok.OrOr:tok \|
	118	"\|" @Tok.Or:tok \|
	119	"--" @Tok.MinusMinus:tok \|
	120	"-=" @Tok.MinusAssign:tok \|
	121	"-" @Tok.Minus:tok \|
	122	"++" @Tok.PlusPlus:tok \|
	123	"+=" @Tok.PlusAssign:tok \|
	124	"+" @Tok.Plus:tok \|
	125	"<=" @Tok.LessEquals:tok \|
	126	"<<" @Tok.LessLess:tok \|
	127	"<<=" @Tok.LessLessEquals:tok \|
	128	"<>" @Tok.LessGreater:tok \|
	129	"<>=" @Tok.LessGreaterEquals:tok \|
	130	"<" @Tok.Less:tok \|
	131	">>>" @Tok.GreaterGreaterGreater:tok \|
	132	">>>=" @Tok.GreaterGreaterGreaterEquals:tok\|
	133	">>=" @Tok.GreaterGreaterEquals:tok \|
	134	">>" @Tok.GreaterGreater:tok \|
	135	">=" @Tok.GreaterEquals:tok \|
	136	">" @Tok.Greater:tok \|
	137	"!==" @Tok.NotEqualsEquals:tok \|
	138	"!=" @Tok.NotEquals:tok \|
	139	"!<>" @Tok.NotGreaterLess:tok \|
	140	"!<>=" @Tok.NotGreaterLessEquals:tok \|
	141	"!<" @Tok.NotLess:tok \|
	142	"!<=" @Tok.NotLessEquals:tok \|
	143	"!>" @Tok.NotGreater:tok \|
	144	"!>=" @Tok.NotGreaterEquals:tok \|
	145	"!~" @Tok.NotCat:tok \|
	146	"!" @Tok.Not:tok \|
	147	"(" @Tok.OpenParen:tok \|
	148	")" @Tok.CloseParen:tok \|
	149	"[" @Tok.OpenBracket:tok \|
	150	"]" @Tok.CloseBracket:tok \|
	151	"{" @Tok.OpenCurl:tok \|
	152	"}" @Tok.CloseCurl:tok \|
	153	"?" @Tok.Question:tok \|
	154	"," @Tok.Comma:tok \|
	155	";" @Tok.Semi:tok \|
	156	":" @Tok.Colon:tok \|
	157	"$" @Tok.Dollar:tok \|
	158	"===" @Tok.EqualsEqualsEquals:tok \|
	159	"==" @Tok.EqualsEquals:tok \|
	160	"=" @Tok.Equals:tok \|
	161	"*=" @Tok.StarEquals:tok \|
	162	"*" @Tok.Star:tok \|
	163	"%=" @Tok.ModEquals:tok \|
	164	"%" @Tok.Mod:tok \|
	165	"^=" @Tok.InverseEquals:tok \|
	166	"^" @Tok.Inverse:tok \|
	167	"~=" @Tok.CatEquals:tok \|
	168	"~~" @Tok.CatCat:tok \|
	169	"~" @Tok.Cat:tok
	170	;
166	171
167	172
…	…
170	175
171	176	Identifier
172		= Token Token.identifier(name);
	177	= Token Token.identifier(name)
173	178	::= (IdentifierStart {IdentifierChar}):name;
174	179
…	…
252	257	OctalDigit OctalDigit \|
253	258	OctalDigit
254		):digits
	259	):digits;
255	260
256	261	HexString
…	…
260	265	HexStringChar
261	266	= Char hexToChar(digits)
262		::= WhiteSpace HexDigit:~~~value WhiteSpace HexDigit:~value~~;
	267	::= WhiteSpace HexDigit:~digits WhiteSpace HexDigit:~digits;
263	268
264	269	Postfix
265		= TokenSubtype subtype;
	270	= TokenSubtype subtype
266	271	::= "c" @TokenSubtype.Char:subtype \|
267	272	"w" @TokenSubtype.WChar:subtype \|
…	…
292	297
293	298	IntegerSuffix
294		= TokenSubtype subtype;
	299	= TokenSubtype subtype
295	300	::= "L" @TokenSubtype.LongInteger:subtype \|
296	301	"u" @TokenSubtype.UnsignedInteger:subtype \|
…	…
302	307
303	308	Decimal
304		= Integer convertToDecimal(value)
	309	= Integer convertToDecimal(String value)
305	310	::= ("0":~value \| (NonZeroDigit:~value {"_"} {DecimalDigit:~value {"_"}}));
306	311
…	…
350	355
351	356	FloatLiteral
352		~~= bool addFloat(value,type,bool isImaginary)~~
353	357	= Token Token.floatingLiteral(value,subtype,isImaginary)
354	358	::= Float:value [FloatSuffix:subtype] [ImaginarySuffix:isImaginary];
…	…
393	397
394	398	ImaginarySuffix
395		= bool value;
	399	= bool value
396	400	::= "i" @true:value \| @false:value;
397	401
…	…
401	405	Keyword
402	406	= Token Token.keyword(tok)
403		::= ~~("abstract" @~~Abstract:tok \|
404		"alias" @Alia:tok \|
405		"align" @Align:tok \|
406		"asm" @Asm:tok \|
407		"assert" @Assert:tok \|
408		"auto" @Auto:tok \|
409		"body" @Body:tok \|
410		"bool" @Bool:tok \|
411		"break" @Break:tok \|
412		"byte" @Byte:tok \|
413		"case" @Case:tok \|
414		"cast" @Cast:tok \|
415		"catch" @Catch:tok \|
416		"cdouble" @Cdouble:tok \|
417		"cent" @Cent:tok \|
418		"cfloat" @Cfloat:tok \|
419		"char" @Char:tok \|
420		"class" @Cla:tok \|
421		"const" @Const:tok \|
422		"continue" @Continue:tok \|
423		"creal" @Creal:tok \|
424		"dchar" @Dchar:tok \|
425		"debug" @Debug:tok \|
426		"default" @Default:tok \|
427		"delegate" @Delegate:tok \|
428		"delete" @Delete:tok \|
429		"deprecated" @Deprecated:tok \|
430		"do" @Do:tok \|
431		"double" @Double:tok \|
432		"else" @Else:tok \|
433		"enum" @Enum:tok \|
434		"export" @Export:tok \|
435		"extern" @Extern:tok \|
436		"false" @False:tok \|
437		"final" @Final:tok \|
438		"finally" @Finally:tok \|
439		"float" @Float:tok \|
440		"for" @For:tok \|
441		"foreach" @Foreach:tok \|
442		"function" @Function:tok \|
443		"goto" @Goto:tok \|
444		"idouble" @Idouble:tok \|
445		"if" @If:tok \|
446		"ifloat" @Ifloat:tok \|
447		"import" @Import:tok \|
448		"in" @In:tok \|
449		"inout" @Inout:tok \|
450		"int" @Int:tok \|
451		"interface" @Interface:tok \|
452		"invariant" @Invariant:tok \|
453		"ireal" @Ireal:tok \|
454		"is" @Is:tok \|
455		"long" @Long:tok \|
456		"mixin" @Mixin:tok \|
457		"module" @Module:tok \|
458		"new" @New:tok \|
459		"null" @Null:tok \|
460		"out" @Out:tok \|
461		"override" @Override:tok \|
462		"package" @Package:tok \|
463		"pragma" @Pragma:tok \|
464		"private" @Private:tok \|
465		"protected" @Protected:tok \|
466		"public" @Public:tok \|
467		"real" @Real:tok \|
468		"return" @Return:tok \|
469		"scope" @Scope:tok \|
470		"short" @Short:tok \|
471		"static" @Static:tok \|
472		"struct" @Struct:tok \|
473		"super" @Super:tok \|
474		"switch" @Switch:tok \|
475		"synchronized" @Synchronized:tok \|
476		"template" @Template:tok \|
477		"this" @Thi:tok \|
478		"throw" @Throw:tok \|
479		"true" @True:tok \|
480		"try" @Try:tok \|
481		"typedef" @Typedef:tok \|
482		"typeid" @Typeid:tok \|
483		"typeof" @Typeof:tok \|
484		"ubyte" @Ubyte:tok \|
485		"ucent" @Ucent:tok \|
486		"uint" @UInt:tok \|
487		"ulong" @Ulong:tok \|
488		"union" @Union:tok \|
489		"unittest" @Unittest:tok \|
490		"ushort" @Ushort:tok \|
491		"version" @Version:tok \|
492		"void" @Void:tok \|
493		"volatile" @Volatile:tok \|
494		"wchar" @Wchar:tok \|
495		"while" @While:tok \|
496		"with" @With:tok
497		~~):tok~~;
	407	::= "abstract" @Tok.Abstract:tok \|
	408	"alias" @Tok.Alia:tok \|
	409	"align" @Tok.Align:tok \|
	410	"asm" @Tok.Asm:tok \|
	411	"assert" @Tok.Assert:tok \|
	412	"auto" @Tok.Auto:tok \|
	413	"body" @Tok.Body:tok \|
	414	"bool" @Tok.Bool:tok \|
	415	"break" @Tok.Break:tok \|
	416	"byte" @Tok.Byte:tok \|
	417	"case" @Tok.Case:tok \|
	418	"cast" @Tok.Cast:tok \|
	419	"catch" @Tok.Catch:tok \|
	420	"cdouble" @Tok.Cdouble:tok \|
	421	"cent" @Tok.Cent:tok \|
	422	"cfloat" @Tok.Cfloat:tok \|
	423	"char" @Tok.Char:tok \|
	424	"class" @Tok.Cla:tok \|
	425	"const" @Tok.Const:tok \|
	426	"continue" @Tok.Continue:tok \|
	427	"creal" @Tok.Creal:tok \|
	428	"dchar" @Tok.Dchar:tok \|
	429	"debug" @Tok.Debug:tok \|
	430	"default" @Tok.Default:tok \|
	431	"delegate" @Tok.Delegate:tok \|
	432	"delete" @Tok.Delete:tok \|
	433	"deprecated" @Tok.Deprecated:tok \|
	434	"do" @Tok.Do:tok \|
	435	"double" @Tok.Double:tok \|
	436	"else" @Tok.Else:tok \|
	437	"enum" @Tok.Enum:tok \|
	438	"export" @Tok.Export:tok \|
	439	"extern" @Tok.Extern:tok \|
	440	"false" @Tok.False:tok \|
	441	"final" @Tok.Final:tok \|
	442	"finally" @Tok.Finally:tok \|
	443	"float" @Tok.Float:tok \|
	444	"for" @Tok.For:tok \|
	445	"foreach" @Tok.Foreach:tok \|
	446	"function" @Tok.Function:tok \|
	447	"goto" @Tok.Goto:tok \|
	448	"idouble" @Tok.Idouble:tok \|
	449	"if" @Tok.If:tok \|
	450	"ifloat" @Tok.Ifloat:tok \|
	451	"import" @Tok.Import:tok \|
	452	"in" @Tok.In:tok \|
	453	"inout" @Tok.Inout:tok \|
	454	"int" @Tok.Int:tok \|
	455	"interface" @Tok.Interface:tok \|
	456	"invariant" @Tok.Invariant:tok \|
	457	"ireal" @Tok.Ireal:tok \|
	458	"is" @Tok.Is:tok \|
	459	"long" @Tok.Long:tok \|
	460	"mixin" @Tok.Mixin:tok \|
	461	"module" @Tok.Module:tok \|
	462	"new" @Tok.New:tok \|
	463	"null" @Tok.Null:tok \|
	464	"out" @Tok.Out:tok \|
	465	"override" @Tok.Override:tok \|
	466	"package" @Tok.Package:tok \|
	467	"pragma" @Tok.Pragma:tok \|
	468	"private" @Tok.Private:tok \|
	469	"protected" @Tok.Protected:tok \|
	470	"public" @Tok.Public:tok \|
	471	"real" @Tok.Real:tok \|
	472	"return" @Tok.Return:tok \|
	473	"scope" @Tok.Scope:tok \|
	474	"short" @Tok.Short:tok \|
	475	"static" @Tok.Static:tok \|
	476	"struct" @Tok.Struct:tok \|
	477	"super" @Tok.Super:tok \|
	478	"switch" @Tok.Switch:tok \|
	479	"synchronized" @Tok.Synchronized:tok \|
	480	"template" @Tok.Template:tok \|
	481	"this" @Tok.Thi:tok \|
	482	"throw" @Tok.Throw:tok \|
	483	"true" @Tok.True:tok \|
	484	"try" @Tok.Try:tok \|
	485	"typedef" @Tok.Typedef:tok \|
	486	"typeid" @Tok.Typeid:tok \|
	487	"typeof" @Tok.Typeof:tok \|
	488	"ubyte" @Tok.Ubyte:tok \|
	489	"ucent" @Tok.Ucent:tok \|
	490	"uint" @Tok.UInt:tok \|
	491	"ulong" @Tok.Ulong:tok \|
	492	"union" @Tok.Union:tok \|
	493	"unittest" @Tok.Unittest:tok \|
	494	"ushort" @Tok.Ushort:tok \|
	495	"version" @Tok.Version:tok \|
	496	"void" @Tok.Void:tok \|
	497	"volatile" @Tok.Volatile:tok \|
	498	"wchar" @Tok.Wchar:tok \|
	499	"while" @Tok.While:tok \|
	500	"with" @Tok.With:tok
	501	;
498	502
499	503	##### Special Tokens #####
…	…
515	519	##### Others #####
516	520
517		~~# TODO: use the C99 specification for Character instead~~
518		~~Character ::= any;~~
519
520		~~#TODO: add the remainder of named char entities~~
521		~~#http://digitalmars.com/d/entity.html~~
522
523		~~NamedCharacterEntity~~
524		~~::= "quot" \| "amp";~~
525
526	521	Letter ::= letter;
527
528	522	Digit ::= digit;

trunk/enki/library/d/TokenizerBase.d

r232	r233
104	104	return result;
105	105	}
	106
	107	// shim to help with direct assignment of token values
	108	String literal(String value){
	109	return value;
	110	}
106	111	}
107	112

ddl

Navigation

Changeset 233

Legend:

trunk/enki/enki_bn.d

trunk/enki/library/d/Token.d

trunk/enki/library/d/Tokenizer.bnf

trunk/enki/library/d/TokenizerBase.d

Download in other formats:

r232	r233
28	28
29	29	private import enki.types;
30		~~private import ddoc.Token;~~
31	30
32	31	alias ulong Integer; //TODO: turn into a more advanced type