Changeset 235

Timestamp:

08/11/06 11:12:17 (2 years ago)

Author:

pragma

Message:

Huge changeset to push Enki towards v1.2.
This update includes the start of a D lexer/parser

Files:

• trunk/enki/BaseParser.d (modified) (1 diff)
• trunk/enki/Directive.d (modified) (14 diffs)
• trunk/enki/EnkiBackend.d (modified) (8 diffs)
• trunk/enki/EnkiParser.d (modified) (6 diffs)
• trunk/enki/Expression.d (modified) (1 diff)
• trunk/enki/IParser.d (modified) (2 diffs)
• trunk/enki/bootstrap.d (modified) (4 diffs)
• trunk/enki/enki.bnf (modified) (4 diffs)
• trunk/enki/enki.d (modified) (2 diffs)
• trunk/enki/enki_bn.d (modified) (1 diff)
• trunk/enki/library/d/CharEntity.bnf (modified) (1 diff)
• trunk/enki/library/d/Character.bnf (modified) (1 diff)
• trunk/enki/library/d/Entity.d (added)
• trunk/enki/library/d/Keyword.bnf (added)
• trunk/enki/library/d/Keyword.d (modified) (1 diff)
• trunk/enki/library/d/Operator.d (modified) (1 diff)
• trunk/enki/library/d/OtherToken.bnf (added)
• trunk/enki/library/d/Parser.bnf (added)
• trunk/enki/library/d/ParserBackend.d (added)
• trunk/enki/library/d/Pragma.bnf (added)
• trunk/enki/library/d/Token.d (modified) (11 diffs)
• trunk/enki/library/d/Tokenizer.bnf (modified) (8 diffs)
• trunk/enki/library/d/TokenizerBase.d (modified) (5 diffs)
• trunk/enki/library/d/ddoc (added)
• trunk/enki/library/d/ddoc/DDocParser.d (added)
• trunk/enki/library/d/ddoc/Ddoc.bnf (added)
• trunk/enki/library/d/parser (added)
• trunk/enki/library/d/parser/Asm.bnf (added)
• trunk/enki/library/d/parser/Asm.d (added)
• trunk/enki/library/d/parser/Attribute.bnf (added)
• trunk/enki/library/d/parser/Attribute.d (added)
• trunk/enki/library/d/parser/Class.bnf (added)
• trunk/enki/library/d/parser/Class.d (added)
• trunk/enki/library/d/parser/Conditional.bnf (added)
• trunk/enki/library/d/parser/Conditional.d (added)
• trunk/enki/library/d/parser/Declaration.bnf (added)
• trunk/enki/library/d/parser/Declaration.d (added)
• trunk/enki/library/d/parser/Enum.bnf (added)
• trunk/enki/library/d/parser/Enum.d (added)
• trunk/enki/library/d/parser/Expression.bnf (added)
• trunk/enki/library/d/parser/Expression.d (added)
• trunk/enki/library/d/parser/Function.bnf (added)
• trunk/enki/library/d/parser/Function.d (added)
• trunk/enki/library/d/parser/Import.bnf (added)
• trunk/enki/library/d/parser/Import.d (added)
• trunk/enki/library/d/parser/Interface.bnf (added)
• trunk/enki/library/d/parser/Interface.d (added)
• trunk/enki/library/d/parser/Mixin.bnf (added)
• trunk/enki/library/d/parser/Mixin.d (added)
• trunk/enki/library/d/parser/Module.bnf (added)
• trunk/enki/library/d/parser/Module.d (added)
• trunk/enki/library/d/parser/Statement.d (added)
• trunk/enki/library/d/parser/Statements.bnf (added)
• trunk/enki/library/d/parser/Struct.bnf (added)
• trunk/enki/library/d/parser/Struct.d (added)
• trunk/enki/library/d/parser/Template.bnf (added)
• trunk/enki/library/d/parser/Template.d (added)
• trunk/enki/library/d/parser/x86 (added)
• trunk/enki/library/d/parser/x86/Opcode.bnf (added)
• trunk/enki/library/d/parser/x86/Opcode.d (added)
• trunk/enki/library/d/parser/x86/Register.bnf (added)
• trunk/enki/library/d/parser/x86/Register.d (added)
• trunk/enki/library/xml/Namespace.bnf (added)
• trunk/enki/library/xml/XMLParser.bnf (modified) (1 diff)
• trunk/enki/library/xpath/xpath10.bnf (modified) (5 diffs)

trunk/enki/BaseParser.d

@@ -47 +47 @@
-    }
-    
-ivate
-ivate
-ivate
+otected
+otected
+otected
-    
-    public this(){      

trunk/enki/Directive.d

@@ -31 +31 @@
-debug private import std.stdio;
-
+/*
+    RootOnly:
+        .baseclass
+        .classname
+        .module
+        .typelib
+        .parsetype
+        .utf
+        .boilerplate
+        
+    FirstPass:
+        .import
+        .define
+        .include
+        .header
+        
+    SecondPass:
+        .alias
+        .code
+*/
+
+
-abstract class Directive : SyntaxLine, IRenderable{
+    enum: uint{ 
+        RootOnly,   
+        FirstPass,
+        SecondPass,
+    }       
+    
+    public uint type(){
+        return Directive.SecondPass;
+    }
+    
-    public void semanticPass(BaseEnkiParser root){
-        //do nothing
@@ -48 +80 @@
-    public void render(CodeGenerator generator,Statement passterm,Statement failterm){
-        // do nothing
-  
+
-}
-
@@ -54 +86 @@
-    String imp;
-    
+    public uint type(){
+        return Directive.FirstPass;
+    }   
+        
-    public this(String imp){
-        this.imp = imp;
@@ -69 +105 @@
-        return "Import Directive";
-    }
+
-}
-
@@ -74 +111 @@
-    String name;
-    
+    public uint type(){
+        return Directive.RootOnly;
+    }   
+    
-    public this(String name){
-        this.name = name;
@@ -93 +134 @@
-class ClassnameDirective : Directive{
-    String name;
+    
+    public uint type(){
+        return Directive.RootOnly;
+    }
-    
-    public this(String name){
@@ -116 +161 @@
-    String description;
-    bool isTerminal;
+    
+    public uint type(){
+        return Directive.FirstPass;
+    }
-    
-    public this(String returnType,String name,bool isTerminal,String description){
@@ -153 +202 @@
-class IncludeDirective : Directive{
-    String filename;
+    
+    public uint type(){
+        return Directive.FirstPass;
+    }
-    
-    public this(String filename){
@@ -195 +248 @@
-    String rule;
-    String ruleAlias;
-
+
+
+
+
+
-    public this(String rule,String ruleAlias){
-        this.rule = rule;
@@ -216 +273 @@
-class ModuleDirective : Directive{
-    String moduleName;
-
+
+
+
+
+
-    public this(String moduleName){
-        this.moduleName = moduleName;
@@ -237 +298 @@
-class CodeDirective : Directive{
-    String code;
+    
+    public uint type(){
+        return Directive.SecondPass;
+    }
-        
-    public this(String code){
@@ -261 +326 @@
-class TypelibDirective : Directive{
-    String moduleName;
-
+
+
+
+
+
-    public this(String moduleName){
-        this.moduleName = moduleName;
@@ -281 +350 @@
-class ParseTypeDirective : Directive{
-    String typeName;
-
+
+
+
+
+
-    public this(String typeName){
-        this.typeName = typeName;
@@ -298 +371 @@
-    }   
-}
+
+
+class BoilerplateDirective : Directive{
+    String code;
+    
+    public uint type(){
+        return Directive.RootOnly;
+    }
+        
+    public this(String code){
+        this.code = code;
+    }
+    
+    public void semanticPass(BaseEnkiParser root){
+        root.setBoilerplate(code);
+    }
+    
+    public String toBNF(){
+        return ".boilerplate{{{" ~ code ~ "}}}\n";
+    }
+    
+    public String toString(){
+        return "Boilerplate Directive";
+    }   
+}
+
+class HeaderDirective : Directive{
+    String code;
+    
+    public uint type(){
+        return Directive.FirstPass;
+    }
+        
+    public this(String code){
+        this.code = code;
+    }
+    
+    public void semanticPass(BaseEnkiParser root){
+        root.setHeader(code);
+    }
+    
+    public String toBNF(){
+        return ".header{{{" ~ code ~ "}}}\n";
+    }
+    
+    public String toString(){
+        return "Header Directive";
+    }
+}
+
+class UTFDirective : Directive{
+    String value;
+    
+    public uint type(){
+        return Directive.RootOnly;
+    }
+        
+    public this(String value){
+        this.value = value;
+    }
+    
+    public void semanticPass(BaseEnkiParser root){
+        switch(value){
+        case "8":
+        case "32":
+        case "16":
+            break;
+        default:
+            throw new Exception("The value '" ~ value ~ "' is not a valid UTF parameter.  Expected '8', '16' or '32'");
+        }
+        root.setUTF(value);     
+    }
+    
+    public String toBNF(){
+        return ".utf(\"" ~ value ~ "\");\n";
+    }   
+    
+    public String toString(){
+        return "UTF Directive";
+    }
+}

trunk/enki/EnkiBackend.d

@@ -64 +64 @@
-    String typelib;
-    String parseType;
+    String boilerplate;
+    String header;
+    String utf;
-    UserProduction[] userProductions;
-
@@ -71 +74 @@
-        this.typelib = "enki.types";
-        this.parseType = "String";
+        this.utf = "8";
-    }
-    
@@ -78 +82 @@
-    
-    public void add(BaseEnkiParser other){
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
-    }
-    
@@ -104 +112 @@
-    }
-    
+    public void setBoilerplate(String value){
+        boilerplate ~= value;
+    }   
+    
+    public void setHeader(String value){
+        header ~= value;
+    }
+        
-    public void defineUserProduction(String returnType,String name,String description){
-        userProductions ~= UserProduction(returnType,name,description);
@@ -119 +135 @@
-    public String getParseType(){
-        return parseType;
+    }
+    
+    public void setUTF(String value){
+        utf = value;
-    }
-    
@@ -177 +197 @@
-    
-    public void semanticPass(){
-
+
+
+
+
+
+
+
+
+
+
-        foreach(line; lines){
-            Rule rule = cast(Rule)line;
-            if(rule){
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-    }
-        
@@ -194 +235 @@
-        auto CodeGenerator generator = new CodeGenerator(this.parseType);
-        with(generator){
-
+
+
+
+
+
+
+
-            if(moduleName){
-                emit("module " ~ moduleName ~ ";");
-            }
+            emit("version(build) pragma(export_version,EnkiUTF" ~ utf ~ ");");
+            emit("");
-            emit("debug private import std.stdio;");
-            
@@ -207 +256 @@
-            }
-            emit("");
+            
+            // header section
+            emit(header);
+            emit("");           
+            
+            // parser class definition
-            if(baseclass){
-                emit("class " ~ classname ~ " : " ~ baseclass ~ "{");

trunk/enki/EnkiParser.d

@@ -1 @@
-
-
-
-
-
-
-
-
-
-
+
-
-/+
-, BCS
+
-
-    Permission is hereby granted, free of charge, to any person
@@ -34 +25 @@
-    OTHER DEALINGS IN THE SOFTWARE.
-+/
+
+module enki.EnkiParser;
+version(build) pragma(export_version,EnkiUTF8);
+
+debug private import std.stdio;
+private import enki.types;
+private import enki.EnkiBackend;
+private import enki.Rule;
+private import enki.Expression;
+private import enki.Directive;
+
+
+
+class EnkiParser : BaseEnkiParser{
+
-    /*
-    
@@ -2124 +2130 @@
-Directive
-    = Directive dir
-
+| BoilerplateDirective:~dir | HeaderDirective:~dir | UTFDirective:~dir 
-
-    */
@@ -2140 +2146 @@
-            {//Expression
-                uint start158 = position;
-
+ || (parse_BoilerplateDirective().assignCat!(Directive)(bind_dir)) || (parse_HeaderDirective().assignCat!(Directive)(bind_dir)) || (parse_UTFDirective().assignCat!(Directive)(bind_dir))
-                    clearErrors();
-                }else{
- or ParseType
+, ParseTypeDirective, BoilerplateDirective, HeaderDirective or UTF
-                    position = start158;
-                    goto mismatch170;
@@ -2596 +2602 @@
-    /*
-    
+BoilerplateDirective
+    = new BoilerplateDirective(String code)
+    ::=  "boilerplate" ws "{{{" { any}:code "}}}";
+
+    */
+    public ResultT!(BoilerplateDirective) parse_BoilerplateDirective(){
+        debug writefln("parse_BoilerplateDirective()");
+        uint start183 = position;
+        String bind_code;
+        
+        
+        {//Expression
+            uint start184 = position;
+            if(!(terminal("boilerplate").success)){
+                goto mismatch200;
+            }
+            if(!(parse_ws().success)){
+                goto mismatch200;
+            }
+            if(!(terminal("{{{").success)){
+                goto mismatch200;
+            }
+            {//ZeroOrMoreExpr
+                uint start185 = position;
+                uint termPos;
+            loop201:
+                termPos = position;
+                if(terminal("}}}").success){
+                    goto loopend202;
+                }
+                {//Expression
+                    uint start186 = position;
+                    if((parse_any().success)){
+                        clearErrors();
+                        goto loop201;
+                    }else{
+                        setError("Expected any.");
+                        position = start186;
+                        goto mismatch200;
+                    }
+                }
+            loopend202:
+                smartAssign!(String,String)(bind_code,sliceData(start185,termPos));
+                {/*do nothing*/}
+            }
+            goto match199;
+        mismatch200:
+            {/*do nothing*/}
+            position = start184;
+            goto mismatch198;
+        match199:
+            clearErrors();
+            goto match197;
+        }
+    match197:
+        debug writefln("parse_BoilerplateDirective() PASS");
+        ResultT!(BoilerplateDirective) passed = ResultT!(BoilerplateDirective)(new BoilerplateDirective(bind_code));
+        return passed;
+    mismatch198:
+        position = start183;
+        ResultT!(BoilerplateDirective) failed = ResultT!(BoilerplateDirective)();
+        return failed;
+    }
+
+    /*
+    
+HeaderDirective
+    = new HeaderDirective(String code)
+    ::=  "header" ws "{{{" { any}:code "}}}";
+
+    */
+    public ResultT!(HeaderDirective) parse_HeaderDirective(){
+        debug writefln("parse_HeaderDirective()");
+        uint start187 = position;
+        String bind_code;
+        
+        
+        {//Expression
+            uint start188 = position;
+            if(!(terminal("header").success)){
+                goto mismatch206;
+            }
+            if(!(parse_ws().success)){
+                goto mismatch206;
+            }
+            if(!(terminal("{{{").success)){
+                goto mismatch206;
+            }
+            {//ZeroOrMoreExpr
+                uint start189 = position;
+                uint termPos;
+            loop207:
+                termPos = position;
+                if(terminal("}}}").success){
+                    goto loopend208;
+                }
+                {//Expression
+                    uint start190 = position;
+                    if((parse_any().success)){
+                        clearErrors();
+                        goto loop207;
+                    }else{
+                        setError("Expected any.");
+                        position = start190;
+                        goto mismatch206;
+                    }
+                }
+            loopend208:
+                smartAssign!(String,String)(bind_code,sliceData(start189,termPos));
+                {/*do nothing*/}
+            }
+            goto match205;
+        mismatch206:
+            {/*do nothing*/}
+            position = start188;
+            goto mismatch204;
+        match205:
+            clearErrors();
+            goto match203;
+        }
+    match203:
+        debug writefln("parse_HeaderDirective() PASS");
+        ResultT!(HeaderDirective) passed = ResultT!(HeaderDirective)(new HeaderDirective(bind_code));
+        return passed;
+    mismatch204:
+        position = start187;
+        ResultT!(HeaderDirective) failed = ResultT!(HeaderDirective)();
+        return failed;
+    }
+
+    /*
+    
+UTFDirective
+    = new UTFDirective(String value)
+    ::=  "utf" ws "(" ws DirectiveArg:value  ws ")" ws ";";
+
+    */
+    public ResultT!(UTFDirective) parse_UTFDirective(){
+        debug writefln("parse_UTFDirective()");
+        uint start191 = position;
+        String bind_value;
+        
+        
+        {//Expression
+            uint start192 = position;
+            if((terminal("utf").success && parse_ws().success && terminal("(").success && parse_ws().success && parse_DirectiveArg().assign!(String)(bind_value) && parse_ws().success && terminal(")").success && parse_ws().success && terminal(";").success)){
+                clearErrors();
+                goto match209;
+            }else{
+                position = start192;
+                goto mismatch210;
+            }
+        }
+    match209:
+        debug writefln("parse_UTFDirective() PASS");
+        ResultT!(UTFDirective) passed = ResultT!(UTFDirective)(new UTFDirective(bind_value));
+        return passed;
+    mismatch210:
+        position = start191;
+        ResultT!(UTFDirective) failed = ResultT!(UTFDirective)();
+        return failed;
+    }
+
+    /*
+    
-DirectiveArg
-    = String arg
@@ -2603 +2774 @@
-    public ResultT!(String) parse_DirectiveArg(){
-        debug writefln("parse_DirectiveArg()");
-8
+9
-        String bind_arg;
-        
-        
-        {//Expression
-8
+9
-            if((parse_Identifier().assign!(String)(bind_arg)) || (parse_String().assign!(String)(bind_arg))){
-                clearErrors();
-197
+211
-            }else{
-                setError("Expected Identifier or String.");
-8
-198
-
-
-197
+9
+212
+
+
+211
-        debug writefln("parse_DirectiveArg() PASS");
-        return ResultT!(String)(bind_arg);
-198
-8
+212
+9
-        return ResultT!(String)();
-    }

trunk/enki/Expression.d

@@ -169 +169 @@
-        }
-        else{
+            debug writefln("isShort.length %d terms.length",isShort.length,terms.length);
-            assert(isShort.length == terms.length);
-            with(generator){

trunk/enki/IParser.d

@@ -40 +40 @@
-    public void clearErrors();
-    public String getErrorReport();
+    public bool hasErrors();
-    
-    protected ResultString regexp(String str);
@@ -46 +47 @@
-    protected void position(uint newPos);
-    protected String sliceData(uint start,uint end);
+    public ResultString terminal(uint ch);
+    public ResultString range(uint start,uint end);
-}

trunk/enki/bootstrap.d

@@ -71 +71 @@
-    SyntaxLine[] lines;
-            
-Cod
+Boilerplat
-    Copyright (c) 2006 Eric Anderton
-
@@ -116 +116 @@
-    lines ~= new DefineDirective("String","sp",true,"Space(s)");
-    lines ~= new DefineDirective("String","ws",false,"Whitespace");
+    
+    lines ~= empty;             
+    lines ~= new ParseTypeDirective("String");
+    lines ~= new UTFDirective("8");
-    
-    lines ~= empty;             
@@ -877 +881 @@
-                        makeTerm(new Production("CodeDirective",new Binding(true,"dir"))),
-                        makeTerm(new Production("TypelibDirective",new Binding(true,"dir"))),
-
+
+
+
+
-                    )
-                ),
@@ -1067 +1074 @@
-        )
-    );  
+        
+    lines ~= new Rule("BoilerplateDirective",
+        new ClassPredicate("BoilerplateDirective", new Param("code")),
+        new Expression(
+            new Terminal("boilerplate",null),
+            new Production("ws",null),
+            new Terminal("{{{",null),
+            new ZeroOrMoreExpr(
+                new Expression(
+                    new Production("any",null)
+                ),
+                new Binding(false,"code"),
+                new Terminal("}}}",null)
+            )
+        )
+    );  
+        
+    lines ~= new Rule("HeaderDirective",
+        new ClassPredicate("HeaderDirective", new Param("code")),
+        new Expression(
+            new Terminal("header",null),
+            new Production("ws",null),
+            new Terminal("{{{",null),
+            new ZeroOrMoreExpr(
+                new Expression(
+                    new Production("any",null)
+                ),
+                new Binding(false,"code"),
+                new Terminal("}}}",null)
+            )
+        )
+    );
+    
+    lines ~= new Rule("UTFDirective",
+        new ClassPredicate("UTFDirective", new Param("value")),
+        new Expression(
+            new Terminal("utf",null),
+            new Production("ws",null),
+            new Terminal("(",null),
+            new Production("ws",null),
+            new Production("DirectiveArg",new Binding(false,"value")),
+            new Production("ws",null),
+            new Terminal(")",null),
+            new Production("ws",null),
+            new Terminal(";",null)          
+        )
+    );
-    
-    lines ~= new Rule("DirectiveArg",

trunk/enki/enki.bnf

@@ -1 @@
-cod
-, BCS
+boilerplat
+
-
-    Permission is hereby granted, free of charge, to any person
@@ -43 +43 @@
-.define("String","ws","false","Whitespace");
-
+.parsetype("String");
+.utf("8");
+
-.baseclass("BaseEnkiParser");
-.classname("EnkiParser");
@@ -215 +218 @@
-Directive
-    = Directive dir
-
+| BoilerplateDirective:~dir | HeaderDirective:~dir | UTFDirective:~dir 
-
-ImportDirective
@@ -257 +260 @@
-    ::=  "parsetype" ws "(" ws DirectiveArg:typeName  ws ")" ws ";";
-
+BoilerplateDirective
+    = new BoilerplateDirective(String code)
+    ::=  "boilerplate" ws "{{{" { any}:code "}}}";
+
+HeaderDirective
+    = new HeaderDirective(String code)
+    ::=  "header" ws "{{{" { any}:code "}}}";
+
+UTFDirective
+    = new UTFDirective(String value)
+    ::=  "utf" ws "(" ws DirectiveArg:value  ws ")" ws ";";
+
-DirectiveArg
-    = String arg

trunk/enki/enki.d

@@ -35 +35 @@
-
-char[] helpText = 
-0
+2
-Copyright(c) 2006 Eric Anderton
-
@@ -46 +46 @@
-  -f<file>  Specify output filename 
-  -b<file>  Specify output ebnf filename
-
+ (output everything to console)
-  
-The -b option is used primarily as a sanity check of

trunk/enki/enki_bn.d

@@ -2 +2 @@
-// This file is automatically maintained by the BUILD utility,
-// Please refrain from manually editing it.
-56
+77

trunk/enki/library/d/CharEntity.bnf

@@ +1 @@
+.boilerplate{{{/+
+    Copyright (c) 2006 Eric Anderton
+
+    Permission is hereby granted, free of charge, to any person
+    obtaining a copy of this software and associated documentation
+    files (the "Software"), to deal in the Software without
+    restriction, including without limitation the rights to use,
+    copy, modify, merge, publish, distribute, sublicense, and/or
+    sell copies of the Software, and to permit persons to whom the
+    Software is furnished to do so, subject to the following
+    conditions:
+
+    The above copyright notice and this permission notice shall be
+    included in all copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+