kicked out FESI parser and implemented parsing with rhino dependency.

all token-handling/file-parsing moved to DocFunction.java, all raw file
handling to Util

src/helma/doc/DocApplication.java

@@ -21,7 +21,7 @@ import helma.main.Server;
 import helma.util.SystemProperties;
 import java.io.*;
 import java.util.*;
-
+import java.awt.*;
 /**
  * 
  */
@@ -60,18 +60,34 @@ public class DocApplication extends DocDirElement {
      * @param args ...
      */
     public static void main(String[] args) {
-        //		DocApplication app;
+        System.out.println("this is helma.doc");
-        //		app = new DocApplication (args[0], args[1]);
+        DocApplication app;
-        //		app.readApplication ();
+        app = new DocApplication (args[0], args[1]);
+        app.readApplication ();
+
         //		DocPrototype el = DocPrototype.newInstance (new File(args[0]));
         //		el.readFiles ();
         //		DocFunction func = DocFunction.newTemplate (new File(args[0]));
-        //		DocFunction func = DocFunction.newAction (new File(args[0]));
-        DocFunction[] func = DocFunction.newFunctions(new File(args[0]));
 
-        //		DocSkin skin = DocSkin.newInstance (new File (args[0]));
+//        DocFunction func = DocFunction.newAction (new File(args[0]));
-        //		System.out.println (func.getContent ());
+
-        //		System.out.println ("\n\n\ncomment = " + func.getComment ());
+//        DocFunction[] func = DocFunction.newFunctions(new File(args[0]));
+//        for (int i=0; i<func.length; i++) {
+//            System.out.println("=============================================");     
+//            System.out.println("function " + func[i].name);
+//            System.out.println("comment = " + func[i].comment + "<<<");
+//            String[] arr = func[i].listParameters();
+//            for (int j=0; j<arr.length; j++) {
+//                System.out.println (arr[j]);
+//            }
+//            System.out.println ("\ncontent:\n" + func[i].content + "<<<");
+//            System.out.println ("\n");
+//        }
+
+//		  DocSkin skin = DocSkin.newInstance (new File (args[0]));
+
+//        System.out.println (func.getContent ());
+//        System.out.println ("\n\n\ncomment = " + func.getComment ());
     }
 
     /**
@@ -79,7 +95,10 @@ public class DocApplication extends DocDirElement {
      */
     private void readProps() {
         File propsFile = new File(location, "app.properties");
-        SystemProperties serverProps = Server.getServer().getProperties();
+        SystemProperties serverProps = null;
+        if (Server.getServer()!=null) {
+            serverProps = Server.getServer().getProperties();
+        }
         SystemProperties appProps = new SystemProperties(propsFile.getAbsolutePath(),
                                                          serverProps);
 

src/helma/doc/DocDirElement.java

@@ -16,7 +16,6 @@
 
 package helma.doc;
 
-import FESI.Parser.*;
 import java.io.*;
 import java.util.*;
 
@@ -42,7 +41,7 @@ public abstract class DocDirElement extends DocElement {
             File f = new File(location, DOCFILES[i]);
 
             if (f.exists()) {
-                String rawComment = readFile(f);
+                String rawComment = Util.readFile(f);
 
                 parseComment(rawComment);
 

src/helma/doc/DocElement.java

@@ -184,11 +184,13 @@ public abstract class DocElement implements IPathElement {
         parameters.add(param);
     }
 
+
     /**
      * parse rawComment, render DocTags
      */
     void parseComment(String rawComment) {
         try {
+            rawComment = rawComment.trim();
             StringTokenizer tok = new StringTokenizer(rawComment, "\n", true);
             int BLANK = 0;
             int TEXT = 1;
@@ -239,7 +241,7 @@ public abstract class DocElement implements IPathElement {
 
             // store the last element, if there was at least one element ...
             if (mode == TEXT) {
-                comment = buf.toString();
+                comment = buf.toString().trim();
             } else if (mode == TAGS) {
                 addTag(buf.toString());
             }
@@ -248,27 +250,6 @@ public abstract class DocElement implements IPathElement {
         }
     }
 
-    /**
-     * utility: read a complete file into a string
-     */
-    protected static String readFile(File file) {
-        try {
-            StringBuffer buf = new StringBuffer();
-            BufferedReader in = new BufferedReader(new FileReader(file));
-            String line = in.readLine();
-
-            while (line != null) {
-                buf.append(line + "\n");
-                line = in.readLine();
-            }
-
-            in.close();
-
-            return buf.toString();
-        } catch (IOException e) {
-            return ("");
-        }
-    }
 
     /**
      *

src/helma/doc/DocFileElement.java

@@ -16,97 +16,19 @@
 
 package helma.doc;
 
-import FESI.Parser.*;
+import java.io.File;
-import java.io.*;
-import java.util.*;
 
 /**
- * 
+ * abstract class for extracting doc information from files.
+ * not used at the moment but left in for further extensions-
  */
 public abstract class DocFileElement extends DocElement {
     protected DocFileElement(String name, File location, int type) {
         super(name, location, type);
     }
 
-    /**
-     * extracts the function name from a file. basically chops the given suffix
-     * and throws an error if the file name doesn't fit.
-     */
-    static protected String nameFromFile(File f, String suffix)
-                                  throws DocException {
-        String filename = f.getName();
 
-        if (!filename.endsWith(suffix)) {
-            throw new DocException("file doesn't have suffix " + suffix + ": " +
-                                   f.toString());
-        }
 
-        return filename.substring(0, filename.lastIndexOf(suffix));
-    }
 
-    /**
-     * creates fesi token manager for a given file.
-     */
-    static protected EcmaScriptTokenManager createTokenManager(File f) {
-        try {
-            ASCII_CharStream is = new ASCII_CharStream(new FileReader(f), 1, 1);
-            EcmaScriptTokenManager mgr = new EcmaScriptTokenManager(is, 0);
 
-            return mgr;
-        } catch (FileNotFoundException fnfe) {
-            fnfe.printStackTrace();
-            throw new DocException(fnfe.toString());
-        }
-    }
-
-    /**
-     * extracts a part of the source code, used to get the code for a
-     * single function from a function file. sets the field "content".
-     */
-    protected void parseSource(File sourceFile, int beginLine, int beginColumn,
-                               int endLine, int endColumn) {
-        StringBuffer buf = new StringBuffer();
-        int ct = 0;
-
-        try {
-            BufferedReader in = new BufferedReader(new FileReader(sourceFile));
-            String line = "";
-
-            while (line != null) {
-                line = in.readLine();
-
-                if (line == null) {
-                    break;
-                }
-
-                ct++;
-
-                if (ct == beginLine) {
-                    buf.append(line.substring(beginColumn - 1, line.length()) + "\n");
-                } else if ((ct > beginLine) && (ct < endLine)) {
-                    buf.append(line + "\n");
-                } else if (ct == endLine) {
-                    buf.append(line.substring(0, endColumn));
-                }
-            }
-        } catch (Exception e) {
-            debug(e.getMessage());
-        }
-
-        content = buf.toString();
-    }
-
-    /**
-     * connects all available specialTokens starting at the given token.
-     */
-    protected void parseCommentFromToken(Token tok) {
-        StringBuffer buf = new StringBuffer();
-
-        while (tok.specialToken != null) {
-            buf.append(tok.specialToken.toString());
-            tok = tok.specialToken;
-        }
-
-        parseComment(buf.toString().trim());
-    }
 }

src/helma/doc/DocFunction.java

@@ -16,15 +16,18 @@
 
 package helma.doc;
 
-import FESI.Parser.*;
 import helma.framework.IPathElement;
+import java.awt.Point;
+import java.lang.reflect.Constructor;
 import java.io.*;
-import java.util.*;
+import java.util.Vector;
+import org.mozilla.javascript.TokenStream;
 
 /**
  * 
  */
 public class DocFunction extends DocFileElement {
+
     protected DocFunction(String name, File location, DocElement parent, int type) {
         super(name, location, type);
         this.parent = parent;
@@ -41,30 +44,20 @@ public class DocFunction extends DocFileElement {
      * creates a new DocFunction object of type ACTION connected to another DocElement
      */
     public static DocFunction newAction(File location, DocElement parent) {
-        String name = nameFromFile(location, ".hac");
+        String name = Util.nameFromFile(location, ".hac");
         DocFunction func = new DocFunction(name, location, parent, ACTION);
-
+        String rawComment = "";
-        func.parseActionFile();
+        try {
-
+            TokenStream ts = getTokenStream (location);
-        return func;
+            Point p = getPoint (ts);
-    }
+            ts.getToken();
-
+            rawComment = Util.getStringFromFile(location, p, getPoint(ts));
-    /**
+        } catch (IOException io) {
-     * creates a new independent DocFunction object of type TEMPLATE
+            io.printStackTrace();
-     */
+            throw new DocException (io.toString());
-    public static DocFunction newTemplate(File location) {
+        }
-        return newTemplate(location, null);
+        func.parseComment(rawComment);
-    }
+        func.content = Util.readFile(location);
-
-    /**
-     * creates a new DocFunction object of type TEMPLATE connected to another DocElement
-     */
-    public static DocFunction newTemplate(File location, DocElement parent) {
-        String name = nameFromFile(location, ".hsp");
-        DocFunction func = new DocFunction(name, location, parent, TEMPLATE);
-
-        func.parseTemplateFile();
-
         return func;
     }
 
@@ -75,106 +68,126 @@ public class DocFunction extends DocFileElement {
         return newFunctions(location, null);
     }
 
+
     /**
      * reads a function file and creates DocFunction objects of type FUNCTION
      * connected to another DocElement.
      */
     public static DocFunction[] newFunctions(File location, DocElement parent) {
         Vector vec = new Vector();
-        EcmaScriptTokenManager mgr = createTokenManager(location);
+        try {
-        Token tok = mgr.getNextToken();
+            // the function file:
+            TokenStream ts = getTokenStream (location);
+            DocFunction curFunction = null;
+            Point curFunctionStart = null;
+            while (!ts.eof()) {
 
-        while (tok.kind != 0) {
+                // store the position of the last token 
-            if (tok.kind == EcmaScriptConstants.FUNCTION) {
+                Point endOfLastToken = getPoint (ts);
-                // store the start position of the function:
+                // new token
-                int beginLine = tok.beginLine;
+                int tok = ts.getToken();
-                int beginColumn = tok.beginColumn;
 
-                // the name is stored in the next token:
+                // if we're currently parsing a functionbody and come to the start
-                String funcName = mgr.getNextToken().toString();
+                // of the next function or eof -> read function body
-
+                if (curFunction != null && (tok== ts.FUNCTION || ts.eof())) {
-                // create the function object
+                    curFunction.content = "function " + Util.getStringFromFile(location, curFunctionStart, endOfLastToken);
-                DocFunction func;
-
-                if (funcName.endsWith("_action")) {
-                    func = new DocFunction(funcName, location, parent, ACTION);
-                } else if (funcName.endsWith("_macro")) {
-                    func = new DocFunction(funcName, location, parent, MACRO);
-                } else {
-                    func = new DocFunction(funcName, location, parent, FUNCTION);
                 }
 
-                // parse the comment from the special token(s) before this token:
+                if (tok == ts.FUNCTION) {
-                func.parseCommentFromToken(tok);
+                    // store the function start for parsing the function body later
-
+                    curFunctionStart = getPoint (ts); 
-                // find the parameters of this function, but only if it's
+                    // get and chop the comment
-                // neither macro nor action:
+                    String rawComment = Util.getStringFromFile(location, endOfLastToken, getPoint (ts)).trim ();
-                if (func.type == FUNCTION) {
+                    if (rawComment.endsWith("function")) {
-                    while ((tok.kind != 0) && (tok.kind != EcmaScriptConstants.RPAREN)) {
+                        rawComment = rawComment.substring (0, rawComment.length()-8).trim();
-                        if (tok.kind == EcmaScriptConstants.IDENTIFIER) {
-                            func.addParameter(tok.image);
-                        }
-
-                        tok = mgr.getNextToken();
                     }
-                } else {
+                    // position stream at function name token
-                    tok = mgr.getNextToken();
+                    tok = ts.getToken();
-                }
+                    // get the name and create the function object
+                    String name = ts.getString();
+                    curFunction = newFunction (name, location, parent);
+                    curFunction.parseComment (rawComment);
+                    vec.add (curFunction);
 
-                // now find the end of the function:
+                    // subloop on the tokenstream: find the parameters of a function
-                int endLine = 0;
+                    // only if it's a function (and not a macro or an action)
+                    if (curFunction.type == FUNCTION) {
+                        while (!ts.eof() && tok != ts.RP) {
+                            // store the position of the last token 
+                            endOfLastToken = getPoint (ts);
+                            // new token
+                            tok = ts.getToken();
+                            if (tok==ts.NAME) {
+                                curFunction.addParameter (ts.getString());
+                            }
+                        }
+                    }
+                } // end if
 
-                // now find the end of the function:
-                int endColumn = 0;
-
-                while ((tok.kind != 0) && (tok.kind != EcmaScriptConstants.FUNCTION)) {
-                    endLine = tok.endLine;
-                    endColumn = tok.endColumn;
-                    tok = mgr.getNextToken();
-                }
-
-                // now we know the exact position of the function in the file,
-                // re-read it and extract the source code:
-                func.parseSource(location, beginLine, beginColumn, endLine, endColumn);
-                vec.add(func);
             }
 
-            if (tok.kind != EcmaScriptConstants.FUNCTION) {
+        } catch (IOException io) {
-                tok = mgr.getNextToken();
+            io.printStackTrace();
-            }
+            throw new DocException (io.toString());
         }
-
         return (DocFunction[]) vec.toArray(new DocFunction[0]);
     }
 
-    /**
-     * reads the content of a .hac file and parses the comment before the first
-     * javascript element
-     */
-    private void parseActionFile() {
-        EcmaScriptTokenManager mgr = createTokenManager(location);
-        Token tok = mgr.getNextToken();
 
-        parseCommentFromToken(tok);
+    private static DocFunction newFunction (String funcName, File location, DocElement parent) {
-        content = readFile(location);
+        if (funcName.endsWith("_action")) {
+            return new DocFunction(funcName, location, parent, ACTION);
+        } else if (funcName.endsWith("_macro")) {
+            return new DocFunction(funcName, location, parent, MACRO);
+        } else {
+            return new DocFunction(funcName, location, parent, FUNCTION);
+        }
     }
 
+
     /**
-     * reads the content of a .hsp file and parses the comment before the first
+     * creates a rhino token stream for a given file
-     * javascript element (only if file starts with >%-tag!).
      */
-    private void parseTemplateFile() {
+    protected static TokenStream getTokenStream (File f) {
-        content = readFile(location);
+        FileReader reader = null;
+        try {
+            reader = new FileReader(f);
+        } catch (FileNotFoundException fnfe) {
+            fnfe.printStackTrace();
+            throw new DocException (fnfe.toString());
+        }
+        String name = f.getName();
+        int line = 0;
 
-        StringReader str = new StringReader(content.substring(content.indexOf("<%") + 2,
+        // FIXME for some reason this doesn't compile with winxp/j2sdk141_03
-                                                              content.indexOf("%>")));
+        // return new TokenStream (reader, null, name, line);
-        ASCII_CharStream ascii = new ASCII_CharStream(str, 1, 1);
-        EcmaScriptTokenManager mgr = new EcmaScriptTokenManager(ascii);
-        Token tok = mgr.getNextToken();
 
-        parseCommentFromToken(tok);
+        // so we have to use reflection:
+        try {
+            Class c = Class.forName ("org.mozilla.javascript.TokenStream");
+            Constructor[] constr = c.getDeclaredConstructors();
+            Object[] params = new Object[4];
+            params[0] = reader;
+            params[1] = null;
+            params[2] = name;
+            params[3] = new Integer(line);
+            return (TokenStream) constr[0].newInstance(params);
+        } catch (Exception anything) {
+            anything.printStackTrace();
+            throw new DocException (anything.toString());
+        }
     }
 
+
+    /**
+     * returns a pointer to the current position in the TokenStream
+     */
+    protected static Point getPoint (TokenStream ts) {
+        return new Point (ts.getOffset(), ts.getLineno());
+    }
+
+
+
     /**
      * from helma.framework.IPathElement. All macros, templates, actions etc
      * have the same prototype.

src/helma/doc/DocProperties.java

@@ -31,7 +31,7 @@ public class DocProperties extends DocFileElement {
                      throws DocException {
         super(location.getName(), location, PROPERTIES);
         this.parent = parent;
-        content = readFile(location);
+        content = Util.readFile(location);
         props = new SystemProperties();
 
         try {

src/helma/doc/DocPrototype.java

@@ -16,7 +16,6 @@
 
 package helma.doc;
 
-import FESI.Parser.*;
 import helma.framework.IPathElement;
 import java.io.*;
 import java.util.*;
@@ -128,8 +127,6 @@ public class DocPrototype extends DocDirElement {
                     continue;
                 } else if (arr[i].endsWith(".hac")) {
                     addChild(DocFunction.newAction(f, this));
-                } else if (arr[i].endsWith(".hsp")) {
-                    addChild(DocFunction.newTemplate(f, this));
                 } else if (arr[i].endsWith(".js")) {
                     DocElement[] elements = DocFunction.newFunctions(f, this);
 

src/helma/doc/DocSkin.java

@@ -27,7 +27,7 @@ public class DocSkin extends DocFileElement {
     protected DocSkin(String name, File location, DocElement parent) {
         super(name, location, SKIN);
         this.parent = parent;
-        content = readFile(location);
+        content = Util.readFile(location);
         parseHandlers();
     }
 
@@ -42,7 +42,7 @@ public class DocSkin extends DocFileElement {
      * creates a new DocSkin object connected to another DocElement
      */
     public static DocSkin newInstance(File location, DocElement parent) {
-        String skinname = nameFromFile(location, ".skin");
+        String skinname = Util.nameFromFile(location, ".skin");
         DocSkin skin = new DocSkin(skinname, location, parent);
 
         return skin;

src/helma/doc/Util.java

@@ -16,21 +16,23 @@
 
 package helma.doc;
 
+import java.awt.Point;
 import java.io.*;
-import java.util.*;
+
 
 /**
  * 
  */
 public final class Util {
     /**
+     * chops a string from comment-delimiters
      *
+     * @param line a line of raw text
      *
-     * @param line ...
+     * @return chopped string
-     *
-     * @return ...
      */
     public static String chopDelimiters(String line) {
+        line = line.trim();
         if (line == null) {
             return null;
         } else if (line.startsWith("/**")) {
@@ -47,4 +49,129 @@ public final class Util {
             return line;
         }
     }
+
+    /**
+     * reads a complete file
+     *
+     * @param file
+     *
+     * @return string with content of file
+     */
+    public static String readFile(File file) {
+        try {
+            StringBuffer buf = new StringBuffer();
+            BufferedReader in = new BufferedReader(new FileReader(file));
+            String line = in.readLine();
+
+            while (line != null) {
+                buf.append(line + "\n");
+                line = in.readLine();
+            }
+
+            in.close();
+
+            return buf.toString();
+        } catch (IOException e) {
+            return ("");
+        }
+    }
+
+
+    /**
+     * reads a part of a file defined by two points
+
+     * @param start of string to extract defined by column x and row y
+
+     * @param end of string to extract
+
+     * @return string
+     */
+    public static String getStringFromFile (File sourceFile, Point start, Point end) {
+        StringBuffer buf = new StringBuffer();
+        int ct = 0;
+        try {
+            BufferedReader in = new BufferedReader(new FileReader(sourceFile));
+            String line = "";
+            while (line != null) {
+                line = in.readLine();
+                if (line == null) {
+                    break;
+                }
+                if ((ct > start.y) && (ct < end.y)) {
+                    buf.append(line + "\n");
+                } else if (ct == start.y) {
+                    if (start.y==end.y) {
+                        buf.append (line.substring (start.x, end.x));
+                        break;
+                    } else {
+                        buf.append(line.substring(start.x, line.length()) + "\n");
+                    }
+                } else if (ct == end.y) {
+                    buf.append(line.substring(0, end.x));
+                    break;
+                }
+                ct++;
+            }
+        } catch (Exception e) {
+            e.printStackTrace();
+            System.out.println(e.getMessage());
+        }
+        return buf.toString().trim();
+    }
+
+
+    /**
+     * method to debug file/stream-handling with Point objects. extracts the line p
+     * points to and prints it with a pointer to the given column
+     *
+     * @param sourceFile
+     * @param p x-value is used for column, y for row
+     * @param debugStr string prefixed to output
+     */
+    public static void debugLineFromFile (File sourceFile, Point p, String debugStr) {
+        try {
+            BufferedReader in = new BufferedReader(new FileReader(sourceFile));
+            String line = "";
+            int ct = 0;
+            while (line != null) {
+                line = in.readLine ();
+                if (line==null) {
+                    System.out.println ("eof reached");
+                    break;
+                }
+                if (ct==p.y) {
+                    System.out.println (debugStr + ": " + line);
+                    for (int i=0; i<(debugStr.length()+1+p.x); i++) {
+                        System.out.print (" ");
+                    }
+                    System.out.println ("^");
+                    break;
+                }
+                ct++;
+            }
+        } catch (Exception e) {
+            e.printStackTrace();
+            System.out.println(e.getMessage());
+        }
+    }
+
+
+
+    /**
+     * extracts the function name from a file. basically chops the given suffix
+     * and throws an error if the file name doesn't fit.
+     */
+    public static String nameFromFile(File f, String suffix)
+                                  throws DocException {
+        String filename = f.getName();
+
+        if (!filename.endsWith(suffix)) {
+            throw new DocException("file doesn't have suffix " + suffix + ": " +
+                                   f.toString());
+        }
+
+        return filename.substring(0, filename.lastIndexOf(suffix));
+    }
+
+
 }