* Rewrite DocFunction to work with current Rhino 1.6 snapshot.

- TokenStream is no longer public, implement Parser.parseTokens()
  - Parse public nested methods in JS constructors
  - adapt to slightly changed sequence of returned tokens.
* Do not implement IPathElement in Server, Application and HelmaDoc classes.
  Implement the necessary methods in the manage application instead.
* Add DocResourceElement.getStartLine() method to get element's position
  within the containing resource
* Rename ScriptingEngine.getIntrospector() to getDoc() and declare DocApplication
  as return value.

src/helma/doc/DocApplication.java

@@ -16,7 +16,6 @@
 
 package helma.doc;
 
-import helma.framework.IPathElement;
 import helma.framework.core.Application;
 import helma.framework.core.Prototype;
 import helma.main.Server;
@@ -140,9 +139,9 @@ public class DocApplication extends DocElement {
      * from helma.framework.IPathElement, overridden with
      * Server.getServer() to work in manage-application
      */
-    public IPathElement getParentElement() {
+    public Object getParentElement() {
         Server s = helma.main.Server.getServer();
 
-        return s.getChildElement(this.name);
+        return s.getApplication(this.name);
     }
 }

src/helma/doc/DocElement.java

@@ -16,14 +16,12 @@
 
 package helma.doc;
 
-import helma.framework.IPathElement;
-
 import java.util.*;
 
 /**
  * 
  */
-public abstract class DocElement implements IPathElement {
+public abstract class DocElement {
     public static final int APPLICATION = 0;
     public static final int PROTOTYPE = 1;
     public static final int ACTION = 2;
@@ -316,9 +314,9 @@ public abstract class DocElement implements IPathElement {
      * from helma.framework.IPathElement. Retrieves a child from the
      * children map.
      */
-    public IPathElement getChildElement(String name) {
+    public DocElement getChildElement(String name) {
         try {
-            return (IPathElement) children.get(name);
+            return (DocElement) children.get(name);
         } catch (ClassCastException cce) {
             debug(cce.toString());
             cce.printStackTrace();
@@ -331,7 +329,7 @@ public abstract class DocElement implements IPathElement {
      * from helma.framework.IPathElement. Returns the parent object
      * of this instance if assigned.
      */
-    public IPathElement getParentElement() {
+    public Object getParentElement() {
         return parent;
     }
 

src/helma/doc/DocFunction.java

@@ -23,18 +23,22 @@ import java.awt.Point;
 import java.io.IOException;
 import java.io.InputStreamReader;
 import java.io.Reader;
-import java.util.Vector;
+import java.util.List;
+import java.util.ArrayList;
 
 import org.mozilla.javascript.*;
 
 /**
- * 
+ *
  */
 public class DocFunction extends DocResourceElement {
 
-    protected DocFunction(String name, Resource res, DocElement parent, int type) {
+    private int startLine;
+
+    protected DocFunction(String name, Resource res, DocElement parent, int type, int lineno) {
         super(name, res, type);
         this.parent = parent;
+        this.startLine = lineno;
     }
 
     /**
@@ -50,12 +54,10 @@ public class DocFunction extends DocResourceElement {
     public static DocFunction newAction(Resource res, DocElement parent) throws IOException {
         String name = res.getBaseName();
         String[] lines = StringUtils.splitLines(res.getContent());
-        DocFunction func = new DocFunction(name, res, parent, ACTION);
+        DocFunction func = new DocFunction(name, res, parent, ACTION, 1);
         String rawComment = "";
-        TokenStreamInjector ts = getTokenStream (res);
+        Token[] tokens = parseTokens(res);
-        Point p = getPoint (ts);
+        rawComment = Util.extractString(lines, getPoint(tokens[0]), getPoint(tokens[1]));
-        ts.getToken();
-        rawComment = Util.extractString(lines, p, getPoint(ts));
         rawComment = Util.chopComment (rawComment);
         func.parseComment(rawComment);
         func.content = res.getContent();
@@ -75,207 +77,210 @@ public class DocFunction extends DocResourceElement {
      * connected to another DocElement.
      */
     public static DocFunction[] newFunctions(Resource res, DocElement parent)
-                throws IOException {
+            throws IOException {
-
+        String[] lines = StringUtils.splitLines(res.getContent());
-        Vector vec = new Vector();
+        Token[] tokens = parseTokens(res);
-
+        List list = new ArrayList();
-            String content = res.getContent();
+        scanFunctions(lines, tokens, list, res, parent, 0, tokens.length);
-            String[] lines = StringUtils.splitLines(content);
+        return (DocFunction[]) list.toArray(new DocFunction[0]);
-            String comment = "";
-
-            int token     = Token.EMPTY;
-            int lastToken = Token.EMPTY;
-            Point marker;
-
-            String lastNameString = null;
-            String functionName   = null;
-            String context        = null;
-
-            TokenStreamInjector ts = getTokenStream (content);
-
-            while (!ts.eof()) {
-
-                // store the position of the last token
-                marker = getPoint (ts);
-                // store last token
-                lastToken = token;
-
-                // now get a new token
-                // regular expression syntax is troublesome for the TokenStream
-                // we can safely ignore syntax errors in regular expressions here
-                try {
-                    token = ts.getToken();
-                } catch(Exception anything) {
-                    continue;
-                }
-
-                if (token == Token.EOL) {
-
-                    String c = Util.extractString(lines, marker, getPoint(ts));
-                    if (c.startsWith("/**"))
-                        comment = c;
-
-                } else if (token == Token.LC) {
-
-                    // when we come across a left brace outside of a function,
-                    // we store the current string of the stream, it might be
-                    // a function object declaration
-                    // e.g. HttpClient = { func1:function()...}
-                    context = ts.getString();
-
-                } else if (token == Token.RC && context != null) {
-
-                    // when we come across a right brace outside of a function,
-                    // we reset the current context cache
-                    context = null;
-
-                } else if (token == Token.NAME) {
-
-                    // store all names, the last one before a function
-                    // declaration may be used as its name
-
-                    if (lastToken != Token.DOT) {
-
-                        lastNameString = ts.getString();
-
-                        // this may be the start of a name chain declaring a function
-                        // e.g. Number.prototype.functionName = function() { }
-                        marker = getPoint(ts);
-                        marker.x -= (ts.getString().length() + 1);
-
-                    } else {
-
-                        // token in front of the name was a dot, so we connect the
-                        // names that way
-                        lastNameString += "." + ts.getString();
-
-                    }
-
-                } else if (token == Token.FUNCTION) {
-
-                    // store the end of the function word
-                    Point p = getPoint(ts);
-
-                    // look at the next token:
-                    int peekToken = ts.peekToken();
-
-                    // depending of the style of the declaration we already have all we need 
-                    // or need to fetch the name from the next token:
-                    if (peekToken == Token.NAME) {
-
-                        // if the token after FUNCTION is NAME, it's the usual function
-                        // declaration like this: function abc() {}
-
-                        // set the pointer for the start of the actual function body
-                        // to the letter f of the function word
-                        marker = p;
-                        marker.x -= 9;
-
-                        // set stream to next token, so that name of the
-                        // function is the stream's current string
-                        token = ts.getToken();
-                        functionName = ts.getString();
-                    } else {
-
-                        // it's a different kind of function declaration.
-                        // the function name is the last found NAME-token
-                        // if context is set, prepend it to the function name
-                        functionName = (context != null) ? context + "." + lastNameString : lastNameString;
-
-                    }
-
-                    // create the function object
-                    DocFunction theFunction = newFunction (functionName, res, parent);
-                    theFunction.parseComment (comment);
-                    vec.add (theFunction);
-
-                    // subloop on the tokenstream: find the parameters of a function
-                    while (!ts.eof() && token != Token.RP) {
-                        token = ts.getToken();
-                        if (token==Token.NAME && theFunction.type == FUNCTION) {
-                            // add names of parameter only for functions, not for macros or actions
-                            theFunction.addParameter (ts.getString());
-                        }
-                    }
-
-                    // subloop on the tokenstream: find the closing right bracket of the function
-                    token = ts.getToken();
-                    int level = (token == Token.LC) ? 1 : 0;
-                    while (!ts.eof() && level > 0) {
-                        // regular expression syntax is troublesome for the TokenStream
-                        // we don't need them here, so we just ignore such an error
-                        try {
-                            token = ts.getToken();
-                        } catch(Exception anything) {
-                            continue;
-                        }
-                        if (token == Token.LC) {
-                            level++;
-                        } else if (token == Token.RC) {
-                            level--;
-                        }
-                    }
-
-                    theFunction.content = Util.extractString(lines, marker, getPoint(ts));
-                    comment = "";
-
-                } // end if
-            } // end while
-
-
-        return (DocFunction[]) vec.toArray(new DocFunction[0]);
     }
 
+    private static void scanFunctions(String[] lines, Token[] tokens, List list,
+                                      Resource res, DocElement parent, int start, int end) {
+        // Token token     = null;
+        Token lastToken = new Token(Token.EMPTY, "", 0, 0);
+        // Point marker;
 
-    private static DocFunction newFunction (String funcName, Resource res, DocElement parent) {
+        String lastNameString = null;
+        String functionName   = null;
+        String context        = null;
+        String comment = "";
+
+        for (int i = start; i  < end - 1; i++) {
+
+            // store the position of the last token
+            Point marker = getPoint(lastToken);
+            // now get a new token
+            Token token = tokens[i];
+            // flag for dropping private functions
+            boolean dropFunction = false;
+
+            if (token.type == Token.EOL) {
+
+                String c = Util.extractString(lines, marker, getPoint(token));
+                if (c.startsWith("/**"))
+                    comment = c;
+
+            } else if (token.type == Token.LC) {
+
+                // when we come across a left brace outside of a function,
+                // we store the current string of the stream, it might be
+                // a function object declaration
+                // e.g. HttpClient = { func1:function()...}
+                context = token.string;
+
+            } else if (token.type == Token.RC && context != null) {
+
+                // when we come across a right brace outside of a function,
+                // we reset the current context cache
+                context = null;
+
+            } else if (token.type == Token.THIS) {
+
+                lastNameString = parent.getName();
+                // this may be the start of a name chain declaring a function
+                // e.g. Number.prototype.functionName = function() { }
+                // marker = getPoint(token);
+                // marker.x -= (5);
+
+            } else if (token.type == Token.NAME) {
+
+                // store all names, the last one before a function
+                // declaration may be used as its name
+
+                if (lastToken.type != Token.DOT) {
+
+                    lastNameString = token.string;
+
+                    // this may be the start of a name chain declaring a function
+                    // e.g. Number.prototype.functionName = function() { }
+                    marker = getPoint(token);
+                    marker.x -= (token.string.length() + 1);
+
+                } else {
+
+                    // token in front of the name was a dot, so we connect the
+                    // names that way
+                    lastNameString += "." + token.string;
+
+                }
+
+            } else if (token.type == Token.FUNCTION) {
+
+                // store the end of the function word
+                Point p = getPoint(token);
+
+                // look at the next token:
+                Token peekToken = tokens[i + 1];
+
+                // depending of the style of the declaration we already have all we need
+                // or need to fetch the name from the next token:
+                if (peekToken.type == Token.NAME) {
+
+                    // if the token after FUNCTION is NAME, it's the usual function
+                    // declaration like this: function abc() {}
+
+                    // set the pointer for the start of the actual function body
+                    // to the letter f of the function word
+                    marker = p;
+                    marker.x -= 9;
+
+                    // set stream to next token, so that name of the
+                    // function is the stream's current string
+                    token = tokens[++i];
+                    functionName = token.string;
+                } else {
+
+                    // it's a different kind of function declaration.
+                    // the function name is the last found NAME-token
+                    // if context is set, prepend it to the function name
+                    functionName = (context != null) ? context + "." + lastNameString : lastNameString;
+
+                }
+
+                DocFunction theFunction = null;
+                if (!dropFunction) {
+                    // create the function object
+                    DocElement par = parent instanceof DocFunction ? parent.parent : parent;
+                    theFunction = newFunction (functionName, res, par, token.lineno + 1);
+                    theFunction.parseComment (comment);
+                    list.add (theFunction);
+                }
+                // reset comment
+                comment = "";
+
+                // subloop on the tokenstream: find the parameters of a function
+                while (i < end && token.type != Token.RP) {
+                    token = tokens[++i];
+                    if (token.type == Token.NAME && theFunction.type == FUNCTION) {
+                        // add names of parameter only for functions, not for macros or actions
+                        theFunction.addParameter (token.string);
+                    }
+                }
+
+                // subloop on the tokenstream: find the closing right bracket of the function
+                token = tokens[++i];
+                int j = i + 1;
+                int level = (token.type == Token.LC) ? 1 : 0;
+                while (i < end && level > 0) {
+                    // regular expression syntax is troublesome for the TokenStream
+                    // we don't need them here, so we just ignore such an error
+                    try {
+                        token = tokens[++i];
+                    } catch(Exception anything) {
+                        continue;
+                    }
+                    if (token.type == Token.LC) {
+                        level++;
+                    } else if (token.type == Token.RC) {
+                        level--;
+                    }
+                }
+
+                if (dropFunction)
+                    continue;
+
+                // parse function body for nested functions
+                scanFunctions(lines, tokens, list, res, theFunction, j, i);
+                // set the function body, starting at the beginning of the first line
+                marker.x = 0;
+                theFunction.content = Util.extractString(lines, marker, getPoint(token));
+
+            } // end if
+
+            lastToken = token;
+
+        } // end while
+
+    }
+
+    private static DocFunction newFunction (String funcName, Resource res, DocElement parent, int lineno) {
         if (funcName.endsWith("_action")) {
-            return new DocFunction(funcName, res, parent, ACTION);
+            return new DocFunction(funcName, res, parent, ACTION, lineno);
         } else if (funcName.endsWith("_macro")) {
-            return new DocFunction(funcName, res, parent, MACRO);
+            return new DocFunction(funcName, res, parent, MACRO, lineno);
         } else {
-            return new DocFunction(funcName, res, parent, FUNCTION);
+            return new DocFunction(funcName, res, parent, FUNCTION, lineno);
         }
     }
 
     /**
      * Creates a rhino token stream for a given file.
-     * @param src the JS source, either as Resource or String object
+     * @param res the JS Resource
      * @return a TokenStream wrapper
-     * @throws IOException if an I/O exception was raised
+     * @throws java.io.IOException if an I/O exception was raised
      */
-    protected static TokenStreamInjector getTokenStream (Object src) throws IOException {
+    protected static Token[] parseTokens(Resource res) throws IOException {
-        // TODO the TokenStreamInjector is really just a hack, and we shouldn't
+        Reader reader = new InputStreamReader(res.getInputStream());
-        // interact with the rhino TokenStream class directly. The proper way to
-        // go would be to use the public Parser class to parse the input, and walk
-        // through the parse tree and extract the comments manually.
-        // As a result of our approach, the TokenStream member in our Parser instance
-        // will be null, resulting in a NullPointerException when an error is
-        // encountered. For the time being, this is something we can live with.
-        Reader reader = null;
-        String content = null;
-        if (src instanceof Resource) {
-            reader = new InputStreamReader(((Resource) src).getInputStream());
-        } else if (src instanceof String) {
-            content = (String) src;
-        } else {
-            throw new IllegalArgumentException("src must be either a Resource or a String");
-        }
         CompilerEnvirons compilerEnv = new CompilerEnvirons();
         compilerEnv.initFromContext(Context.getCurrentContext());
+        compilerEnv.setGenerateDebugInfo(true);
+        compilerEnv.setGeneratingSource(true);
+        compilerEnv.setOptimizationLevel(-1);
         ErrorReporter errorReporter = Context.getCurrentContext().getErrorReporter();
         Parser parser = new Parser(compilerEnv, errorReporter);
-        return new TokenStreamInjector (parser, reader, content, 0);
+        return parser.parseTokens(reader, res.getName(),  0);
     }
 
     /**
      * Returns a pointer to the current position in the TokenStream
-     * @param ts the TokenStream
+     * @param token the TokenStream
      * @return the current position
      */
-    protected static Point getPoint (TokenStreamInjector ts) {
+    protected static Point getPoint (Token token) {
-        return new Point (ts.getOffset(), ts.getLineno());
+        return new Point (token.offset, token.lineno);
     }
-    
+
     /**
      * from helma.framework.IPathElement. All macros, templates, actions etc
      * have the same prototype.
@@ -283,4 +288,12 @@ public class DocFunction extends DocResourceElement {
     public java.lang.String getPrototype() {
         return "docfunction";
     }
+
+    /**
+     * Get the first line of this function within the containing resource.
+     * @return the first line of the function
+     */
+    public int getStartLine() {
+        return startLine;
+    }
 }

src/helma/doc/DocResourceElement.java

@@ -39,4 +39,12 @@ public abstract class DocResourceElement extends DocElement {
     public String toString() {
         return resource.getName();
     }
+
+    /**
+     * Get the line number this element starts in. Defaults to 0.
+     * @return the first line of this element within its resource
+     */
+    public int getStartLine() {
+        return 0;
+    }
 }

src/helma/doc/Util.java

@@ -88,6 +88,22 @@ public final class Util {
         return buf.toString().trim();
     }
 
+    /**
+     * Extract a part of a file defined by two points from a String array
+     * @param lines an array of lines
+     * @param start of string to extract defined by column x and row y
+     * @param end of string to extract
+     * @return string
+     */
+    public static String extractString (String[] lines, int start, int end) {
+        StringBuffer buf = new StringBuffer();
+        int to = Math.min(end + 1, lines.length);
+        for (int i = start; i < to; i++) {
+            buf.append(lines[i]);
+            buf.append("\n");
+        }
+        return buf.toString().trim();
+    }
 
     /**
      * method to debug file/stream-handling with Point objects. extracts the line p

src/helma/framework/core/Application.java

@@ -24,6 +24,8 @@ import helma.main.Server;
 import helma.objectmodel.*;
 import helma.objectmodel.db.*;
 import helma.util.*;
+import helma.doc.DocApplication;
+
 import java.io.*;
 import java.lang.reflect.*;
 import java.rmi.*;
@@ -39,7 +41,7 @@ import java.util.ArrayList;
  * requests from the Web server or XML-RPC port and dispatches them to
  * the evaluators.
  */
-public final class Application implements IPathElement, Runnable {
+public final class Application implements Runnable {
     // the name of this application
     private String name;
 
@@ -436,7 +438,7 @@ public final class Application implements IPathElement, Runnable {
             eval = getEvaluator();
             eval.invokeInternal(null, "onStart", RequestEvaluator.EMPTY_ARGS);
         } catch (Exception xcept) {
-            logError("Error in " + name + "onStart()", xcept);
+            logError("Error in " + name + ".onStart()", xcept);
         } finally {
             releaseEvaluator(eval);
         }
@@ -1342,49 +1344,17 @@ public final class Application implements IPathElement, Runnable {
         }
     }
 
-    //////////////////////////////////////////////////////////////////////////////////////////////////////////
+    public DocApplication getDoc() {
-    ///   The following methods are the IPathElement interface for this application.
+        RequestEvaluator eval = null;
-    ///   this is useful for scripting and url-building in the base-app.
+        try {
-    //////////////////////////////////////////////////////////////////////////////////////////////////////////
+            eval = getEvaluator();
-    public String getElementName() {
+            return eval.scriptingEngine.getDoc();
-        return name;
+        } catch (Exception xcept) {
-    }
+            logError("Error in getDoc() for " + name, xcept);
-
+            return null;
-    /**
+        } finally {
-     *
+            releaseEvaluator(eval);
-     *
-     * @param name ...
-     *
-     * @return ...
-     */
-    public IPathElement getChildElement(String name) {
-        // as Prototype and the helma.scripting-classes don't offer enough information
-        // we use the classes from helma.doc-pacakge for introspection.
-        // the first time an url like /appname/api/ is parsed, the application is read again
-        // parsed for comments and exposed as an IPathElement
-        if (name.equals("api") && allThreads.size() > 0) {
-            return ((RequestEvaluator) allThreads.get(0)).scriptingEngine.getIntrospector();
         }
-
-        return null;
-    }
-
-    /**
-     *
-     *
-     * @return ...
-     */
-    public IPathElement getParentElement() {
-        return helma.main.Server.getServer();
-    }
-
-    /**
-     *
-     *
-     * @return ...
-     */
-    public String getPrototype() {
-        return "application";
     }
 
     ////////////////////////////////////////////////////////////////////////

src/helma/main/Server.java

@@ -40,7 +40,7 @@ import helma.util.ResourceProperties;
 /**
  * Helma server main class.
  */
-public class Server implements IPathElement, Runnable {
+public class Server implements Runnable {
     // version string
     public static final String version = "1.6.x (__builddate__)";
 
@@ -884,35 +884,6 @@ public class Server implements IPathElement, Runnable {
         appManager.stop(name);
     }
 
-    /**
-     * method from helma.framework.IPathElement
-     */
-    public String getElementName() {
-        return "root";
-    }
-
-    /**
-     * method from helma.framework.IPathElement,
-     * returning active applications
-     */
-    public IPathElement getChildElement(String name) {
-        return appManager.getApplication(name);
-    }
-
-    /**
-     * method from helma.framework.IPathElement
-     */
-    public IPathElement getParentElement() {
-        return null;
-    }
-
-    /**
-     * method from helma.framework.IPathElement
-     */
-    public String getPrototype() {
-        return "root";
-    }
-
     static class HelmaLogSink implements LogSink {
                         
         public String getOptions() {

src/helma/scripting/ScriptingEngine.java

@@ -20,6 +20,8 @@ import helma.framework.IPathElement;
 import helma.framework.repository.Resource;
 import helma.framework.core.Application;
 import helma.framework.core.RequestEvaluator;
+import helma.doc.DocApplication;
+
 import java.io.OutputStream;
 import java.io.IOException;
 import java.io.InputStream;
@@ -121,7 +123,7 @@ public interface ScriptingEngine {
      *  In order to be compatible with the standard Helma management application, this
      *  class should be compatible with helma.doc.DocApplication.
      */
-    public IPathElement getIntrospector();
+    public DocApplication getDoc();
 
     /**
      * Provide object serialization for this engine's scripted objects. If no special

src/helma/scripting/rhino/RhinoEngine.java

@@ -479,7 +479,7 @@ public class RhinoEngine implements ScriptingEngine {
     /**
      * Get an introspector to this engine.
      */
-    public IPathElement getIntrospector() {
+    public DocApplication getDoc() {
         if (doc == null) {
             try {
                 doc = new DocApplication(app);