From d818de0d0a9486584dff6542ee8676df672250a4 Mon Sep 17 00:00:00 2001 From: hns Date: Mon, 20 Nov 2006 10:54:02 +0000 Subject: [PATCH] * Rewrite DocFunction to work with current Rhino 1.6 snapshot. - TokenStream is no longer public, implement Parser.parseTokens() - Parse public nested methods in JS constructors - adapt to slightly changed sequence of returned tokens. * Do not implement IPathElement in Server, Application and HelmaDoc classes. Implement the necessary methods in the manage application instead. * Add DocResourceElement.getStartLine() method to get element's position within the containing resource * Rename ScriptingEngine.getIntrospector() to getDoc() and declare DocApplication as return value. --- src/helma/doc/DocApplication.java | 5 +- src/helma/doc/DocElement.java | 10 +- src/helma/doc/DocFunction.java | 383 +++++++++++---------- src/helma/doc/DocResourceElement.java | 8 + src/helma/doc/Util.java | 16 + src/helma/framework/core/Application.java | 58 +--- src/helma/main/Server.java | 31 +- src/helma/scripting/ScriptingEngine.java | 4 +- src/helma/scripting/rhino/RhinoEngine.java | 2 +- 9 files changed, 247 insertions(+), 270 deletions(-) diff --git a/src/helma/doc/DocApplication.java b/src/helma/doc/DocApplication.java index 0c5bfd3c..54345ab8 100644 --- a/src/helma/doc/DocApplication.java +++ b/src/helma/doc/DocApplication.java @@ -16,7 +16,6 @@ package helma.doc; -import helma.framework.IPathElement; import helma.framework.core.Application; import helma.framework.core.Prototype; import helma.main.Server; @@ -140,9 +139,9 @@ public class DocApplication extends DocElement { * from helma.framework.IPathElement, overridden with * Server.getServer() to work in manage-application */ - public IPathElement getParentElement() { + public Object getParentElement() { Server s = helma.main.Server.getServer(); - return s.getChildElement(this.name); + return s.getApplication(this.name); } } diff --git a/src/helma/doc/DocElement.java b/src/helma/doc/DocElement.java index b0363cc8..c46057b1 100644 --- a/src/helma/doc/DocElement.java +++ b/src/helma/doc/DocElement.java @@ -16,14 +16,12 @@ package helma.doc; -import helma.framework.IPathElement; - import java.util.*; /** * */ -public abstract class DocElement implements IPathElement { +public abstract class DocElement { public static final int APPLICATION = 0; public static final int PROTOTYPE = 1; public static final int ACTION = 2; @@ -316,9 +314,9 @@ public abstract class DocElement implements IPathElement { * from helma.framework.IPathElement. Retrieves a child from the * children map. */ - public IPathElement getChildElement(String name) { + public DocElement getChildElement(String name) { try { - return (IPathElement) children.get(name); + return (DocElement) children.get(name); } catch (ClassCastException cce) { debug(cce.toString()); cce.printStackTrace(); @@ -331,7 +329,7 @@ public abstract class DocElement implements IPathElement { * from helma.framework.IPathElement. Returns the parent object * of this instance if assigned. */ - public IPathElement getParentElement() { + public Object getParentElement() { return parent; } diff --git a/src/helma/doc/DocFunction.java b/src/helma/doc/DocFunction.java index e5a39956..8b707469 100644 --- a/src/helma/doc/DocFunction.java +++ b/src/helma/doc/DocFunction.java @@ -23,18 +23,22 @@ import java.awt.Point; import java.io.IOException; import java.io.InputStreamReader; import java.io.Reader; -import java.util.Vector; +import java.util.List; +import java.util.ArrayList; import org.mozilla.javascript.*; /** - * + * */ public class DocFunction extends DocResourceElement { - protected DocFunction(String name, Resource res, DocElement parent, int type) { + private int startLine; + + protected DocFunction(String name, Resource res, DocElement parent, int type, int lineno) { super(name, res, type); this.parent = parent; + this.startLine = lineno; } /** @@ -50,12 +54,10 @@ public class DocFunction extends DocResourceElement { public static DocFunction newAction(Resource res, DocElement parent) throws IOException { String name = res.getBaseName(); String[] lines = StringUtils.splitLines(res.getContent()); - DocFunction func = new DocFunction(name, res, parent, ACTION); + DocFunction func = new DocFunction(name, res, parent, ACTION, 1); String rawComment = ""; - TokenStreamInjector ts = getTokenStream (res); - Point p = getPoint (ts); - ts.getToken(); - rawComment = Util.extractString(lines, p, getPoint(ts)); + Token[] tokens = parseTokens(res); + rawComment = Util.extractString(lines, getPoint(tokens[0]), getPoint(tokens[1])); rawComment = Util.chopComment (rawComment); func.parseComment(rawComment); func.content = res.getContent(); @@ -75,207 +77,210 @@ public class DocFunction extends DocResourceElement { * connected to another DocElement. */ public static DocFunction[] newFunctions(Resource res, DocElement parent) - throws IOException { - - Vector vec = new Vector(); - - String content = res.getContent(); - String[] lines = StringUtils.splitLines(content); - String comment = ""; - - int token = Token.EMPTY; - int lastToken = Token.EMPTY; - Point marker; - - String lastNameString = null; - String functionName = null; - String context = null; - - TokenStreamInjector ts = getTokenStream (content); - - while (!ts.eof()) { - - // store the position of the last token - marker = getPoint (ts); - // store last token - lastToken = token; - - // now get a new token - // regular expression syntax is troublesome for the TokenStream - // we can safely ignore syntax errors in regular expressions here - try { - token = ts.getToken(); - } catch(Exception anything) { - continue; - } - - if (token == Token.EOL) { - - String c = Util.extractString(lines, marker, getPoint(ts)); - if (c.startsWith("/**")) - comment = c; - - } else if (token == Token.LC) { - - // when we come across a left brace outside of a function, - // we store the current string of the stream, it might be - // a function object declaration - // e.g. HttpClient = { func1:function()...} - context = ts.getString(); - - } else if (token == Token.RC && context != null) { - - // when we come across a right brace outside of a function, - // we reset the current context cache - context = null; - - } else if (token == Token.NAME) { - - // store all names, the last one before a function - // declaration may be used as its name - - if (lastToken != Token.DOT) { - - lastNameString = ts.getString(); - - // this may be the start of a name chain declaring a function - // e.g. Number.prototype.functionName = function() { } - marker = getPoint(ts); - marker.x -= (ts.getString().length() + 1); - - } else { - - // token in front of the name was a dot, so we connect the - // names that way - lastNameString += "." + ts.getString(); - - } - - } else if (token == Token.FUNCTION) { - - // store the end of the function word - Point p = getPoint(ts); - - // look at the next token: - int peekToken = ts.peekToken(); - - // depending of the style of the declaration we already have all we need - // or need to fetch the name from the next token: - if (peekToken == Token.NAME) { - - // if the token after FUNCTION is NAME, it's the usual function - // declaration like this: function abc() {} - - // set the pointer for the start of the actual function body - // to the letter f of the function word - marker = p; - marker.x -= 9; - - // set stream to next token, so that name of the - // function is the stream's current string - token = ts.getToken(); - functionName = ts.getString(); - } else { - - // it's a different kind of function declaration. - // the function name is the last found NAME-token - // if context is set, prepend it to the function name - functionName = (context != null) ? context + "." + lastNameString : lastNameString; - - } - - // create the function object - DocFunction theFunction = newFunction (functionName, res, parent); - theFunction.parseComment (comment); - vec.add (theFunction); - - // subloop on the tokenstream: find the parameters of a function - while (!ts.eof() && token != Token.RP) { - token = ts.getToken(); - if (token==Token.NAME && theFunction.type == FUNCTION) { - // add names of parameter only for functions, not for macros or actions - theFunction.addParameter (ts.getString()); - } - } - - // subloop on the tokenstream: find the closing right bracket of the function - token = ts.getToken(); - int level = (token == Token.LC) ? 1 : 0; - while (!ts.eof() && level > 0) { - // regular expression syntax is troublesome for the TokenStream - // we don't need them here, so we just ignore such an error - try { - token = ts.getToken(); - } catch(Exception anything) { - continue; - } - if (token == Token.LC) { - level++; - } else if (token == Token.RC) { - level--; - } - } - - theFunction.content = Util.extractString(lines, marker, getPoint(ts)); - comment = ""; - - } // end if - } // end while - - - return (DocFunction[]) vec.toArray(new DocFunction[0]); + throws IOException { + String[] lines = StringUtils.splitLines(res.getContent()); + Token[] tokens = parseTokens(res); + List list = new ArrayList(); + scanFunctions(lines, tokens, list, res, parent, 0, tokens.length); + return (DocFunction[]) list.toArray(new DocFunction[0]); } + private static void scanFunctions(String[] lines, Token[] tokens, List list, + Resource res, DocElement parent, int start, int end) { + // Token token = null; + Token lastToken = new Token(Token.EMPTY, "", 0, 0); + // Point marker; - private static DocFunction newFunction (String funcName, Resource res, DocElement parent) { + String lastNameString = null; + String functionName = null; + String context = null; + String comment = ""; + + for (int i = start; i < end - 1; i++) { + + // store the position of the last token + Point marker = getPoint(lastToken); + // now get a new token + Token token = tokens[i]; + // flag for dropping private functions + boolean dropFunction = false; + + if (token.type == Token.EOL) { + + String c = Util.extractString(lines, marker, getPoint(token)); + if (c.startsWith("/**")) + comment = c; + + } else if (token.type == Token.LC) { + + // when we come across a left brace outside of a function, + // we store the current string of the stream, it might be + // a function object declaration + // e.g. HttpClient = { func1:function()...} + context = token.string; + + } else if (token.type == Token.RC && context != null) { + + // when we come across a right brace outside of a function, + // we reset the current context cache + context = null; + + } else if (token.type == Token.THIS) { + + lastNameString = parent.getName(); + // this may be the start of a name chain declaring a function + // e.g. Number.prototype.functionName = function() { } + // marker = getPoint(token); + // marker.x -= (5); + + } else if (token.type == Token.NAME) { + + // store all names, the last one before a function + // declaration may be used as its name + + if (lastToken.type != Token.DOT) { + + lastNameString = token.string; + + // this may be the start of a name chain declaring a function + // e.g. Number.prototype.functionName = function() { } + marker = getPoint(token); + marker.x -= (token.string.length() + 1); + + } else { + + // token in front of the name was a dot, so we connect the + // names that way + lastNameString += "." + token.string; + + } + + } else if (token.type == Token.FUNCTION) { + + // store the end of the function word + Point p = getPoint(token); + + // look at the next token: + Token peekToken = tokens[i + 1]; + + // depending of the style of the declaration we already have all we need + // or need to fetch the name from the next token: + if (peekToken.type == Token.NAME) { + + // if the token after FUNCTION is NAME, it's the usual function + // declaration like this: function abc() {} + + // set the pointer for the start of the actual function body + // to the letter f of the function word + marker = p; + marker.x -= 9; + + // set stream to next token, so that name of the + // function is the stream's current string + token = tokens[++i]; + functionName = token.string; + } else { + + // it's a different kind of function declaration. + // the function name is the last found NAME-token + // if context is set, prepend it to the function name + functionName = (context != null) ? context + "." + lastNameString : lastNameString; + + } + + DocFunction theFunction = null; + if (!dropFunction) { + // create the function object + DocElement par = parent instanceof DocFunction ? parent.parent : parent; + theFunction = newFunction (functionName, res, par, token.lineno + 1); + theFunction.parseComment (comment); + list.add (theFunction); + } + // reset comment + comment = ""; + + // subloop on the tokenstream: find the parameters of a function + while (i < end && token.type != Token.RP) { + token = tokens[++i]; + if (token.type == Token.NAME && theFunction.type == FUNCTION) { + // add names of parameter only for functions, not for macros or actions + theFunction.addParameter (token.string); + } + } + + // subloop on the tokenstream: find the closing right bracket of the function + token = tokens[++i]; + int j = i + 1; + int level = (token.type == Token.LC) ? 1 : 0; + while (i < end && level > 0) { + // regular expression syntax is troublesome for the TokenStream + // we don't need them here, so we just ignore such an error + try { + token = tokens[++i]; + } catch(Exception anything) { + continue; + } + if (token.type == Token.LC) { + level++; + } else if (token.type == Token.RC) { + level--; + } + } + + if (dropFunction) + continue; + + // parse function body for nested functions + scanFunctions(lines, tokens, list, res, theFunction, j, i); + // set the function body, starting at the beginning of the first line + marker.x = 0; + theFunction.content = Util.extractString(lines, marker, getPoint(token)); + + } // end if + + lastToken = token; + + } // end while + + } + + private static DocFunction newFunction (String funcName, Resource res, DocElement parent, int lineno) { if (funcName.endsWith("_action")) { - return new DocFunction(funcName, res, parent, ACTION); + return new DocFunction(funcName, res, parent, ACTION, lineno); } else if (funcName.endsWith("_macro")) { - return new DocFunction(funcName, res, parent, MACRO); + return new DocFunction(funcName, res, parent, MACRO, lineno); } else { - return new DocFunction(funcName, res, parent, FUNCTION); + return new DocFunction(funcName, res, parent, FUNCTION, lineno); } } /** * Creates a rhino token stream for a given file. - * @param src the JS source, either as Resource or String object + * @param res the JS Resource * @return a TokenStream wrapper - * @throws IOException if an I/O exception was raised + * @throws java.io.IOException if an I/O exception was raised */ - protected static TokenStreamInjector getTokenStream (Object src) throws IOException { - // TODO the TokenStreamInjector is really just a hack, and we shouldn't - // interact with the rhino TokenStream class directly. The proper way to - // go would be to use the public Parser class to parse the input, and walk - // through the parse tree and extract the comments manually. - // As a result of our approach, the TokenStream member in our Parser instance - // will be null, resulting in a NullPointerException when an error is - // encountered. For the time being, this is something we can live with. - Reader reader = null; - String content = null; - if (src instanceof Resource) { - reader = new InputStreamReader(((Resource) src).getInputStream()); - } else if (src instanceof String) { - content = (String) src; - } else { - throw new IllegalArgumentException("src must be either a Resource or a String"); - } + protected static Token[] parseTokens(Resource res) throws IOException { + Reader reader = new InputStreamReader(res.getInputStream()); CompilerEnvirons compilerEnv = new CompilerEnvirons(); compilerEnv.initFromContext(Context.getCurrentContext()); + compilerEnv.setGenerateDebugInfo(true); + compilerEnv.setGeneratingSource(true); + compilerEnv.setOptimizationLevel(-1); ErrorReporter errorReporter = Context.getCurrentContext().getErrorReporter(); Parser parser = new Parser(compilerEnv, errorReporter); - return new TokenStreamInjector (parser, reader, content, 0); + return parser.parseTokens(reader, res.getName(), 0); } /** * Returns a pointer to the current position in the TokenStream - * @param ts the TokenStream + * @param token the TokenStream * @return the current position */ - protected static Point getPoint (TokenStreamInjector ts) { - return new Point (ts.getOffset(), ts.getLineno()); + protected static Point getPoint (Token token) { + return new Point (token.offset, token.lineno); } - + /** * from helma.framework.IPathElement. All macros, templates, actions etc * have the same prototype. @@ -283,4 +288,12 @@ public class DocFunction extends DocResourceElement { public java.lang.String getPrototype() { return "docfunction"; } + + /** + * Get the first line of this function within the containing resource. + * @return the first line of the function + */ + public int getStartLine() { + return startLine; + } } diff --git a/src/helma/doc/DocResourceElement.java b/src/helma/doc/DocResourceElement.java index 3ee13a1c..559163c4 100644 --- a/src/helma/doc/DocResourceElement.java +++ b/src/helma/doc/DocResourceElement.java @@ -39,4 +39,12 @@ public abstract class DocResourceElement extends DocElement { public String toString() { return resource.getName(); } + + /** + * Get the line number this element starts in. Defaults to 0. + * @return the first line of this element within its resource + */ + public int getStartLine() { + return 0; + } } diff --git a/src/helma/doc/Util.java b/src/helma/doc/Util.java index 8c71590e..08760342 100644 --- a/src/helma/doc/Util.java +++ b/src/helma/doc/Util.java @@ -88,6 +88,22 @@ public final class Util { return buf.toString().trim(); } + /** + * Extract a part of a file defined by two points from a String array + * @param lines an array of lines + * @param start of string to extract defined by column x and row y + * @param end of string to extract + * @return string + */ + public static String extractString (String[] lines, int start, int end) { + StringBuffer buf = new StringBuffer(); + int to = Math.min(end + 1, lines.length); + for (int i = start; i < to; i++) { + buf.append(lines[i]); + buf.append("\n"); + } + return buf.toString().trim(); + } /** * method to debug file/stream-handling with Point objects. extracts the line p diff --git a/src/helma/framework/core/Application.java b/src/helma/framework/core/Application.java index 8b17dc62..3011168d 100644 --- a/src/helma/framework/core/Application.java +++ b/src/helma/framework/core/Application.java @@ -24,6 +24,8 @@ import helma.main.Server; import helma.objectmodel.*; import helma.objectmodel.db.*; import helma.util.*; +import helma.doc.DocApplication; + import java.io.*; import java.lang.reflect.*; import java.rmi.*; @@ -39,7 +41,7 @@ import java.util.ArrayList; * requests from the Web server or XML-RPC port and dispatches them to * the evaluators. */ -public final class Application implements IPathElement, Runnable { +public final class Application implements Runnable { // the name of this application private String name; @@ -436,7 +438,7 @@ public final class Application implements IPathElement, Runnable { eval = getEvaluator(); eval.invokeInternal(null, "onStart", RequestEvaluator.EMPTY_ARGS); } catch (Exception xcept) { - logError("Error in " + name + "onStart()", xcept); + logError("Error in " + name + ".onStart()", xcept); } finally { releaseEvaluator(eval); } @@ -1342,49 +1344,17 @@ public final class Application implements IPathElement, Runnable { } } - ////////////////////////////////////////////////////////////////////////////////////////////////////////// - /// The following methods are the IPathElement interface for this application. - /// this is useful for scripting and url-building in the base-app. - ////////////////////////////////////////////////////////////////////////////////////////////////////////// - public String getElementName() { - return name; - } - - /** - * - * - * @param name ... - * - * @return ... - */ - public IPathElement getChildElement(String name) { - // as Prototype and the helma.scripting-classes don't offer enough information - // we use the classes from helma.doc-pacakge for introspection. - // the first time an url like /appname/api/ is parsed, the application is read again - // parsed for comments and exposed as an IPathElement - if (name.equals("api") && allThreads.size() > 0) { - return ((RequestEvaluator) allThreads.get(0)).scriptingEngine.getIntrospector(); + public DocApplication getDoc() { + RequestEvaluator eval = null; + try { + eval = getEvaluator(); + return eval.scriptingEngine.getDoc(); + } catch (Exception xcept) { + logError("Error in getDoc() for " + name, xcept); + return null; + } finally { + releaseEvaluator(eval); } - - return null; - } - - /** - * - * - * @return ... - */ - public IPathElement getParentElement() { - return helma.main.Server.getServer(); - } - - /** - * - * - * @return ... - */ - public String getPrototype() { - return "application"; } //////////////////////////////////////////////////////////////////////// diff --git a/src/helma/main/Server.java b/src/helma/main/Server.java index 32ff67bf..2bb9c02a 100644 --- a/src/helma/main/Server.java +++ b/src/helma/main/Server.java @@ -40,7 +40,7 @@ import helma.util.ResourceProperties; /** * Helma server main class. */ -public class Server implements IPathElement, Runnable { +public class Server implements Runnable { // version string public static final String version = "1.6.x (__builddate__)"; @@ -884,35 +884,6 @@ public class Server implements IPathElement, Runnable { appManager.stop(name); } - /** - * method from helma.framework.IPathElement - */ - public String getElementName() { - return "root"; - } - - /** - * method from helma.framework.IPathElement, - * returning active applications - */ - public IPathElement getChildElement(String name) { - return appManager.getApplication(name); - } - - /** - * method from helma.framework.IPathElement - */ - public IPathElement getParentElement() { - return null; - } - - /** - * method from helma.framework.IPathElement - */ - public String getPrototype() { - return "root"; - } - static class HelmaLogSink implements LogSink { public String getOptions() { diff --git a/src/helma/scripting/ScriptingEngine.java b/src/helma/scripting/ScriptingEngine.java index e8aa4f4e..0441325e 100644 --- a/src/helma/scripting/ScriptingEngine.java +++ b/src/helma/scripting/ScriptingEngine.java @@ -20,6 +20,8 @@ import helma.framework.IPathElement; import helma.framework.repository.Resource; import helma.framework.core.Application; import helma.framework.core.RequestEvaluator; +import helma.doc.DocApplication; + import java.io.OutputStream; import java.io.IOException; import java.io.InputStream; @@ -121,7 +123,7 @@ public interface ScriptingEngine { * In order to be compatible with the standard Helma management application, this * class should be compatible with helma.doc.DocApplication. */ - public IPathElement getIntrospector(); + public DocApplication getDoc(); /** * Provide object serialization for this engine's scripted objects. If no special diff --git a/src/helma/scripting/rhino/RhinoEngine.java b/src/helma/scripting/rhino/RhinoEngine.java index 465332d5..fcb4d17b 100644 --- a/src/helma/scripting/rhino/RhinoEngine.java +++ b/src/helma/scripting/rhino/RhinoEngine.java @@ -479,7 +479,7 @@ public class RhinoEngine implements ScriptingEngine { /** * Get an introspector to this engine. */ - public IPathElement getIntrospector() { + public DocApplication getDoc() { if (doc == null) { try { doc = new DocApplication(app);