From 2a4d03ac13aedffdbf3ebf10adcc9bd9f9442e81 Mon Sep 17 00:00:00 2001
From: hns <hannesw@gmail.com>
Date: Thu, 12 Sep 2002 17:18:45 +0000
Subject: [PATCH] Implemented experimental prefetchChildren method on nodes.
 Try calling prefetchChildren(start, length) on internal node objects. Works
 also on nodes with groupby-collections.

Null columns from the DB are now set to Properties, which required some fixes
when converting a string (or other) property to a node reference.
---
 src/helma/objectmodel/db/Node.java        |  62 ++++++----
 src/helma/objectmodel/db/NodeManager.java | 132 +++++++++++++++++++---
 src/helma/objectmodel/db/Property.java    |  35 ++++--
 src/helma/objectmodel/db/Relation.java    |  21 +++-
 4 files changed, 205 insertions(+), 45 deletions(-)

diff --git a/src/helma/objectmodel/db/Node.java b/src/helma/objectmodel/db/Node.java
index 811999cc..3233b9ac 100644
--- a/src/helma/objectmodel/db/Node.java
+++ b/src/helma/objectmodel/db/Node.java
@@ -127,16 +127,16 @@ public final class Node implements INode, Serializable {
     /**
      * used by Xml deserialization
      */
-     public void setPropMap (Hashtable propMap)	{
+    public void setPropMap (Hashtable propMap)	{
 	this.propMap = propMap;
-     }
+    }
 
     /**
      * used by Xml deserialization
      */
-     public void setSubnodes (List subnodes)	{
+    public void setSubnodes (List subnodes)	{
 	this.subnodes = subnodes;
-}
+    }
 
     private transient String prototype;
 
@@ -282,12 +282,14 @@ public final class Node implements INode, Serializable {
 
 	    Value val = rec.getValue (rel.getDbField ());
 
-	    if (val.isNull ())
-	        continue;
+	    // if (val.isNull ())
+	    //     continue;
 
 	    Property newprop = new Property (rel.propName, this);
 
-	    switch (val.type ()) {
+	    if (val.isNull ())
+	        newprop.setStringValue (null);
+	    else switch (val.type ()) {
 
 	        case Types.BIT:
 	            newprop.setBooleanValue (val.asBoolean());
@@ -335,7 +337,9 @@ public final class Node implements INode, Serializable {
 	            break;
 
 	        case Types.NULL:
-	            continue;
+	            newprop.setStringValue (null);
+	            break;
+	            // continue;
 
 	        default:
 	            newprop.setStringValue (val.asString());
@@ -345,15 +349,17 @@ public final class Node implements INode, Serializable {
 	    if(propMap == null)
 	        propMap = new Hashtable ();
 	    propMap.put (rel.propName.toLowerCase(), newprop);
-	    // mark property as clean, since it's fresh from the db
-	    newprop.dirty = false;
 
 	    // if the property is a pointer to another node, change the property type to NODE
 	    if (rel.reftype == Relation.REFERENCE && rel.usesPrimaryKey ()) {
 	        // FIXME: References to anything other than the primary key are not supported
-	        newprop.nhandle = new NodeHandle (new DbKey (rel.otherType, newprop.getStringValue ()));
-	        newprop.type = IProperty.NODE;
+	        newprop.convertToNodeReference (rel.otherType);
+	        // newprop.nhandle = new NodeHandle (new DbKey (rel.otherType, newprop.getStringValue ()));
+	        // newprop.type = IProperty.NODE;
 	    }
+
+	    // mark property as clean, since it's fresh from the db
+	    newprop.dirty = false;
 	}
 	// again set created and lastmodified. This is because
 	// lastmodified has been been updated, and we want both values to
@@ -916,7 +922,7 @@ public final class Node implements INode, Serializable {
 	Node retval = null;
 
 	if (subid != null) {
-	
+
 	    loadNodes ();
 	    if (subnodes == null || subnodes.size() == 0)
 	        return null;
@@ -1109,7 +1115,7 @@ public final class Node implements INode, Serializable {
 	if (propMap != null) {
 	    for (Enumeration e2 = propMap.elements (); e2.hasMoreElements (); ) {
 	        Property p = (Property) e2.nextElement ();
-	        if (p != null && p.type == Property.NODE)
+	        if (p != null && p.getType() == Property.NODE)
 	            p.unregisterNode ();
 	    }
 	}
@@ -1215,6 +1221,23 @@ public final class Node implements INode, Serializable {
 	}
     }
 
+    public void prefetchChildren (int startIndex, int length) throws Exception {
+	if (length < 1)
+	    return;
+	if (startIndex < 0)
+	    return;
+	loadNodes ();
+	if (subnodes == null)
+	    return;
+	int l = Math.min (subnodes.size()-startIndex, length);
+	if (l < 1)
+	    return;
+	Key[] keys = new Key[l];
+	for (int i=startIndex; i<startIndex+l; i++)
+	    keys[i] = ((NodeHandle) subnodes.get (i)).getKey ();
+	nmgr.nmgr.prefetchNodes (this, dbmap.getSubnodeRelation (), keys);
+    }
+
     public Enumeration getSubnodes () {
 	loadNodes ();
 	class Enum implements Enumeration {
@@ -1290,10 +1313,9 @@ public final class Node implements INode, Serializable {
 	// See if this could be a relationally linked node which still doesn't know
 	// (i.e, still thinks it's just the key as a string)
 	DbMapping pmap = dbmap == null ? null : dbmap.getExactPropertyMapping (propname);
-	if (pmap != null && prop != null && prop.type != IProperty.NODE) {
-	    // this is a relational node stored by id but we still think it's just a string. fix it
-	    prop.nhandle = new NodeHandle (new DbKey (pmap, prop.getStringValue ()));
-	    prop.type = IProperty.NODE;
+	if (pmap != null && prop != null && prop.getType() != IProperty.NODE) {
+	    // this is a relational node stored by id but we still think it's just a string. Fix it
+	    prop.convertToNodeReference (pmap);
 	}
 
 	// the property does not exist in our propmap - see if we can create it on the fly,
@@ -1645,7 +1667,7 @@ public final class Node implements INode, Serializable {
 
 	Property prop = propMap == null ? null : (Property) propMap.get (p2);
 	if (prop != null) {
-	    if (prop.type == IProperty.NODE && n.getHandle ().equals (prop.nhandle)) {
+	    if (prop.getType() == IProperty.NODE && n.getHandle ().equals (prop.getNodeHandle())) {
 	        // nothing to do, just clean up locks and return
 	        if (state == CLEAN) clearWriteLock ();
 	        if (n.state == CLEAN) n.clearWriteLock ();
@@ -1705,7 +1727,7 @@ public final class Node implements INode, Serializable {
 	    Property p = (Property) propMap.remove (propname.toLowerCase ());
 	    if (p != null) {
 	        checkWriteLock ();
-	        if (p.type == Property.NODE)
+	        if (p.getType() == Property.NODE)
 	            p.unregisterNode ();
 	        // Server.throwNodeEvent (new NodeEvent (this, NodeEvent.PROPERTIES_CHANGED));
 	        lastmodified = System.currentTimeMillis ();
diff --git a/src/helma/objectmodel/db/NodeManager.java b/src/helma/objectmodel/db/NodeManager.java
index 9fb91a3b..8d939a16 100644
--- a/src/helma/objectmodel/db/NodeManager.java
+++ b/src/helma/objectmodel/db/NodeManager.java
@@ -159,7 +159,8 @@ public final class NodeManager {
 
 
     /**
-    *  Get a node by key.
+    *  Get a node by key. This is called from a node that already holds
+    *  a reference to another node via a NodeHandle/Key.
     */
     public Node getNode (Key key) throws Exception {
 
@@ -217,6 +218,8 @@ public final class NodeManager {
 
     /**
     *  Get a node by relation, using the home node, the relation and a key to apply.
+    *  In contrast to getNode (Key key), this is usually called when we don't yet know
+    *  whether such a node exists.
     */
     public Node getNode (Node home, String kstr, Relation rel) throws Exception {
 
@@ -361,7 +364,7 @@ public final class NodeManager {
     */
     public void insertNode (IDatabase db, ITransaction txn, Node node) throws Exception {
 
-	Transactor tx = (Transactor) Thread.currentThread ();
+	// Transactor tx = (Transactor) Thread.currentThread ();
 	// tx.timer.beginEvent ("insertNode "+node);
 
 	DbMapping dbm = node.getDbMapping ();
@@ -369,7 +372,7 @@ public final class NodeManager {
 	if (dbm == null || !dbm.isRelational ()) {
 	    db.saveNode (txn, node.getID (), node);
 	} else {
-	    app.logEvent ("inserting relational node: "+node.getID ());
+	    // app.logEvent ("inserting relational node: "+node.getID ());
 	    TableDataSet tds = null;
 	    try {
 	        tds = new TableDataSet (dbm.getConnection (), dbm.getSchema (), dbm.getKeyDef ());
@@ -439,7 +442,7 @@ public final class NodeManager {
     */
     public void updateNode (IDatabase db, ITransaction txn, Node node) throws Exception {
 
-	Transactor tx = (Transactor) Thread.currentThread ();
+	// Transactor tx = (Transactor) Thread.currentThread ();
 	// tx.timer.beginEvent ("updateNode "+node);
 
 	DbMapping dbm = node.getDbMapping ();
@@ -542,7 +545,7 @@ public final class NodeManager {
     */
     public void deleteNode (IDatabase db, ITransaction txn, Node node) throws Exception {
 
-	Transactor tx = (Transactor) Thread.currentThread ();
+	// Transactor tx = (Transactor) Thread.currentThread ();
 	// tx.timer.beginEvent ("deleteNode "+node);
 
 	DbMapping dbm = node.getDbMapping ();
@@ -572,7 +575,7 @@ public final class NodeManager {
      */
     public synchronized String generateMaxID (DbMapping map) throws Exception {
 
-	Transactor tx = (Transactor) Thread.currentThread ();
+	// Transactor tx = (Transactor) Thread.currentThread ();
 	// tx.timer.beginEvent ("generateID "+map);
 
 	QueryDataSet qds = null;
@@ -606,7 +609,7 @@ public final class NodeManager {
      */
     public String generateID (DbMapping map) throws Exception {
 
-	Transactor tx = (Transactor) Thread.currentThread ();
+	// Transactor tx = (Transactor) Thread.currentThread ();
 	// tx.timer.beginEvent ("generateID "+map);
 
 	QueryDataSet qds = null;
@@ -633,7 +636,7 @@ public final class NodeManager {
      */
     public List getNodeIDs (Node home, Relation rel) throws Exception {
 
-	Transactor tx = (Transactor) Thread.currentThread ();
+	// Transactor tx = (Transactor) Thread.currentThread ();
 	// tx.timer.beginEvent ("getNodeIDs "+home);
 
 	if (rel == null || rel.otherType == null || !rel.otherType.isRelational ()) {
@@ -710,7 +713,7 @@ public final class NodeManager {
 	if (rel.groupby != null)
 	    return getNodeIDs (home, rel);
 
-	Transactor tx = (Transactor) Thread.currentThread ();
+	// Transactor tx = (Transactor) Thread.currentThread ();
 	// tx.timer.beginEvent ("getNodes "+home);
 
 	if (rel == null || rel.otherType == null || !rel.otherType.isRelational ()) {
@@ -732,7 +735,7 @@ public final class NodeManager {
 	            if (rel.getOrder () != null)
 	                tds.order (rel.getOrder ());
 	        }
-	
+
 	        if (logSql)
 	           app.logEvent ("### getNodes: "+tds.getSelectString());
 
@@ -740,7 +743,7 @@ public final class NodeManager {
 	            tds.fetchRecords (rel.maxSize);
 	        else
 	            tds.fetchRecords ();
-	
+
 	        for (int i=0; i<tds.size (); i++) {
 	            // create new Nodes.
 	            Record rec = tds.getRecord (i);
@@ -766,6 +769,109 @@ public final class NodeManager {
 	}
     }
 
+    /**
+     *
+     */
+    public void prefetchNodes (Node home, Relation rel, Key[] keys) throws Exception {
+
+	DbMapping dbm = rel.otherType;
+	if (dbm == null || !dbm.isRelational ()) {
+	    // this does nothing for objects in the embedded database
+	    return;
+	} else {
+	    int missing = cache.containsKeys (keys);
+	    if (missing > 0) {
+	        TableDataSet tds =  new TableDataSet (dbm.getConnection (),
+					dbm.getSchema (),
+					dbm.getKeyDef ());
+	        try {
+	            String idfield = rel.groupby != null ? rel.groupby : dbm.getIDField ();
+	            boolean needsQuotes = dbm.needsQuotes (idfield);
+	            StringBuffer whereBuffer = new StringBuffer (idfield);
+	            whereBuffer.append (" in (");
+	            boolean first = true;
+	            for (int i=0; i<keys.length; i++) {
+	                if (keys[i] != null) {
+	                    if (!first)
+	                        whereBuffer.append (',');
+	                    else
+	                        first = false;
+	                    if (needsQuotes) {
+	                        whereBuffer.append ("'");
+	                        whereBuffer.append (escape (keys[i].getID ()));
+	                        whereBuffer.append ("'");
+	                    } else {
+	                        whereBuffer.append (keys[i].getID ());
+	                    }
+	                }
+	            }
+	            whereBuffer.append (')');
+	            if (rel.groupby != null) {
+	                whereBuffer.insert (0, rel.renderConstraints (home, home.getNonVirtualParent ()));
+	            }
+	            tds.where (whereBuffer.toString ());
+
+	            if (logSql)
+	               app.logEvent ("### prefetchNodes: "+tds.getSelectString());
+
+	            tds.fetchRecords ();
+
+	            String groupbyProp = null;
+	            HashMap groupbySubnodes = null;
+	            if (rel.groupby != null) {
+	                groupbyProp = dbm.columnNameToProperty (rel.groupby);
+	                groupbySubnodes = new HashMap();
+	            }
+
+	            for (int i=0; i<tds.size (); i++) {
+	                // create new Nodes.
+	                Record rec = tds.getRecord (i);
+	                Node node = new Node (dbm, rec, safe);
+	                Key primKey = node.getKey ();
+
+	                // for grouped nodes, collect subnode lists for the intermediary
+	                // group nodes.
+	                if (groupbyProp != null) {
+	                    String groupbyName = node.getString (groupbyProp, false);
+	                    List sn = (List) groupbySubnodes.get (groupbyName);
+	                    if (sn == null) {
+	                        sn = new ExternalizableVector ();
+	                        groupbySubnodes.put (groupbyName, sn);
+	                    }
+	                    sn.add (new NodeHandle (primKey));
+	                }
+
+	                // register new nodes with the cache. If an up-to-date copy
+	                // existed in the cache, use that.
+	                synchronized (cache) {
+	                    Node oldnode = (Node) cache.put (primKey, node);
+	                    if (oldnode != null && oldnode.getState() != INode.INVALID) {
+	                        cache.put (primKey, oldnode);
+	                    }
+	                }
+	            }
+	            // If these are grouped nodes, build the intermediary group nodes
+	            // with the subnod lists we created
+	            if (groupbyProp != null) {
+	                for (Iterator i=groupbySubnodes.keySet().iterator(); i.hasNext(); ) {
+	                    String groupname = (String) i.next();
+	                    if (groupname == null) continue;
+	                    Node groupnode = home.getGroupbySubnode (groupname, true);
+	                    cache.put (groupnode.getKey(), groupnode);
+	                    groupnode.setSubnodes ((List) groupbySubnodes.get(groupname));
+	                    groupnode.lastSubnodeFetch = System.currentTimeMillis ();
+	                }
+	            }
+	        } finally {
+	            if (tds != null)  try {
+	                tds.close ();
+	            } catch (Exception ignore) {}
+	        }
+	    }
+	}
+    }
+
+
 
     /**
      * Count the nodes contained in the child collection of the home node
@@ -773,7 +879,7 @@ public final class NodeManager {
      */
     public int countNodes (Node home, Relation rel) throws Exception {
 
-	Transactor tx = (Transactor) Thread.currentThread ();
+	// Transactor tx = (Transactor) Thread.currentThread ();
 	// tx.timer.beginEvent ("countNodes "+home);
 
 	if (rel == null || rel.otherType == null || !rel.otherType.isRelational ()) {
@@ -822,7 +928,7 @@ public final class NodeManager {
      */
     public Vector getPropertyNames (Node home, Relation rel) throws Exception {
 
-	Transactor tx = (Transactor) Thread.currentThread ();
+	// Transactor tx = (Transactor) Thread.currentThread ();
 	// tx.timer.beginEvent ("getNodeIDs "+home);
 
 	if (rel == null || rel.otherType == null || !rel.otherType.isRelational ()) {
diff --git a/src/helma/objectmodel/db/Property.java b/src/helma/objectmodel/db/Property.java
index 5a1f0dfd..c6d887d9 100644
--- a/src/helma/objectmodel/db/Property.java
+++ b/src/helma/objectmodel/db/Property.java
@@ -10,24 +10,24 @@ import java.text.*;
 import helma.objectmodel.*;
 
 /**
- * A property implementation for Nodes stored inside a database. Basically 
+ * A property implementation for Nodes stored inside a database. Basically
  * the same as for transient nodes, with a few hooks added.
  */
 public final class Property implements IProperty, Serializable, Cloneable {
 
 
-    protected String propname;
-    protected Node node;
+    private String propname;
+    private Node node;
 
-    protected String svalue;
-    protected boolean bvalue;
-    protected long lvalue;
-    protected double dvalue;
+    private String svalue;
+    private boolean bvalue;
+    private long lvalue;
+    private double dvalue;
     // protected String nvalueID;
-    protected NodeHandle nhandle;
-    protected Object jvalue;
+    private NodeHandle nhandle;
+    private Object jvalue;
 
-    protected int type;
+    private int type;
 
     transient boolean dirty;
 
@@ -221,6 +221,19 @@ public final class Property implements IProperty, Serializable, Cloneable {
 	dirty = true;
     }
 
+    public NodeHandle getNodeHandle () {
+	return nhandle;
+    }
+    
+    public void convertToNodeReference (DbMapping dbm) {
+	String id = getStringValue ();
+	if (id == null)
+	    nhandle = null;
+	else 
+	    nhandle = new NodeHandle (new DbKey (dbm, id));
+	type = NODE;
+    }
+
     public void setJavaObjectValue (Object value) {
 	if (type == NODE)
 	    unregisterNode ();
@@ -281,7 +294,7 @@ public final class Property implements IProperty, Serializable, Cloneable {
 	case FLOAT:
 	    return Double.toString (dvalue);
 	case NODE:
-	    return nhandle.getID ();
+	    return nhandle == null ? null : nhandle.getID ();
 	case JAVAOBJECT:
 	    return jvalue == null ? null : jvalue.toString ();
 	}
diff --git a/src/helma/objectmodel/db/Relation.java b/src/helma/objectmodel/db/Relation.java
index 4ec33af7..adcb2feb 100644
--- a/src/helma/objectmodel/db/Relation.java
+++ b/src/helma/objectmodel/db/Relation.java
@@ -342,6 +342,8 @@ public final class Relation {
 	vr.filter = filter;
 	vr.constraints = constraints;
 	vr.addConstraint (new Constraint (null, null, groupby, true));
+	vr.aggressiveLoading = aggressiveLoading;
+	vr.aggressiveCaching = aggressiveCaching;
 	return vr;
     }
 
@@ -393,7 +395,9 @@ public final class Relation {
 	    q.append (filter);
 	}
 	if (groupby != null) {
-	    q.append (" GROUP BY "+groupby);
+	    q.append (prefix);
+	    q.append (groupby);
+	    q.append (" IS NOT NULL GROUP BY "+groupby);
 	    if (useOrder && groupbyorder != null)
 	        q.append (" ORDER BY "+groupbyorder);
 	} else if (useOrder && order != null)
@@ -401,6 +405,21 @@ public final class Relation {
 	return q.toString ();
     }
 
+    public String renderConstraints (INode home, INode nonvirtual) throws SQLException {
+	StringBuffer q = new StringBuffer ();
+	String suffix = " AND ";
+	for (int i=0; i<constraints.length; i++) {
+	    constraints[i].addToQuery (q, home, nonvirtual);
+	    q.append (suffix);
+	}
+	if (filter != null) {
+	    q.append (filter);
+	    q.append (suffix);
+	}
+	return q.toString ();
+    }
+
+
     /**
      * Get the order section to use for this relation
      */