Implement aggressive loading of object references via left joins

src/helma/objectmodel/db/DbMapping.java

@@ -77,6 +77,9 @@ public final class DbMapping implements Updatable {
     // Map of db columns by name
     HashMap columnMap;
 
+    // Array of aggressively loaded references
+    Relation[] joins;
+
     // pre-rendered select statement
     String selectString = null;
     String insertString = null;
@@ -256,6 +259,7 @@ public final class DbMapping implements Updatable {
 
         HashMap p2d = new HashMap();
         HashMap d2p = new HashMap();
+        ArrayList joinList = new ArrayList();
 
         for (Enumeration e = props.keys(); e.hasMoreElements();) {
             String propName = (String) e.nextElement();
@@ -293,6 +297,13 @@ public final class DbMapping implements Updatable {
                         }
                     }
 
+                    // check if a reference is aggressively fetched
+                    if ((rel.reftype == Relation.REFERENCE ||
+                             rel.reftype == Relation.COMPLEX_REFERENCE) &&
+                             rel.aggressiveLoading) {
+                        joinList.add(rel);
+                    }
+
                     // app.logEvent ("Mapping "+propName+" -> "+dbField);
                 }
             } catch (Exception x) {
@@ -303,6 +314,9 @@ public final class DbMapping implements Updatable {
         prop2db = p2d;
         db2prop = d2p;
 
+        joins = new Relation[joinList.size()];
+        joins = (Relation[]) joinList.toArray(joins);
+
         String subnodeMapping = props.getProperty("_children");
 
         if (subnodeMapping != null) {
@@ -844,9 +858,9 @@ public final class DbMapping implements Updatable {
                 Relation rel = columnNameToRelation(colName);
 
                 DbColumn col = new DbColumn(colName, meta.getColumnType(i + 1), rel, this);
-                if (col.isMapped()) {
+                // if (col.isMapped()) {
                     list.add(col);
-                }
+                // }
             }
             columns = new DbColumn[list.size()];
             columns = (DbColumn[]) list.toArray(columns);
@@ -855,6 +869,13 @@ public final class DbMapping implements Updatable {
         return columns;
     }
 
+    /**
+     *  Return the array of relations that are fetched with objects of this type.
+     */
+    public Relation[] getJoins() {
+        return joins;
+    }
+
     /**
      *
      *
@@ -909,7 +930,7 @@ public final class DbMapping implements Updatable {
 
         StringBuffer s = new StringBuffer("SELECT ");
 
-        DbColumn[] cols = columns;
+        /* DbColumn[] cols = columns;
 
         if (cols == null) {
             cols = getColumns();
@@ -922,11 +943,28 @@ public final class DbMapping implements Updatable {
             }
         }
 
+        for (int i = 0; i < joins.length; i++) {
+        } */
+
+        s.append ("*");
+
         s.append(" FROM ");
 
         s.append(getTableName());
         s.append(" ");
 
+        for (int i = 0; i < joins.length; i++) {
+            if (!joins[i].otherType.isRelational()) {
+                continue;
+            }
+            s.append("LEFT JOIN ");
+            s.append(joins[i].otherType.getTableName());
+            s.append(" AS _HLM_");
+            s.append(joins[i].propName);
+            s.append(" ON ");
+            joins[i].renderJoinConstraints(s);
+        }
+
         // cache rendered string for later calls.
         selectString = s.toString();
 

src/helma/objectmodel/db/Node.java

@@ -76,7 +76,7 @@ public final class Node implements INode, Serializable {
     transient private int state;
 
     /**
-     * This constructor is only used for instances of the NullNode subclass. Do not use for ordinary Nodes.
+     * This constructor is only used for NullNode instance. Do not use for ordinary Nodes.
      */
     Node() {
         created = lastmodified = System.currentTimeMillis();
@@ -145,197 +145,28 @@ public final class Node implements INode, Serializable {
     }
 
     /**
-     * Constructor used for nodes being stored in a relational database table.
+     * Initializer used for nodes being stored in a relational database table.
      */
-    public Node(DbMapping dbm, ResultSet rs, DbColumn[] columns, WrappedNodeManager nmgr)
+    public void init(DbMapping dbm, String id, String name, String protoName,
-            throws SQLException, IOException {
+                Hashtable propMap, WrappedNodeManager nmgr) {
         this.nmgr = nmgr;
 
         // see what prototype/DbMapping this object should use
-        dbmap = dbm;
+        this.dbmap = dbm;
+        // set the prototype name
+        this.prototype = protoName;
 
-        created = lastmodified = System.currentTimeMillis();
+        this.id = id;
-
-        for (int i = 0; i < columns.length; i++) {
-
-            // set prototype?
-            if (columns[i].isPrototypeField()) {
-                String protoName = rs.getString(i+1);
-
-                if (protoName != null) {
-                    dbmap = nmgr.getDbMapping(protoName);
-
-                    if (dbmap == null) {
-                        // invalid prototype name!
-                        System.err.println("Warning: Invalid prototype name: " + protoName +
-                                       " - using default");
-                        dbmap = dbm;
-                    }
-                }
-            }
-
-            // set id?
-            if (columns[i].isIdField()) {
-                id = rs.getString(i+1);
-            }
-
-            // set name?
-            if (columns[i].isNameField()) {
-                name = rs.getString(i+1);
-            }
-
-            Relation rel = columns[i].getRelation();
-
-            if ((rel == null) ||
-                    ((rel.reftype != Relation.PRIMITIVE) &&
-                    (rel.reftype != Relation.REFERENCE))) {
-                continue;
-            }
-
-            Property newprop = new Property(rel.propName, this);
-
-            switch (columns[i].getType()) {
-                case Types.BIT:
-                    newprop.setBooleanValue(rs.getBoolean(i+1));
-
-                    break;
-
-                case Types.TINYINT:
-                case Types.BIGINT:
-                case Types.SMALLINT:
-                case Types.INTEGER:
-                    newprop.setIntegerValue(rs.getLong(i+1));
-
-                    break;
-
-                case Types.REAL:
-                case Types.FLOAT:
-                case Types.DOUBLE:
-                    newprop.setFloatValue(rs.getDouble(i+1));
-
-                    break;
-
-                case Types.DECIMAL:
-                case Types.NUMERIC:
-
-                    BigDecimal num = rs.getBigDecimal(i+1);
-
-                    if (num == null) {
-                        break;
-                    }
-
-                    if (num.scale() > 0) {
-                        newprop.setFloatValue(num.doubleValue());
-                    } else {
-                        newprop.setIntegerValue(num.longValue());
-                    }
-
-                    break;
-
-                case Types.VARBINARY:
-                case Types.BINARY:
-                    newprop.setStringValue(rs.getString(i+1));
-
-                    break;
-
-                case Types.LONGVARBINARY:
-                case Types.LONGVARCHAR:
-
-                    try {
-                        newprop.setStringValue(rs.getString(i+1));
-                    } catch (SQLException x) {
-                        Reader in = rs.getCharacterStream(i+1);
-                        char[] buffer = new char[2048];
-                        int read = 0;
-                        int r = 0;
-
-                        while ((r = in.read(buffer, read, buffer.length - read)) > -1) {
-                            read += r;
-
-                            if (read == buffer.length) {
-                                // grow input buffer
-                                char[] newBuffer = new char[buffer.length * 2];
-
-                                System.arraycopy(buffer, 0, newBuffer, 0, buffer.length);
-                                buffer = newBuffer;
-                            }
-                        }
-
-                        newprop.setStringValue(new String(buffer, 0, read));
-                    }
-
-                    break;
-
-                case Types.CHAR:
-                case Types.VARCHAR:
-                case Types.OTHER:
-                    newprop.setStringValue(rs.getString(i+1));
-
-                    break;
-
-                case Types.DATE:
-                    newprop.setDateValue(rs.getDate(i+1));
-
-                    break;
-
-                case Types.TIME:
-                    newprop.setDateValue(rs.getTime(i+1));
-
-                    break;
-
-                case Types.TIMESTAMP:
-                    newprop.setDateValue(rs.getTimestamp(i+1));
-
-                    break;
-
-                case Types.NULL:
-                    newprop.setStringValue(null);
-
-                    break;
-
-                // continue;
-                default:
-                    newprop.setStringValue(rs.getString(i+1));
-
-                    break;
-            }
-
-            if (rs.wasNull()) {
-                newprop.setStringValue(null);
-            }
-
-            if (propMap == null) {
-                propMap = new Hashtable();
-            }
-
-            propMap.put(rel.propName.toLowerCase(), newprop);
-
-            // if the property is a pointer to another node, change the property type to NODE
-            if ((rel.reftype == Relation.REFERENCE) && rel.usesPrimaryKey()) {
-                // FIXME: References to anything other than the primary key are not supported
-                newprop.convertToNodeReference(rel.otherType);
-
-                // newprop.nhandle = new NodeHandle (new DbKey (rel.otherType, newprop.getStringValue ()));
-                // newprop.type = IProperty.NODE;
-            }
-
-            // mark property as clean, since it's fresh from the db
-            newprop.dirty = false;
-        }
-
-        // set the prototype from the dbmap,
-        // which was possibly modified while reading the resultset
-        setPrototype(dbmap.getTypeName());
 
+        this.name = name;
         // If name was not set from resultset, create a synthetical name now.
         if ((name == null) || (name.length() == 0)) {
-            name = dbmap.getTypeName() + " " + id;
+            this.name = dbmap.getTypeName() + " " + id;
         }
 
-        // again set created and lastmodified. This is because
+        this.propMap = propMap;
-        // lastmodified has been been updated, and we want both values to
+
-        // be identical to show that the node hasn't been changed since
+        // set lastmodified and created timestamps and mark as clean
-        // it was first created.
         created = lastmodified = System.currentTimeMillis();
         markAs(CLEAN);
     }

src/helma/objectmodel/db/NodeManager.java

@@ -19,6 +19,7 @@ package helma.objectmodel.db;
 import helma.framework.core.Application;
 import helma.objectmodel.*;
 import helma.util.CacheMap;
+import java.math.BigDecimal;
 import java.io.*;
 import java.sql.*;
 import java.util.*;
@@ -972,6 +973,7 @@ public final class NodeManager {
     public List getNodeIDs(Node home, Relation rel) throws Exception {
         // Transactor tx = (Transactor) Thread.currentThread ();
         // tx.timer.beginEvent ("getNodeIDs "+home);
+
         if ((rel == null) || (rel.otherType == null) || !rel.otherType.isRelational()) {
             // this should never be called for embedded nodes
             throw new RuntimeException("NodeMgr.getNodeIDs called for non-relational node " +
@@ -1114,7 +1116,10 @@ public final class NodeManager {
 
                 while (rs.next()) {
                     // create new Nodes.
-                    Node node = new Node(rel.otherType, rs, columns, safe);
+                    Node node = createNode(rel.otherType, rs, columns, 0);
+                    if (node == null) {
+                        continue;
+                    }
                     Key primKey = node.getKey();
 
                     retval.add(new NodeHandle(primKey));
@@ -1159,6 +1164,7 @@ public final class NodeManager {
                 Connection con = dbm.getConnection();
                 Statement stmt = con.createStatement();
                 DbColumn[] columns = dbm.getColumns();
+                Relation[] joins = dbm.getJoins();
                 StringBuffer q = dbm.getSelect();
 
                 try {
@@ -1166,6 +1172,8 @@ public final class NodeManager {
                     boolean needsQuotes = dbm.needsQuotes(idfield);
 
                     q.append("WHERE ");
+                    q.append(dbm.getTableName());
+                    q.append(".");
                     q.append(idfield);
                     q.append(" IN (");
 
@@ -1224,7 +1232,10 @@ public final class NodeManager {
 
                     while (rs.next()) {
                         // create new Nodes.
-                        Node node = new Node(dbm, rs, columns, safe);
+                        Node node = createNode(dbm, rs, columns, 0);
+                        if (node == null) {
+                            continue;
+                        }
                         Key primKey = node.getKey();
 
                         // for grouped nodes, collect subnode lists for the intermediary
@@ -1272,6 +1283,28 @@ public final class NodeManager {
                                 }
                             }
                         }
+
+                        int resultSetOffset = columns.length;
+                        // create joined objects
+                        for (int i = 0; i < joins.length; i++) {
+                            DbMapping jdbm = joins[i].otherType;
+                            node = createNode(jdbm, rs, jdbm.getColumns(), resultSetOffset);
+                            if (node != null) {
+                                primKey = node.getKey();
+                                // register new nodes with the cache. If an up-to-date copy
+                                // existed in the cache, use that.
+                                synchronized (cache) {
+                                    Node oldnode = (Node) cache.put(primKey, node);
+
+                                    if ((oldnode != null) &&
+                                            (oldnode.getState() != INode.INVALID)) {
+                                        // found an ok version in the cache, use it.
+                                        cache.put(primKey, oldnode);
+                                    }
+                                }
+                            }
+                            resultSetOffset += jdbm.getColumns().length;
+                        }
                     }
 
                     // If these are grouped nodes, build the intermediary group nodes
@@ -1292,6 +1325,8 @@ public final class NodeManager {
                             groupnode.lastSubnodeFetch = System.currentTimeMillis();
                         }
                     }
+                } catch (Exception x) {
+                    System.err.println ("ERROR IN PREFETCHNODES: "+x);
                 } finally {
                     if (stmt != null) {
                         try {
@@ -1455,6 +1490,8 @@ public final class NodeManager {
                 StringBuffer q = dbm.getSelect();
 
                 q.append("WHERE ");
+                q.append(dbm.getTableName());
+                q.append(".");
                 q.append(idfield);
                 q.append(" = ");
 
@@ -1476,7 +1513,7 @@ public final class NodeManager {
                     return null;
                 }
 
-                node = new Node(dbm, rs, columns, safe);
+                node = createNode(dbm, rs, columns, 0);
 
                 if (rs.next()) {
                     throw new RuntimeException("More than one value returned by query.");
@@ -1538,12 +1575,15 @@ public final class NodeManager {
                 if (home.getSubnodeRelation() != null && !rel.isComplexReference()) {
                     // combine our key with the constraints in the manually set subnode relation
                     q.append("WHERE ");
+                    q.append(dbm.getTableName());
+                    q.append(".");
                     q.append(rel.accessName);
                     q.append(" = '");
                     q.append(escape(kstr));
                     q.append("'");
-                    q.append(" AND ");
+                    q.append(" AND (");
                     q.append(home.getSubnodeRelation().trim().substring(5));
+                    q.append(")");
                 } else {
                     q.append(rel.buildQuery(home, home.getNonVirtualParent(), kstr,
                                             "WHERE ", false));
@@ -1561,7 +1601,7 @@ public final class NodeManager {
                     return null;
                 }
 
-                node = new Node(rel.otherType, rs, columns, safe);
+                node = createNode(rel.otherType, rs, columns, 0);
 
                 if (rs.next()) {
                     throw new RuntimeException("More than one value returned by query.");
@@ -1589,6 +1629,193 @@ public final class NodeManager {
         return node;
     }
 
+    /**
+     *  Create a new Node from a ResultSet.
+     */
+    public Node createNode(DbMapping dbm, ResultSet rs, DbColumn[] columns, int offset)
+                throws SQLException, IOException {
+        Hashtable propMap = new Hashtable();
+        String id = null;
+        String name = null;
+        String protoName = dbm.getTypeName();
+        DbMapping dbmap = dbm;
+
+        Node node = new Node();
+
+        for (int i = 0; i < columns.length; i++) {
+
+            // set prototype?
+            if (columns[i].isPrototypeField()) {
+                protoName = rs.getString(i+1+offset);
+
+                if (protoName != null) {
+                    dbmap = getDbMapping(protoName);
+
+                    if (dbmap == null) {
+                        // invalid prototype name!
+                        System.err.println("Warning: Invalid prototype name: " + protoName +
+                                       " - using default");
+                        dbmap = dbm;
+                    }
+                }
+            }
+
+            // set id?
+            if (columns[i].isIdField()) {
+                id = rs.getString(i+1+offset);
+                // if id == null, the object doesn't actually exist - return null
+                if (id == null) {
+                    return null;
+                }
+            }
+
+            // set name?
+            if (columns[i].isNameField()) {
+                name = rs.getString(i+1+offset);
+            }
+
+            Relation rel = columns[i].getRelation();
+
+            if ((rel == null) ||
+                    ((rel.reftype != Relation.PRIMITIVE) &&
+                    (rel.reftype != Relation.REFERENCE))) {
+                continue;
+            }
+
+            Property newprop = new Property(rel.propName, node);
+
+            switch (columns[i].getType()) {
+                case Types.BIT:
+                    newprop.setBooleanValue(rs.getBoolean(i+1+offset));
+
+                    break;
+
+                case Types.TINYINT:
+                case Types.BIGINT:
+                case Types.SMALLINT:
+                case Types.INTEGER:
+                    newprop.setIntegerValue(rs.getLong(i+1+offset));
+
+                    break;
+
+                case Types.REAL:
+                case Types.FLOAT:
+                case Types.DOUBLE:
+                    newprop.setFloatValue(rs.getDouble(i+1+offset));
+
+                    break;
+
+                case Types.DECIMAL:
+                case Types.NUMERIC:
+
+                    BigDecimal num = rs.getBigDecimal(i+1+offset);
+
+                    if (num == null) {
+                        break;
+                    }
+
+                    if (num.scale() > 0) {
+                        newprop.setFloatValue(num.doubleValue());
+                    } else {
+                        newprop.setIntegerValue(num.longValue());
+                    }
+
+                    break;
+
+                case Types.VARBINARY:
+                case Types.BINARY:
+                    newprop.setStringValue(rs.getString(i+1+offset));
+
+                    break;
+
+                case Types.LONGVARBINARY:
+                case Types.LONGVARCHAR:
+
+                    try {
+                        newprop.setStringValue(rs.getString(i+1+offset));
+                    } catch (SQLException x) {
+                        Reader in = rs.getCharacterStream(i+1+offset);
+                        char[] buffer = new char[2048];
+                        int read = 0;
+                        int r = 0;
+
+                        while ((r = in.read(buffer, read, buffer.length - read)) > -1) {
+                            read += r;
+
+                            if (read == buffer.length) {
+                                // grow input buffer
+                                char[] newBuffer = new char[buffer.length * 2];
+
+                                System.arraycopy(buffer, 0, newBuffer, 0, buffer.length);
+                                buffer = newBuffer;
+                            }
+                        }
+
+                        newprop.setStringValue(new String(buffer, 0, read));
+                    }
+
+                    break;
+
+                case Types.CHAR:
+                case Types.VARCHAR:
+                case Types.OTHER:
+                    newprop.setStringValue(rs.getString(i+1+offset));
+
+                    break;
+
+                case Types.DATE:
+                    newprop.setDateValue(rs.getDate(i+1+offset));
+
+                    break;
+
+                case Types.TIME:
+                    newprop.setDateValue(rs.getTime(i+1+offset));
+
+                    break;
+
+                case Types.TIMESTAMP:
+                    newprop.setDateValue(rs.getTimestamp(i+1+offset));
+
+                    break;
+
+                case Types.NULL:
+                    newprop.setStringValue(null);
+
+                    break;
+
+                // continue;
+                default:
+                    newprop.setStringValue(rs.getString(i+1+offset));
+
+                    break;
+            }
+
+            if (rs.wasNull()) {
+                newprop.setStringValue(null);
+            }
+
+            propMap.put(rel.propName.toLowerCase(), newprop);
+
+            // if the property is a pointer to another node, change the property type to NODE
+            if ((rel.reftype == Relation.REFERENCE) && rel.usesPrimaryKey()) {
+                // FIXME: References to anything other than the primary key are not supported
+                newprop.convertToNodeReference(rel.otherType);
+            }
+
+            // mark property as clean, since it's fresh from the db
+            newprop.dirty = false;
+        }
+
+        if (id == null) {
+            return null;
+        }
+
+        node.init(dbmap, id, name, protoName, propMap, safe);
+
+        return node;
+    }
+
+
     /**
      * Get a DbMapping for a given prototype name. This is just a proxy
      * method to the app's getDbMapping() method.

src/helma/objectmodel/db/Relation.java

@@ -626,6 +626,8 @@ public final class Relation {
 
             String accessColumn = (accessName == null) ? otherType.getIDField() : accessName;
 
+            q.append(otherType.getTableName());
+            q.append(".");
             q.append(accessColumn);
             q.append(" = ");
 
@@ -678,21 +680,39 @@ public final class Relation {
     public String renderConstraints(INode home, INode nonvirtual)
                              throws SQLException {
         StringBuffer q = new StringBuffer();
-        String suffix = " AND ";
+        String prefix = " AND ";
 
         for (int i = 0; i < constraints.length; i++) {
+            q.append(prefix);
             constraints[i].addToQuery(q, home, nonvirtual);
-            q.append(suffix);
         }
 
         if (filter != null) {
+            q.append(prefix);
             q.append(filter);
-            q.append(suffix);
         }
 
         return q.toString();
     }
 
+    public void renderJoinConstraints(StringBuffer select) {
+        for (int i = 0; i < constraints.length; i++) {
+            select.append(ownType.getTableName());
+            select.append(".");
+            select.append(constraints[i].localName);
+            select.append(" = _HLM_");
+            select.append(propName);
+            select.append(".");
+            select.append(constraints[i].foreignName);
+            if (i == constraints.length-1) {
+                select.append(" ");
+            } else {
+                select.append(" AND ");
+            }
+        }
+
+    }
+
     /**
      * Get the order section to use for this relation
      */
@@ -966,6 +986,8 @@ public final class Relation {
                 local = ref.getString(homeprop);
             }
 
+            q.append(otherType.getTableName());
+            q.append(".");
             q.append(foreignName);
             q.append(" = ");