From 469399e972dca5ba43e364cfaa95a6696b8b6cec Mon Sep 17 00:00:00 2001 From: hns Date: Wed, 28 May 2003 13:33:23 +0000 Subject: [PATCH] Implement aggressive loading of object references via left joins --- src/helma/objectmodel/db/DbMapping.java | 44 +++- src/helma/objectmodel/db/Node.java | 195 ++---------------- src/helma/objectmodel/db/NodeManager.java | 237 +++++++++++++++++++++- src/helma/objectmodel/db/Relation.java | 28 ++- 4 files changed, 311 insertions(+), 193 deletions(-) diff --git a/src/helma/objectmodel/db/DbMapping.java b/src/helma/objectmodel/db/DbMapping.java index 6e894df7..38ca5824 100644 --- a/src/helma/objectmodel/db/DbMapping.java +++ b/src/helma/objectmodel/db/DbMapping.java @@ -77,6 +77,9 @@ public final class DbMapping implements Updatable { // Map of db columns by name HashMap columnMap; + // Array of aggressively loaded references + Relation[] joins; + // pre-rendered select statement String selectString = null; String insertString = null; @@ -256,6 +259,7 @@ public final class DbMapping implements Updatable { HashMap p2d = new HashMap(); HashMap d2p = new HashMap(); + ArrayList joinList = new ArrayList(); for (Enumeration e = props.keys(); e.hasMoreElements();) { String propName = (String) e.nextElement(); @@ -293,6 +297,13 @@ public final class DbMapping implements Updatable { } } + // check if a reference is aggressively fetched + if ((rel.reftype == Relation.REFERENCE || + rel.reftype == Relation.COMPLEX_REFERENCE) && + rel.aggressiveLoading) { + joinList.add(rel); + } + // app.logEvent ("Mapping "+propName+" -> "+dbField); } } catch (Exception x) { @@ -303,6 +314,9 @@ public final class DbMapping implements Updatable { prop2db = p2d; db2prop = d2p; + joins = new Relation[joinList.size()]; + joins = (Relation[]) joinList.toArray(joins); + String subnodeMapping = props.getProperty("_children"); if (subnodeMapping != null) { @@ -844,9 +858,9 @@ public final class DbMapping implements Updatable { Relation rel = columnNameToRelation(colName); DbColumn col = new DbColumn(colName, meta.getColumnType(i + 1), rel, this); - if (col.isMapped()) { + // if (col.isMapped()) { list.add(col); - } + // } } columns = new DbColumn[list.size()]; columns = (DbColumn[]) list.toArray(columns); @@ -855,6 +869,13 @@ public final class DbMapping implements Updatable { return columns; } + /** + * Return the array of relations that are fetched with objects of this type. + */ + public Relation[] getJoins() { + return joins; + } + /** * * @@ -909,7 +930,7 @@ public final class DbMapping implements Updatable { StringBuffer s = new StringBuffer("SELECT "); - DbColumn[] cols = columns; + /* DbColumn[] cols = columns; if (cols == null) { cols = getColumns(); @@ -922,11 +943,28 @@ public final class DbMapping implements Updatable { } } + for (int i = 0; i < joins.length; i++) { + } */ + + s.append ("*"); + s.append(" FROM "); s.append(getTableName()); s.append(" "); + for (int i = 0; i < joins.length; i++) { + if (!joins[i].otherType.isRelational()) { + continue; + } + s.append("LEFT JOIN "); + s.append(joins[i].otherType.getTableName()); + s.append(" AS _HLM_"); + s.append(joins[i].propName); + s.append(" ON "); + joins[i].renderJoinConstraints(s); + } + // cache rendered string for later calls. selectString = s.toString(); diff --git a/src/helma/objectmodel/db/Node.java b/src/helma/objectmodel/db/Node.java index ccc8da7d..1d733b31 100644 --- a/src/helma/objectmodel/db/Node.java +++ b/src/helma/objectmodel/db/Node.java @@ -76,7 +76,7 @@ public final class Node implements INode, Serializable { transient private int state; /** - * This constructor is only used for instances of the NullNode subclass. Do not use for ordinary Nodes. + * This constructor is only used for NullNode instance. Do not use for ordinary Nodes. */ Node() { created = lastmodified = System.currentTimeMillis(); @@ -145,197 +145,28 @@ public final class Node implements INode, Serializable { } /** - * Constructor used for nodes being stored in a relational database table. + * Initializer used for nodes being stored in a relational database table. */ - public Node(DbMapping dbm, ResultSet rs, DbColumn[] columns, WrappedNodeManager nmgr) - throws SQLException, IOException { + public void init(DbMapping dbm, String id, String name, String protoName, + Hashtable propMap, WrappedNodeManager nmgr) { this.nmgr = nmgr; // see what prototype/DbMapping this object should use - dbmap = dbm; + this.dbmap = dbm; + // set the prototype name + this.prototype = protoName; - created = lastmodified = System.currentTimeMillis(); - - for (int i = 0; i < columns.length; i++) { - - // set prototype? - if (columns[i].isPrototypeField()) { - String protoName = rs.getString(i+1); - - if (protoName != null) { - dbmap = nmgr.getDbMapping(protoName); - - if (dbmap == null) { - // invalid prototype name! - System.err.println("Warning: Invalid prototype name: " + protoName + - " - using default"); - dbmap = dbm; - } - } - } - - // set id? - if (columns[i].isIdField()) { - id = rs.getString(i+1); - } - - // set name? - if (columns[i].isNameField()) { - name = rs.getString(i+1); - } - - Relation rel = columns[i].getRelation(); - - if ((rel == null) || - ((rel.reftype != Relation.PRIMITIVE) && - (rel.reftype != Relation.REFERENCE))) { - continue; - } - - Property newprop = new Property(rel.propName, this); - - switch (columns[i].getType()) { - case Types.BIT: - newprop.setBooleanValue(rs.getBoolean(i+1)); - - break; - - case Types.TINYINT: - case Types.BIGINT: - case Types.SMALLINT: - case Types.INTEGER: - newprop.setIntegerValue(rs.getLong(i+1)); - - break; - - case Types.REAL: - case Types.FLOAT: - case Types.DOUBLE: - newprop.setFloatValue(rs.getDouble(i+1)); - - break; - - case Types.DECIMAL: - case Types.NUMERIC: - - BigDecimal num = rs.getBigDecimal(i+1); - - if (num == null) { - break; - } - - if (num.scale() > 0) { - newprop.setFloatValue(num.doubleValue()); - } else { - newprop.setIntegerValue(num.longValue()); - } - - break; - - case Types.VARBINARY: - case Types.BINARY: - newprop.setStringValue(rs.getString(i+1)); - - break; - - case Types.LONGVARBINARY: - case Types.LONGVARCHAR: - - try { - newprop.setStringValue(rs.getString(i+1)); - } catch (SQLException x) { - Reader in = rs.getCharacterStream(i+1); - char[] buffer = new char[2048]; - int read = 0; - int r = 0; - - while ((r = in.read(buffer, read, buffer.length - read)) > -1) { - read += r; - - if (read == buffer.length) { - // grow input buffer - char[] newBuffer = new char[buffer.length * 2]; - - System.arraycopy(buffer, 0, newBuffer, 0, buffer.length); - buffer = newBuffer; - } - } - - newprop.setStringValue(new String(buffer, 0, read)); - } - - break; - - case Types.CHAR: - case Types.VARCHAR: - case Types.OTHER: - newprop.setStringValue(rs.getString(i+1)); - - break; - - case Types.DATE: - newprop.setDateValue(rs.getDate(i+1)); - - break; - - case Types.TIME: - newprop.setDateValue(rs.getTime(i+1)); - - break; - - case Types.TIMESTAMP: - newprop.setDateValue(rs.getTimestamp(i+1)); - - break; - - case Types.NULL: - newprop.setStringValue(null); - - break; - - // continue; - default: - newprop.setStringValue(rs.getString(i+1)); - - break; - } - - if (rs.wasNull()) { - newprop.setStringValue(null); - } - - if (propMap == null) { - propMap = new Hashtable(); - } - - propMap.put(rel.propName.toLowerCase(), newprop); - - // if the property is a pointer to another node, change the property type to NODE - if ((rel.reftype == Relation.REFERENCE) && rel.usesPrimaryKey()) { - // FIXME: References to anything other than the primary key are not supported - newprop.convertToNodeReference(rel.otherType); - - // newprop.nhandle = new NodeHandle (new DbKey (rel.otherType, newprop.getStringValue ())); - // newprop.type = IProperty.NODE; - } - - // mark property as clean, since it's fresh from the db - newprop.dirty = false; - } - - // set the prototype from the dbmap, - // which was possibly modified while reading the resultset - setPrototype(dbmap.getTypeName()); + this.id = id; + this.name = name; // If name was not set from resultset, create a synthetical name now. if ((name == null) || (name.length() == 0)) { - name = dbmap.getTypeName() + " " + id; + this.name = dbmap.getTypeName() + " " + id; } - // again set created and lastmodified. This is because - // lastmodified has been been updated, and we want both values to - // be identical to show that the node hasn't been changed since - // it was first created. + this.propMap = propMap; + + // set lastmodified and created timestamps and mark as clean created = lastmodified = System.currentTimeMillis(); markAs(CLEAN); } diff --git a/src/helma/objectmodel/db/NodeManager.java b/src/helma/objectmodel/db/NodeManager.java index b2607ef9..55a799a5 100644 --- a/src/helma/objectmodel/db/NodeManager.java +++ b/src/helma/objectmodel/db/NodeManager.java @@ -19,6 +19,7 @@ package helma.objectmodel.db; import helma.framework.core.Application; import helma.objectmodel.*; import helma.util.CacheMap; +import java.math.BigDecimal; import java.io.*; import java.sql.*; import java.util.*; @@ -972,6 +973,7 @@ public final class NodeManager { public List getNodeIDs(Node home, Relation rel) throws Exception { // Transactor tx = (Transactor) Thread.currentThread (); // tx.timer.beginEvent ("getNodeIDs "+home); + if ((rel == null) || (rel.otherType == null) || !rel.otherType.isRelational()) { // this should never be called for embedded nodes throw new RuntimeException("NodeMgr.getNodeIDs called for non-relational node " + @@ -1114,7 +1116,10 @@ public final class NodeManager { while (rs.next()) { // create new Nodes. - Node node = new Node(rel.otherType, rs, columns, safe); + Node node = createNode(rel.otherType, rs, columns, 0); + if (node == null) { + continue; + } Key primKey = node.getKey(); retval.add(new NodeHandle(primKey)); @@ -1159,6 +1164,7 @@ public final class NodeManager { Connection con = dbm.getConnection(); Statement stmt = con.createStatement(); DbColumn[] columns = dbm.getColumns(); + Relation[] joins = dbm.getJoins(); StringBuffer q = dbm.getSelect(); try { @@ -1166,6 +1172,8 @@ public final class NodeManager { boolean needsQuotes = dbm.needsQuotes(idfield); q.append("WHERE "); + q.append(dbm.getTableName()); + q.append("."); q.append(idfield); q.append(" IN ("); @@ -1224,7 +1232,10 @@ public final class NodeManager { while (rs.next()) { // create new Nodes. - Node node = new Node(dbm, rs, columns, safe); + Node node = createNode(dbm, rs, columns, 0); + if (node == null) { + continue; + } Key primKey = node.getKey(); // for grouped nodes, collect subnode lists for the intermediary @@ -1272,6 +1283,28 @@ public final class NodeManager { } } } + + int resultSetOffset = columns.length; + // create joined objects + for (int i = 0; i < joins.length; i++) { + DbMapping jdbm = joins[i].otherType; + node = createNode(jdbm, rs, jdbm.getColumns(), resultSetOffset); + if (node != null) { + primKey = node.getKey(); + // register new nodes with the cache. If an up-to-date copy + // existed in the cache, use that. + synchronized (cache) { + Node oldnode = (Node) cache.put(primKey, node); + + if ((oldnode != null) && + (oldnode.getState() != INode.INVALID)) { + // found an ok version in the cache, use it. + cache.put(primKey, oldnode); + } + } + } + resultSetOffset += jdbm.getColumns().length; + } } // If these are grouped nodes, build the intermediary group nodes @@ -1292,6 +1325,8 @@ public final class NodeManager { groupnode.lastSubnodeFetch = System.currentTimeMillis(); } } + } catch (Exception x) { + System.err.println ("ERROR IN PREFETCHNODES: "+x); } finally { if (stmt != null) { try { @@ -1455,6 +1490,8 @@ public final class NodeManager { StringBuffer q = dbm.getSelect(); q.append("WHERE "); + q.append(dbm.getTableName()); + q.append("."); q.append(idfield); q.append(" = "); @@ -1476,7 +1513,7 @@ public final class NodeManager { return null; } - node = new Node(dbm, rs, columns, safe); + node = createNode(dbm, rs, columns, 0); if (rs.next()) { throw new RuntimeException("More than one value returned by query."); @@ -1538,12 +1575,15 @@ public final class NodeManager { if (home.getSubnodeRelation() != null && !rel.isComplexReference()) { // combine our key with the constraints in the manually set subnode relation q.append("WHERE "); + q.append(dbm.getTableName()); + q.append("."); q.append(rel.accessName); q.append(" = '"); q.append(escape(kstr)); q.append("'"); - q.append(" AND "); + q.append(" AND ("); q.append(home.getSubnodeRelation().trim().substring(5)); + q.append(")"); } else { q.append(rel.buildQuery(home, home.getNonVirtualParent(), kstr, "WHERE ", false)); @@ -1561,7 +1601,7 @@ public final class NodeManager { return null; } - node = new Node(rel.otherType, rs, columns, safe); + node = createNode(rel.otherType, rs, columns, 0); if (rs.next()) { throw new RuntimeException("More than one value returned by query."); @@ -1589,6 +1629,193 @@ public final class NodeManager { return node; } + /** + * Create a new Node from a ResultSet. + */ + public Node createNode(DbMapping dbm, ResultSet rs, DbColumn[] columns, int offset) + throws SQLException, IOException { + Hashtable propMap = new Hashtable(); + String id = null; + String name = null; + String protoName = dbm.getTypeName(); + DbMapping dbmap = dbm; + + Node node = new Node(); + + for (int i = 0; i < columns.length; i++) { + + // set prototype? + if (columns[i].isPrototypeField()) { + protoName = rs.getString(i+1+offset); + + if (protoName != null) { + dbmap = getDbMapping(protoName); + + if (dbmap == null) { + // invalid prototype name! + System.err.println("Warning: Invalid prototype name: " + protoName + + " - using default"); + dbmap = dbm; + } + } + } + + // set id? + if (columns[i].isIdField()) { + id = rs.getString(i+1+offset); + // if id == null, the object doesn't actually exist - return null + if (id == null) { + return null; + } + } + + // set name? + if (columns[i].isNameField()) { + name = rs.getString(i+1+offset); + } + + Relation rel = columns[i].getRelation(); + + if ((rel == null) || + ((rel.reftype != Relation.PRIMITIVE) && + (rel.reftype != Relation.REFERENCE))) { + continue; + } + + Property newprop = new Property(rel.propName, node); + + switch (columns[i].getType()) { + case Types.BIT: + newprop.setBooleanValue(rs.getBoolean(i+1+offset)); + + break; + + case Types.TINYINT: + case Types.BIGINT: + case Types.SMALLINT: + case Types.INTEGER: + newprop.setIntegerValue(rs.getLong(i+1+offset)); + + break; + + case Types.REAL: + case Types.FLOAT: + case Types.DOUBLE: + newprop.setFloatValue(rs.getDouble(i+1+offset)); + + break; + + case Types.DECIMAL: + case Types.NUMERIC: + + BigDecimal num = rs.getBigDecimal(i+1+offset); + + if (num == null) { + break; + } + + if (num.scale() > 0) { + newprop.setFloatValue(num.doubleValue()); + } else { + newprop.setIntegerValue(num.longValue()); + } + + break; + + case Types.VARBINARY: + case Types.BINARY: + newprop.setStringValue(rs.getString(i+1+offset)); + + break; + + case Types.LONGVARBINARY: + case Types.LONGVARCHAR: + + try { + newprop.setStringValue(rs.getString(i+1+offset)); + } catch (SQLException x) { + Reader in = rs.getCharacterStream(i+1+offset); + char[] buffer = new char[2048]; + int read = 0; + int r = 0; + + while ((r = in.read(buffer, read, buffer.length - read)) > -1) { + read += r; + + if (read == buffer.length) { + // grow input buffer + char[] newBuffer = new char[buffer.length * 2]; + + System.arraycopy(buffer, 0, newBuffer, 0, buffer.length); + buffer = newBuffer; + } + } + + newprop.setStringValue(new String(buffer, 0, read)); + } + + break; + + case Types.CHAR: + case Types.VARCHAR: + case Types.OTHER: + newprop.setStringValue(rs.getString(i+1+offset)); + + break; + + case Types.DATE: + newprop.setDateValue(rs.getDate(i+1+offset)); + + break; + + case Types.TIME: + newprop.setDateValue(rs.getTime(i+1+offset)); + + break; + + case Types.TIMESTAMP: + newprop.setDateValue(rs.getTimestamp(i+1+offset)); + + break; + + case Types.NULL: + newprop.setStringValue(null); + + break; + + // continue; + default: + newprop.setStringValue(rs.getString(i+1+offset)); + + break; + } + + if (rs.wasNull()) { + newprop.setStringValue(null); + } + + propMap.put(rel.propName.toLowerCase(), newprop); + + // if the property is a pointer to another node, change the property type to NODE + if ((rel.reftype == Relation.REFERENCE) && rel.usesPrimaryKey()) { + // FIXME: References to anything other than the primary key are not supported + newprop.convertToNodeReference(rel.otherType); + } + + // mark property as clean, since it's fresh from the db + newprop.dirty = false; + } + + if (id == null) { + return null; + } + + node.init(dbmap, id, name, protoName, propMap, safe); + + return node; + } + + /** * Get a DbMapping for a given prototype name. This is just a proxy * method to the app's getDbMapping() method. diff --git a/src/helma/objectmodel/db/Relation.java b/src/helma/objectmodel/db/Relation.java index 972aa8ca..eda70d14 100644 --- a/src/helma/objectmodel/db/Relation.java +++ b/src/helma/objectmodel/db/Relation.java @@ -626,6 +626,8 @@ public final class Relation { String accessColumn = (accessName == null) ? otherType.getIDField() : accessName; + q.append(otherType.getTableName()); + q.append("."); q.append(accessColumn); q.append(" = "); @@ -678,21 +680,39 @@ public final class Relation { public String renderConstraints(INode home, INode nonvirtual) throws SQLException { StringBuffer q = new StringBuffer(); - String suffix = " AND "; + String prefix = " AND "; for (int i = 0; i < constraints.length; i++) { + q.append(prefix); constraints[i].addToQuery(q, home, nonvirtual); - q.append(suffix); } if (filter != null) { + q.append(prefix); q.append(filter); - q.append(suffix); } return q.toString(); } + public void renderJoinConstraints(StringBuffer select) { + for (int i = 0; i < constraints.length; i++) { + select.append(ownType.getTableName()); + select.append("."); + select.append(constraints[i].localName); + select.append(" = _HLM_"); + select.append(propName); + select.append("."); + select.append(constraints[i].foreignName); + if (i == constraints.length-1) { + select.append(" "); + } else { + select.append(" AND "); + } + } + + } + /** * Get the order section to use for this relation */ @@ -966,6 +986,8 @@ public final class Relation { local = ref.getString(homeprop); } + q.append(otherType.getTableName()); + q.append("."); q.append(foreignName); q.append(" = ");