* Do not pass file uploads in memory as byte arrays.

* Implement file upload monitoring support.
* Do not swallow exceptions in GET/POST parameter parsing
* Store any bytes remaining after GET/POST parameter parsing
  into req.data.http_get_remainder and http_post_remainder, respectively.

src/helma/framework/UploadStatus.java

@@ -0,0 +1,82 @@
+/*
+ * Helma License Notice
+ *
+ * The contents of this file are subject to the Helma License
+ * Version 2.0 (the "License"). You may not use this file except in
+ * compliance with the License. A copy of the License is available at
+ * http://adele.helma.org/download/helma/license.txt
+ *
+ * Copyright 2007 Helma Software. All Rights Reserved.
+ *
+ * $RCSfile$
+ * $Author$
+ * $Revision$
+ * $Date$
+ */
+
+package helma.framework;
+
+import java.io.Serializable;
+
+public class UploadStatus implements Serializable {
+
+    long current = 0;
+    long total = 0;
+    int itemsRead = 0;
+    String error = null;
+    long lastModified;
+
+    public UploadStatus() {
+        lastModified = System.currentTimeMillis();
+    }
+
+    public void update(long bytesRead, long contentLength, int itemsRead) {
+        this.current = bytesRead;
+        this.total = contentLength;
+        this.itemsRead = itemsRead;
+        lastModified = System.currentTimeMillis();
+    }
+
+    public void setError(String error) {
+        this.error = error;
+        lastModified = System.currentTimeMillis();
+    }
+
+    public String getError() {
+        return error;
+    }
+
+    public long getCurrent() {
+        return current;
+    }
+
+    public long getTotal() {
+        return total;
+    }
+
+    public int getItemsRead() {
+        return itemsRead;
+    }
+
+    public boolean isDisposable() {
+        // Make upload status disposable if it hasn't been modified for the last
+        // 10 minutes, regardless of whether the upload has finished or not
+        return System.currentTimeMillis() - lastModified > 60000;
+    }
+
+    public String toString() {
+        StringBuffer buffer = new StringBuffer("{current: ").append(current)
+                .append(", total: ").append(total)
+                .append(", itemsRead: ").append(itemsRead)
+                .append(", error: ");
+        if (error == null) {
+            buffer.append("null");
+        } else {
+            buffer.append("\"");
+            buffer.append(error.replaceAll("\"", "\\\\\""));
+            buffer.append("\"");
+        }
+        return buffer.append("}").toString();
+    }
+
+}

src/helma/framework/core/Application.java

@@ -1549,6 +1549,7 @@ public final class Application implements Runnable {
                 while (it.hasNext()) {
                     Session session = (Session) it.next();
 
+                    session.pruneUploads();
                     if ((now - session.lastTouched()) > (sessionTimeout * 60000)) {
                         NodeHandle userhandle = session.userHandle;
 
@@ -1792,6 +1793,25 @@ public final class Application implements Runnable {
         return proto.getDbMapping();
     }
 
+    /**
+     * Return the current upload status.
+     * @param req the upload RequestTrans
+     * @return the current upload status.
+     */
+    public UploadStatus getUploadStatus(RequestTrans req) {
+        String uploadId = (String) req.get("upload_id");
+        if (uploadId == null)
+            return null;
+
+        String sessionId = req.getSession();
+        Session session = getSession(sessionId);
+        if (session == null)
+            return null;
+        return session.createUpload(uploadId);
+    }
+
+
+
     private synchronized void updateProperties() {
         // if so property file has been updated, re-read props.
         if (props.lastModified() > lastPropertyRead) {

src/helma/framework/core/Session.java

@@ -19,6 +19,7 @@ package helma.framework.core;
 import helma.objectmodel.*;
 import helma.objectmodel.db.*;
 import helma.framework.ResponseTrans;
+import helma.framework.UploadStatus;
 
 import java.io.*;
 import java.util.*;
@@ -51,6 +52,8 @@ public class Session implements Serializable {
     protected String message;
     protected StringBuffer debugBuffer;
 
+    protected HashMap uploads = null;
+
     /**
      * Creates a new Session object.
      *
@@ -322,4 +325,38 @@ public class Session implements Serializable {
     public void setDebugBuffer(StringBuffer buffer) {
         debugBuffer = buffer;
     }
+
+    protected UploadStatus createUpload(String uploadId) {
+        if (uploads == null) {
+            uploads = new HashMap();
+        }
+        UploadStatus status = new UploadStatus();
+        uploads.put(uploadId, status);
+        return status;
+    }
+
+    protected UploadStatus getUpload(String uploadId) {
+        UploadStatus status = null;
+        if (uploads == null) {
+            uploads = new HashMap();
+        } else {
+            status = (UploadStatus) uploads.get(uploadId);
+        }
+        if (status == null) {
+            status = new UploadStatus();
+            uploads.put(uploadId, status);
+        }
+        return status;
+    }
+
+    protected void pruneUploads() {
+        if (uploads == null || uploads.isEmpty())
+            return;
+        for (Iterator it = uploads.values().iterator(); it.hasNext();) {
+            UploadStatus status = (UploadStatus) it.next();
+            if (status.isDisposable()) {
+                it.remove();
+            }
+        }
+    }
 }

src/helma/framework/core/SessionBean.java

@@ -17,6 +17,8 @@
 package helma.framework.core;
 
 import helma.objectmodel.INode;
+import helma.framework.UploadStatus;
+
 import java.io.Serializable;
 import java.util.Date;
 
@@ -204,4 +206,13 @@ public class SessionBean implements Serializable {
         session.message = msg;
     }
 
+    /**
+     * Get an upload status for the current user session.
+     * @param uploadId the upload id
+     * @return the upload status
+     */
+    public UploadStatus getUploadStatus(String uploadId) {
+        return session.getUpload(uploadId);
+    }
+
 }

src/helma/servlet/AbstractServletClient.java

@@ -30,6 +30,8 @@ import javax.servlet.http.*;
 import org.apache.commons.fileupload.FileItem;
 import org.apache.commons.fileupload.FileUploadBase;
 import org.apache.commons.fileupload.disk.DiskFileItemFactory;
+import org.apache.commons.fileupload.FileUpload;
+import org.apache.commons.fileupload.ProgressListener;
 import org.apache.commons.fileupload.servlet.ServletFileUpload;
 import org.apache.commons.fileupload.servlet.ServletRequestContext;
 
@@ -146,76 +148,6 @@ public abstract class AbstractServletClient extends HttpServlet {
                 encoding = getApplication().getCharset();
             }
 
-            // read and set http parameters
-            Map parameters = parseParameters(request, encoding);
-
-            for (Iterator i = parameters.entrySet().iterator(); i.hasNext();) {
-                Map.Entry entry = (Map.Entry) i.next();
-                String key = (String) entry.getKey();
-                String[] values = (String[]) entry.getValue();
-
-                if ((values != null) && (values.length > 0)) {
-                    reqtrans.set(key, values[0]); // set to single string value
-
-                    if (values.length > 1) {
-                        reqtrans.set(key + "_array", values); // set string array
-                    }
-                }
-            }
-
-            try {
-                ServletRequestContext reqcx = new ServletRequestContext(request);
-                if (ServletFileUpload.isMultipartContent(reqcx)) {
-                    // File Upload
-                    ServletFileUpload upload = new ServletFileUpload(new DiskFileItemFactory());
-
-                    upload.setSizeMax(uploadLimit * 1024);
-                    upload.setHeaderEncoding(encoding);
-
-                    List uploads = upload.parseRequest(request);
-                    Iterator it = uploads.iterator();
-
-                    while (it.hasNext()) {
-                        FileItem item = (FileItem) it.next();
-                        String name = item.getFieldName();
-                        Object value = null;
-                        // check if this is an ordinary HTML form element or a file upload
-                        if (item.isFormField()) {
-                            value =  item.getString(encoding);
-                        } else {
-                            value = new MimePart(item.getName(),
-                                    item.get(),
-                                    item.getContentType());
-                        }
-                        item.delete();
-                        // if multiple values exist for this name, append to _array
-                        if (reqtrans.get(name) != null) {
-                            appendFormValue(reqtrans, name, value);
-                        } else {
-                            reqtrans.set(name, value);
-                        }
-                    }
-
-                }
-            } catch (Exception upx) {
-                log("Error in file upload", upx);
-                if (uploadSoftfail) {
-                    String msg = upx.getMessage();
-                    if (msg == null || msg.length() == 0) {
-                        msg = upx.toString();
-                    }
-                    reqtrans.set("helma_upload_error", msg);
-                } else if (upx instanceof FileUploadBase.SizeLimitExceededException) {
-                    sendError(response, HttpServletResponse.SC_REQUEST_ENTITY_TOO_LARGE,
-                            "File upload size exceeds limit of " + uploadLimit + "kB");
-                    return;
-                } else {
-                    sendError(response, HttpServletResponse.SC_INTERNAL_SERVER_ERROR,
-                            "Error in file upload: " + upx);
-                    return;
-                }
-            }
-
             // read cookies
             Cookie[] reqCookies = request.getCookies();
 
@@ -310,8 +242,109 @@ public abstract class AbstractServletClient extends HttpServlet {
                 reqtrans.set("authorization", authorization);
             }
 
+            // read and set http parameters
+            Map parameters = parseParameters(request, encoding);
+
+            if (parameters != null) {
+                for (Iterator i = parameters.entrySet().iterator(); i.hasNext();) {
+                    Map.Entry entry = (Map.Entry) i.next();
+                    String key = (String) entry.getKey();
+                    String[] values = (String[]) entry.getValue();
+
+                    if ((values != null) && (values.length > 0)) {
+                        // set to single string value
+                        reqtrans.set(key, values[0]);
+
+                        if (values.length > 1) {
+                            // set string array
+                            reqtrans.set(key + "_array", values);
+                        }
+                    }
+                }
+            }
+
+            List uploads = null;
+            ServletRequestContext reqcx = new ServletRequestContext(request);
+
+            if (ServletFileUpload.isMultipartContent(reqcx)) {
+                // get session for upload progress monitoring
+                final UploadStatus uploadStatus = getApplication().getUploadStatus(reqtrans);
+                try {
+                    // handle file upload
+                    DiskFileItemFactory factory = new DiskFileItemFactory();
+                    FileUpload upload = new FileUpload(factory);
+                    // use upload limit for individual file size, but also set a limit on overall size
+                    upload.setFileSizeMax(uploadLimit * 1024);
+                    upload.setSizeMax(uploadLimit * 1024 * 10);
+
+                    // register upload tracker with user's session
+                    if (uploadStatus != null) {
+                        upload.setProgressListener(new ProgressListener() {
+                            public void update(long bytesRead, long contentLength, int itemsRead) {
+                                uploadStatus.update(bytesRead, contentLength, itemsRead);
+                            }
+                        });
+                    }
+
+                    uploads = upload.parseRequest(reqcx);
+                    Iterator it = uploads.iterator();
+
+                    while (it.hasNext()) {
+                        FileItem item = (FileItem) it.next();
+                        String name = item.getFieldName();
+                        Object value;
+                        // check if this is an ordinary HTML form element or a file upload
+                        if (item.isFormField()) {
+                            value =  item.getString(encoding);
+                        } else {
+                            value = new MimePart(item);
+                        }
+                        // if multiple values exist for this name, append to _array
+                        if (reqtrans.get(name) != null) {
+                            appendFormValue(reqtrans, name, value);
+                        } else {
+                            reqtrans.set(name, value);
+                        }
+                    }
+
+                } catch (Exception upx) {
+                    log("Error in file upload", upx);
+                    String message;
+                    if (upx instanceof FileUploadBase.SizeLimitExceededException) {
+                        message = "File upload size exceeds limit of " + uploadLimit + " kB";
+                    } else {
+                        message = upx.getMessage();
+                        if (message == null || message.length() == 0) {
+                            message = upx.toString();
+                        }
+                    }
+                    if (uploadStatus != null) {
+                        uploadStatus.setError(message);
+                    }
+
+                    if (uploadSoftfail || uploadStatus != null) {
+                        reqtrans.set("helma_upload_error", message);
+                    } else if (upx instanceof FileUploadBase.SizeLimitExceededException) {
+                        sendError(response, HttpServletResponse.SC_REQUEST_ENTITY_TOO_LARGE,
+                                "Error in file upload: " + message);
+                        return;
+                    } else {
+                        sendError(response, HttpServletResponse.SC_INTERNAL_SERVER_ERROR,
+                                "Error in file upload: " + message);
+                        return;
+                    }
+                }
+            }
+
             ResponseTrans restrans = getApplication().execute(reqtrans);
 
+            // delete uploads if any
+            if (uploads != null) {
+                for (int i = 0; i < uploads.size(); i++) {
+                    ((FileItem) uploads.get(i)).delete();
+                }
+            }
+
             // if the response was already written and committed by the application
             // we can skip this part and return
             if (response.isCommitted()) {
@@ -511,7 +544,7 @@ public abstract class AbstractServletClient extends HttpServlet {
             byte buffer[] = new byte[bufferSize];
             int l;
     
-            while (length>0) {
+            while (length > 0) {
                 if (length < bufferSize)
                     l = in.read(buffer, 0, length);
                 else
@@ -683,46 +716,45 @@ public abstract class AbstractServletClient extends HttpServlet {
         map.put(name, newValues);
     }
 
-    protected Map parseParameters(HttpServletRequest request, String encoding) {
+    protected Map parseParameters(HttpServletRequest request, String encoding)
+            throws IOException {
+        // check if there are any parameters before we get started
+        String queryString = request.getQueryString();
+        String contentType = request.getContentType();
+        boolean isFormPost = "post".equals(request.getMethod().toLowerCase())
+                && contentType != null
+                && contentType.toLowerCase().startsWith("application/x-www-form-urlencoded");
+
+        if (queryString == null && !isFormPost) {
+            return null;
+        }
+
         HashMap parameters = new HashMap();
 
         // Parse any query string parameters from the request
-        String queryString = request.getQueryString();
         if (queryString != null) {
-            try {
+            parseParameters(parameters, queryString.getBytes(), encoding, false);
-                parseParameters(parameters, queryString.getBytes(), encoding);
-            } catch (Exception e) {
-                log("Error parsing query string", e);
-            }
         }
 
         // Parse any posted parameters in the input stream
-        String contentType = request.getContentType();
+        if (isFormPost) {
-        if ("post".equals(request.getMethod().toLowerCase()) 
+            int max = request.getContentLength();
-                && contentType != null
+            int len = 0;
-                && contentType.toLowerCase()
+            byte[] buf = new byte[max];
-                        .startsWith("application/x-www-form-urlencoded")) {
+            ServletInputStream is = request.getInputStream();
-            try {
-                int max = request.getContentLength();
-                int len = 0;
-                byte[] buf = new byte[max];
-                ServletInputStream is = request.getInputStream();
 
-                while (len < max) {
+            while (len < max) {
-                    int next = is.read(buf, len, max - len);
+                int next = is.read(buf, len, max - len);
 
-                    if (next < 0) {
+                if (next < 0) {
-                        break;
+                    break;
-                    }
-
-                    len += next;
                 }
 
-                // is.close();
+                len += next;
-                parseParameters(parameters, buf, encoding);
-            } catch (IOException e) {
-                log("Error reading POST body", e);
             }
+
+            // is.close();
+            parseParameters(parameters, buf, encoding, true);
         }
 
         return parameters;
@@ -747,7 +779,7 @@ public abstract class AbstractServletClient extends HttpServlet {
      *
      * @exception UnsupportedEncodingException if the data is malformed
      */
-    public static void parseParameters(Map map, byte[] data, String encoding)
+    public static void parseParameters(Map map, byte[] data, String encoding, boolean isPost)
                                 throws UnsupportedEncodingException {
         if ((data != null) && (data.length > 0)) {
             int ix = 0;
@@ -793,10 +825,18 @@ public abstract class AbstractServletClient extends HttpServlet {
                 }
             }
 
-            //The last value does not end in '&'.  So save it now.
             if (key != null) {
+                // The last value does not end in '&'.  So save it now.
                 value = new String(data, 0, ox, encoding);
                 putMapEntry(map, key, value);
+            } else if (ox > 0) {
+                // Store any residual bytes in req.data.http_post_remainder
+                value = new String(data, 0, ox, encoding);
+                if (isPost) {
+                    putMapEntry(map, "http_post_remainder", value);
+                } else {
+                    putMapEntry(map, "http_get_remainder", value);
+                }
             }
         }
     }