//
// Jala Project [http://opensvn.csie.org/traccgi/jala]
//
// Copyright 2004 ORF Online und Teletext GmbH
//
// Licensed under the Apache License, Version 2.0 (the ``License'');
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an ``AS IS'' BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// $Revision$
// $LastChangedBy$
// $LastChangedDate$
// $HeadURL$
//


/**
 * @fileoverview Fields and methods of the jala.IndexManager class.
 */


// Define the global namespace for Jala modules
if (!global.jala) {
   global.jala = {};
}


/**
 * HelmaLib dependencies
 */
app.addRepository("modules/helma/Search.js");
app.addRepository("modules/helma/File.js");

/**
 * Constructs a new IndexManager object.
 * @class This class basically sits on top of a helma.Search.Index instance
 * and provides methods for adding, removing and optimizing the underlying index.
 * All methods generate jobs that are put into an internal queue which is
 * processed asynchronously by a separate worker thread. This means all calls
 * to add(), remove() and optimize() will return immediately, but the changes to
 * the index will be done within a short delay. Please keep in mind to change the
 * status of this IndexManager instance to REBUILDING before starting to rebuild
 * the index, as this ensures that all add/remove/optimize jobs will stay in the
 * queue and will only be processed after switching the status back to NORMAL.
 * This ensures that objects that have been modified during a rebuilding process
 * are re-indexed properly afterwards.
 * @param {String} name The name of the index, which is the name of the directory
 * the index already resides or will be created in.
 * @param {helma.File} dir The base directory where this index's directory
 * is already existing or will be created in. If not specified a RAM directory
 * is used.
 * @param {String} lang The language of the documents in this index. This leads
 * to the proper Lucene analyzer being used for indexing documents.
 * @constructor
 * @see helma.Search.createIndex
 */
jala.IndexManager = function IndexManager(name, dir, lang) {

   /**
    * Private variable containing the worker thread
    * @private
    */
   var thread = null;

   /**
    * Private flag indicating that the worker thread should stop
    * @type Boolean
    * @private
    */
   var interrupted = false;

   /**
    * Private variable containing the index managed by
    * this IndexManager instance.
    * @private 
    */
   var index = null;

   /**
    * Private variable containing a status indicator.
    * @type Number
    * @private
    */
   var status = jala.IndexManager.NORMAL;

   /**
    * Synchronized linked list that functions as a queue for
    * asynchronous processing of index manipulation jobs.
    * @type java.util.LinkedList
    * @private
    * @see jala.IndexManager.Job
    */
   var queue = java.util.Collections.synchronizedList(new java.util.LinkedList());

   /**
    * The name of the unique identifier field in the index. Defaults to "id".
    * @type String
    * @private
    */
   var idFieldname = "id";

   /**
    * The index directory
    * @type Packages.org.apache.lucene.store.Directory
    * @private
    */
   var indexDirectory = null;
   
   /**
    * The searcher utilized by {@link #search}
    * @type jala.IndexManager.Searcher
    * @private
    */
   var searcher = null;

   /**
    * Returns the directory of the underlying index
    * @returns The directory of the underlying index
    * @type Packages.org.apache.lucene.store.Directory
    */
   this.getDirectory = function() {
      return indexDirectory;
   };

   /**
    * Returns the underlying index.
    * @returns The index this queue is working on.
    * @type helma.Search.Index
    */
   this.getIndex = function() {
      return index;
   };

   /**
    * Returns the status of this manager.
    * @returns The status of this index manager.
    * @type Number
    * @see #NORMAL
    * @see #REBUILDING
    */
   this.getStatus = function() {
      return status;
   };

   /**
    * Modifies the status of this manager, which has implications
    * on how index modifying jobs are handled. If the status
    * is {@link #REBUILDING}, all jobs are queued until the status
    * is set back to {@link #NORMAL}.
    * @param {Number} s The new status of this manager.
    * @see #NORMAL
    * @see #REBUILDING
    */
   this.setStatus = function(s) {
      status = s;
      return;
   };

   /**
    * Returns the queue this index manager is using.
    * @returns The queue.
    * @type java.util.LinkedList
    */
   this.getQueue = function() {
      return queue;
   };

   /**
    * Returns the name of the index manger, which
    * is equal to the name of the underlying index
    * @returns The name of the index manager
    * @type String
    */
   this.getName = function() {
      return name;
   };

   /**
    * Returns the name of the field containing the unique identifier
    * of document objects in the index wrapped by this IndexManager.
    * Defaults to "id".
    * @returns The name of the id field in the index
    * @type String
    * @see #setIdFieldname 
    */
   this.getIdFieldname = function() {
      return idFieldname;
   };

   /**
    * Sets the name of the field containing the unique identifier
    * of document objects in the index wrapped by this IndexManager.
    * @see #getIdFieldname 
    */
   this.setIdFieldname = function(name) {
      idFieldname = name;
      return;
   };

   /**
    * Returns true if the underlying index is currently optimized.
    * @returns True in case the index is optimized, false otherwise.
    * @type Boolean
    */
   this.hasOptimizingJob = function() {
      for (var i=0; i<queue.size(); i++) {
         if (queue.get(i).type == jala.IndexManager.Job.OPTIMIZE) {
            return true;
         }
      }
      return false;
   };

   /**
    * Returns true if the underlying index is currently rebuilding.
    * @returns True in case the index is rebuilding, false otherwise.
    * @type Boolean
    */
   this.isRebuilding = function() {
      return status == jala.IndexManager.REBUILDING;
   };

   /**
    * Starts the IndexManager worker thread that processes the job queue
    */
   this.start = function() {
      if (!this.isRunning()) {
         interrupted = false;
         thread = app.invokeAsync(this, function() {
            while (interrupted === false) {
               if (this.getStatus() != jala.IndexManager.REBUILDING && !queue.isEmpty()) {
                  var job = queue.remove(0);
                  if (this.processJob(job) === false) {
                     // processing job failed, check if we should re-add
                     if (job.errors < jala.IndexManager.MAXTRIES) {
                        // increment error counter and put back into queue
                        job.errors += 1;
                        queue.add(job);
                     } else {
                        this.log("error", "error during queue flush: tried " +
                                 jala.IndexManager.MAXTRIES + " times to handle " +
                                 job.type + " job " + ", giving up.");
                     }
                  }
                  this.log("debug", "remaining jobs " + queue.size());
                  // if no more jobs are waiting, optimize the index and re-open
                  // the index searcher to make changes visible
                  if (queue.isEmpty()) {
                     var start = java.lang.System.currentTimeMillis();
                     try {
                        this.getIndex().optimize();
                        this.log("optimized index in " + jala.IndexManager.getRuntime(start) + " ms");
                        this.initSearcher();
                     } catch (e) {
                        this.log("error", "Unable to optimize index or re-open searcher, reason: " + e.toString());
                     }
                  }
               } else {
                  // wait for 100ms before checking again
                  java.lang.Thread.sleep(100);
               }
            }
            return true;
         }, [], -1);
         this.log("started successfully");
      } else {
         this.log("already running");
      }
      return;
   };
   
   /**
    * Stops this IndexManager instance. This function waits for 10 seconds
    * maximum for the worker thread to stop.
    * @returns True if the worker thread stopped successfully, false otherwise
    * @type Boolean
    */
   this.stop = function() {
      interrupted = true;
      var result;
      if ((result = this.isRunning()) === true) {
         if ((result = thread.waitForResult(10000)) === true) {
            thread = null;
            this.log("stopped successfully");
         } else {
            result = false;
            this.log("error", "unable to stop");
         }
      } else {
         this.log("info", "already stopped");
      }
      return result;
   };

   /**
    * Returns true if this IndexManager instance is running
    * @returns True if this IndexManager instance is running, false otherwise.
    * @type Boolean
    */
   this.isRunning = function() {
      if (thread != null) {
         return thread.running;
      }
      return false;
   };

   /**
    * Read only reference containing the running status of this IndexManager
    * @type Boolean
    */
   this.running; // for api documentation only, is overwritten by getter below
   this.__defineGetter__("running", function() {
       return this.isRunning()
   });

   /**
    * Read only reference containing the number of pending jobs
    * @type Number
    */
   this.pending; // for api documentation only, is overwritten by getter below
   this.__defineGetter__("pending", function() {
       return queue.size()
   });

   /**
    * Initializes the searcher
    * @private
    */
   this.initSearcher = function() {
      searcher = new Packages.org.apache.lucene.search.IndexSearcher(indexDirectory);
      return;
   };

   /**
    * Returns the searcher of this index manager
    * @returns The searcher of this index manager
    * @type org.apache.lucene.search.IndexSearcher
    * @private
    */
   this.getSearcher = function() {
      if (searcher === null) {
         this.initSearcher();
      }
      return searcher;
   };
   
   /**
    * Main constructor body. Initializes the underlying index.
    */
   var search = new helma.Search();
   var analyzer = helma.Search.getAnalyzer(lang);
   if (dir != null) {
      indexDirectory = search.getDirectory(new helma.File(dir, name));
      this.log("created/mounted " + indexDirectory);
   } else {
      indexDirectory = search.getRAMDirectory();
      this.log("created new RAM directory");
   }
   index = search.createIndex(indexDirectory, analyzer);

   return this;
};

/**
 * Constant defining the maximum number of tries to add/remove
 * an object to/from the underlying index.
 * @type Number
 * @final
 */
jala.IndexManager.MAXTRIES = 10;

/**
 * Constant defining normal mode of this index manager.
 * @type Number
 * @final
 */
jala.IndexManager.NORMAL = 1;

/**
 * Constant defining rebuilding mode of this index manager.
 * @type Number
 * @final
 */
jala.IndexManager.REBUILDING = 2;

/**
 * Returns the milliseconds elapsed between the current timestamp
 * and the one passed as argument.
 * @returns The elapsed time in millis.
 * @type Number
 * @private
 */
jala.IndexManager.getRuntime = function(millis) {
   return java.lang.System.currentTimeMillis() - millis;
};

/** @ignore */
jala.IndexManager.prototype.toString = function() {
   return "[" + this.constructor.name + " '" + this.getName() + "' (" +
          this.pending + " objects queued)]";
};

/**
 * Helper function that prefixes every log message with
 * the name of the IndexManager.
 * @param {String} level An optional logging level. Accepted values
 * @param {String} msg The log message
 * are "debug", "info", "warn" and "error".
 */
jala.IndexManager.prototype.log = function(/* msg, level */) {
   var level = "info", message;
   if (arguments.length == 2) {
      level = arguments[0];
      message = arguments[1];
   } else {
      message = arguments[0];
   }
   app.logger[level]("[" + this.constructor.name + " '" +
                     this.getName() + "'] " + message);
   return;
};

/**
 * Static helper method that returns the value of the "id"
 * field of a document object.
 * @param {helma.Search.Document} doc The document whose id
 * should be returned.
 * @private
 */
jala.IndexManager.prototype.getDocumentId = function(doc) {
   try {
      return doc.getField(this.getIdFieldname()).value;
   } catch (e) {
      // ignore
   }
   return null;
};

/**
 * Queues the document object passed as argument for addition to the underlying
 * index. This includes that all existing documents with the same identifier will
 * be removed before the object passed as argument is added.
 * @param {helma.Search.Document} doc The document object that should be
 * added to the underlying index.
 * @returns True if the job was added successfully to the internal queue,
 * false otherwise.
 * @type Boolean
 * @see helma.Search.Document
 */
jala.IndexManager.prototype.add = function(doc) {
   var id;
   if (!doc) {
      this.log("error", "missing document object to add");
      return false;
   } else if ((id = this.getDocumentId(doc)) == null) {
      this.log("error", "document doesn't contain an Id field '" +
               this.getIdFieldname() + "'");
      return false;
   }
   // the job's callback function which actually adds the document to the index
   var callback = function() {
      var start = java.lang.System.currentTimeMillis();
      this.getIndex().updateDocument(doc, this.getIdFieldname());
      this.log("debug", "added document with Id " + id +
               " to index in " + jala.IndexManager.getRuntime(start) + " ms");
      return;
   }
   var job = new jala.IndexManager.Job(jala.IndexManager.Job.ADD, callback);
   this.getQueue().add(job);
   this.log("debug", "queued adding document " + id + " to index");
   return true;
};

/**
 * Queues the removal of all index documents whose identifier value ("id" by default)
 * matches the number passed as argument.
 * @param {Number} id The identifier value
 * @returns True if the removal job was added successfully to the queue, false
 * otherwise.
 * @type Boolean
 */
jala.IndexManager.prototype.remove = function(id) {
   if (id === null || isNaN(id)) {
      this.log("error", "missing or invalid document id to remove");
      return false;
   }
   // the job's callback function which actually removes all documents
   // with the given id from the index
   var callback = function() {
      var start = java.lang.System.currentTimeMillis();
      this.getIndex().removeDocument(this.getIdFieldname(), parseInt(id, 10));
      this.log("debug", "removed document with Id " + id +
               " from index in " + jala.IndexManager.getRuntime(start) + " ms");
   };
   var job = new jala.IndexManager.Job(jala.IndexManager.Job.REMOVE, callback);
   this.getQueue().add(job);
   this.log("debug", "queued removal of document with Id " + id);
   return true;
};

/**
 * Queues the optimization of the underlying index. Normally there is no need
 * to call this method explicitly, as the index will be optimized after all
 * queued jobs have been processed.
 * @returns True if the optimizing job was added, false otherwise, which means
 * that there is already an optimizing job waiting in the queue.
 * @type Boolean
 */
jala.IndexManager.prototype.optimize = function() {
   if (this.hasOptimizingJob()) {
      return false;
   }
   var callback = function() {
      var start = java.lang.System.currentTimeMillis();
      this.getIndex().optimize();
      this.log("optimized index in " + jala.IndexManager.getRuntime(start) + " ms");
      // re-open index searcher, so that changes are seen
      this.initSearcher();
      return;
   };
   var job = new jala.IndexManager.Job(jala.IndexManager.Job.OPTIMIZE, callback);
   this.getQueue().add(job);
   this.log("debug", "queued index optimization");
   return true;
};

/**
 * Processes a single queued job
 * @param {Object} job
 * @private
 */
jala.IndexManager.prototype.processJob = function(job) {
   this.log("debug", job.type + " job has been in queue for " +
            jala.IndexManager.getRuntime(job.createtime.getTime()) +
            " ms, processing now...");
   try {
      job.callback.call(this);
   } catch (e) {
      this.log("error", "Exception while processing job " + job.type + ": " + e);
      return false;
   }
   return true;
};

/**
 * Searches the underlying index using the searcher of this index manager
 * @param {helma.Search.Query|org.apache.lucene.search.Query} query The query
 * to execute. Can be either an instance of helma.Search.Query, or an instance
 * of org.apache.lucene.search.Query
 * @param {helma.Search.QueryFilter|org.apache.lucene.search.Filter} filter
 * An optional query filter
 * @param {Array} sortFields An optional array containing
 * org.apache.lucene.search.SortField instances to use for sorting the result
 * @returns A HitCollection containing the search results
 * @type helma.Search.HitCollection
 */
jala.IndexManager.prototype.search = function(query, filter, sortFields) {
   var pkg = Packages.org.apache.lucene;
   if (query == null || (!(query instanceof helma.Search.Query) &&
            !(query instanceof pkg.search.Query))) {
      throw "jala.IndexManager search(): missing or invalid query";
   } else if (query instanceof helma.Search.Query) {
      // unwrap query
      query = query.getQuery();
   }
   if (filter != null && filter instanceof helma.Search.QueryFilter) {
      // unwrap filter
      filter = filter.getFilter();
   }

   var searcher = this.getSearcher();
   var analyzer = this.getIndex().getAnalyzer();
   var hits;
   if (sortFields != null && sortFields.length > 0) {
      // convert the array with sortfields to a java array
      var arr = java.lang.reflect.Array.newInstance(pkg.search.SortField, sortFields.length);
      sortFields.forEach(function(sortField, idx) {
         arr[idx] = sortField;
      });
      var sort = pkg.search.Sort(arr);
      if (filter) {
         hits = searcher.search(query, filter, sort);
      } else {
         hits = searcher.search(query, sort);
      }
   } else if (filter) {
      hits = searcher.search(query, filter);
   } else {
      hits = searcher.search(query);
   }
   this.log("debug", "Query: " + query.toString());
   return new helma.Search.HitCollection(hits);
};

/**
 * Parses the query string passed as argument into a lucene Query instance
 * @param {String} queryStr The query string to parse
 * @param {Array} fields An array containing the names of the files to search in
 * @param {Object} boostMap An optional object containing properties whose name denotes
 * the name of the field to boost in the query, and the value the boost value.
 * @returns The query
 * @type org.apache.lucene.search.Query
 */
jala.IndexManager.prototype.parseQuery = function(queryStr, fields, boostMap) {
   if (queryStr == null || typeof(queryStr) !== "string") {
      throw "IndexManager.parseQuery(): missing or invalid query string";
   }
   if (fields == null || fields.constructor !== Array || fields.length < 1) {
      throw "IndexManager.parseQuery(): missing fields argument";
   }
   var query = null;
   var analyzer = this.getIndex().getAnalyzer();
   var pkg = Packages.org.apache.lucene;
   var map = null;
   if (boostMap != null) {
      // convert the javascript object into a HashMap
      map = new java.util.HashMap();
      for (var name in boostMap) {
         map.put(name, new java.lang.Float(boostMap[name]));
      }
   }
   var parser;
   try {
      if (fields.length > 1) {
         parser = new pkg.queryParser.MultiFieldQueryParser(fields, analyzer, map);
      } else {
         parser = new pkg.queryParser.QueryParser(fields, analyzer);
      }
      query = parser.parse(queryStr);
   } catch (e) {
      // ignore, but write a message to debug log
      app.logger.debug("Unable to construct search query '" + queryStr +
                       "', reason: " + e);
   }
   return query;
};

/**
 * Parses the query passed as argument and returns a caching filter. If an array
 * with more than one query strings is passed as argument, this method constructs
 * a boolean query filter where all queries in the array must match.
 * @param {String|Array} query Either a query string, or an array containing
 * one or more query strings
 * @param {org.apache.lucene.analysis.Analyzer} analyzer Optional analyzer
 * to use when parsing the filter query
 * @returns A caching query filter
 * @type org.apache.lucene.search.CachingWrapperFilter
 */
jala.IndexManager.prototype.parseQueryFilter = function(query, analyzer) {
   var filter = null;
   if (query != null) {
      var pkg = Packages.org.apache.lucene;
      // use the index' analyzer if none has been specified
      if (analyzer == null) {
         analyzer = this.getIndex().getAnalyzer();
      }
      var parser = new pkg.queryParser.QueryParser("", analyzer);
      var filterQuery;
      try {
         if (query.constructor === Array) {
            if (query.length > 1) {
               filterQuery = new pkg.search.BooleanQuery();
               query.forEach(function(queryStr){
                  filterQuery.add(parser.parse(queryStr), pkg.search.BooleanClause.Occur.MUST);
               }, this);
            } else {
               filterQuery = parser.parse(query[0]);
            }
         } else {
            filterQuery = parser.parse(query);
         }
         filter = new pkg.search.CachingWrapperFilter(new pkg.search.QueryWrapperFilter(filterQuery));
      } catch (e) {
         app.logger.debug("Unable to parse query filter '" + query + "', reason: " + e);
      }
   }
   return filter;
};



/*********************
 *****   J O B   *****
 *********************/


/**
 * Creates a new Job instance.
 * @class Instances of this class represent a single index
 * manipulation job to be processed by the index manager.
 * @param {Number} id The Id of the job
 * @param {Number} type The type of job, which can be either
 * jala.IndexManager.Job.ADD, jala.IndexManager.Job.REMOVE
 * or jala.IndexManager.Job.OPTIMIZE.
 * @param {Object} data The data needed to process the job.
 * @returns A newly created Job instance.
 * @constructor
 * @see jala.IndexManager.Job
 */
jala.IndexManager.Job = function(type, callback) {
   /**
    * The type of the job
    * @type Number
    */
   this.type = type;

   /**
    * The data needed to process this job. For adding jobs this property
    * must contain the {@link helma.Search.Document} instance to add to
    * the index. For removal job this property must contain the unique identifier
    * of the document that should be removed from the index. For optimizing
    * jobs this property is null.
    */
   this.callback = callback;

   /**
    * An internal error counter which is increased whenever processing
    * the job failed.
    * @type Number
    * @see jala.IndexManager.MAXTRIES
    */
   this.errors = 0;
   
   /**
    * The date and time at which this job was created.
    * @type Date
    */
   this.createtime = new Date();

   return this;
};

/** @ignore */
jala.IndexManager.Job.prototype.toString = function() {
   return "[Job (type: " + this.type + ")]";
};

/**
 * Constant defining an add job
 * @type Number
 * @final
 */
jala.IndexManager.Job.ADD = "add";

/**
 * Constant defining a removal job
 * @type Number
 * @final
 */
jala.IndexManager.Job.REMOVE = "remove";

/**
 * Constant defining an optimizing job
 * @type Number
 * @final
 */
jala.IndexManager.Job.OPTIMIZE = "optimize";