# Source: jala/code/IndexManager.js

    //
    // Jala Project [http://opensvn.csie.org/traccgi/jala]
    //
    // Copyright 2004 ORF Online und Teletext GmbH
    //
    // Licensed under the Apache License, Version 2.0 (the ``License'');
    // you may not use this file except in compliance with the License.
    // You may obtain a copy of the License at
    //
    //    http://www.apache.org/licenses/LICENSE-2.0
    //
    // Unless required by applicable law or agreed to in writing, software
    // distributed under the License is distributed on an ``AS IS'' BASIS,
    // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    // See the License for the specific language governing permissions and
    // limitations under the License.
    //
    // $Revision$
    // $LastChangedBy$
    // $LastChangedDate$
    // $HeadURL$
    //

    /**
     * @fileoverview Fields and methods of the jala.IndexManager class.
     */

    // Define the global namespace for Jala modules
    if (!global.jala) {
       global.jala = {};
    }

    /**
     * HelmaLib dependencies
     */
    app.addRepository("modules/helma/Search.js");
    app.addRepository("modules/helma/File.js");

    /**
     * Constructs a new IndexManager object.
     * @class This class basically sits on top of a helma.Search.Index instance
     * and provides methods for adding, removing and optimizing the underlying index.
     * All methods generate jobs that are put into an internal queue which is
     * processed asynchronously by a separate worker thread. This means all calls
     * to add(), remove() and optimize() will return immediately, but the changes to
     * the index will be done within a short delay. Please keep in mind to change the
     * status of this IndexManager instance to REBUILDING before starting to rebuild
     * the index, as this ensures that all add/remove/optimize jobs will stay in the
     * queue and will only be processed after switching the status back to NORMAL.
     * This ensures that objects that have been modified during a rebuilding process
     * are re-indexed properly afterwards.
     * @param {String} name The name of the index, which is the name of the directory
     * the index already resides or will be created in.
     * @param {helma.File} dir The base directory where this index's directory
     * is already existing or will be created in. If not specified a RAM directory
     * is used.
     * @param {String} lang The language of the documents in this index. This leads
     * to the proper Lucene analyzer being used for indexing documents.
     * @constructor
     * @see helma.Search.createIndex
     */
    jala.IndexManager = function IndexManager(name, dir, lang) {

       /**
        * Private variable containing the worker thread
        * @private
        */
       var thread = null;

       /**
        * Private flag indicating that the worker thread should stop
        * @type Boolean
        * @private
        */
       var interrupted = false;

       /**
        * Private variable containing the index managed by
        * this IndexManager instance.
        * @private
        */
       var index = null;

       /**
        * Private variable containing a status indicator.
        * @type Number
        * @private
        */
       var status = jala.IndexManager.NORMAL;

       /**
        * Synchronized linked list that functions as a queue for
        * asynchronous processing of index manipulation jobs.
        * @type java.util.LinkedList
        * @private
        * @see jala.IndexManager.Job
        */
       var queue = java.util.Collections.synchronizedList(new java.util.LinkedList());

       /**
        * The name of the unique identifier field in the index. Defaults to "id".
        * @type String
        * @private
        */
       var idFieldname = "id";

       /**
        * The index directory
        * @type Packages.org.apache.lucene.store.Directory
        * @private
        */
       var indexDirectory = null;

       /**
        * The searcher utilized by {@link #search}
        * @type jala.IndexManager.Searcher
        * @private
        */
       var searcher = null;

       /**
        * Returns the directory of the underlying index
        * @returns The directory of the underlying index
        * @type Packages.org.apache.lucene.store.Directory
        */
       this.getDirectory = function() {
          return indexDirectory;
       };

       /**
        * Returns the underlying index.
        * @returns The index this queue is working on.
        * @type helma.Search.Index
        */
       this.getIndex = function() {
          return index;
       };

       /**
        * Returns the status of this manager.
        * @returns The status of this index manager.
        * @type Number
        * @see #NORMAL
        * @see #REBUILDING
        */
       this.getStatus = function() {
          return status;
       };

       /**
        * Modifies the status of this manager, which has implications
        * on how index modifying jobs are handled. If the status
        * is {@link #REBUILDING}, all jobs are queued until the status
        * is set back to {@link #NORMAL}.
        * @param {Number} s The new status of this manager.
        * @see #NORMAL
        * @see #REBUILDING
        */
       this.setStatus = function(s) {
          status = s;
          return;
       };

       /**
        * Returns the queue this index manager is using.
        * @returns The queue.
        * @type java.util.LinkedList
        */
       this.getQueue = function() {
          return queue;
       };

       /**
        * Returns the name of the index manger, which
        * is equal to the name of the underlying index
        * @returns The name of the index manager
        * @type String
        */
       this.getName = function() {
          return name;
       };

       /**
        * Returns the name of the field containing the unique identifier
        * of document objects in the index wrapped by this IndexManager.
        * Defaults to "id".
        * @returns The name of the id field in the index
        * @type String
        * @see #setIdFieldname
        */
       this.getIdFieldname = function() {
          return idFieldname;
       };

       /**
        * Sets the name of the field containing the unique identifier
        * of document objects in the index wrapped by this IndexManager.
        * @see #getIdFieldname
        */
       this.setIdFieldname = function(name) {
          idFieldname = name;
          return;
       };

       /**
        * Returns true if the underlying index is currently optimized.
        * @returns True in case the index is optimized, false otherwise.
        * @type Boolean
        */
       this.hasOptimizingJob = function() {
          for (var i=0; i<queue.size(); i++) {
             if (queue.get(i).type == jala.IndexManager.Job.OPTIMIZE) {
                return true;
             }
          }
          return false;
       };

       /**
        * Returns true if the underlying index is currently rebuilding.
        * @returns True in case the index is rebuilding, false otherwise.
        * @type Boolean
        */
       this.isRebuilding = function() {
          return status == jala.IndexManager.REBUILDING;
       };

       /**
        * Starts the IndexManager worker thread that processes the job queue
        */
       this.start = function() {
          if (!this.isRunning()) {
             interrupted = false;
             thread = app.invokeAsync(this, function() {
                while (interrupted === false) {
                   if (this.getStatus() != jala.IndexManager.REBUILDING && !queue.isEmpty()) {
                      var job = queue.remove(0);
                      if (this.processJob(job) === false) {
                         // processing job failed, check if we should re-add
                         if (job.errors < jala.IndexManager.MAXTRIES) {
                            // increment error counter and put back into queue
                            job.errors += 1;
                            queue.add(job);
                         } else {
                            this.log("error", "error during queue flush: tried " +
                                     jala.IndexManager.MAXTRIES + " times to handle " +
                                     job.type + " job " + ", giving up.");
                         }
                      }
                      this.log("debug", "remaining jobs " + queue.size());
                      // if no more jobs are waiting, optimize the index and re-open
                      // the index searcher to make changes visible
                      if (queue.isEmpty()) {
                         var start = java.lang.System.currentTimeMillis();
                         try {
                            this.getIndex().optimize();
                            this.log("optimized index in " + jala.IndexManager.getRuntime(start) + " ms");
                            this.initSearcher();
                         } catch (e) {
                            this.log("error", "Unable to optimize index or re-open searcher, reason: " + e.toString());
                         }
                      }
                   } else {
                      // wait for 100ms before checking again
                      java.lang.Thread.sleep(100);
                   }
                }
                return true;
             }, [], -1);
             this.log("started successfully");
          } else {
             this.log("already running");
          }
          return;
       };

       /**
        * Stops this IndexManager instance. This function waits for 10 seconds
        * maximum for the worker thread to stop.
        * @returns True if the worker thread stopped successfully, false otherwise
        * @type Boolean
        */
       this.stop = function() {
          interrupted = true;
          var result;
          if ((result = this.isRunning()) === true) {
             if ((result = thread.waitForResult(10000)) === true) {
                thread = null;
                this.log("stopped successfully");
             } else {
                result = false;
                this.log("error", "unable to stop");
             }
          } else {
             this.log("info", "already stopped");
          }
          return result;
       };

       /**
        * Returns true if this IndexManager instance is running
        * @returns True if this IndexManager instance is running, false otherwise.
        * @type Boolean
        */
       this.isRunning = function() {
          if (thread != null) {
             return thread.running;
          }
          return false;
       };

       /**
        * Read only reference containing the running status of this IndexManager
        * @type Boolean
        */
       this.running; // for api documentation only, is overwritten by getter below
       this.__defineGetter__("running", function() {
           return this.isRunning()
       });

       /**
        * Read only reference containing the number of pending jobs
        * @type Number
        */
       this.pending; // for api documentation only, is overwritten by getter below
       this.__defineGetter__("pending", function() {
           return queue.size()
       });

       /**
        * Initializes the searcher
        * @private
        */
       this.initSearcher = function() {
          searcher = new Packages.org.apache.lucene.search.IndexSearcher(indexDirectory);
          return;
       };

       /**
        * Returns the searcher of this index manager
        * @returns The searcher of this index manager
        * @type org.apache.lucene.search.IndexSearcher
        * @private
        */
       this.getSearcher = function() {
          if (searcher === null) {
             this.initSearcher();
          }
          return searcher;
       };

       /**
        * Main constructor body. Initializes the underlying index.
        */
       var search = new helma.Search();
       var analyzer = helma.Search.getAnalyzer(lang);
       if (dir != null) {
          indexDirectory = search.getDirectory(new helma.File(dir, name));
          this.log("created/mounted " + indexDirectory);
       } else {
          indexDirectory = search.getRAMDirectory();
          this.log("created new RAM directory");
       }
       index = search.createIndex(indexDirectory, analyzer);

       return this;
    };

    /**
     * Constant defining the maximum number of tries to add/remove
     * an object to/from the underlying index.
     * @type Number
     * @final
     */
    jala.IndexManager.MAXTRIES = 10;

    /**
     * Constant defining normal mode of this index manager.
     * @type Number
     * @final
     */
    jala.IndexManager.NORMAL = 1;

    /**
     * Constant defining rebuilding mode of this index manager.
     * @type Number
     * @final
     */
    jala.IndexManager.REBUILDING = 2;

    /**
     * Returns the milliseconds elapsed between the current timestamp
     * and the one passed as argument.
     * @returns The elapsed time in millis.
     * @type Number
     * @private
     */
    jala.IndexManager.getRuntime = function(millis) {
       return java.lang.System.currentTimeMillis() - millis;
    };

    /** @ignore */
    jala.IndexManager.prototype.toString = function() {
       return "[" + this.constructor.name + " '" + this.getName() + "' (" +
              this.pending + " objects queued)]";
    };

    /**
     * Helper function that prefixes every log message with
     * the name of the IndexManager.
     * @param {String} level An optional logging level. Accepted values
     * @param {String} msg The log message
     * are "debug", "info", "warn" and "error".
     */
    jala.IndexManager.prototype.log = function(/* msg, level */) {
       var level = "info", message;
       if (arguments.length == 2) {
          level = arguments[0];
          message = arguments[1];
       } else {
          message = arguments[0];
       }
       app.logger[level]("[" + this.constructor.name + " '" +
                         this.getName() + "'] " + message);
       return;
    };

    /**
     * Static helper method that returns the value of the "id"
     * field of a document object.
     * @param {helma.Search.Document} doc The document whose id
     * should be returned.
     * @private
     */
    jala.IndexManager.prototype.getDocumentId = function(doc) {
       try {
          return doc.getField(this.getIdFieldname()).value;
       } catch (e) {
          // ignore
       }
       return null;
    };

    /**
     * Queues the document object passed as argument for addition to the underlying
     * index. This includes that all existing documents with the same identifier will
     * be removed before the object passed as argument is added.
     * @param {helma.Search.Document} doc The document object that should be
     * added to the underlying index.
     * @returns True if the job was added successfully to the internal queue,
     * false otherwise.
     * @type Boolean
     * @see helma.Search.Document
     */
    jala.IndexManager.prototype.add = function(doc) {
       var id;
       if (!doc) {
          this.log("error", "missing document object to add");
          return false;
       } else if ((id = this.getDocumentId(doc)) == null) {
          this.log("error", "document doesn't contain an Id field '" +
                   this.getIdFieldname() + "'");
          return false;
       }
       // the job's callback function which actually adds the document to the index
       var callback = function() {
          var start = java.lang.System.currentTimeMillis();
          this.getIndex().updateDocument(doc, this.getIdFieldname());
          this.log("debug", "added document with Id " + id +
                   " to index in " + jala.IndexManager.getRuntime(start) + " ms");
          return;
       }
       var job = new jala.IndexManager.Job(jala.IndexManager.Job.ADD, callback);
       this.getQueue().add(job);
       this.log("debug", "queued adding document " + id + " to index");
       return true;
    };

    /**
     * Queues the removal of all index documents whose identifier value ("id" by default)
     * matches the number passed as argument.
     * @param {Number} id The identifier value
     * @returns True if the removal job was added successfully to the queue, false
     * otherwise.
     * @type Boolean
     */
    jala.IndexManager.prototype.remove = function(id) {
       if (id === null || isNaN(id)) {
          this.log("error", "missing or invalid document id to remove");
          return false;
       }
       // the job's callback function which actually removes all documents
       // with the given id from the index
       var callback = function() {
          var start = java.lang.System.currentTimeMillis();
          this.getIndex().removeDocument(this.getIdFieldname(), parseInt(id, 10));
          this.log("debug", "removed document with Id " + id +
                   " from index in " + jala.IndexManager.getRuntime(start) + " ms");
       };
       var job = new jala.IndexManager.Job(jala.IndexManager.Job.REMOVE, callback);
       this.getQueue().add(job);
       this.log("debug", "queued removal of document with Id " + id);
       return true;
    };

    /**
     * Queues the optimization of the underlying index. Normally there is no need
     * to call this method explicitly, as the index will be optimized after all
     * queued jobs have been processed.
     * @returns True if the optimizing job was added, false otherwise, which means
     * that there is already an optimizing job waiting in the queue.
     * @type Boolean
     */
    jala.IndexManager.prototype.optimize = function() {
       if (this.hasOptimizingJob()) {
          return false;
       }
       var callback = function() {
          var start = java.lang.System.currentTimeMillis();
          this.getIndex().optimize();
          this.log("optimized index in " + jala.IndexManager.getRuntime(start) + " ms");
          // re-open index searcher, so that changes are seen
          this.initSearcher();
          return;
       };
       var job = new jala.IndexManager.Job(jala.IndexManager.Job.OPTIMIZE, callback);
       this.getQueue().add(job);
       this.log("debug", "queued index optimization");
       return true;
    };

    /**
     * Processes a single queued job
     * @param {Object} job
     * @private
     */
    jala.IndexManager.prototype.processJob = function(job) {
       this.log("debug", job.type + " job has been in queue for " +
                jala.IndexManager.getRuntime(job.createtime.getTime()) +
                " ms, processing now...");
       try {
          job.callback.call(this);
       } catch (e) {
          this.log("error", "Exception while processing job " + job.type + ": " + e);
          return false;
       }
       return true;
    };

    /**
     * Searches the underlying index using the searcher of this index manager
     * @param {helma.Search.Query|org.apache.lucene.search.Query} query The query
     * to execute. Can be either an instance of helma.Search.Query, or an instance
     * of org.apache.lucene.search.Query
     * @param {helma.Search.QueryFilter|org.apache.lucene.search.Filter} filter
     * An optional query filter
     * @param {Array} sortFields An optional array containing
     * org.apache.lucene.search.SortField instances to use for sorting the result
     * @returns A HitCollection containing the search results
     * @type helma.Search.HitCollection
     */
    jala.IndexManager.prototype.search = function(query, filter, sortFields) {
       var pkg = Packages.org.apache.lucene;
       if (query == null || (!(query instanceof helma.Search.Query) &&
                !(query instanceof pkg.search.Query))) {
          throw "jala.IndexManager search(): missing or invalid query";
       } else if (query instanceof helma.Search.Query) {
          // unwrap query
          query = query.getQuery();
       }
       if (filter != null && filter instanceof helma.Search.QueryFilter) {
          // unwrap filter
          filter = filter.getFilter();
       }

       var searcher = this.getSearcher();
       var analyzer = this.getIndex().getAnalyzer();
       var hits;
       if (sortFields != null && sortFields.length > 0) {
          // convert the array with sortfields to a java array
          var arr = java.lang.reflect.Array.newInstance(pkg.search.SortField, sortFields.length);
          sortFields.forEach(function(sortField, idx) {
             arr[idx] = sortField;
          });
          var sort = pkg.search.Sort(arr);
          if (filter) {
             hits = searcher.search(query, filter, sort);
          } else {
             hits = searcher.search(query, sort);
          }
       } else if (filter) {
          hits = searcher.search(query, filter);
       } else {
          hits = searcher.search(query);
       }
       this.log("debug", "Query: " + query.toString());
       return new helma.Search.HitCollection(hits);
    };

    /**
     * Parses the query string passed as argument into a lucene Query instance
     * @param {String} queryStr The query string to parse
     * @param {Array} fields An array containing the names of the files to search in
     * @param {Object} boostMap An optional object containing properties whose name denotes
     * the name of the field to boost in the query, and the value the boost value.
     * @returns The query
     * @type org.apache.lucene.search.Query
     */
    jala.IndexManager.prototype.parseQuery = function(queryStr, fields, boostMap) {
       if (queryStr == null || typeof(queryStr) !== "string") {
          throw "IndexManager.parseQuery(): missing or invalid query string";
       }
       if (fields == null || fields.constructor !== Array || fields.length < 1) {
          throw "IndexManager.parseQuery(): missing fields argument";
       }
       var query = null;
       var analyzer = this.getIndex().getAnalyzer();
       var pkg = Packages.org.apache.lucene;
       var map = null;
       if (boostMap != null) {
          // convert the javascript object into a HashMap
          map = new java.util.HashMap();
          for (var name in boostMap) {
             map.put(name, new java.lang.Float(boostMap[name]));
          }
       }
       var parser;
       try {
          if (fields.length > 1) {
             parser = new pkg.queryParser.MultiFieldQueryParser(fields, analyzer, map);
          } else {
             parser = new pkg.queryParser.QueryParser(fields, analyzer);
          }
          query = parser.parse(queryStr);
       } catch (e) {
          // ignore, but write a message to debug log
          app.logger.debug("Unable to construct search query '" + queryStr +
                           "', reason: " + e);
       }
       return query;
    };

    /**
     * Parses the query passed as argument and returns a caching filter. If an array
     * with more than one query strings is passed as argument, this method constructs
     * a boolean query filter where all queries in the array must match.
     * @param {String|Array} query Either a query string, or an array containing
     * one or more query strings
     * @param {org.apache.lucene.analysis.Analyzer} analyzer Optional analyzer
     * to use when parsing the filter query
     * @returns A caching query filter
     * @type org.apache.lucene.search.CachingWrapperFilter
     */
    jala.IndexManager.prototype.parseQueryFilter = function(query, analyzer) {
       var filter = null;
       if (query != null) {
          var pkg = Packages.org.apache.lucene;
          // use the index' analyzer if none has been specified
          if (analyzer == null) {
             analyzer = this.getIndex().getAnalyzer();
          }
          var parser = new pkg.queryParser.QueryParser("", analyzer);
          var filterQuery;
          try {
             if (query.constructor === Array) {
                if (query.length > 1) {
                   filterQuery = new pkg.search.BooleanQuery();
                   query.forEach(function(queryStr){
                      filterQuery.add(parser.parse(queryStr), pkg.search.BooleanClause.Occur.MUST);
                   }, this);
                } else {
                   filterQuery = parser.parse(query[0]);
                }
             } else {
                filterQuery = parser.parse(query);
             }
             filter = new pkg.search.CachingWrapperFilter(new pkg.search.QueryWrapperFilter(filterQuery));
          } catch (e) {
             app.logger.debug("Unable to parse query filter '" + query + "', reason: " + e);
          }
       }
       return filter;
    };

    /*********************
     *****   J O B   *****
     *********************/

    /**
     * Creates a new Job instance.
     * @class Instances of this class represent a single index
     * manipulation job to be processed by the index manager.
     * @param {Number} id The Id of the job
     * @param {Number} type The type of job, which can be either
     * jala.IndexManager.Job.ADD, jala.IndexManager.Job.REMOVE
     * or jala.IndexManager.Job.OPTIMIZE.
     * @param {Object} data The data needed to process the job.
     * @returns A newly created Job instance.
     * @constructor
     * @see jala.IndexManager.Job
     */
    jala.IndexManager.Job = function(type, callback) {
       /**
        * The type of the job
        * @type Number
        */
       this.type = type;

       /**
        * The data needed to process this job. For adding jobs this property
        * must contain the {@link helma.Search.Document} instance to add to
        * the index. For removal job this property must contain the unique identifier
        * of the document that should be removed from the index. For optimizing
        * jobs this property is null.
        */
       this.callback = callback;

       /**
        * An internal error counter which is increased whenever processing
        * the job failed.
        * @type Number
        * @see jala.IndexManager.MAXTRIES
        */
       this.errors = 0;

       /**
        * The date and time at which this job was created.
        * @type Date
        */
       this.createtime = new Date();

       return this;
    };

    /** @ignore */
    jala.IndexManager.Job.prototype.toString = function() {
       return "[Job (type: " + this.type + ")]";
    };

    /**
     * Constant defining an add job
     * @type Number
     * @final
     */
    jala.IndexManager.Job.ADD = "add";

    /**
     * Constant defining a removal job
     * @type Number
     * @final
     */
    jala.IndexManager.Job.REMOVE = "remove";

    /**
     * Constant defining an optimizing job
     * @type Number
     * @final
     */
    jala.IndexManager.Job.OPTIMIZE = "optimize";