Made LucenePage stateless

James Moger

2012-03-13 7d97c619b1c067f971cfe0fda2ca8a56eef1a316

 src/com/gitblit/utils/LuceneUtils.java

@@ -1,4 +1,21 @@
/*

 * Copyright 2012 gitblit.com.

 *

 * Licensed under the Apache License, Version 2.0 (the "License");

 * you may not use this file except in compliance with the License.

 * You may obtain a copy of the License at

 *

 *     http://www.apache.org/licenses/LICENSE-2.0

 *

 * Unless required by applicable law or agreed to in writing, software

 * distributed under the License is distributed on an "AS IS" BASIS,

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

 * See the License for the specific language governing permissions and

 * limitations under the License.

 */

package com.gitblit.utils;



import static org.eclipse.jgit.treewalk.filter.TreeFilter.ANY_DIFF;



import java.io.ByteArrayOutputStream;

import java.io.File;

@@ -8,14 +25,18 @@
import java.util.ArrayList;

import java.util.Arrays;

import java.util.Collections;

import java.util.Comparator;

import java.util.HashMap;

import java.util.LinkedHashSet;

import java.util.List;

import java.util.Map;

import java.util.Set;

import java.util.TreeMap;

import java.util.TreeSet;

import java.util.concurrent.ConcurrentHashMap;



import org.apache.lucene.analysis.Analyzer;

import org.apache.lucene.analysis.TokenStream;

import org.apache.lucene.analysis.standard.StandardAnalyzer;

import org.apache.lucene.document.DateTools;

import org.apache.lucene.document.DateTools.Resolution;

@@ -36,23 +57,30 @@
import org.apache.lucene.search.Query;

import org.apache.lucene.search.ScoreDoc;

import org.apache.lucene.search.TopScoreDocCollector;

import org.apache.lucene.search.highlight.Fragmenter;

import org.apache.lucene.search.highlight.Highlighter;

import org.apache.lucene.search.highlight.InvalidTokenOffsetsException;

import org.apache.lucene.search.highlight.QueryScorer;

import org.apache.lucene.search.highlight.SimpleHTMLFormatter;

import org.apache.lucene.search.highlight.SimpleSpanFragmenter;

import org.apache.lucene.search.highlight.TokenSources;

import org.apache.lucene.store.Directory;

import org.apache.lucene.store.FSDirectory;

import org.apache.lucene.util.Version;

import org.eclipse.jgit.diff.DiffEntry.ChangeType;

import org.eclipse.jgit.lib.Constants;

import org.eclipse.jgit.lib.FileMode;

import org.eclipse.jgit.lib.ObjectId;

import org.eclipse.jgit.lib.ObjectLoader;

import org.eclipse.jgit.lib.ObjectReader;

import org.eclipse.jgit.lib.Repository;

import org.eclipse.jgit.revwalk.RevCommit;

import org.eclipse.jgit.revwalk.RevObject;

import org.eclipse.jgit.revwalk.RevTree;

import org.eclipse.jgit.revwalk.RevWalk;

import org.eclipse.jgit.storage.file.FileBasedConfig;

import org.eclipse.jgit.treewalk.EmptyTreeIterator;

import org.eclipse.jgit.treewalk.TreeWalk;

import org.eclipse.jgit.util.FS;



import com.gitblit.GitBlit;

import com.gitblit.models.IssueModel;

import com.gitblit.models.IssueModel.Attachment;

import com.gitblit.models.PathModel.PathChangeModel;

@@ -88,7 +116,9 @@
   private static final int INDEX_VERSION = 1;



   private static final String FIELD_OBJECT_TYPE = "type";

   private static final String FIELD_OBJECT_ID = "id";

   private static final String FIELD_ISSUE = "issue";

   private static final String FIELD_PATH = "path";

   private static final String FIELD_COMMIT = "commit";

   private static final String FIELD_BRANCH = "branch";

   private static final String FIELD_REPOSITORY = "repository";

   private static final String FIELD_SUMMARY = "summary";

@@ -96,6 +126,7 @@
   private static final String FIELD_AUTHOR = "author";

   private static final String FIELD_COMMITTER = "committer";

   private static final String FIELD_DATE = "date";

   private static final String FIELD_TAG = "tag";

   private static final String FIELD_LABEL = "label";

   private static final String FIELD_ATTACHMENT = "attachment";



@@ -109,27 +140,47 @@
   private static final Map<File, IndexSearcher> SEARCHERS = new ConcurrentHashMap<File, IndexSearcher>();

   private static final Map<File, IndexWriter> WRITERS = new ConcurrentHashMap<File, IndexWriter>();



   private static final String LUCENE_DIR = "lucene";

   private static final String CONF_FILE = "lucene.conf";

   private static final String CONF_INDEX = "index";

   private static final String CONF_VERSION = "version";

   private static final String CONF_ALIAS = "aliases";

   private static final String CONF_BRANCH = "branches";



	
   /**

    * Returns the name of the repository.

    * Returns the author for the commit, if this information is available.

    * 

    * @param repository

    * @return the repository name

    * @param commit

    * @return an author or unknown

    */

   private static String getName(Repository repository) {

      String rootPath = GitBlit.getRepositoriesFolder().getAbsolutePath();

      if (repository.isBare()) {

         return StringUtils.getRelativePath(rootPath, repository.getDirectory()

               .getAbsolutePath());

      } else {

         return StringUtils.getRelativePath(rootPath, repository.getDirectory().getParentFile()

               .getAbsolutePath());

   private static String getAuthor(RevCommit commit) {

      String name = "unknown";

      try {

         name = commit.getAuthorIdent().getName();

         if (StringUtils.isEmpty(name)) {

            name = commit.getAuthorIdent().getEmailAddress();

         }

      } catch (NullPointerException n) {						
      }

      return name;

   }

	
   /**

    * Returns the committer for the commit, if this information is available.

    * 
    * @param commit

    * @return an committer or unknown

    */

   private static String getCommitter(RevCommit commit) {

      String name = "unknown";

      try {

         name = commit.getCommitterIdent().getName();

         if (StringUtils.isEmpty(name)) {

            name = commit.getCommitterIdent().getEmailAddress();

         }

      } catch (NullPointerException n) {						
      }

      return name;

   }



   /**

@@ -182,7 +233,7 @@
    */

   public static boolean deleteIndex(Repository repository) {

      try {

         File luceneIndex = new File(repository.getDirectory(), "lucene");

         File luceneIndex = new File(repository.getDirectory(), LUCENE_DIR);

         if (luceneIndex.exists()) {

            org.eclipse.jgit.util.FileUtils.delete(luceneIndex,

                  org.eclipse.jgit.util.FileUtils.RECURSIVE);

@@ -201,16 +252,16 @@
    * This completely indexes the repository and will destroy any existing

    * index.

    * 

    * @param repositoryName

    * @param repository

    * @return IndexResult

    */

   public static IndexResult reindex(Repository repository) {

   public static IndexResult reindex(String repositoryName, Repository repository) {

      IndexResult result = new IndexResult();

      if (!LuceneUtils.deleteIndex(repository)) {

         return result;

      }

      try {

         String repositoryName = getName(repository);

      try {			
         FileBasedConfig config = getConfig(repository);

         Set<String> indexedCommits = new TreeSet<String>();

         IndexWriter writer = getIndexWriter(repository, true);

@@ -226,113 +277,176 @@
            }

            tags.get(tag.getReferencedObjectId().getName()).add(tag.displayName);

         }

			
         ObjectReader reader = repository.newObjectReader();



         // walk through each branch

         // get the local branches

         List<RefModel> branches = JGitUtils.getLocalBranches(repository, true, -1);

			
         // sort them by most recently updated

         Collections.sort(branches, new Comparator<RefModel>() {

            @Override

            public int compare(RefModel ref1, RefModel ref2) {

               return ref2.getDate().compareTo(ref1.getDate());

            }

         });

			
         // reorder default branch to first position

         RefModel defaultBranch = null;

         ObjectId defaultBranchId = JGitUtils.getDefaultBranch(repository);

         for (RefModel branch :  branches) {

            if (branch.getObjectId().equals(defaultBranchId)) {

               defaultBranch = branch;					
               break;

            }

         }

         branches.remove(defaultBranch);

         branches.add(0, defaultBranch);

			
         // walk through each branch

         for (RefModel branch : branches) {

            if (excludedBranches.contains(branch.getName())) {

               continue;

            }



            String branchName = branch.getName();

            RevWalk revWalk = new RevWalk(repository);

            RevCommit rev = revWalk.parseCommit(branch.getObjectId());

            RevWalk revWalk = new RevWalk(reader);

            RevCommit tip = revWalk.parseCommit(branch.getObjectId());

            String tipId = tip.getId().getName();



            String keyName = getBranchKey(branchName);

            config.setString(CONF_ALIAS, null, keyName, branchName);

            config.setString(CONF_BRANCH, null, keyName, rev.getName());

            config.setString(CONF_BRANCH, null, keyName, tipId);



            // index the blob contents of the tree

            TreeWalk treeWalk = new TreeWalk(repository);

            treeWalk.addTree(tip.getTree());

            treeWalk.setRecursive(true);								
				
            Map<String, ObjectId> paths = new TreeMap<String, ObjectId>();

            while (treeWalk.next()) {

               paths.put(treeWalk.getPathString(), treeWalk.getObjectId(0));

            }				


            ByteArrayOutputStream os = new ByteArrayOutputStream();

            byte[] tmp = new byte[32767];

            TreeWalk treeWalk = new TreeWalk(repository);

            treeWalk.addTree(rev.getTree());

            treeWalk.setRecursive(true);

            String revDate = DateTools.timeToString(rev.getCommitTime() * 1000L,

                  Resolution.MINUTE);

            while (treeWalk.next()) {

               Document doc = new Document();

               doc.add(new Field(FIELD_OBJECT_TYPE, ObjectType.blob.name(), Store.YES,

                     Index.NOT_ANALYZED_NO_NORMS));

               doc.add(new Field(FIELD_REPOSITORY, repositoryName, Store.YES,

                     Index.NOT_ANALYZED));

               doc.add(new Field(FIELD_BRANCH, branchName, Store.YES, Index.NOT_ANALYZED));

               doc.add(new Field(FIELD_OBJECT_ID, treeWalk.getPathString(), Store.YES,

                     Index.NOT_ANALYZED));

               doc.add(new Field(FIELD_DATE, revDate, Store.YES, Index.NO));

               doc.add(new Field(FIELD_AUTHOR, rev.getAuthorIdent().getName(), Store.YES,

                     Index.NOT_ANALYZED_NO_NORMS));

               doc.add(new Field(FIELD_COMMITTER, rev.getCommitterIdent().getName(),

                     Store.YES, Index.NOT_ANALYZED_NO_NORMS));

               doc.add(new Field(FIELD_LABEL, branch.getName(), Store.YES, Index.ANALYZED));



               // determine extension to compare to the extension

               // blacklist

               String ext = null;

               String name = treeWalk.getPathString().toLowerCase();

               if (name.indexOf('.') > -1) {

                  ext = name.substring(name.lastIndexOf('.') + 1);

            RevWalk commitWalk = new RevWalk(reader);

            commitWalk.markStart(tip);

				
            RevCommit commit;

            while ((paths.size() > 0) && (commit = commitWalk.next()) != null) {

               TreeWalk diffWalk = new TreeWalk(reader);

               int parentCount = commit.getParentCount();

               switch (parentCount) {

               case 0:

                  diffWalk.addTree(new EmptyTreeIterator());

                  break;

               case 1:

                  diffWalk.addTree(getTree(commitWalk, commit.getParent(0)));

                  break;

               default:

                  // skip merge commits

                  continue;

               }



               if (StringUtils.isEmpty(ext) || !excludedExtensions.contains(ext)) {

                  // read the blob content

                  ObjectId entid = treeWalk.getObjectId(0);

                  FileMode entmode = treeWalk.getFileMode(0);

                  RevObject ro = revWalk.lookupAny(entid, entmode.getObjectType());

                  revWalk.parseBody(ro);

                  ObjectLoader ldr = repository.open(ro.getId(), Constants.OBJ_BLOB);

                  InputStream in = ldr.openStream();

                  os.reset();

                  int n = 0;

                  while ((n = in.read(tmp)) > 0) {

                     os.write(tmp, 0, n);

               diffWalk.addTree(getTree(commitWalk, commit));

               diffWalk.setFilter(ANY_DIFF);

               diffWalk.setRecursive(true);

               while ((paths.size() > 0) && diffWalk.next()) {

                  String path = diffWalk.getPathString();

                  if (!paths.containsKey(path)) {

                     continue;

                  }

                  in.close();

                  byte[] content = os.toByteArray();

                  String str = new String(content, "UTF-8");

                  doc.add(new Field(FIELD_CONTENT, str, Store.NO, Index.ANALYZED));

						
                  // remove path from set

                  ObjectId blobId = paths.remove(path);

                  result.blobCount++;

						
                  // index the blob metadata

                  String blobAuthor = getAuthor(commit);

                  String blobCommitter = getCommitter(commit);

                  String blobDate = DateTools.timeToString(commit.getCommitTime() * 1000L,

                        Resolution.MINUTE);

						
                  Document doc = new Document();

                  doc.add(new Field(FIELD_OBJECT_TYPE, ObjectType.blob.name(), Store.YES, Index.NOT_ANALYZED_NO_NORMS));

                  doc.add(new Field(FIELD_REPOSITORY, repositoryName, Store.YES, Index.ANALYZED));

                  doc.add(new Field(FIELD_BRANCH, branchName, Store.YES, Index.ANALYZED));

                  doc.add(new Field(FIELD_COMMIT, commit.getName(), Store.YES, Index.ANALYZED));

                  doc.add(new Field(FIELD_PATH, path, Store.YES, Index.ANALYZED));

                  doc.add(new Field(FIELD_DATE, blobDate, Store.YES, Index.NO));

                  doc.add(new Field(FIELD_AUTHOR, blobAuthor, Store.YES, Index.ANALYZED));

                  doc.add(new Field(FIELD_COMMITTER, blobCommitter, Store.YES, Index.ANALYZED));					


                  // determine extension to compare to the extension

                  // blacklist

                  String ext = null;

                  String name = path.toLowerCase();

                  if (name.indexOf('.') > -1) {

                     ext = name.substring(name.lastIndexOf('.') + 1);

                  }



                  // index the blob content

                  if (StringUtils.isEmpty(ext) || !excludedExtensions.contains(ext)) {							
                     ObjectLoader ldr = repository.open(blobId, Constants.OBJ_BLOB);

                     InputStream in = ldr.openStream();							
                     int n;

                     while ((n = in.read(tmp)) > 0) {

                        os.write(tmp, 0, n);

                     }

                     in.close();

                     byte[] content = os.toByteArray();

                     String str = new String(content, Constants.CHARACTER_ENCODING);

                     doc.add(new Field(FIELD_CONTENT, str, Store.YES, Index.ANALYZED));

                     os.reset();

                  }							
						
                  // add the blob to the index

                  writer.addDocument(doc);

               }

            }



            os.close();

            treeWalk.release();



            // index the head commit object

            String head = rev.getId().getName();

            if (indexedCommits.add(head)) {

               Document doc = createDocument(rev, tags.get(head));

               doc.add(new Field(FIELD_REPOSITORY, repositoryName, Store.YES,

                     Index.NOT_ANALYZED));

               doc.add(new Field(FIELD_BRANCH, branchName, Store.YES, Index.NOT_ANALYZED));

            // index the tip commit object

            if (indexedCommits.add(tipId)) {

               Document doc = createDocument(tip, tags.get(tipId));

               doc.add(new Field(FIELD_REPOSITORY, repositoryName, Store.YES, Index.ANALYZED));

               doc.add(new Field(FIELD_BRANCH, branchName, Store.YES, Index.ANALYZED));

               writer.addDocument(doc);

               result.commitCount += 1;

               result.branchCount += 1;

            }



            // traverse the log and index the previous commit objects

            revWalk.markStart(rev);

            while ((rev = revWalk.next()) != null) {

            RevWalk historyWalk = new RevWalk(reader);

            historyWalk.markStart(historyWalk.parseCommit(tip.getId()));

            RevCommit rev;

            while ((rev = historyWalk.next()) != null) {

               String hash = rev.getId().getName();

               if (indexedCommits.add(hash)) {

                  Document doc = createDocument(rev, tags.get(hash));

                  doc.add(new Field(FIELD_REPOSITORY, repositoryName, Store.YES,

                        Index.NOT_ANALYZED));

                  doc.add(new Field(FIELD_BRANCH, branchName, Store.YES, Index.NOT_ANALYZED));

                  doc.add(new Field(FIELD_REPOSITORY, repositoryName, Store.YES, Index.ANALYZED));

                  doc.add(new Field(FIELD_BRANCH, branchName, Store.YES, Index.ANALYZED));

                  writer.addDocument(doc);

                  result.commitCount += 1;

               }

            }



            // finished

            revWalk.dispose();

         }



         // finished

         reader.release();

			
         // this repository has a gb-issues branch, index all issues

         if (IssueUtils.getIssuesBranch(repository) != null) {

            List<IssueModel> issues = IssueUtils.getIssues(repository, null);

            if (issues.size() > 0) {

               result.branchCount += 1;

            }

            for (IssueModel issue : issues) {

               result.issueCount++;

               Document doc = createDocument(issue);

               doc.add(new Field(FIELD_REPOSITORY, repositoryName, Store.YES,

                     Index.NOT_ANALYZED));

               doc.add(new Field(FIELD_REPOSITORY, repositoryName, Store.YES, Index.ANALYZED));

               writer.addDocument(doc);

            }

         }

@@ -348,18 +462,39 @@
      }

      return result;

   }

	
   /**

    * Get the tree associated with the given commit.

    *

    * @param walk

    * @param commit

    * @return tree

    * @throws IOException

    */

   protected static RevTree getTree(final RevWalk walk, final RevCommit commit)

         throws IOException {

      final RevTree tree = commit.getTree();

      if (tree != null) {

         return tree;

      }

      walk.parseHeaders(commit);

      return commit.getTree();

   }



   /**

    * Incrementally update the index with the specified commit for the

    * repository.

    * 

    * @param repositoryName

    * @param repository

    * @param branch

    *            the fully qualified branch name (e.g. refs/heads/master)

    * @param commit

    * @return true, if successful

    */

   public static boolean index(Repository repository, String branch, RevCommit commit) {

   private static IndexResult index(String repositoryName, Repository repository, 
         String branch, RevCommit commit) {

      IndexResult result = new IndexResult();

      try {

         if (excludedBranches.contains(branch)) {

            if (IssueUtils.GB_ISSUES.equals(branch)) {

@@ -367,43 +502,44 @@
               String issueId = commit.getShortMessage().substring(2).trim();

               IssueModel issue = IssueUtils.getIssue(repository, issueId);

               if (issue == null) {

                  // delete the old issue from the index, if exists

                  // issue was deleted, remove from index

                  IndexWriter writer = getIndexWriter(repository, false);

                  writer.deleteDocuments(

                        new Term(FIELD_OBJECT_TYPE, ObjectType.issue.name()), new Term(

                              FIELD_OBJECT_ID, issueId));

                              FIELD_ISSUE, issueId));

                  writer.commit();

                  return true;

                  result.success = true;

                  return result;

               }

               return index(repository, issue);

               result.success = index(repositoryName, repository, issue);

               result.issueCount++;

               return result;

					
            }

            return false;

            return result;

         }

         List<PathChangeModel> changedPaths = JGitUtils.getFilesInCommit(repository, commit);

         String repositoryName = getName(repository);

         String revDate = DateTools.timeToString(commit.getCommitTime() * 1000L,

               Resolution.MINUTE);

         IndexWriter writer = getIndexWriter(repository, false);

         for (PathChangeModel path : changedPaths) {

            // delete the indexed blob

            writer.deleteDocuments(new Term(FIELD_OBJECT_TYPE, ObjectType.blob.name()),

                  new Term(FIELD_BRANCH, branch), new Term(FIELD_OBJECT_ID, path.path));

                  new Term(FIELD_BRANCH, branch), new Term(FIELD_PATH, path.path));



            // re-index the blob

            if (!ChangeType.DELETE.equals(path.changeType)) {

               result.blobCount++;

               Document doc = new Document();

               doc.add(new Field(FIELD_OBJECT_TYPE, ObjectType.blob.name(), Store.YES,

                     Index.NOT_ANALYZED_NO_NORMS));

               doc.add(new Field(FIELD_REPOSITORY, repositoryName, Store.YES,

                     Index.NOT_ANALYZED));

               doc.add(new Field(FIELD_BRANCH, branch, Store.YES, Index.NOT_ANALYZED));

               doc.add(new Field(FIELD_OBJECT_ID, path.path, Store.YES, Index.NOT_ANALYZED));

               doc.add(new Field(FIELD_REPOSITORY, repositoryName, Store.YES, Index.ANALYZED));

               doc.add(new Field(FIELD_BRANCH, branch, Store.YES, Index.ANALYZED));

               doc.add(new Field(FIELD_COMMIT, commit.getName(), Store.YES, Index.ANALYZED));

               doc.add(new Field(FIELD_PATH, path.path, Store.YES, Index.ANALYZED));

               doc.add(new Field(FIELD_DATE, revDate, Store.YES, Index.NO));

               doc.add(new Field(FIELD_AUTHOR, commit.getAuthorIdent().getName(), Store.YES,

                     Index.NOT_ANALYZED_NO_NORMS));

               doc.add(new Field(FIELD_COMMITTER, commit.getCommitterIdent().getName(),

                     Store.YES, Index.NOT_ANALYZED_NO_NORMS));

               doc.add(new Field(FIELD_LABEL, branch, Store.YES, Index.ANALYZED));

               doc.add(new Field(FIELD_AUTHOR, getAuthor(commit), Store.YES, Index.ANALYZED));

               doc.add(new Field(FIELD_COMMITTER, getCommitter(commit), Store.YES, Index.ANALYZED));



               // determine extension to compare to the extension

               // blacklist

@@ -417,7 +553,7 @@
                  // read the blob content

                  String str = JGitUtils.getStringContent(repository, commit.getTree(),

                        path.path);

                  doc.add(new Field(FIELD_CONTENT, str, Store.NO, Index.ANALYZED));

                  doc.add(new Field(FIELD_CONTENT, str, Store.YES, Index.ANALYZED));

                  writer.addDocument(doc);

               }

            }

@@ -425,11 +561,12 @@
         writer.commit();



         Document doc = createDocument(commit, null);

         return index(repository, doc);

         result.commitCount++;

         result.success = index(repositoryName, repository, doc);

      } catch (Exception e) {

         e.printStackTrace();

      }

      return false;

      return result;

   }



   /**

@@ -440,16 +577,16 @@
    * @param issue

    * @return true, if successful

    */

   public static boolean index(Repository repository, IssueModel issue) {

   public static boolean index(String repositoryName, Repository repository, IssueModel issue) {

      try {

         // delete the old issue from the index, if exists

         IndexWriter writer = getIndexWriter(repository, false);

         writer.deleteDocuments(new Term(FIELD_OBJECT_TYPE, ObjectType.issue.name()), new Term(

               FIELD_OBJECT_ID, String.valueOf(issue.id)));

               FIELD_ISSUE, String.valueOf(issue.id)));

         writer.commit();



         Document doc = createDocument(issue);

         return index(repository, doc);

         return index(repositoryName, repository, doc);

      } catch (Exception e) {

         e.printStackTrace();

      }

@@ -459,10 +596,11 @@
   /**

    * Updates a repository index incrementally from the last indexed commits.

    * 

    * @param repositoryName

    * @param repository

    * @return IndexResult

    */

   public static IndexResult updateIndex(Repository repository) {

   public static IndexResult updateIndex(String repositoryName, Repository repository) {

      IndexResult result = new IndexResult();

      try {

         FileBasedConfig config = getConfig(repository);

@@ -511,11 +649,14 @@
               revs = JGitUtils.getRevLog(repository, lastCommit, branchName);

            }



            // reverse the list of commits so we start with the first commit

            if (revs.size() > 0) {

               result.branchCount += 1;

            }

				
            // reverse the list of commits so we start with the first commit				
            Collections.reverse(revs);

            for (RevCommit commit : revs) {

               index(repository, branchName, commit);

               result.commitCount += 1;

               result.add(index(repositoryName, repository, branchName, commit));					
            }



            // update the config

@@ -550,12 +691,12 @@
   private static Document createDocument(IssueModel issue) {

      Document doc = new Document();

      doc.add(new Field(FIELD_OBJECT_TYPE, ObjectType.issue.name(), Store.YES,

            Field.Index.NOT_ANALYZED_NO_NORMS));

      doc.add(new Field(FIELD_OBJECT_ID, issue.id, Store.YES, Index.NOT_ANALYZED));

      doc.add(new Field(FIELD_BRANCH, IssueUtils.GB_ISSUES, Store.YES, Index.NOT_ANALYZED));

            Field.Index.NOT_ANALYZED));

      doc.add(new Field(FIELD_ISSUE, issue.id, Store.YES, Index.ANALYZED));

      doc.add(new Field(FIELD_BRANCH, IssueUtils.GB_ISSUES, Store.YES, Index.ANALYZED));

      doc.add(new Field(FIELD_DATE, DateTools.dateToString(issue.created, Resolution.MINUTE),

            Store.YES, Field.Index.NO));

      doc.add(new Field(FIELD_AUTHOR, issue.reporter, Store.YES, Index.NOT_ANALYZED_NO_NORMS));

      doc.add(new Field(FIELD_AUTHOR, issue.reporter, Store.YES, Index.ANALYZED));

      List<String> attachments = new ArrayList<String>();

      for (Attachment attachment : issue.getAttachments()) {

         attachments.add(attachment.name.toLowerCase());

@@ -563,7 +704,7 @@
      doc.add(new Field(FIELD_ATTACHMENT, StringUtils.flattenStrings(attachments), Store.YES,

            Index.ANALYZED));

      doc.add(new Field(FIELD_SUMMARY, issue.summary, Store.YES, Index.ANALYZED));

      doc.add(new Field(FIELD_CONTENT, issue.toString(), Store.NO, Index.ANALYZED));

      doc.add(new Field(FIELD_CONTENT, issue.toString(), Store.YES, Index.ANALYZED));

      doc.add(new Field(FIELD_LABEL, StringUtils.flattenStrings(issue.getLabels()), Store.YES,

            Index.ANALYZED));

      return doc;

@@ -579,19 +720,16 @@
   private static Document createDocument(RevCommit commit, List<String> tags) {

      Document doc = new Document();

      doc.add(new Field(FIELD_OBJECT_TYPE, ObjectType.commit.name(), Store.YES,

            Index.NOT_ANALYZED_NO_NORMS));

      doc.add(new Field(FIELD_OBJECT_ID, commit.getName(), Store.YES, Index.NOT_ANALYZED));

            Index.NOT_ANALYZED));

      doc.add(new Field(FIELD_COMMIT, commit.getName(), Store.YES, Index.ANALYZED));

      doc.add(new Field(FIELD_DATE, DateTools.timeToString(commit.getCommitTime() * 1000L,

            Resolution.MINUTE), Store.YES, Index.NO));

      doc.add(new Field(FIELD_AUTHOR, commit.getCommitterIdent().getName(), Store.YES,

            Index.NOT_ANALYZED_NO_NORMS));

      doc.add(new Field(FIELD_AUTHOR, getAuthor(commit), Store.YES, Index.ANALYZED));

      doc.add(new Field(FIELD_COMMITTER, getCommitter(commit), Store.YES, Index.ANALYZED));

      doc.add(new Field(FIELD_SUMMARY, commit.getShortMessage(), Store.YES, Index.ANALYZED));

      doc.add(new Field(FIELD_CONTENT, commit.getFullMessage(), Store.NO, Index.ANALYZED));

      doc.add(new Field(FIELD_CONTENT, commit.getFullMessage(), Store.YES, Index.ANALYZED));

      if (!ArrayUtils.isEmpty(tags)) {

         if (!ArrayUtils.isEmpty(tags)) {

            doc.add(new Field(FIELD_LABEL, StringUtils.flattenStrings(tags), Store.YES,

                  Index.ANALYZED));

         }

         doc.add(new Field(FIELD_TAG, StringUtils.flattenStrings(tags), Store.YES, Index.ANALYZED));

      }

      return doc;

   }

@@ -599,13 +737,13 @@
   /**

    * Incrementally index an object for the repository.

    * 

    * @param repositoryName

    * @param repository

    * @param doc

    * @return true, if successful

    */

   private static boolean index(Repository repository, Document doc) {

      try {

         String repositoryName = getName(repository);

   private static boolean index(String repositoryName, Repository repository, Document doc) {

      try {			
         doc.add(new Field(FIELD_REPOSITORY, repositoryName, Store.YES, Index.NOT_ANALYZED));

         IndexWriter writer = getIndexWriter(repository, false);

         writer.addDocument(doc);

@@ -622,13 +760,18 @@
      SearchResult result = new SearchResult();

      result.score = score;

      result.date = DateTools.stringToDate(doc.get(FIELD_DATE));

      result.summary = doc.get(FIELD_SUMMARY);

      result.summary = doc.get(FIELD_SUMMARY);		
      result.author = doc.get(FIELD_AUTHOR);

      result.committer = doc.get(FIELD_COMMITTER);

      result.type = ObjectType.fromName(doc.get(FIELD_OBJECT_TYPE));

      result.repository = doc.get(FIELD_REPOSITORY);

      result.branch = doc.get(FIELD_BRANCH);

      result.id = doc.get(FIELD_OBJECT_ID);

      result.commitId = doc.get(FIELD_COMMIT);

      result.issueId = doc.get(FIELD_ISSUE);

      result.path = doc.get(FIELD_PATH);

      if (doc.get(FIELD_TAG) != null) {

         result.tags = StringUtils.getStringsFromValue(doc.get(FIELD_TAG));

      }

      if (doc.get(FIELD_LABEL) != null) {

         result.labels = StringUtils.getStringsFromValue(doc.get(FIELD_LABEL));

      }

@@ -672,7 +815,7 @@
   private static IndexWriter getIndexWriter(Repository repository, boolean forceCreate)

         throws IOException {

      IndexWriter indexWriter = WRITERS.get(repository.getDirectory());

      File indexFolder = new File(repository.getDirectory(), "lucene");

      File indexFolder = new File(repository.getDirectory(), LUCENE_DIR);

      Directory directory = FSDirectory.open(indexFolder);

      if (forceCreate || !indexFolder.exists()) {

         // if the writer is going to blow away the existing index and create

@@ -759,12 +902,45 @@
            int docId = hits[i].doc;

            Document doc = searcher.doc(docId);

            SearchResult result = createSearchResult(doc, hits[i].score);

            String content = doc.get(FIELD_CONTENT);

            result.fragment = getHighlightedFragment(analyzer, query, content);

            results.add(result);

         }

      } catch (Exception e) {

         e.printStackTrace();

      }

      return new ArrayList<SearchResult>(results);

   }

	
   private static String getHighlightedFragment(Analyzer analyzer, Query query,

         String content) throws IOException, InvalidTokenOffsetsException {

      content = content == null ? "":StringUtils.escapeForHtml(content, false);	
      TokenStream stream = TokenSources.getTokenStream("content", content, analyzer);

      QueryScorer scorer = new QueryScorer(query, "content");

      Fragmenter fragmenter = new SimpleSpanFragmenter(scorer, 150);



      SimpleHTMLFormatter formatter = new SimpleHTMLFormatter("<span class=\"highlight\">", "</span>");

      Highlighter highlighter = new Highlighter(formatter, scorer);

		
      highlighter.setTextFragmenter(fragmenter);

      String [] fragments = highlighter.getBestFragments(stream, content, 5);

      if (ArrayUtils.isEmpty(fragments)) {

         return content;

      }

      if (fragments.length == 1) {

         return "<pre>" + fragments[0] + "</pre>";

      }

      StringBuilder sb = new StringBuilder();

      for (int i = 0, len = fragments.length; i < len; i++) {

         String fragment = fragments[i].trim();			
         sb.append("<pre>");

         sb.append(fragment);

         sb.append("</pre>");

         if (i < len - 1) {

            sb.append("<span class=\"ellipses\">...</span><br/>");

         }

      }

      return sb.toString();

   }



   /**

@@ -794,6 +970,16 @@


   public static class IndexResult {

      public boolean success;

      public int branchCount;

      public int commitCount;

      public int blobCount;

      public int issueCount;

		
      public void add(IndexResult result) {

         this.branchCount += result.branchCount;

         this.commitCount += result.commitCount;

         this.blobCount += result.blobCount;

         this.issueCount += result.issueCount;			
      }

   }

}