From 43e32ec71a508e1bb68b247fdca74f64bcf629b3 Mon Sep 17 00:00:00 2001
From: James Moger <james.moger@gitblit.com>
Date: Tue, 28 Feb 2012 22:41:31 -0500
Subject: [PATCH] Draft integration of Lucene search mechanism and web ui
---
src/com/gitblit/GitBlit.java | 4 +-
src/com/gitblit/wicket/pages/RootPage.java | 3 +
src/com/gitblit/utils/LuceneUtils.java | 44 ++++++++++++++-------
distrib/gitblit.properties | 20 ++++++++++
src/com/gitblit/LuceneExecutor.java | 27 ++++++++-----
resources/gitblit.css | 12 ++++++
6 files changed, 83 insertions(+), 27 deletions(-)
diff --git a/distrib/gitblit.properties b/distrib/gitblit.properties
index 2ac6598..ede5f59 100644
--- a/distrib/gitblit.properties
+++ b/distrib/gitblit.properties
@@ -91,6 +91,26 @@
# SINCE 0.8.0
groovy.postReceiveScripts =
+# If true, a Lucene index will be generated and maintained for each repository.
+# Lucene search replaces brute-force Git repository traversal.
+#
+# SINCE 0.9.0
+# RESTART REQUIRED
+lucene.enable = false
+
+# If *lucene.pollingMode* = true, Gitblit will periodically check all repositories
+# for branch updates.
+# If *lucene.pollingMode* = false, repositories will only be indexed on pushes
+# to Gitblit.
+#
+# Regardless of this setting, Gitblit will check all repositories for branch
+# updates 1 minute after startup. Indexes will automatically be built for any
+# repository that is missing its index or if an index version change is detected.
+#
+# SINCE 0.9.0
+# RESTART REQUIRED
+lucene.pollingMode = false
+
#
# Authentication Settings
#
diff --git a/resources/gitblit.css b/resources/gitblit.css
index 6ad45fb..bfa1c20 100644
--- a/resources/gitblit.css
+++ b/resources/gitblit.css
@@ -266,6 +266,18 @@
padding: 2px;
}
+div.searchResult {
+ padding:5px;
+}
+
+div.searchResult .summary {
+ font-weight: bold;
+}
+
+div.searchResult .branch {
+ color: #008000;
+}
+
div.header, div.commitHeader, table.repositories th {
background-color:#e0e0e0;
background-repeat:repeat-x;
diff --git a/src/com/gitblit/GitBlit.java b/src/com/gitblit/GitBlit.java
index a7bfad4..f6691dc 100644
--- a/src/com/gitblit/GitBlit.java
+++ b/src/com/gitblit/GitBlit.java
@@ -1824,8 +1824,8 @@
}
luceneExecutor = new LuceneExecutor(settings);
if (luceneExecutor.isReady()) {
- logger.info("Lucene executor is scheduled to process the repository queue every 10 minutes.");
- scheduledExecutor.scheduleAtFixedRate(luceneExecutor, 1, 10, TimeUnit.MINUTES);
+ logger.info("Lucene executor is scheduled to process the repository queue every 2 minutes.");
+ scheduledExecutor.scheduleAtFixedRate(luceneExecutor, 1, 2, TimeUnit.MINUTES);
} else {
logger.warn("Lucene executor is disabled.");
}
diff --git a/src/com/gitblit/LuceneExecutor.java b/src/com/gitblit/LuceneExecutor.java
index e4624ef..b4e5134 100644
--- a/src/com/gitblit/LuceneExecutor.java
+++ b/src/com/gitblit/LuceneExecutor.java
@@ -29,6 +29,7 @@
import com.gitblit.models.RepositoryModel;
import com.gitblit.utils.JGitUtils;
import com.gitblit.utils.LuceneUtils;
+import com.gitblit.utils.LuceneUtils.IndexResult;
/**
* The Lucene executor handles indexing repositories synchronously and
@@ -53,8 +54,8 @@
public LuceneExecutor(IStoredSettings settings) {
this.settings = settings;
- this.isLuceneEnabled = settings.getBoolean("lucene.enableLucene", false);
- this.isPollingMode = settings.getBoolean("lucene.pollingMode", false);
+ this.isLuceneEnabled = settings.getBoolean(Keys.lucene.enable, false);
+ this.isPollingMode = settings.getBoolean(Keys.lucene.pollingMode, false);
}
/**
@@ -144,11 +145,14 @@
if (LuceneUtils.shouldReindex(repository)) {
// (re)build the entire index
long start = System.currentTimeMillis();
- boolean success = LuceneUtils.reindex(repository);
+ IndexResult result = LuceneUtils.reindex(repository);
long duration = System.currentTimeMillis() - start;
- if (success) {
- String msg = "Built {0} Lucene index in {1} msecs";
- logger.info(MessageFormat.format(msg, repositoryName, duration));
+ if (result.success) {
+ if (result.commitCount > 0) {
+ String msg = "Built {0} Lucene index from {1} commits in {2} msecs";
+ logger.info(MessageFormat.format(msg, repositoryName,
+ result.commitCount, duration));
+ }
} else {
String msg = "Could not build {0} Lucene index!";
logger.error(MessageFormat.format(msg, repositoryName));
@@ -156,11 +160,14 @@
} else {
// update the index with latest commits
long start = System.currentTimeMillis();
- boolean success = LuceneUtils.updateIndex(repository);
+ IndexResult result = LuceneUtils.updateIndex(repository);
long duration = System.currentTimeMillis() - start;
- if (success) {
- String msg = "Updated {0} Lucene index in {1} msecs";
- logger.info(MessageFormat.format(msg, repositoryName, duration));
+ if (result.success) {
+ if (result.commitCount > 0) {
+ String msg = "Updated {0} Lucene index with {1} commits in {2} msecs";
+ logger.info(MessageFormat.format(msg, repositoryName,
+ result.commitCount, duration));
+ }
} else {
String msg = "Could not update {0} Lucene index!";
logger.error(MessageFormat.format(msg, repositoryName));
diff --git a/src/com/gitblit/utils/LuceneUtils.java b/src/com/gitblit/utils/LuceneUtils.java
index d463cdf..e824236 100644
--- a/src/com/gitblit/utils/LuceneUtils.java
+++ b/src/com/gitblit/utils/LuceneUtils.java
@@ -52,6 +52,7 @@
import org.eclipse.jgit.treewalk.TreeWalk;
import org.eclipse.jgit.util.FS;
+import com.gitblit.GitBlit;
import com.gitblit.models.IssueModel;
import com.gitblit.models.IssueModel.Attachment;
import com.gitblit.models.PathModel.PathChangeModel;
@@ -121,10 +122,13 @@
* @return the repository name
*/
private static String getName(Repository repository) {
+ String rootPath = GitBlit.getRepositoriesFolder().getAbsolutePath();
if (repository.isBare()) {
- return repository.getDirectory().getName();
+ return StringUtils.getRelativePath(rootPath, repository.getDirectory()
+ .getAbsolutePath());
} else {
- return repository.getDirectory().getParentFile().getName();
+ return StringUtils.getRelativePath(rootPath, repository.getDirectory().getParentFile()
+ .getAbsolutePath());
}
}
@@ -198,11 +202,12 @@
* index.
*
* @param repository
- * @return true if the indexing has succeeded
+ * @return IndexResult
*/
- public static boolean reindex(Repository repository) {
+ public static IndexResult reindex(Repository repository) {
+ IndexResult result = new IndexResult();
if (!LuceneUtils.deleteIndex(repository)) {
- return false;
+ return result;
}
try {
String repositoryName = getName(repository);
@@ -300,6 +305,7 @@
Index.NOT_ANALYZED));
doc.add(new Field(FIELD_BRANCH, branchName, Store.YES, Index.NOT_ANALYZED));
writer.addDocument(doc);
+ result.commitCount += 1;
}
// traverse the log and index the previous commit objects
@@ -312,6 +318,7 @@
Index.NOT_ANALYZED));
doc.add(new Field(FIELD_BRANCH, branchName, Store.YES, Index.NOT_ANALYZED));
writer.addDocument(doc);
+ result.commitCount += 1;
}
}
@@ -335,11 +342,11 @@
config.save();
resetIndexSearcher(repository);
writer.commit();
- return true;
+ result.success = true;
} catch (Exception e) {
e.printStackTrace();
}
- return false;
+ return result;
}
/**
@@ -453,9 +460,10 @@
* Updates a repository index incrementally from the last indexed commits.
*
* @param repository
+ * @return IndexResult
*/
- public static boolean updateIndex(Repository repository) {
- boolean success = false;
+ public static IndexResult updateIndex(Repository repository) {
+ IndexResult result = new IndexResult();
try {
FileBasedConfig config = getConfig(repository);
config.load();
@@ -475,13 +483,13 @@
// detect branch deletion
// first assume all branches are deleted and then remove each
- // existing branch from deletedBranches during indexing
+ // existing branch from deletedBranches during indexing
Set<String> deletedBranches = new TreeSet<String>();
for (String alias : config.getNames(CONF_ALIAS)) {
String branch = config.getString(CONF_ALIAS, null, alias);
deletedBranches.add(branch);
}
-
+
// walk through each branches
List<RefModel> branches = JGitUtils.getLocalBranches(repository, true, -1);
for (RefModel branch : branches) {
@@ -490,7 +498,7 @@
// remove this branch from the deletedBranches set
deletedBranches.remove(branchName);
- // determine last commit
+ // determine last commit
String keyName = getBranchKey(branchName);
String lastCommit = config.getString(CONF_BRANCH, null, keyName);
@@ -507,6 +515,7 @@
Collections.reverse(revs);
for (RevCommit commit : revs) {
index(repository, branchName, commit);
+ result.commitCount += 1;
}
// update the config
@@ -515,7 +524,7 @@
config.setString(CONF_BRANCH, null, keyName, branch.getObjectId().getName());
config.save();
}
-
+
// the deletedBranches set will normally be empty by this point
// unless a branch really was deleted and no longer exists
if (deletedBranches.size() > 0) {
@@ -525,11 +534,11 @@
writer.commit();
}
}
- success = true;
+ result.success = true;
} catch (Throwable t) {
t.printStackTrace();
}
- return success;
+ return result;
}
/**
@@ -782,4 +791,9 @@
}
SEARCHERS.clear();
}
+
+ public static class IndexResult {
+ public boolean success;
+ public int commitCount;
+ }
}
diff --git a/src/com/gitblit/wicket/pages/RootPage.java b/src/com/gitblit/wicket/pages/RootPage.java
index bad0140..d81fae1 100644
--- a/src/com/gitblit/wicket/pages/RootPage.java
+++ b/src/com/gitblit/wicket/pages/RootPage.java
@@ -101,6 +101,9 @@
pages.add(new PageRegistration("gb.repositories", RepositoriesPage.class,
getRootPageParameters()));
pages.add(new PageRegistration("gb.activity", ActivityPage.class, getRootPageParameters()));
+ if (GitBlit.getBoolean(Keys.lucene.enable, false)) {
+ pages.add(new PageRegistration("gb.search", LucenePage.class));
+ }
if (showAdmin) {
pages.add(new PageRegistration("gb.users", UsersPage.class));
}
--
Gitblit v1.9.1