From d4b95298902c8cea1411fc696ed80028b6091aa7 Mon Sep 17 00:00:00 2001
From: Rafael Cavazin <rafaelcavazin@gmail.com>
Date: Thu, 06 Dec 2012 11:25:01 -0500
Subject: [PATCH] Update from upstream/master
---
src/com/gitblit/LuceneExecutor.java | 85 ++++++++++++++++++++++++++++++++----------
1 files changed, 65 insertions(+), 20 deletions(-)
diff --git a/src/com/gitblit/LuceneExecutor.java b/src/com/gitblit/LuceneExecutor.java
index c702dcc..0e4baae 100644
--- a/src/com/gitblit/LuceneExecutor.java
+++ b/src/com/gitblit/LuceneExecutor.java
@@ -69,6 +69,7 @@
import org.apache.lucene.util.Version;
import org.eclipse.jgit.diff.DiffEntry.ChangeType;
import org.eclipse.jgit.lib.Constants;
+import org.eclipse.jgit.lib.FileMode;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.ObjectLoader;
import org.eclipse.jgit.lib.ObjectReader;
@@ -105,7 +106,7 @@
public class LuceneExecutor implements Runnable {
- private static final int INDEX_VERSION = 3;
+ private static final int INDEX_VERSION = 5;
private static final String FIELD_OBJECT_TYPE = "type";
private static final String FIELD_ISSUE = "issue";
@@ -166,10 +167,21 @@
String exts = storedSettings.getString(Keys.web.luceneIgnoreExtensions, luceneIgnoreExtensions);
excludedExtensions = new TreeSet<String>(StringUtils.getStringsFromValue(exts));
+ if (GitBlit.self().isCollectingGarbage()) {
+ // busy collecting garbage, try again later
+ return;
+ }
+
for (String repositoryName: GitBlit.self().getRepositoryList()) {
RepositoryModel model = GitBlit.self().getRepositoryModel(repositoryName);
if (model.hasCommits && !ArrayUtils.isEmpty(model.indexedBranches)) {
Repository repository = GitBlit.self().getRepository(model.name);
+ if (repository == null) {
+ if (GitBlit.self().isCollectingGarbage(model.name)) {
+ logger.info(MessageFormat.format("Skipping Lucene index of {0}, busy garbage collecting", repositoryName));
+ }
+ continue;
+ }
index(model, repository);
repository.close();
System.gc();
@@ -285,7 +297,7 @@
close(repositoryName);
// delete the index folder
- File repositoryFolder = new File(repositoriesFolder, repositoryName);
+ File repositoryFolder = FileKey.resolve(new File(repositoriesFolder, repositoryName), FS.DETECTED);
File luceneIndex = new File(repositoryFolder, LUCENE_DIR);
if (luceneIndex.exists()) {
org.eclipse.jgit.util.FileUtils.delete(luceneIndex,
@@ -301,7 +313,6 @@
throw new RuntimeException(e);
}
}
-
/**
* Returns the author for the commit, if this information is available.
@@ -413,7 +424,8 @@
if (!deleteIndex(model.name)) {
return result;
}
- try {
+ try {
+ String [] encodings = storedSettings.getStrings(Keys.web.blobEncodings).toArray(new String[0]);
FileBasedConfig config = getConfig(repository);
Set<String> indexedCommits = new TreeSet<String>();
IndexWriter writer = getIndexWriter(model.name);
@@ -493,7 +505,10 @@
Map<String, ObjectId> paths = new TreeMap<String, ObjectId>();
while (treeWalk.next()) {
- paths.put(treeWalk.getPathString(), treeWalk.getObjectId(0));
+ // ensure path is not in a submodule
+ if (treeWalk.getFileMode(0) != FileMode.GITLINK) {
+ paths.put(treeWalk.getPathString(), treeWalk.getObjectId(0));
+ }
}
ByteArrayOutputStream os = new ByteArrayOutputStream();
@@ -563,7 +578,7 @@
}
in.close();
byte[] content = os.toByteArray();
- String str = new String(content, Constants.CHARACTER_ENCODING);
+ String str = StringUtils.decodeString(content, encodings);
doc.add(new Field(FIELD_CONTENT, str, Store.YES, Index.ANALYZED));
os.reset();
}
@@ -648,6 +663,9 @@
Resolution.MINUTE);
IndexWriter writer = getIndexWriter(repositoryName);
for (PathChangeModel path : changedPaths) {
+ if (path.isSubmodule()) {
+ continue;
+ }
// delete the indexed blob
deleteBlob(repositoryName, branch, path.name);
@@ -676,8 +694,10 @@
// read the blob content
String str = JGitUtils.getStringContent(repository, commit.getTree(),
path.path, encodings);
- doc.add(new Field(FIELD_CONTENT, str, Store.YES, Index.ANALYZED));
- writer.addDocument(doc);
+ if (str != null) {
+ doc.add(new Field(FIELD_CONTENT, str, Store.YES, Index.ANALYZED));
+ writer.addDocument(doc);
+ }
}
}
}
@@ -728,8 +748,9 @@
* @param repositoryName
* @param issueId
* @throws Exception
+ * @return true, if deleted, false if no record was deleted
*/
- private void deleteIssue(String repositoryName, String issueId) throws Exception {
+ private boolean deleteIssue(String repositoryName, String issueId) throws Exception {
BooleanQuery query = new BooleanQuery();
Term objectTerm = new Term(FIELD_OBJECT_TYPE, SearchObjectType.issue.name());
query.add(new TermQuery(objectTerm), Occur.MUST);
@@ -737,8 +758,17 @@
query.add(new TermQuery(issueidTerm), Occur.MUST);
IndexWriter writer = getIndexWriter(repositoryName);
+ int numDocsBefore = writer.numDocs();
writer.deleteDocuments(query);
writer.commit();
+ int numDocsAfter = writer.numDocs();
+ if (numDocsBefore == numDocsAfter) {
+ logger.debug(MessageFormat.format("no records found to delete {0}", query.toString()));
+ return false;
+ } else {
+ logger.debug(MessageFormat.format("deleted {0} records with {1}", numDocsBefore - numDocsAfter, query.toString()));
+ return true;
+ }
}
/**
@@ -748,19 +778,29 @@
* @param branch
* @param path
* @throws Exception
+ * @return true, if deleted, false if no record was deleted
*/
- private void deleteBlob(String repositoryName, String branch, String path) throws Exception {
- BooleanQuery query = new BooleanQuery();
- Term objectTerm = new Term(FIELD_OBJECT_TYPE, SearchObjectType.blob.name());
- query.add(new TermQuery(objectTerm), Occur.MUST);
- Term branchTerm = new Term(FIELD_BRANCH, branch);
- query.add(new TermQuery(branchTerm), Occur.MUST);
- Term pathTerm = new Term(FIELD_PATH, path);
- query.add(new TermQuery(pathTerm), Occur.MUST);
+ public boolean deleteBlob(String repositoryName, String branch, String path) throws Exception {
+ String pattern = MessageFormat.format("{0}:'{'0} AND {1}:\"'{'1'}'\" AND {2}:\"'{'2'}'\"", FIELD_OBJECT_TYPE, FIELD_BRANCH, FIELD_PATH);
+ String q = MessageFormat.format(pattern, SearchObjectType.blob.name(), branch, path);
+ BooleanQuery query = new BooleanQuery();
+ StandardAnalyzer analyzer = new StandardAnalyzer(LUCENE_VERSION);
+ QueryParser qp = new QueryParser(LUCENE_VERSION, FIELD_SUMMARY, analyzer);
+ query.add(qp.parse(q), Occur.MUST);
+
IndexWriter writer = getIndexWriter(repositoryName);
- writer.deleteDocuments(query);
+ int numDocsBefore = writer.numDocs();
+ writer.deleteDocuments(query);
writer.commit();
+ int numDocsAfter = writer.numDocs();
+ if (numDocsBefore == numDocsAfter) {
+ logger.debug(MessageFormat.format("no records found to delete {0}", query.toString()));
+ return false;
+ } else {
+ logger.debug(MessageFormat.format("deleted {0} records with {1}", numDocsBefore - numDocsAfter, query.toString()));
+ return true;
+ }
}
/**
@@ -881,7 +921,9 @@
IssueModel issue = IssueUtils.getIssue(repository, issueId);
if (issue == null) {
// issue was deleted, remove from index
- deleteIssue(model.name, issueId);
+ if (!deleteIssue(model.name, issueId)) {
+ logger.error(MessageFormat.format("Failed to delete issue {0} from Lucene index!", issueId));
+ }
} else {
// issue was updated
index(model.name, issue);
@@ -1119,7 +1161,7 @@
qp = new QueryParser(LUCENE_VERSION, FIELD_CONTENT, analyzer);
qp.setAllowLeadingWildcard(true);
query.add(qp.parse(text), Occur.SHOULD);
-
+
IndexSearcher searcher;
if (repositories.length == 1) {
// single repository search
@@ -1135,7 +1177,10 @@
MultiSourceReader reader = new MultiSourceReader(rdrs);
searcher = new IndexSearcher(reader);
}
+
Query rewrittenQuery = searcher.rewrite(query);
+ logger.debug(rewrittenQuery.toString());
+
TopScoreDocCollector collector = TopScoreDocCollector.create(5000, true);
searcher.search(rewrittenQuery, collector);
int offset = Math.max(0, (page - 1) * pageSize);
--
Gitblit v1.9.1