From 3c4ce1d366641158927cee8b4e47191ed681b524 Mon Sep 17 00:00:00 2001
From: James Moger <james.moger@gitblit.com>
Date: Mon, 24 Jun 2013 14:04:46 -0400
Subject: [PATCH] Improve performance of cache filling
---
src/main/java/com/gitblit/utils/CommitCache.java | 6 +-
src/main/java/com/gitblit/utils/RefLogUtils.java | 4 ++
src/main/java/com/gitblit/wicket/pages/DashboardPage.java | 16 ++++---
src/main/java/com/gitblit/utils/ActivityUtils.java | 4 ++
src/main/java/com/gitblit/GitBlit.java | 42 +++++++++++++++++++-
5 files changed, 59 insertions(+), 13 deletions(-)
diff --git a/src/main/java/com/gitblit/GitBlit.java b/src/main/java/com/gitblit/GitBlit.java
index 6fd168a..eabaa3f 100644
--- a/src/main/java/com/gitblit/GitBlit.java
+++ b/src/main/java/com/gitblit/GitBlit.java
@@ -102,6 +102,7 @@
import com.gitblit.models.GitClientApplication;
import com.gitblit.models.Metric;
import com.gitblit.models.ProjectModel;
+import com.gitblit.models.RefModel;
import com.gitblit.models.RegistrantAccessPermission;
import com.gitblit.models.RepositoryModel;
import com.gitblit.models.RepositoryUrl;
@@ -3402,9 +3403,8 @@
configureJGit();
configureFanout();
configureGitDaemon();
-
- CommitCache.instance().setCacheDays(settings.getInteger(Keys.web.activityCacheDays, 14));
-
+ configureCommitCache();
+
ContainerUtils.CVE_2007_0450.test();
}
@@ -3514,6 +3514,42 @@
}
}
+ protected void configureCommitCache() {
+ int daysToCache = settings.getInteger(Keys.web.activityCacheDays, 14);
+ if (daysToCache <= 0) {
+ logger.info("commit cache disabled");
+ } else {
+ long start = System.nanoTime();
+ long repoCount = 0;
+ long commitCount = 0;
+ logger.info(MessageFormat.format("preparing {0} day commit cache. please wait...", daysToCache));
+ CommitCache.instance().setCacheDays(daysToCache);
+ Date cutoff = CommitCache.instance().getCutoffDate();
+ for (String repositoryName : getRepositoryList()) {
+ RepositoryModel model = getRepositoryModel(repositoryName);
+ if (model.hasCommits && model.lastChange.after(cutoff)) {
+ repoCount++;
+ Repository repository = getRepository(repositoryName);
+ for (RefModel ref : JGitUtils.getLocalBranches(repository, true, -1)) {
+ if (!ref.getDate().after(cutoff)) {
+ // branch not recently updated
+ continue;
+ }
+ List<?> commits = CommitCache.instance().getCommits(repositoryName, repository, ref.getName());
+ if (commits.size() > 0) {
+ logger.info(MessageFormat.format(" cached {0} commits for {1}:{2}",
+ commits.size(), repositoryName, ref.getName()));
+ commitCount += commits.size();
+ }
+ }
+ repository.close();
+ }
+ }
+ logger.info(MessageFormat.format("built {0} day commit cache of {1} commits across {2} repositories in {3} msecs",
+ daysToCache, commitCount, repoCount, TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start)));
+ }
+ }
+
protected final Logger getLogger() {
return logger;
}
diff --git a/src/main/java/com/gitblit/utils/ActivityUtils.java b/src/main/java/com/gitblit/utils/ActivityUtils.java
index fa74350..c4e9587 100644
--- a/src/main/java/com/gitblit/utils/ActivityUtils.java
+++ b/src/main/java/com/gitblit/utils/ActivityUtils.java
@@ -105,6 +105,10 @@
if (StringUtils.isEmpty(objectId)) {
for (RefModel local : JGitUtils.getLocalBranches(
repository, true, -1)) {
+ if (!local.getDate().after(thresholdDate)) {
+ // branch not recently updated
+ continue;
+ }
branches.add(local.getName());
}
} else {
diff --git a/src/main/java/com/gitblit/utils/CommitCache.java b/src/main/java/com/gitblit/utils/CommitCache.java
index e188ff9..9db5f0c 100644
--- a/src/main/java/com/gitblit/utils/CommitCache.java
+++ b/src/main/java/com/gitblit/utils/CommitCache.java
@@ -67,7 +67,7 @@
*
* @return
*/
- protected Date getCacheCutoffDate() {
+ public Date getCutoffDate() {
final Calendar cal = Calendar.getInstance();
cal.setTimeInMillis(System.currentTimeMillis());
cal.set(Calendar.HOUR_OF_DAY, 0);
@@ -118,7 +118,7 @@
* @return a list of commits
*/
public List<RepositoryCommit> getCommits(String repositoryName, Repository repository, String branch) {
- return getCommits(repositoryName, repository, branch, getCacheCutoffDate());
+ return getCommits(repositoryName, repository, branch, getCutoffDate());
}
/**
@@ -134,7 +134,7 @@
*/
public List<RepositoryCommit> getCommits(String repositoryName, Repository repository, String branch, Date sinceDate) {
long start = System.nanoTime();
- Date cacheCutoffDate = getCacheCutoffDate();
+ Date cacheCutoffDate = getCutoffDate();
List<RepositoryCommit> list;
if (cacheDays > 0 && (sinceDate.getTime() >= cacheCutoffDate.getTime())) {
// request fits within the cache window
diff --git a/src/main/java/com/gitblit/utils/RefLogUtils.java b/src/main/java/com/gitblit/utils/RefLogUtils.java
index 73b7146..0dd6652 100644
--- a/src/main/java/com/gitblit/utils/RefLogUtils.java
+++ b/src/main/java/com/gitblit/utils/RefLogUtils.java
@@ -548,6 +548,10 @@
Map<String, DailyLogEntry> dailydigests = new HashMap<String, DailyLogEntry>();
String linearParent = null;
for (RefModel local : JGitUtils.getLocalBranches(repository, true, -1)) {
+ if (!local.getDate().after(minimumDate)) {
+ // branch not recently updated
+ continue;
+ }
String branch = local.getName();
List<RepositoryCommit> commits = CommitCache.instance().getCommits(repositoryName, repository, branch, minimumDate);
for (RepositoryCommit commit : commits) {
diff --git a/src/main/java/com/gitblit/wicket/pages/DashboardPage.java b/src/main/java/com/gitblit/wicket/pages/DashboardPage.java
index db3092d..68a4279 100644
--- a/src/main/java/com/gitblit/wicket/pages/DashboardPage.java
+++ b/src/main/java/com/gitblit/wicket/pages/DashboardPage.java
@@ -16,9 +16,7 @@
package com.gitblit.wicket.pages;
import java.io.Serializable;
-import java.text.DateFormat;
import java.text.MessageFormat;
-import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Collection;
@@ -55,7 +53,6 @@
import com.gitblit.wicket.charting.GoogleChart;
import com.gitblit.wicket.charting.GoogleCharts;
import com.gitblit.wicket.charting.GooglePieChart;
-import com.gitblit.wicket.ng.NgController;
import com.gitblit.wicket.panels.DigestsPanel;
import com.gitblit.wicket.panels.LinkPanel;
@@ -83,10 +80,15 @@
// create daily commit digest feed
List<DailyLogEntry> digests = new ArrayList<DailyLogEntry>();
for (RepositoryModel model : repositories) {
- Repository repository = GitBlit.self().getRepository(model.name);
- List<DailyLogEntry> entries = RefLogUtils.getDailyLogByRef(model.name, repository, minimumDate, timezone);
- digests.addAll(entries);
- repository.close();
+ if (model.isCollectingGarbage) {
+ continue;
+ }
+ if (model.hasCommits && model.lastChange.after(minimumDate)) {
+ Repository repository = GitBlit.self().getRepository(model.name);
+ List<DailyLogEntry> entries = RefLogUtils.getDailyLogByRef(model.name, repository, minimumDate, timezone);
+ digests.addAll(entries);
+ repository.close();
+ }
}
Fragment activityFragment = new Fragment("activity", "activityFragment", this);
--
Gitblit v1.9.1