From 4d44cf806ddfa8d051f2d6b1289fa3b67b0daf2e Mon Sep 17 00:00:00 2001
From: James Moger <james.moger@gitblit.com>
Date: Fri, 21 Oct 2011 17:00:07 -0400
Subject: [PATCH] Cache repository sizes and default metrics for performance boost
---
src/com/gitblit/models/ObjectCache.java | 85 ++++++++++++++++++++++++++++
src/com/gitblit/GitBlit.java | 56 +++++++++++++++++-
docs/04_releases.mkd | 2
docs/00_index.mkd | 2
src/com/gitblit/wicket/pages/SummaryPage.java | 3
5 files changed, 143 insertions(+), 5 deletions(-)
diff --git a/docs/00_index.mkd b/docs/00_index.mkd
index ee5faaf..6a2d0d1 100644
--- a/docs/00_index.mkd
+++ b/docs/00_index.mkd
@@ -31,6 +31,8 @@
- improved: updated ui with Twitter's Bootstrap CSS toolkit
**New:** *web.loginMessage = gitblit*
+- improved: repositories list performance by caching repository sizes (issue 27)
+- improved: summary page performance by caching metric calculations (issue 25)
- added: authenticated JSON RPC mechanism
**New:** *web.enableRpcServlet = true*
**New:** *web.enableRpcAdministration = false*
diff --git a/docs/04_releases.mkd b/docs/04_releases.mkd
index fad61d3..ba09075 100644
--- a/docs/04_releases.mkd
+++ b/docs/04_releases.mkd
@@ -5,6 +5,8 @@
- improved: updated ui with Twitter's Bootstrap CSS toolkit
**New:** *web.loginMessage = gitblit*
+- improved: repositories list performance by caching repository sizes (issue 27)
+- improved: summary page performance by caching metric calculations (issue 25)
- added: authenticated JSON RPC mechanism
**New:** *web.enableRpcServlet = true*
**New:** *web.enableRpcAdministration = false*
diff --git a/src/com/gitblit/GitBlit.java b/src/com/gitblit/GitBlit.java
index 50eeb9e..51d5612 100644
--- a/src/com/gitblit/GitBlit.java
+++ b/src/com/gitblit/GitBlit.java
@@ -60,12 +60,15 @@
import com.gitblit.models.FederationModel;
import com.gitblit.models.FederationProposal;
import com.gitblit.models.FederationSet;
+import com.gitblit.models.Metric;
+import com.gitblit.models.ObjectCache;
import com.gitblit.models.RepositoryModel;
import com.gitblit.models.UserModel;
import com.gitblit.utils.ByteFormat;
import com.gitblit.utils.FederationUtils;
import com.gitblit.utils.JGitUtils;
import com.gitblit.utils.JsonUtils;
+import com.gitblit.utils.MetricUtils;
import com.gitblit.utils.StringUtils;
/**
@@ -96,6 +99,10 @@
.synchronizedList(new ArrayList<FederationModel>());
private final Map<String, FederationModel> federationPullResults = new ConcurrentHashMap<String, FederationModel>();
+
+ private final ObjectCache<Long> repositorySizeCache = new ObjectCache<Long>();
+
+ private final ObjectCache<List<Metric>> repositoryMetricsCache = new ObjectCache<List<Metric>>();
private RepositoryResolver<Void> repositoryResolver;
@@ -419,6 +426,16 @@
}
/**
+ * Clears all the cached data for the specified repository.
+ *
+ * @param repositoryName
+ */
+ public void clearRepositoryCache(String repositoryName) {
+ repositorySizeCache.remove(repositoryName);
+ repositoryMetricsCache.remove(repositoryName);
+ }
+
+ /**
* Returns the list of all repositories available to Gitblit. This method
* does not consider user access permissions.
*
@@ -550,14 +567,22 @@
}
/**
- * Returns the size in bytes of the repository.
+ * Returns the size in bytes of the repository. Gitblit caches the
+ * repository sizes to reduce the performance penalty of recursive
+ * calculation. The cache is updated if the repository has been changed
+ * since the last calculation.
*
* @param model
* @return size in bytes
*/
public long calculateSize(RepositoryModel model) {
+ if (repositorySizeCache.hasCurrent(model.name, model.lastChange)) {
+ return repositorySizeCache.getObject(model.name);
+ }
File gitDir = FileKey.resolve(new File(repositoriesFolder, model.name), FS.DETECTED);
- return com.gitblit.utils.FileUtils.folderSize(gitDir);
+ long size = com.gitblit.utils.FileUtils.folderSize(gitDir);
+ repositorySizeCache.updateObject(model.name, model.lastChange, size);
+ return size;
}
/**
@@ -596,7 +621,26 @@
}
/**
- * Returns the gitblit string vlaue for the specified key. If key is not
+ * Returns the metrics for the default branch of the specified repository.
+ * This method builds a metrics cache. The cache is updated if the
+ * repository is updated. A new copy of the metrics list is returned on each
+ * call so that modifications to the list are non-destructive.
+ *
+ * @param model
+ * @param repository
+ * @return a new array list of metrics
+ */
+ public List<Metric> getRepositoryDefaultMetrics(RepositoryModel model, Repository repository) {
+ if (repositoryMetricsCache.hasCurrent(model.name, model.lastChange)) {
+ return new ArrayList<Metric>(repositoryMetricsCache.getObject(model.name));
+ }
+ List<Metric> metrics = MetricUtils.getDateMetrics(repository, null, true, null);
+ repositoryMetricsCache.updateObject(model.name, model.lastChange, metrics);
+ return new ArrayList<Metric>(metrics);
+ }
+
+ /**
+ * Returns the gitblit string value for the specified key. If key is not
* set, returns defaultValue.
*
* @param config
@@ -678,6 +722,9 @@
"Failed to rename repository permissions ''{0}'' to ''{1}''.",
repositoryName, repository.name));
}
+
+ // clear the cache
+ clearRepositoryCache(repositoryName);
}
// load repository
@@ -758,6 +805,9 @@
return true;
}
}
+
+ // clear the repository cache
+ clearRepositoryCache(repositoryName);
} catch (Throwable t) {
logger.error(MessageFormat.format("Failed to delete repository {0}", repositoryName), t);
}
diff --git a/src/com/gitblit/models/ObjectCache.java b/src/com/gitblit/models/ObjectCache.java
new file mode 100644
index 0000000..48ede90
--- /dev/null
+++ b/src/com/gitblit/models/ObjectCache.java
@@ -0,0 +1,85 @@
+/*
+ * Copyright 2011 gitblit.com.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.gitblit.models;
+
+import java.io.Serializable;
+import java.util.Date;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * Reusable object cache.
+ *
+ * @author James Moger
+ *
+ */
+public class ObjectCache<X> implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ private final Map<String, CachedObject<X>> cache = new ConcurrentHashMap<String, CachedObject<X>>();
+
+ private class CachedObject<Y> {
+
+ public final String name;
+
+ private volatile Date date;
+
+ private volatile Y object;
+
+ CachedObject(String name) {
+ this.name = name;
+ date = new Date(0);
+ }
+
+ @Override
+ public String toString() {
+ return getClass().getSimpleName() + ": " + name;
+ }
+ }
+
+ public boolean hasCurrent(String name, Date date) {
+ return cache.containsKey(name) && cache.get(name).date.compareTo(date) == 0;
+ }
+
+ public Date getDate(String name) {
+ return cache.get(name).date;
+ }
+
+ public X getObject(String name) {
+ return cache.get(name).object;
+ }
+
+ public void updateObject(String name, X object) {
+ this.updateObject(name, new Date(), object);
+ }
+
+ public void updateObject(String name, Date date, X object) {
+ CachedObject<X> obj;
+ if (cache.containsKey(name)) {
+ obj = cache.get(name);
+ } else {
+ obj = new CachedObject<X>(name);
+ cache.put(name, obj);
+ }
+ obj.date = date;
+ obj.object = object;
+ }
+
+ public Object remove(String name) {
+ return cache.remove(name).object;
+ }
+}
diff --git a/src/com/gitblit/wicket/pages/SummaryPage.java b/src/com/gitblit/wicket/pages/SummaryPage.java
index 9fd90c5..50810a1 100644
--- a/src/com/gitblit/wicket/pages/SummaryPage.java
+++ b/src/com/gitblit/wicket/pages/SummaryPage.java
@@ -45,7 +45,6 @@
import com.gitblit.models.PathModel;
import com.gitblit.utils.JGitUtils;
import com.gitblit.utils.MarkdownUtils;
-import com.gitblit.utils.MetricUtils;
import com.gitblit.utils.StringUtils;
import com.gitblit.utils.TimeUtils;
import com.gitblit.wicket.WicketUtils;
@@ -68,7 +67,7 @@
List<Metric> metrics = null;
Metric metricsTotal = null;
if (GitBlit.getBoolean(Keys.web.generateActivityGraph, true)) {
- metrics = MetricUtils.getDateMetrics(r, null, true, null);
+ metrics = GitBlit.self().getRepositoryDefaultMetrics(getRepositoryModel(), r);
metricsTotal = metrics.remove(0);
}
--
Gitblit v1.9.1