From 47867891efc2aa996fa78f7c224e46d65dc04457 Mon Sep 17 00:00:00 2001
From: James Moger <james.moger@gitblit.com>
Date: Wed, 06 Jun 2012 23:40:30 -0400
Subject: [PATCH] Expose JGit's runtime configuration settings (issue 93)

---
 src/com/gitblit/GitBlit.java               |   26 ++
 docs/04_releases.mkd                       |    7 
 distrib/gitblit.properties                 |  326 +++++++++++++++++++++++------------
 src/com/gitblit/IStoredSettings.java       |   55 ++++++
 src/com/gitblit/utils/FileUtils.java       |   65 +++++++
 tests/com/gitblit/tests/FileUtilsTest.java |   28 +++
 6 files changed, 391 insertions(+), 116 deletions(-)

diff --git a/distrib/gitblit.properties b/distrib/gitblit.properties
index 58833c0..5292a91 100644
--- a/distrib/gitblit.properties
+++ b/distrib/gitblit.properties
@@ -47,6 +47,98 @@
 # SINCE 1.0.0
 git.defaultAccessRestriction = NONE
 
+# Number of bytes of a pack file to load into memory in a single read operation.
+# This is the "page size" of the JGit buffer cache, used for all pack access
+# operations. All disk IO occurs as single window reads. Setting this too large
+# may cause the process to load more data than is required; setting this too small
+# may increase the frequency of read() system calls.
+#
+# Default on JGit is 8 KiB on all platforms.
+#
+# Common unit suffixes of k, m, or g are supported.
+#
+# SINCE 1.0.0
+# RESTART REQUIRED
+git.packedGitWindowSize = 8k
+
+# Maximum number of bytes to load and cache in memory from pack files. If JGit
+# needs to access more than this many bytes it will unload less frequently used
+# windows to reclaim memory space within the process. As this buffer must be shared
+# with the rest of the JVM heap, it should be a fraction of the total memory available.
+#
+# The JGit team recommends setting this value larger than the size of your biggest
+# repository. This ensures you can serve most requests from memory.
+#
+# Default on JGit is 10 MiB on all platforms.
+#
+# Common unit suffixes of k, m, or g are supported.
+#
+# SINCE 1.0.0
+# RESTART REQUIRED
+git.packedGitLimit = 10m
+
+# Maximum number of bytes to reserve for caching base objects that multiple deltafied
+# objects reference. By storing the entire decompressed base object in a cache Git
+# is able to avoid unpacking and decompressing frequently used base objects multiple times.
+#
+# Default on JGit is 10 MiB on all platforms. You probably do not need to adjust
+# this value.
+#
+# Common unit suffixes of k, m, or g are supported.
+#
+# SINCE 1.0.0
+# RESTART REQUIRED
+git.deltaBaseCacheLimit = 10m
+
+# Maximum number of pack files to have open at once. A pack file must be opened
+# in order for any of its data to be available in a cached window.
+#
+# If you increase this to a larger setting you may need to also adjust the ulimit
+# on file descriptors for the host JVM, as Gitblit needs additional file descriptors
+# available for network sockets and other repository data manipulation.
+#
+# Default on JGit is 128 file descriptors on all platforms.
+#
+# SINCE 1.0.0
+# RESTART REQUIRED
+git.packedGitOpenFiles = 128
+
+# Largest object size, in bytes, that JGit will allocate as a contiguous byte
+# array. Any file revision larger than this threshold will have to be streamed,
+# typically requiring the use of temporary files under $GIT_DIR/objects to implement
+# psuedo-random access during delta decompression.
+#
+# Servers with very high traffic should set this to be larger than the size of
+# their common big files. For example a server managing the Android platform
+# typically has to deal with ~10-12 MiB XML files, so 15 m would be a reasonable
+# setting in that environment. Setting this too high may cause the JVM to run out
+# of heap space when handling very big binary files, such as device firmware or
+# CD-ROM ISO images. Make sure to adjust your JVM heap accordingly. 
+#
+# Default is 50 MiB on all platforms.
+#
+# Common unit suffixes of k, m, or g are supported.
+#
+# SINCE 1.0.0
+# RESTART REQUIRED
+git.streamFileThreshold = 50m
+
+# When true, JGit will use mmap() rather than malloc()+read() to load data from
+# pack files.  The use of mmap can be problematic on some JVMs as the garbage
+# collector must deduce that a memory mapped segment is no longer in use before
+# a call to munmap() can be made by the JVM native code.
+#
+# In server applications (such as Gitblit) that need to access many pack files,
+# setting this to true risks artificially running out of virtual address space, 
+# as the garbage collector cannot reclaim unused mapped spaces fast enough.
+#
+# Default on JGit is false. Although potentially slower, it yields much more
+# predictable behavior.
+#
+# SINCE 1.0.0
+# RESTART REQUIRED
+git.packedGitMmap = false
+
 #
 # Groovy Integration
 #
@@ -164,121 +256,6 @@
 #
 # SINCE 0.5.0 
 realm.minPasswordLength = 5
-
-# URL of the LDAP server.
-#
-# SINCE 1.0.0
-realm.ldap.server = ldap://localhost
-
-# Login username for LDAP searches.
-# If this value is unspecified, anonymous LDAP login will be used.
-# 
-# e.g. mydomain\\username
-#
-# SINCE 1.0.0
-realm.ldap.username = cn=Directory Manager
-
-# Login password for LDAP searches.
-#
-# SINCE 1.0.0
-realm.ldap.password = password
-
-# The LdapUserService must be backed by another user service for standard user
-# and team management.
-# default: users.conf
-#
-# SINCE 1.0.0
-# RESTART REQUIRED
-realm.ldap.backingUserService = users.conf
-
-# Delegate team membership control to LDAP.
-#
-# If true, team user memberships will be specified by LDAP groups.  This will
-# disable team selection in Edit User and user selection in Edit Team.
-#
-# If false, LDAP will only be used for authentication and Gitblit will maintain
-# team memberships with the *realm.ldap.backingUserService*.
-#
-# SINCE 1.0.0
-realm.ldap.maintainTeams = false
-
-# Root node for all LDAP users
-#
-# This is the root node from which subtree user searches will begin.
-# If blank, Gitblit will search ALL nodes.
-#
-# SINCE 1.0.0
-realm.ldap.accountBase = OU=Users,OU=UserControl,OU=MyOrganization,DC=MyDomain
-
-# Filter criteria for LDAP users
-#
-# Query pattern to use when searching for a user account. This may be any valid 
-# LDAP query expression, including the standard (&) and (|) operators.
-#
-# Variables may be injected via the ${variableName} syntax.
-# Recognized variables are:
-#    ${username} - The text entered as the user name
-#
-# SINCE 1.0.0
-realm.ldap.accountPattern = (&(objectClass=person)(sAMAccountName=${username}))
-
-# Root node for all LDAP groups to be used as Gitblit Teams
-#
-# This is the root node from which subtree team searches will begin.
-# If blank, Gitblit will search ALL nodes.  
-#
-# SINCE 1.0.0
-realm.ldap.groupBase = OU=Groups,OU=UserControl,OU=MyOrganization,DC=MyDomain
-
-# Filter criteria for LDAP groups
-#
-# Query pattern to use when searching for a team. This may be any valid 
-# LDAP query expression, including the standard (&) and (|) operators.
-#
-# Variables may be injected via the ${variableName} syntax.
-# Recognized variables are:
-#    ${username} - The text entered as the user name
-#    ${dn} - The Distinguished Name of the user logged in
-#
-# All attributes from the LDAP User record are available. For example, if a user
-# has an attribute "fullName" set to "John", "(fn=${fullName})" will be 
-# translated to "(fn=John)".
-#
-# SINCE 1.0.0
-realm.ldap.groupMemberPattern = (&(objectClass=group)(member=${dn}))
-
-# LDAP users or groups that should be given administrator privileges.
-#
-# Teams are specified with a leading '@' character.  Groups with spaces in the
-# name can be entered as "@team name".
-#
-# e.g. realm.ldap.admins = john @git_admins "@git admins"
-#
-# SPACE-DELIMITED
-# SINCE 1.0.0
-realm.ldap.admins = @Git_Admins
-
-# Attribute(s) on the USER record that indicate their display (or full) name.
-# Leave blank for no mapping available in LDAP.
-#
-# This may be a single attribute, or a string of multiple attributes.  Examples:
-#  displayName - Uses the attribute 'displayName' on the user record
-#  ${personalTitle}. ${givenName} ${surname} - Will concatenate the 3 
-#       attributes together, with a '.' after personalTitle
-#
-# SINCE 1.0.0
-realm.ldap.displayName = displayName
-
-# Attribute(s) on the USER record that indicate their email address.
-# Leave blank for no mapping available in LDAP.
-#
-# This may be a single attribute, or a string of multiple attributes.  Examples:
-#  email - Uses the attribute 'email' on the user record
-#  ${givenName}.${surname}@gitblit.com -Will concatenate the 2 attributes
-#       together with a '.' and '@' creating something like first.last@gitblit.com 
-#
-# SINCE 1.0.0
-realm.ldap.email = email
 
 #
 # Gitblit Web Settings
@@ -754,6 +731,125 @@
 #federation.example1.mergeAccounts = true
 
 #
+# Advanced Realm Settings
+#
+
+# URL of the LDAP server.
+#
+# SINCE 1.0.0
+realm.ldap.server = ldap://localhost
+
+# Login username for LDAP searches.
+# If this value is unspecified, anonymous LDAP login will be used.
+# 
+# e.g. mydomain\\username
+#
+# SINCE 1.0.0
+realm.ldap.username = cn=Directory Manager
+
+# Login password for LDAP searches.
+#
+# SINCE 1.0.0
+realm.ldap.password = password
+
+# The LdapUserService must be backed by another user service for standard user
+# and team management.
+# default: users.conf
+#
+# SINCE 1.0.0
+# RESTART REQUIRED
+realm.ldap.backingUserService = users.conf
+
+# Delegate team membership control to LDAP.
+#
+# If true, team user memberships will be specified by LDAP groups.  This will
+# disable team selection in Edit User and user selection in Edit Team.
+#
+# If false, LDAP will only be used for authentication and Gitblit will maintain
+# team memberships with the *realm.ldap.backingUserService*.
+#
+# SINCE 1.0.0
+realm.ldap.maintainTeams = false
+
+# Root node for all LDAP users
+#
+# This is the root node from which subtree user searches will begin.
+# If blank, Gitblit will search ALL nodes.
+#
+# SINCE 1.0.0
+realm.ldap.accountBase = OU=Users,OU=UserControl,OU=MyOrganization,DC=MyDomain
+
+# Filter criteria for LDAP users
+#
+# Query pattern to use when searching for a user account. This may be any valid 
+# LDAP query expression, including the standard (&) and (|) operators.
+#
+# Variables may be injected via the ${variableName} syntax.
+# Recognized variables are:
+#    ${username} - The text entered as the user name
+#
+# SINCE 1.0.0
+realm.ldap.accountPattern = (&(objectClass=person)(sAMAccountName=${username}))
+
+# Root node for all LDAP groups to be used as Gitblit Teams
+#
+# This is the root node from which subtree team searches will begin.
+# If blank, Gitblit will search ALL nodes.  
+#
+# SINCE 1.0.0
+realm.ldap.groupBase = OU=Groups,OU=UserControl,OU=MyOrganization,DC=MyDomain
+
+# Filter criteria for LDAP groups
+#
+# Query pattern to use when searching for a team. This may be any valid 
+# LDAP query expression, including the standard (&) and (|) operators.
+#
+# Variables may be injected via the ${variableName} syntax.
+# Recognized variables are:
+#    ${username} - The text entered as the user name
+#    ${dn} - The Distinguished Name of the user logged in
+#
+# All attributes from the LDAP User record are available. For example, if a user
+# has an attribute "fullName" set to "John", "(fn=${fullName})" will be 
+# translated to "(fn=John)".
+#
+# SINCE 1.0.0
+realm.ldap.groupMemberPattern = (&(objectClass=group)(member=${dn}))
+
+# LDAP users or groups that should be given administrator privileges.
+#
+# Teams are specified with a leading '@' character.  Groups with spaces in the
+# name can be entered as "@team name".
+#
+# e.g. realm.ldap.admins = john @git_admins "@git admins"
+#
+# SPACE-DELIMITED
+# SINCE 1.0.0
+realm.ldap.admins = @Git_Admins
+
+# Attribute(s) on the USER record that indicate their display (or full) name.
+# Leave blank for no mapping available in LDAP.
+#
+# This may be a single attribute, or a string of multiple attributes.  Examples:
+#  displayName - Uses the attribute 'displayName' on the user record
+#  ${personalTitle}. ${givenName} ${surname} - Will concatenate the 3 
+#       attributes together, with a '.' after personalTitle
+#
+# SINCE 1.0.0
+realm.ldap.displayName = displayName
+
+# Attribute(s) on the USER record that indicate their email address.
+# Leave blank for no mapping available in LDAP.
+#
+# This may be a single attribute, or a string of multiple attributes.  Examples:
+#  email - Uses the attribute 'email' on the user record
+#  ${givenName}.${surname}@gitblit.com -Will concatenate the 2 attributes
+#       together with a '.' and '@' creating something like first.last@gitblit.com 
+#
+# SINCE 1.0.0
+realm.ldap.email = email
+
+#
 # Server Settings
 #
 
diff --git a/docs/04_releases.mkd b/docs/04_releases.mkd
index 9e61a82..d20000b 100644
--- a/docs/04_releases.mkd
+++ b/docs/04_releases.mkd
@@ -16,6 +16,13 @@
 
 #### additions
 
+- Exposed JGit's internal configuration settings in gitblit.properties/web.xml (issue 93)  
+    **New:** *git.packedGitWindowSize = 8k*  
+    **New:** *git.packedGitLimit = 10m*  
+    **New:** *git.deltaBaseCacheLimit = 10m*  
+    **New:** *git.packedGitOpenFiles = 128*  
+    **New:** *git.streamFileThreshold = 50m*  
+    **New:** *git.packedGitMmap = false*  
 - Added default access restriction.  Applies to new repositories and repositories that have not been configured with Gitblit. (issue 88)  
     **New:** *git.defaultAccessRestriction = NONE*  
 - Added LDAP User Service with many new *realm.ldap* keys (Github/jcrygier)
diff --git a/src/com/gitblit/GitBlit.java b/src/com/gitblit/GitBlit.java
index 969dc53..f96340a 100644
--- a/src/com/gitblit/GitBlit.java
+++ b/src/com/gitblit/GitBlit.java
@@ -56,6 +56,8 @@
 import org.eclipse.jgit.lib.Repository;
 import org.eclipse.jgit.lib.RepositoryCache.FileKey;
 import org.eclipse.jgit.lib.StoredConfig;
+import org.eclipse.jgit.storage.file.WindowCache;
+import org.eclipse.jgit.storage.file.WindowCacheConfig;
 import org.eclipse.jgit.transport.resolver.FileResolver;
 import org.eclipse.jgit.transport.resolver.RepositoryResolver;
 import org.eclipse.jgit.transport.resolver.ServiceNotAuthorizedException;
@@ -1929,7 +1931,29 @@
 		scheduledExecutor.scheduleAtFixedRate(luceneExecutor, 1, 2, TimeUnit.MINUTES);
 		if (startFederation) {
 			configureFederation();
-		}		
+		}
+		
+		// Configure JGit
+		WindowCacheConfig cfg = new WindowCacheConfig();
+		
+		cfg.setPackedGitWindowSize(settings.getFilesize(Keys.git.packedGitWindowSize, cfg.getPackedGitWindowSize()));
+		cfg.setPackedGitLimit(settings.getFilesize(Keys.git.packedGitLimit, cfg.getPackedGitLimit()));
+		cfg.setDeltaBaseCacheLimit(settings.getFilesize(Keys.git.deltaBaseCacheLimit, cfg.getDeltaBaseCacheLimit()));
+		cfg.setPackedGitOpenFiles(settings.getFilesize(Keys.git.packedGitOpenFiles, cfg.getPackedGitOpenFiles()));
+		cfg.setStreamFileThreshold(settings.getFilesize(Keys.git.streamFileThreshold, cfg.getStreamFileThreshold()));
+		cfg.setPackedGitMMAP(settings.getBoolean(Keys.git.packedGitMmap, cfg.isPackedGitMMAP()));
+		
+		try {
+			WindowCache.reconfigure(cfg);
+			logger.debug(MessageFormat.format("{0} = {1,number,0}", Keys.git.packedGitWindowSize, cfg.getPackedGitWindowSize()));
+			logger.debug(MessageFormat.format("{0} = {1,number,0}", Keys.git.packedGitLimit, cfg.getPackedGitLimit()));
+			logger.debug(MessageFormat.format("{0} = {1,number,0}", Keys.git.deltaBaseCacheLimit, cfg.getDeltaBaseCacheLimit()));
+			logger.debug(MessageFormat.format("{0} = {1,number,0}", Keys.git.packedGitOpenFiles, cfg.getPackedGitOpenFiles()));
+			logger.debug(MessageFormat.format("{0} = {1,number,0}", Keys.git.streamFileThreshold, cfg.getStreamFileThreshold()));
+			logger.debug(MessageFormat.format("{0} = {1}", Keys.git.packedGitMmap, cfg.isPackedGitMMAP()));
+		} catch (IllegalArgumentException e) {
+			logger.error("Failed to configure JGit parameters!", e);
+		}
 	}
 	
 	private void logTimezone(String type, TimeZone zone) {
diff --git a/src/com/gitblit/IStoredSettings.java b/src/com/gitblit/IStoredSettings.java
index e060091..790f8b6 100644
--- a/src/com/gitblit/IStoredSettings.java
+++ b/src/com/gitblit/IStoredSettings.java
@@ -120,6 +120,61 @@
 	}
 
 	/**
+	 * Returns the long value for the specified key. If the key does not
+	 * exist or the value for the key can not be interpreted as an long, the
+	 * defaultValue is returned.
+	 * 
+	 * @param key
+	 * @param defaultValue
+	 * @return key value or defaultValue
+	 */
+	public long getLong(String name, long defaultValue) {
+		Properties props = getSettings();
+		if (props.containsKey(name)) {
+			try {
+				String value = props.getProperty(name);
+				if (!StringUtils.isEmpty(value)) {
+					return Long.parseLong(value.trim());
+				}
+			} catch (NumberFormatException e) {
+				logger.warn("Failed to parse long for " + name + " using default of "
+						+ defaultValue);
+			}
+		}
+		return defaultValue;
+	}
+	
+	/**
+	 * Returns an int filesize from a string value such as 50m or 50mb
+	 * @param name
+	 * @param defaultValue
+	 * @return an int filesize or defaultValue if the key does not exist or can
+	 *         not be parsed
+	 */
+	public int getFilesize(String name, int defaultValue) {
+		String val = getString(name, null);
+		if (StringUtils.isEmpty(val)) {
+			return defaultValue;
+		}
+		return com.gitblit.utils.FileUtils.convertSizeToInt(val, defaultValue);
+	}
+	
+	/**
+	 * Returns an long filesize from a string value such as 50m or 50mb
+	 * @param name
+	 * @param defaultValue
+	 * @return a long filesize or defaultValue if the key does not exist or can
+	 *         not be parsed
+	 */
+	public long getFilesize(String key, long defaultValue) {
+		String val = getString(key, null);
+		if (StringUtils.isEmpty(val)) {
+			return defaultValue;
+		}
+		return com.gitblit.utils.FileUtils.convertSizeToLong(val, defaultValue);
+	}
+
+	/**
 	 * Returns the char value for the specified key. If the key does not exist
 	 * or the value for the key can not be interpreted as a char, the
 	 * defaultValue is returned.
diff --git a/src/com/gitblit/utils/FileUtils.java b/src/com/gitblit/utils/FileUtils.java
index f8d35c8..a14928f 100644
--- a/src/com/gitblit/utils/FileUtils.java
+++ b/src/com/gitblit/utils/FileUtils.java
@@ -34,6 +34,71 @@
  * 
  */
 public class FileUtils {
+	
+	/** 1024 (number of bytes in one kilobyte) */
+	public static final int KB = 1024;
+
+	/** 1024 {@link #KB} (number of bytes in one megabyte) */
+	public static final int MB = 1024 * KB;
+
+	/** 1024 {@link #MB} (number of bytes in one gigabyte) */
+	public static final int GB = 1024 * MB;
+
+	/**
+	 * Returns an int from a string representation of a file size.
+	 * e.g. 50m = 50 megabytes
+	 * 
+	 * @param aString
+	 * @param defaultValue
+	 * @return an int value or the defaultValue if aString can not be parsed
+	 */
+	public static int convertSizeToInt(String aString, int defaultValue) {
+		return (int) convertSizeToLong(aString, defaultValue);
+	}
+	
+	/**
+	 * Returns a long from a string representation of a file size.
+	 * e.g. 50m = 50 megabytes
+	 * 
+	 * @param aString
+	 * @param defaultValue
+	 * @return a long value or the defaultValue if aString can not be parsed
+	 */
+	public static long convertSizeToLong(String aString, long defaultValue) {
+		// trim string and remove all spaces 
+		aString = aString.toLowerCase().trim();
+		StringBuilder sb = new StringBuilder();
+		for (String a : aString.split(" ")) {
+			sb.append(a);
+		}
+		aString = sb.toString();
+		
+		// identify value and unit
+		int idx = 0;
+		int len = aString.length();
+		while (Character.isDigit(aString.charAt(idx))) {
+			idx++;
+			if (idx == len) {
+				break;
+			}
+		}
+		long value = 0;
+		String unit = null;
+		try {
+			value = Long.parseLong(aString.substring(0, idx));
+			unit = aString.substring(idx);
+		} catch (Exception e) {
+			return defaultValue;
+		}
+		if (unit.equals("g") || unit.equals("gb")) {
+			return value * GB;
+		} else if (unit.equals("m") || unit.equals("mb")) {
+			return value * MB;
+		} else if (unit.equals("k") || unit.equals("kb")) {
+			return value * KB;
+		}
+		return defaultValue;
+	}
 
 	/**
 	 * Returns the string content of the specified file.
diff --git a/tests/com/gitblit/tests/FileUtilsTest.java b/tests/com/gitblit/tests/FileUtilsTest.java
index 12161bc..8e5cf8a 100644
--- a/tests/com/gitblit/tests/FileUtilsTest.java
+++ b/tests/com/gitblit/tests/FileUtilsTest.java
@@ -55,4 +55,32 @@
 		size = FileUtils.folderSize(file);
 		assertEquals("size is actually " + size, 11556L, size);
 	}
+	
+	@Test
+	public void testStringSizes() throws Exception {
+		assertEquals(50 * FileUtils.KB, FileUtils.convertSizeToInt("50k", 0));
+		assertEquals(50 * FileUtils.MB, FileUtils.convertSizeToInt("50m", 0));
+		assertEquals(2 * FileUtils.GB, FileUtils.convertSizeToInt("2g", 0));
+
+		assertEquals(50 * FileUtils.KB, FileUtils.convertSizeToInt("50kb", 0));
+		assertEquals(50 * FileUtils.MB, FileUtils.convertSizeToInt("50mb", 0));
+		assertEquals(2 * FileUtils.GB, FileUtils.convertSizeToInt("2gb", 0));
+
+		assertEquals(50L * FileUtils.KB, FileUtils.convertSizeToLong("50k", 0));
+		assertEquals(50L * FileUtils.MB, FileUtils.convertSizeToLong("50m", 0));
+		assertEquals(50L * FileUtils.GB, FileUtils.convertSizeToLong("50g", 0));
+
+		assertEquals(50L * FileUtils.KB, FileUtils.convertSizeToLong("50kb", 0));
+		assertEquals(50L * FileUtils.MB, FileUtils.convertSizeToLong("50mb", 0));
+		assertEquals(50L * FileUtils.GB, FileUtils.convertSizeToLong("50gb", 0));
+		
+		assertEquals(50 * FileUtils.KB, FileUtils.convertSizeToInt("50 k", 0));
+		assertEquals(50 * FileUtils.MB, FileUtils.convertSizeToInt("50 m", 0));
+		assertEquals(2 * FileUtils.GB, FileUtils.convertSizeToInt("2 g", 0));
+
+		assertEquals(50 * FileUtils.KB, FileUtils.convertSizeToInt("50 kb", 0));
+		assertEquals(50 * FileUtils.MB, FileUtils.convertSizeToInt("50 mb", 0));
+		assertEquals(2 * FileUtils.GB, FileUtils.convertSizeToInt("2 gb", 0));
+
+	}
 }
\ No newline at end of file

--
Gitblit v1.9.1