From 0e44acbb2fec928a1606dc60f427a148fff405c9 Mon Sep 17 00:00:00 2001
From: Mohamed Ragab <moragab@gmail.com>
Date: Wed, 02 May 2012 11:15:01 -0400
Subject: [PATCH] Added a script to facilitate setting the proxy host and port and no proxy hosts, and then it concatenates all the java system properties for setting the java proxy configurations and puts the resulting string in an environment variable JAVA_PROXY_CONFIG, modified the scirpts gitblit, gitblit-ubuntu, and gitblit-centos to source the java-proxy-config.sh script and then include the resulting java proxy configuration in the java command
---
src/com/gitblit/FederationPullExecutor.java | 237 ++++++++++++++++++++++++++++++++++++++++++++++++++++------
1 files changed, 210 insertions(+), 27 deletions(-)
diff --git a/src/com/gitblit/FederationPullExecutor.java b/src/com/gitblit/FederationPullExecutor.java
index 127e1fc..432e293 100644
--- a/src/com/gitblit/FederationPullExecutor.java
+++ b/src/com/gitblit/FederationPullExecutor.java
@@ -15,21 +15,31 @@
*/
package com.gitblit;
+import static org.eclipse.jgit.lib.Constants.DOT_GIT_EXT;
+
import java.io.File;
import java.io.FileOutputStream;
+import java.io.IOException;
import java.net.InetAddress;
import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Date;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Properties;
+import java.util.Set;
import java.util.concurrent.TimeUnit;
+import org.eclipse.jgit.api.Git;
+import org.eclipse.jgit.api.ResetCommand;
+import org.eclipse.jgit.api.ResetCommand.ResetType;
+import org.eclipse.jgit.lib.Ref;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.lib.StoredConfig;
+import org.eclipse.jgit.revwalk.RevCommit;
import org.eclipse.jgit.transport.CredentialsProvider;
import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider;
import org.slf4j.Logger;
@@ -37,14 +47,19 @@
import com.gitblit.Constants.FederationPullStatus;
import com.gitblit.Constants.FederationStrategy;
+import com.gitblit.GitBlitException.ForbiddenException;
import com.gitblit.models.FederationModel;
+import com.gitblit.models.RefModel;
import com.gitblit.models.RepositoryModel;
+import com.gitblit.models.TeamModel;
import com.gitblit.models.UserModel;
+import com.gitblit.utils.ArrayUtils;
import com.gitblit.utils.FederationUtils;
+import com.gitblit.utils.FileUtils;
import com.gitblit.utils.JGitUtils;
-import com.gitblit.utils.TimeUtils;
import com.gitblit.utils.JGitUtils.CloneResult;
import com.gitblit.utils.StringUtils;
+import com.gitblit.utils.TimeUtils;
/**
* FederationPullExecutor pulls repository updates and, optionally, user
@@ -56,6 +71,8 @@
private final List<FederationModel> registrations;
+ private final boolean isDaemon;
+
/**
* Constructor for specifying a single federation registration. This
* constructor is used to schedule the next pull execution.
@@ -63,7 +80,7 @@
* @param registration
*/
private FederationPullExecutor(FederationModel registration) {
- this(Arrays.asList(registration));
+ this(Arrays.asList(registration), true);
}
/**
@@ -72,9 +89,13 @@
* on each registrations frequency setting.
*
* @param registrations
+ * @param isDaemon
+ * if true, registrations are rescheduled in perpetuity. if
+ * false, the federation pull operation is executed once.
*/
- public FederationPullExecutor(List<FederationModel> registrations) {
+ public FederationPullExecutor(List<FederationModel> registrations, boolean isDaemon) {
this.registrations = registrations;
+ this.isDaemon = isDaemon;
}
/**
@@ -98,7 +119,7 @@
String message = "Federation pull of " + registration.name + " @ "
+ registration.url + " is now at " + is.name();
GitBlit.self()
- .notifyAdministrators(
+ .sendMailToAdministrators(
"Pull Status of " + registration.name + " is " + is.name(),
message);
}
@@ -108,14 +129,16 @@
"Failed to pull from federated gitblit ({0} @ {1})", registration.name,
registration.url), t);
} finally {
- schedule(registration);
+ if (isDaemon) {
+ schedule(registration);
+ }
}
}
}
/**
* Mirrors a repository and, optionally, the server's users, and/or
- * configuration settings from a remote Gitblit instance.
+ * configuration settings from a origin Gitblit instance.
*
* @param registration
* @throws Exception
@@ -148,6 +171,7 @@
continue;
}
+ // Determine local repository name
String repositoryName;
if (StringUtils.isEmpty(registrationFolder)) {
repositoryName = repository.name;
@@ -155,13 +179,32 @@
repositoryName = registrationFolder + "/" + repository.name;
}
+ if (registration.bare) {
+ // bare repository, ensure .git suffix
+ if (!repositoryName.toLowerCase().endsWith(DOT_GIT_EXT)) {
+ repositoryName += DOT_GIT_EXT;
+ }
+ } else {
+ // normal repository, strip .git suffix
+ if (repositoryName.toLowerCase().endsWith(DOT_GIT_EXT)) {
+ repositoryName = repositoryName.substring(0,
+ repositoryName.indexOf(DOT_GIT_EXT));
+ }
+ }
+
// confirm that the origin of any pre-existing repository matches
// the clone url
+ String fetchHead = null;
Repository existingRepository = GitBlit.self().getRepository(repositoryName);
if (existingRepository != null) {
StoredConfig config = existingRepository.getConfig();
config.load();
String origin = config.getString("remote", "origin", "url");
+ RevCommit commit = JGitUtils.getCommit(existingRepository,
+ org.eclipse.jgit.lib.Constants.FETCH_HEAD);
+ if (commit != null) {
+ fetchHead = commit.getName();
+ }
existingRepository.close();
if (!origin.startsWith(registration.url)) {
logger.warn(MessageFormat
@@ -177,20 +220,90 @@
Constants.FEDERATION_USER, registration.token);
logger.info(MessageFormat.format("Pulling federated repository {0} from {1} @ {2}",
repository.name, registration.name, registration.url));
+
CloneResult result = JGitUtils.cloneRepository(registrationFolderFile, repository.name,
- cloneUrl, credentials);
+ cloneUrl, registration.bare, credentials);
Repository r = GitBlit.self().getRepository(repositoryName);
RepositoryModel rm = GitBlit.self().getRepositoryModel(repositoryName);
+ repository.isFrozen = registration.mirror;
if (result.createdRepository) {
// default local settings
repository.federationStrategy = FederationStrategy.EXCLUDE;
- repository.isFrozen = true;
+ repository.isFrozen = registration.mirror;
+ repository.showRemoteBranches = !registration.mirror;
+ logger.info(MessageFormat.format(" cloning {0}", repository.name));
+ registration.updateStatus(repository, FederationPullStatus.MIRRORED);
} else {
+ // fetch and update
+ boolean fetched = false;
+ RevCommit commit = JGitUtils.getCommit(r, org.eclipse.jgit.lib.Constants.FETCH_HEAD);
+ String newFetchHead = commit.getName();
+ fetched = fetchHead == null || !fetchHead.equals(newFetchHead);
+
+ if (registration.mirror) {
+ // mirror
+ if (fetched) {
+ // find the first branch name that FETCH_HEAD points to
+ List<RefModel> refs = JGitUtils.getAllRefs(r).get(commit.getId());
+ if (!ArrayUtils.isEmpty(refs)) {
+ for (RefModel ref : refs) {
+ if (ref.displayName.startsWith(org.eclipse.jgit.lib.Constants.R_REMOTES)) {
+ newFetchHead = ref.displayName;
+ break;
+ }
+ }
+ }
+ // reset HEAD to the FETCH_HEAD branch.
+ // if no branch was found, reset HEAD to the commit id.
+ Git git = new Git(r);
+ ResetCommand reset = git.reset();
+ reset.setMode(ResetType.SOFT);
+ reset.setRef(newFetchHead);
+ Ref ref = reset.call();
+ logger.info(MessageFormat.format(" resetting HEAD of {0} to {1}",
+ repository.name, ref.getObjectId().getName()));
+ registration.updateStatus(repository, FederationPullStatus.MIRRORED);
+ } else {
+ // indicate no commits pulled
+ registration.updateStatus(repository, FederationPullStatus.NOCHANGE);
+ }
+ } else {
+ // non-mirror
+ if (fetched) {
+ // indicate commits pulled to origin/master
+ registration.updateStatus(repository, FederationPullStatus.PULLED);
+ } else {
+ // indicate no commits pulled
+ registration.updateStatus(repository, FederationPullStatus.NOCHANGE);
+ }
+ }
+
// preserve local settings
repository.isFrozen = rm.isFrozen;
repository.federationStrategy = rm.federationStrategy;
+
+ // merge federation sets
+ Set<String> federationSets = new HashSet<String>();
+ if (rm.federationSets != null) {
+ federationSets.addAll(rm.federationSets);
+ }
+ if (repository.federationSets != null) {
+ federationSets.addAll(repository.federationSets);
+ }
+ repository.federationSets = new ArrayList<String>(federationSets);
+
+ // merge indexed branches
+ Set<String> indexedBranches = new HashSet<String>();
+ if (rm.indexedBranches != null) {
+ indexedBranches.addAll(rm.indexedBranches);
+ }
+ if (repository.indexedBranches != null) {
+ indexedBranches.addAll(repository.indexedBranches);
+ }
+ repository.indexedBranches = new ArrayList<String>(indexedBranches);
+
}
- // only repositories that are actually _cloned_ from the source
+ // only repositories that are actually _cloned_ from the origin
// Gitblit repository are marked as federated. If the origin
// is from somewhere else, these repositories are not considered
// "federated" repositories.
@@ -198,21 +311,24 @@
GitBlit.self().updateConfiguration(r, repository);
r.close();
- registration.updateStatus(repository, FederationPullStatus.PULLED);
}
+
+ IUserService userService = null;
try {
// Pull USERS
+ // TeamModels are automatically pulled because they are contained
+ // within the UserModel. The UserService creates unknown teams
+ // and updates existing teams.
Collection<UserModel> users = FederationUtils.getUsers(registration);
if (users != null && users.size() > 0) {
- File realmFile = new File(registrationFolderFile, registration.name
- + "_users.properties");
+ File realmFile = new File(registrationFolderFile, registration.name + "_users.conf");
realmFile.delete();
- FileUserService userService = new FileUserService(realmFile);
+ userService = new ConfigUserService(realmFile);
for (UserModel user : users) {
userService.updateUserModel(user.username, user);
- // merge the remote permissions and remote accounts into
+ // merge the origin permissions and origin accounts into
// the user accounts of this Gitblit instance
if (registration.mergeAccounts) {
// reparent all repository permissions if the local
@@ -239,16 +355,61 @@
localUser.canAdmin = user.canAdmin;
GitBlit.self().updateUserModel(localUser.username, localUser, false);
}
+
+ for (String teamname : GitBlit.self().getAllTeamnames()) {
+ TeamModel team = GitBlit.self().getTeamModel(teamname);
+ if (user.isTeamMember(teamname) && !team.hasUser(user.username)) {
+ // new team member
+ team.addUser(user.username);
+ GitBlit.self().updateTeamModel(teamname, team, false);
+ } else if (!user.isTeamMember(teamname) && team.hasUser(user.username)) {
+ // remove team member
+ team.removeUser(user.username);
+ GitBlit.self().updateTeamModel(teamname, team, false);
+ }
+
+ // update team repositories
+ TeamModel remoteTeam = user.getTeam(teamname);
+ if (remoteTeam != null && !ArrayUtils.isEmpty(remoteTeam.repositories)) {
+ int before = team.repositories.size();
+ team.addRepositories(remoteTeam.repositories);
+ int after = team.repositories.size();
+ if (after > before) {
+ // repository count changed, update
+ GitBlit.self().updateTeamModel(teamname, team, false);
+ }
+ }
+ }
}
}
}
- } catch (Exception e) {
- // a 403 error code is normal for a PULL_REPOSITORIES token
- if (!e.getMessage().contains("403")) {
- logger.warn(MessageFormat.format(
- "Failed to retrieve USERS from federated gitblit ({0} @ {1})",
- registration.name, registration.url), e);
+ } catch (ForbiddenException e) {
+ // ignore forbidden exceptions
+ } catch (IOException e) {
+ logger.warn(MessageFormat.format(
+ "Failed to retrieve USERS from federated gitblit ({0} @ {1})",
+ registration.name, registration.url), e);
+ }
+
+ try {
+ // Pull TEAMS
+ // We explicitly pull these even though they are embedded in
+ // UserModels because it is possible to use teams to specify
+ // mailing lists or push scripts without specifying users.
+ if (userService != null) {
+ Collection<TeamModel> teams = FederationUtils.getTeams(registration);
+ if (teams != null && teams.size() > 0) {
+ for (TeamModel team : teams) {
+ userService.updateTeamModel(team);
+ }
+ }
}
+ } catch (ForbiddenException e) {
+ // ignore forbidden exceptions
+ } catch (IOException e) {
+ logger.warn(MessageFormat.format(
+ "Failed to retrieve TEAMS from federated gitblit ({0} @ {1})",
+ registration.name, registration.url), e);
}
try {
@@ -262,18 +423,39 @@
properties.store(os, null);
os.close();
}
- } catch (Exception e) {
- // a 403 error code is normal for a PULL_REPOSITORIES token
- if (!e.getMessage().contains("403")) {
- logger.warn(MessageFormat.format(
- "Failed to retrieve SETTINGS from federated gitblit ({0} @ {1})",
- registration.name, registration.url), e);
+ } catch (ForbiddenException e) {
+ // ignore forbidden exceptions
+ } catch (IOException e) {
+ logger.warn(MessageFormat.format(
+ "Failed to retrieve SETTINGS from federated gitblit ({0} @ {1})",
+ registration.name, registration.url), e);
+ }
+
+ try {
+ // Pull SCRIPTS
+ Map<String, String> scripts = FederationUtils.getScripts(registration);
+ if (scripts != null && scripts.size() > 0) {
+ for (Map.Entry<String, String> script : scripts.entrySet()) {
+ String scriptName = script.getKey();
+ if (scriptName.endsWith(".groovy")) {
+ scriptName = scriptName.substring(0, scriptName.indexOf(".groovy"));
+ }
+ File file = new File(registrationFolderFile, registration.name + "_"
+ + scriptName + ".groovy");
+ FileUtils.writeContent(file, script.getValue());
+ }
}
+ } catch (ForbiddenException e) {
+ // ignore forbidden exceptions
+ } catch (IOException e) {
+ logger.warn(MessageFormat.format(
+ "Failed to retrieve SCRIPTS from federated gitblit ({0} @ {1})",
+ registration.name, registration.url), e);
}
}
/**
- * Sends a status acknowledgment to the source Gitblit instance. This
+ * Sends a status acknowledgment to the origin Gitblit instance. This
* includes the results of the federated pull.
*
* @param registration
@@ -290,6 +472,7 @@
federationName = addr.getHostName();
}
FederationUtils.acknowledgeStatus(addr.getHostAddress(), registration);
+ logger.info(MessageFormat.format("Pull status sent to {0}", registration.url));
}
/**
--
Gitblit v1.9.1