Extract Federation, Gitblit and Services manager from GitBlit singleton

James Moger

2013-11-21 269c5043ab8f66f67d5719ac5149a436ca1baa2b

 src/main/java/com/gitblit/FederationPullExecutor.java

@@ -1,533 +1,486 @@
/*

 * Copyright 2011 gitblit.com.

 *

 * Licensed under the Apache License, Version 2.0 (the "License");

 * you may not use this file except in compliance with the License.

 * You may obtain a copy of the License at

 *

 *     http://www.apache.org/licenses/LICENSE-2.0

 *

 * Unless required by applicable law or agreed to in writing, software

 * distributed under the License is distributed on an "AS IS" BASIS,

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

 * See the License for the specific language governing permissions and

 * limitations under the License.

 */

package com.gitblit;



import static org.eclipse.jgit.lib.Constants.DOT_GIT_EXT;



import java.io.File;

import java.io.FileOutputStream;

import java.io.IOException;

import java.net.InetAddress;

import java.text.MessageFormat;

import java.util.ArrayList;

import java.util.Arrays;

import java.util.Collection;

import java.util.Date;

import java.util.HashMap;

import java.util.HashSet;

import java.util.List;

import java.util.Map;

import java.util.Properties;

import java.util.Set;

import java.util.concurrent.TimeUnit;



import org.eclipse.jgit.lib.Repository;

import org.eclipse.jgit.lib.StoredConfig;

import org.eclipse.jgit.revwalk.RevCommit;

import org.eclipse.jgit.transport.CredentialsProvider;

import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider;

import org.slf4j.Logger;

import org.slf4j.LoggerFactory;



import com.gitblit.Constants.AccessPermission;

import com.gitblit.Constants.FederationPullStatus;

import com.gitblit.Constants.FederationStrategy;

import com.gitblit.GitBlitException.ForbiddenException;

import com.gitblit.manager.IGitblitManager;

import com.gitblit.manager.INotificationManager;

import com.gitblit.manager.IRepositoryManager;

import com.gitblit.manager.IRuntimeManager;

import com.gitblit.manager.IUserManager;

import com.gitblit.models.FederationModel;

import com.gitblit.models.RefModel;

import com.gitblit.models.RepositoryModel;

import com.gitblit.models.TeamModel;

import com.gitblit.models.UserModel;

import com.gitblit.utils.ArrayUtils;

import com.gitblit.utils.FederationUtils;

import com.gitblit.utils.FileUtils;

import com.gitblit.utils.JGitUtils;

import com.gitblit.utils.JGitUtils.CloneResult;

import com.gitblit.utils.StringUtils;

import com.gitblit.utils.TimeUtils;



/**

 * FederationPullExecutor pulls repository updates and, optionally, user

 * accounts and server settings from registered Gitblit instances.

 */

public class FederationPullExecutor implements Runnable {



   private final Logger logger = LoggerFactory.getLogger(FederationPullExecutor.class);



   private final List<FederationModel> registrations;



   private final boolean isDaemon;



   /**

    * Constructor for specifying a single federation registration. This

    * constructor is used to schedule the next pull execution.

    *

    * @param registration

    */

   private FederationPullExecutor(FederationModel registration) {

      this(Arrays.asList(registration), true);

   }



   /**

    * Constructor to specify a group of federation registrations. This is

    * normally used at startup to pull and then schedule the next update based

    * on each registrations frequency setting.

    *

    * @param registrations

    * @param isDaemon

    *            if true, registrations are rescheduled in perpetuity. if

    *            false, the federation pull operation is executed once.

    */

   public FederationPullExecutor(List<FederationModel> registrations, boolean isDaemon) {

      this.registrations = registrations;

      this.isDaemon = isDaemon;

   }



   /**

    * Run method for this pull executor.

    */

   @Override

   public void run() {

      for (FederationModel registration : registrations) {

         FederationPullStatus was = registration.getLowestStatus();

         try {

            Date now = new Date(System.currentTimeMillis());

            pull(registration);

            sendStatusAcknowledgment(registration);

            registration.lastPull = now;

            FederationPullStatus is = registration.getLowestStatus();

            if (is.ordinal() < was.ordinal()) {

               // the status for this registration has downgraded

               logger.warn("Federation pull status of {0} is now {1}", registration.name,

                     is.name());

               if (registration.notifyOnError) {

                  String message = "Federation pull of " + registration.name + " @ "

                        + registration.url + " is now at " + is.name();

                  INotificationManager mailManager = GitBlit.getManager(INotificationManager.class);

                  mailManager

                        .sendMailToAdministrators(

                              "Pull Status of " + registration.name + " is " + is.name(),

                              message);

               }

            }

         } catch (Throwable t) {

            logger.error(MessageFormat.format(

                  "Failed to pull from federated gitblit ({0} @ {1})", registration.name,

                  registration.url), t);

         } finally {

            if (isDaemon) {

               schedule(registration);

            }

         }

      }

   }



   /**

    * Mirrors a repository and, optionally, the server's users, and/or

    * configuration settings from a origin Gitblit instance.

    *

    * @param registration

    * @throws Exception

    */

   private void pull(FederationModel registration) throws Exception {

      Map<String, RepositoryModel> repositories = FederationUtils.getRepositories(registration,

            true);

      String registrationFolder = registration.folder.toLowerCase().trim();

      // confirm valid characters in server alias

      Character c = StringUtils.findInvalidCharacter(registrationFolder);

      if (c != null) {

         logger.error(MessageFormat

               .format("Illegal character ''{0}'' in folder name ''{1}'' of federation registration {2}!",

                     c, registrationFolder, registration.name));

         return;

      }

      IRepositoryManager repositoryManager = GitBlit.getManager(IRepositoryManager.class);

      File repositoriesFolder = repositoryManager.getRepositoriesFolder();

      File registrationFolderFile = new File(repositoriesFolder, registrationFolder);

      registrationFolderFile.mkdirs();



      // Clone/Pull the repository

      for (Map.Entry<String, RepositoryModel> entry : repositories.entrySet()) {

         String cloneUrl = entry.getKey();

         RepositoryModel repository = entry.getValue();

         if (!repository.hasCommits) {

            logger.warn(MessageFormat.format(

                  "Skipping federated repository {0} from {1} @ {2}. Repository is EMPTY.",

                  repository.name, registration.name, registration.url));

            registration.updateStatus(repository, FederationPullStatus.SKIPPED);

            continue;

         }



         // Determine local repository name

         String repositoryName;

         if (StringUtils.isEmpty(registrationFolder)) {

            repositoryName = repository.name;

         } else {

            repositoryName = registrationFolder + "/" + repository.name;

         }



         if (registration.bare) {

            // bare repository, ensure .git suffix

            if (!repositoryName.toLowerCase().endsWith(DOT_GIT_EXT)) {

               repositoryName += DOT_GIT_EXT;

            }

         } else {

            // normal repository, strip .git suffix

            if (repositoryName.toLowerCase().endsWith(DOT_GIT_EXT)) {

               repositoryName = repositoryName.substring(0,

                     repositoryName.indexOf(DOT_GIT_EXT));

            }

         }



         // confirm that the origin of any pre-existing repository matches

         // the clone url

         String fetchHead = null;

         Repository existingRepository = repositoryManager.getRepository(repositoryName);



         if (existingRepository == null && repositoryManager.isCollectingGarbage(repositoryName)) {

            logger.warn(MessageFormat.format("Skipping local repository {0}, busy collecting garbage", repositoryName));

            continue;

         }



         if (existingRepository != null) {

            StoredConfig config = existingRepository.getConfig();

            config.load();

            String origin = config.getString("remote", "origin", "url");

            RevCommit commit = JGitUtils.getCommit(existingRepository,

                  org.eclipse.jgit.lib.Constants.FETCH_HEAD);

            if (commit != null) {

               fetchHead = commit.getName();

            }

            existingRepository.close();

            if (!origin.startsWith(registration.url)) {

               logger.warn(MessageFormat

                     .format("Skipping federated repository {0} from {1} @ {2}. Origin does not match, consider EXCLUDING.",

                           repository.name, registration.name, registration.url));

               registration.updateStatus(repository, FederationPullStatus.SKIPPED);

               continue;

            }

         }



         // clone/pull this repository

         CredentialsProvider credentials = new UsernamePasswordCredentialsProvider(

               Constants.FEDERATION_USER, registration.token);

         logger.info(MessageFormat.format("Pulling federated repository {0} from {1} @ {2}",

               repository.name, registration.name, registration.url));



         CloneResult result = JGitUtils.cloneRepository(registrationFolderFile, repository.name,

               cloneUrl, registration.bare, credentials);

         Repository r = repositoryManager.getRepository(repositoryName);

         RepositoryModel rm = repositoryManager.getRepositoryModel(repositoryName);

         repository.isFrozen = registration.mirror;

         if (result.createdRepository) {

            // default local settings

            repository.federationStrategy = FederationStrategy.EXCLUDE;

            repository.isFrozen = registration.mirror;

            repository.showRemoteBranches = !registration.mirror;

            logger.info(MessageFormat.format("     cloning {0}", repository.name));

            registration.updateStatus(repository, FederationPullStatus.MIRRORED);

         } else {

            // fetch and update

            boolean fetched = false;

            RevCommit commit = JGitUtils.getCommit(r, org.eclipse.jgit.lib.Constants.FETCH_HEAD);

            String newFetchHead = commit.getName();

            fetched = fetchHead == null || !fetchHead.equals(newFetchHead);



            if (registration.mirror) {

               // mirror

               if (fetched) {

                  // update local branches to match the remote tracking branches

                  for (RefModel ref : JGitUtils.getRemoteBranches(r, false, -1)) {

                     if (ref.displayName.startsWith("origin/")) {

                        String branch = org.eclipse.jgit.lib.Constants.R_HEADS

                              + ref.displayName.substring(ref.displayName.indexOf('/') + 1);

                        String hash = ref.getReferencedObjectId().getName();



                        JGitUtils.setBranchRef(r, branch, hash);

                        logger.info(MessageFormat.format("     resetting {0} of {1} to {2}", branch,

                              repository.name, hash));

                     }

                  }



                  String newHead;

                  if (StringUtils.isEmpty(repository.HEAD)) {

                     newHead = newFetchHead;

                  } else {

                     newHead = repository.HEAD;

                  }

                  JGitUtils.setHEADtoRef(r, newHead);

                  logger.info(MessageFormat.format("     resetting HEAD of {0} to {1}",

                        repository.name, newHead));

                  registration.updateStatus(repository, FederationPullStatus.MIRRORED);

               } else {

                  // indicate no commits pulled

                  registration.updateStatus(repository, FederationPullStatus.NOCHANGE);

               }

            } else {

               // non-mirror

               if (fetched) {

                  // indicate commits pulled to origin/master

                  registration.updateStatus(repository, FederationPullStatus.PULLED);

               } else {

                  // indicate no commits pulled

                  registration.updateStatus(repository, FederationPullStatus.NOCHANGE);

               }

            }



            // preserve local settings

            repository.isFrozen = rm.isFrozen;

            repository.federationStrategy = rm.federationStrategy;



            // merge federation sets

            Set<String> federationSets = new HashSet<String>();

            if (rm.federationSets != null) {

               federationSets.addAll(rm.federationSets);

            }

            if (repository.federationSets != null) {

               federationSets.addAll(repository.federationSets);

            }

            repository.federationSets = new ArrayList<String>(federationSets);



            // merge indexed branches

            Set<String> indexedBranches = new HashSet<String>();

            if (rm.indexedBranches != null) {

               indexedBranches.addAll(rm.indexedBranches);

            }

            if (repository.indexedBranches != null) {

               indexedBranches.addAll(repository.indexedBranches);

            }

            repository.indexedBranches = new ArrayList<String>(indexedBranches);



         }

         // only repositories that are actually _cloned_ from the origin

         // Gitblit repository are marked as federated. If the origin

         // is from somewhere else, these repositories are not considered

         // "federated" repositories.

         repository.isFederated = cloneUrl.startsWith(registration.url);



         repositoryManager.updateConfiguration(r, repository);

         r.close();

      }



      IUserManager userManager = GitBlit.getManager(IUserManager.class);

      IGitblitManager gitblitManager = GitBlit.getManager(IGitblitManager.class);

      IUserService userService = null;



      try {

         // Pull USERS

         // TeamModels are automatically pulled because they are contained

         // within the UserModel. The UserService creates unknown teams

         // and updates existing teams.

         Collection<UserModel> users = FederationUtils.getUsers(registration);

         if (users != null && users.size() > 0) {

            File realmFile = new File(registrationFolderFile, registration.name + "_users.conf");

            realmFile.delete();

            userService = new ConfigUserService(realmFile);

            for (UserModel user : users) {

               userService.updateUserModel(user.username, user);



               // merge the origin permissions and origin accounts into

               // the user accounts of this Gitblit instance

               if (registration.mergeAccounts) {

                  // reparent all repository permissions if the local

                  // repositories are stored within subfolders

                  if (!StringUtils.isEmpty(registrationFolder)) {

                     if (user.permissions != null) {

                        // pulling from >= 1.2 version

                        Map<String, AccessPermission> copy = new HashMap<String, AccessPermission>(user.permissions);

                        user.permissions.clear();

                        for (Map.Entry<String, AccessPermission> entry : copy.entrySet()) {

                           user.setRepositoryPermission(registrationFolder + "/" + entry.getKey(), entry.getValue());

                        }

                     } else {

                        // pulling from <= 1.1 version

                        List<String> permissions = new ArrayList<String>(user.repositories);

                        user.repositories.clear();

                        for (String permission : permissions) {

                           user.addRepositoryPermission(registrationFolder + "/" + permission);

                        }

                     }

                  }



                  // insert new user or update local user

                  UserModel localUser = userManager.getUserModel(user.username);

                  if (localUser == null) {

                     // create new local user

                     gitblitManager.updateUserModel(user.username, user, true);

                  } else {

                     // update repository permissions of local user

                     if (user.permissions != null) {

                        // pulling from >= 1.2 version

                        Map<String, AccessPermission> copy = new HashMap<String, AccessPermission>(user.permissions);

                        for (Map.Entry<String, AccessPermission> entry : copy.entrySet()) {

                           localUser.setRepositoryPermission(entry.getKey(), entry.getValue());

                        }

                     } else {

                        // pulling from <= 1.1 version

                        for (String repository : user.repositories) {

                           localUser.addRepositoryPermission(repository);

                        }

                     }

                     localUser.password = user.password;

                     localUser.canAdmin = user.canAdmin;

                     gitblitManager.updateUserModel(localUser.username, localUser, false);

                  }



                  for (String teamname : userManager.getAllTeamNames()) {

                     TeamModel team = userManager.getTeamModel(teamname);

                     if (user.isTeamMember(teamname) && !team.hasUser(user.username)) {

                        // new team member

                        team.addUser(user.username);

                        userManager.updateTeamModel(teamname, team);

                     } else if (!user.isTeamMember(teamname) && team.hasUser(user.username)) {

                        // remove team member

                        team.removeUser(user.username);

                        userManager.updateTeamModel(teamname, team);

                     }



                     // update team repositories

                     TeamModel remoteTeam = user.getTeam(teamname);

                     if (remoteTeam != null) {

                        if (remoteTeam.permissions != null) {

                           // pulling from >= 1.2

                           for (Map.Entry<String, AccessPermission> entry : remoteTeam.permissions.entrySet()){

                              team.setRepositoryPermission(entry.getKey(), entry.getValue());

                           }

                           userManager.updateTeamModel(teamname, team);

                        } else if(!ArrayUtils.isEmpty(remoteTeam.repositories)) {

                           // pulling from <= 1.1

                           team.addRepositoryPermissions(remoteTeam.repositories);

                           userManager.updateTeamModel(teamname, team);

                        }

                     }

                  }

               }

            }

         }

      } catch (ForbiddenException e) {

         // ignore forbidden exceptions

      } catch (IOException e) {

         logger.warn(MessageFormat.format(

               "Failed to retrieve USERS from federated gitblit ({0} @ {1})",

               registration.name, registration.url), e);

      }



      try {

         // Pull TEAMS

         // We explicitly pull these even though they are embedded in

         // UserModels because it is possible to use teams to specify

         // mailing lists or push scripts without specifying users.

         if (userService != null) {

            Collection<TeamModel> teams = FederationUtils.getTeams(registration);

            if (teams != null && teams.size() > 0) {

               for (TeamModel team : teams) {

                  userService.updateTeamModel(team);

               }

            }

         }

      } catch (ForbiddenException e) {

         // ignore forbidden exceptions

      } catch (IOException e) {

         logger.warn(MessageFormat.format(

               "Failed to retrieve TEAMS from federated gitblit ({0} @ {1})",

               registration.name, registration.url), e);

      }



      try {

         // Pull SETTINGS

         Map<String, String> settings = FederationUtils.getSettings(registration);

         if (settings != null && settings.size() > 0) {

            Properties properties = new Properties();

            properties.putAll(settings);

            FileOutputStream os = new FileOutputStream(new File(registrationFolderFile,

                  registration.name + "_" + Constants.PROPERTIES_FILE));

            properties.store(os, null);

            os.close();

         }

      } catch (ForbiddenException e) {

         // ignore forbidden exceptions

      } catch (IOException e) {

         logger.warn(MessageFormat.format(

               "Failed to retrieve SETTINGS from federated gitblit ({0} @ {1})",

               registration.name, registration.url), e);

      }



      try {

         // Pull SCRIPTS

         Map<String, String> scripts = FederationUtils.getScripts(registration);

         if (scripts != null && scripts.size() > 0) {

            for (Map.Entry<String, String> script : scripts.entrySet()) {

               String scriptName = script.getKey();

               if (scriptName.endsWith(".groovy")) {

                  scriptName = scriptName.substring(0, scriptName.indexOf(".groovy"));

               }

               File file = new File(registrationFolderFile, registration.name + "_"

                     + scriptName + ".groovy");

               FileUtils.writeContent(file, script.getValue());

            }

         }

      } catch (ForbiddenException e) {

         // ignore forbidden exceptions

      } catch (IOException e) {

         logger.warn(MessageFormat.format(

               "Failed to retrieve SCRIPTS from federated gitblit ({0} @ {1})",

               registration.name, registration.url), e);

      }

   }



   /**

    * Sends a status acknowledgment to the origin Gitblit instance. This

    * includes the results of the federated pull.

    *

    * @param registration

    * @throws Exception

    */

   private void sendStatusAcknowledgment(FederationModel registration) throws Exception {

      if (!registration.sendStatus) {

         // skip status acknowledgment

         return;

      }

      InetAddress addr = InetAddress.getLocalHost();

      IStoredSettings settings = GitBlit.getManager(IRuntimeManager.class).getSettings();

      String federationName = settings.getString(Keys.federation.name, null);

      if (StringUtils.isEmpty(federationName)) {

         federationName = addr.getHostName();

      }

      FederationUtils.acknowledgeStatus(addr.getHostAddress(), registration);

      logger.info(MessageFormat.format("Pull status sent to {0}", registration.url));

   }



   /**

    * Schedules the next check of the federated Gitblit instance.

    *

    * @param registration

    */

   private void schedule(FederationModel registration) {

      // schedule the next pull

      int mins = TimeUtils.convertFrequencyToMinutes(registration.frequency);

      registration.nextPull = new Date(System.currentTimeMillis() + (mins * 60 * 1000L));

      GitBlit.self().executor()

            .schedule(new FederationPullExecutor(registration), mins, TimeUnit.MINUTES);

      logger.info(MessageFormat.format(

            "Next pull of {0} @ {1} scheduled for {2,date,yyyy-MM-dd HH:mm}",

            registration.name, registration.url, registration.nextPull));

   }

}

package com.gitblit;

import static org.eclipse.jgit.lib.Constants.DOT_GIT_EXT;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.InetAddress;
import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;

import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.lib.StoredConfig;
import org.eclipse.jgit.revwalk.RevCommit;
import org.eclipse.jgit.transport.CredentialsProvider;
import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.gitblit.Constants.AccessPermission;
import com.gitblit.Constants.FederationPullStatus;
import com.gitblit.Constants.FederationStrategy;
import com.gitblit.GitBlitException.ForbiddenException;
import com.gitblit.models.FederationModel;
import com.gitblit.models.RefModel;
import com.gitblit.models.RepositoryModel;
import com.gitblit.models.TeamModel;
import com.gitblit.models.UserModel;
import com.gitblit.utils.ArrayUtils;
import com.gitblit.utils.FederationUtils;
import com.gitblit.utils.FileUtils;
import com.gitblit.utils.JGitUtils;
import com.gitblit.utils.JGitUtils.CloneResult;
import com.gitblit.utils.StringUtils;

public abstract class FederationPullExecutor implements Runnable {

   Logger logger = LoggerFactory.getLogger(getClass());

   Gitblit gitblit;

   private final List<FederationModel> registrations;

   /**
    * Constructor for specifying a single federation registration. This
    * constructor is used to schedule the next pull execution.
    *
    * @param provider
    * @param registration
    */
   public FederationPullExecutor(FederationModel registration) {
      this(Arrays.asList(registration));
   }

   /**
    * Constructor to specify a group of federation registrations. This is
    * normally used at startup to pull and then schedule the next update based
    * on each registrations frequency setting.
    *
    * @param provider
    * @param registrations
    * @param isDaemon
    *            if true, registrations are rescheduled in perpetuity. if
    *            false, the federation pull operation is executed once.
    */
   public FederationPullExecutor(List<FederationModel> registrations) {
      this.registrations = registrations;
   }

   public abstract void reschedule(FederationModel registration);

   /**
    * Run method for this pull executor.
    */
   @Override
   public void run() {
      for (FederationModel registration : registrations) {
         FederationPullStatus was = registration.getLowestStatus();
         try {
            Date now = new Date(System.currentTimeMillis());
            pull(registration);
            sendStatusAcknowledgment(registration);
            registration.lastPull = now;
            FederationPullStatus is = registration.getLowestStatus();
            if (is.ordinal() < was.ordinal()) {
               // the status for this registration has downgraded
               logger.warn("Federation pull status of {0} is now {1}", registration.name,
                     is.name());
               if (registration.notifyOnError) {
                  String message = "Federation pull of " + registration.name + " @ "
                        + registration.url + " is now at " + is.name();
                  gitblit.sendMailToAdministrators(
                        "Pull Status of " + registration.name + " is " + is.name(),
                        message);
               }
            }
         } catch (Throwable t) {
            logger.error(MessageFormat.format(
                  "Failed to pull from federated gitblit ({0} @ {1})", registration.name,
                  registration.url), t);
         } finally {
            reschedule(registration);
         }
      }
   }

   /**
    * Mirrors a repository and, optionally, the server's users, and/or
    * configuration settings from a origin Gitblit instance.
    *
    * @param registration
    * @throws Exception
    */
   private void pull(FederationModel registration) throws Exception {
      Map<String, RepositoryModel> repositories = FederationUtils.getRepositories(registration,
            true);
      String registrationFolder = registration.folder.toLowerCase().trim();
      // confirm valid characters in server alias
      Character c = StringUtils.findInvalidCharacter(registrationFolder);
      if (c != null) {
         logger.error(MessageFormat
               .format("Illegal character ''{0}'' in folder name ''{1}'' of federation registration {2}!",
                     c, registrationFolder, registration.name));
         return;
      }
      File repositoriesFolder = gitblit.getRepositoriesFolder();
      File registrationFolderFile = new File(repositoriesFolder, registrationFolder);
      registrationFolderFile.mkdirs();

      // Clone/Pull the repository
      for (Map.Entry<String, RepositoryModel> entry : repositories.entrySet()) {
         String cloneUrl = entry.getKey();
         RepositoryModel repository = entry.getValue();
         if (!repository.hasCommits) {
            logger.warn(MessageFormat.format(
                  "Skipping federated repository {0} from {1} @ {2}. Repository is EMPTY.",
                  repository.name, registration.name, registration.url));
            registration.updateStatus(repository, FederationPullStatus.SKIPPED);
            continue;
         }

         // Determine local repository name
         String repositoryName;
         if (StringUtils.isEmpty(registrationFolder)) {
            repositoryName = repository.name;
         } else {
            repositoryName = registrationFolder + "/" + repository.name;
         }

         if (registration.bare) {
            // bare repository, ensure .git suffix
            if (!repositoryName.toLowerCase().endsWith(DOT_GIT_EXT)) {
               repositoryName += DOT_GIT_EXT;
            }
         } else {
            // normal repository, strip .git suffix
            if (repositoryName.toLowerCase().endsWith(DOT_GIT_EXT)) {
               repositoryName = repositoryName.substring(0,
                     repositoryName.indexOf(DOT_GIT_EXT));
            }
         }

         // confirm that the origin of any pre-existing repository matches
         // the clone url
         String fetchHead = null;
         Repository existingRepository = gitblit.getRepository(repositoryName);

         if (existingRepository == null && gitblit.isCollectingGarbage(repositoryName)) {
            logger.warn(MessageFormat.format("Skipping local repository {0}, busy collecting garbage", repositoryName));
            continue;
         }

         if (existingRepository != null) {
            StoredConfig config = existingRepository.getConfig();
            config.load();
            String origin = config.getString("remote", "origin", "url");
            RevCommit commit = JGitUtils.getCommit(existingRepository,
                  org.eclipse.jgit.lib.Constants.FETCH_HEAD);
            if (commit != null) {
               fetchHead = commit.getName();
            }
            existingRepository.close();
            if (!origin.startsWith(registration.url)) {
               logger.warn(MessageFormat
                     .format("Skipping federated repository {0} from {1} @ {2}. Origin does not match, consider EXCLUDING.",
                           repository.name, registration.name, registration.url));
               registration.updateStatus(repository, FederationPullStatus.SKIPPED);
               continue;
            }
         }

         // clone/pull this repository
         CredentialsProvider credentials = new UsernamePasswordCredentialsProvider(
               Constants.FEDERATION_USER, registration.token);
         logger.info(MessageFormat.format("Pulling federated repository {0} from {1} @ {2}",
               repository.name, registration.name, registration.url));

         CloneResult result = JGitUtils.cloneRepository(registrationFolderFile, repository.name,
               cloneUrl, registration.bare, credentials);
         Repository r = gitblit.getRepository(repositoryName);
         RepositoryModel rm = gitblit.getRepositoryModel(repositoryName);
         repository.isFrozen = registration.mirror;
         if (result.createdRepository) {
            // default local settings
            repository.federationStrategy = FederationStrategy.EXCLUDE;
            repository.isFrozen = registration.mirror;
            repository.showRemoteBranches = !registration.mirror;
            logger.info(MessageFormat.format("     cloning {0}", repository.name));
            registration.updateStatus(repository, FederationPullStatus.MIRRORED);
         } else {
            // fetch and update
            boolean fetched = false;
            RevCommit commit = JGitUtils.getCommit(r, org.eclipse.jgit.lib.Constants.FETCH_HEAD);
            String newFetchHead = commit.getName();
            fetched = fetchHead == null || !fetchHead.equals(newFetchHead);

            if (registration.mirror) {
               // mirror
               if (fetched) {
                  // update local branches to match the remote tracking branches
                  for (RefModel ref : JGitUtils.getRemoteBranches(r, false, -1)) {
                     if (ref.displayName.startsWith("origin/")) {
                        String branch = org.eclipse.jgit.lib.Constants.R_HEADS
                              + ref.displayName.substring(ref.displayName.indexOf('/') + 1);
                        String hash = ref.getReferencedObjectId().getName();

                        JGitUtils.setBranchRef(r, branch, hash);
                        logger.info(MessageFormat.format("     resetting {0} of {1} to {2}", branch,
                              repository.name, hash));
                     }
                  }

                  String newHead;
                  if (StringUtils.isEmpty(repository.HEAD)) {
                     newHead = newFetchHead;
                  } else {
                     newHead = repository.HEAD;
                  }
                  JGitUtils.setHEADtoRef(r, newHead);
                  logger.info(MessageFormat.format("     resetting HEAD of {0} to {1}",
                        repository.name, newHead));
                  registration.updateStatus(repository, FederationPullStatus.MIRRORED);
               } else {
                  // indicate no commits pulled
                  registration.updateStatus(repository, FederationPullStatus.NOCHANGE);
               }
            } else {
               // non-mirror
               if (fetched) {
                  // indicate commits pulled to origin/master
                  registration.updateStatus(repository, FederationPullStatus.PULLED);
               } else {
                  // indicate no commits pulled
                  registration.updateStatus(repository, FederationPullStatus.NOCHANGE);
               }
            }

            // preserve local settings
            repository.isFrozen = rm.isFrozen;
            repository.federationStrategy = rm.federationStrategy;

            // merge federation sets
            Set<String> federationSets = new HashSet<String>();
            if (rm.federationSets != null) {
               federationSets.addAll(rm.federationSets);
            }
            if (repository.federationSets != null) {
               federationSets.addAll(repository.federationSets);
            }
            repository.federationSets = new ArrayList<String>(federationSets);

            // merge indexed branches
            Set<String> indexedBranches = new HashSet<String>();
            if (rm.indexedBranches != null) {
               indexedBranches.addAll(rm.indexedBranches);
            }
            if (repository.indexedBranches != null) {
               indexedBranches.addAll(repository.indexedBranches);
            }
            repository.indexedBranches = new ArrayList<String>(indexedBranches);

         }
         // only repositories that are actually _cloned_ from the origin
         // Gitblit repository are marked as federated. If the origin
         // is from somewhere else, these repositories are not considered
         // "federated" repositories.
         repository.isFederated = cloneUrl.startsWith(registration.url);

         gitblit.updateConfiguration(r, repository);
         r.close();
      }

      IUserService userService = null;

      try {
         // Pull USERS
         // TeamModels are automatically pulled because they are contained
         // within the UserModel. The UserService creates unknown teams
         // and updates existing teams.
         Collection<UserModel> users = FederationUtils.getUsers(registration);
         if (users != null && users.size() > 0) {
            File realmFile = new File(registrationFolderFile, registration.name + "_users.conf");
            realmFile.delete();
            userService = new ConfigUserService(realmFile);
            for (UserModel user : users) {
               userService.updateUserModel(user.username, user);

               // merge the origin permissions and origin accounts into
               // the user accounts of this Gitblit instance
               if (registration.mergeAccounts) {
                  // reparent all repository permissions if the local
                  // repositories are stored within subfolders
                  if (!StringUtils.isEmpty(registrationFolder)) {
                     if (user.permissions != null) {
                        // pulling from >= 1.2 version
                        Map<String, AccessPermission> copy = new HashMap<String, AccessPermission>(user.permissions);
                        user.permissions.clear();
                        for (Map.Entry<String, AccessPermission> entry : copy.entrySet()) {
                           user.setRepositoryPermission(registrationFolder + "/" + entry.getKey(), entry.getValue());
                        }
                     } else {
                        // pulling from <= 1.1 version
                        List<String> permissions = new ArrayList<String>(user.repositories);
                        user.repositories.clear();
                        for (String permission : permissions) {
                           user.addRepositoryPermission(registrationFolder + "/" + permission);
                        }
                     }
                  }

                  // insert new user or update local user
                  UserModel localUser = gitblit.getUserModel(user.username);
                  if (localUser == null) {
                     // create new local user
                     gitblit.updateUserModel(user.username, user, true);
                  } else {
                     // update repository permissions of local user
                     if (user.permissions != null) {
                        // pulling from >= 1.2 version
                        Map<String, AccessPermission> copy = new HashMap<String, AccessPermission>(user.permissions);
                        for (Map.Entry<String, AccessPermission> entry : copy.entrySet()) {
                           localUser.setRepositoryPermission(entry.getKey(), entry.getValue());
                        }
                     } else {
                        // pulling from <= 1.1 version
                        for (String repository : user.repositories) {
                           localUser.addRepositoryPermission(repository);
                        }
                     }
                     localUser.password = user.password;
                     localUser.canAdmin = user.canAdmin;
                     gitblit.updateUserModel(localUser.username, localUser, false);
                  }

                  for (String teamname : gitblit.getAllTeamNames()) {
                     TeamModel team = gitblit.getTeamModel(teamname);
                     if (user.isTeamMember(teamname) && !team.hasUser(user.username)) {
                        // new team member
                        team.addUser(user.username);
                        gitblit.updateTeamModel(teamname, team);
                     } else if (!user.isTeamMember(teamname) && team.hasUser(user.username)) {
                        // remove team member
                        team.removeUser(user.username);
                        gitblit.updateTeamModel(teamname, team);
                     }

                     // update team repositories
                     TeamModel remoteTeam = user.getTeam(teamname);
                     if (remoteTeam != null) {
                        if (remoteTeam.permissions != null) {
                           // pulling from >= 1.2
                           for (Map.Entry<String, AccessPermission> entry : remoteTeam.permissions.entrySet()){
                              team.setRepositoryPermission(entry.getKey(), entry.getValue());
                           }
                           gitblit.updateTeamModel(teamname, team);
                        } else if (!ArrayUtils.isEmpty(remoteTeam.repositories)) {
                           // pulling from <= 1.1
                           team.addRepositoryPermissions(remoteTeam.repositories);
                           gitblit.updateTeamModel(teamname, team);
                        }
                     }
                  }
               }
            }
         }
      } catch (ForbiddenException e) {
         // ignore forbidden exceptions
      } catch (IOException e) {
         logger.warn(MessageFormat.format(
               "Failed to retrieve USERS from federated gitblit ({0} @ {1})",
               registration.name, registration.url), e);
      }

      try {
         // Pull TEAMS
         // We explicitly pull these even though they are embedded in
         // UserModels because it is possible to use teams to specify
         // mailing lists or push scripts without specifying users.
         if (userService != null) {
            Collection<TeamModel> teams = FederationUtils.getTeams(registration);
            if (teams != null && teams.size() > 0) {
               for (TeamModel team : teams) {
                  userService.updateTeamModel(team);
               }
            }
         }
      } catch (ForbiddenException e) {
         // ignore forbidden exceptions
      } catch (IOException e) {
         logger.warn(MessageFormat.format(
               "Failed to retrieve TEAMS from federated gitblit ({0} @ {1})",
               registration.name, registration.url), e);
      }

      try {
         // Pull SETTINGS
         Map<String, String> settings = FederationUtils.getSettings(registration);
         if (settings != null && settings.size() > 0) {
            Properties properties = new Properties();
            properties.putAll(settings);
            FileOutputStream os = new FileOutputStream(new File(registrationFolderFile,
                  registration.name + "_" + Constants.PROPERTIES_FILE));
            properties.store(os, null);
            os.close();
         }
      } catch (ForbiddenException e) {
         // ignore forbidden exceptions
      } catch (IOException e) {
         logger.warn(MessageFormat.format(
               "Failed to retrieve SETTINGS from federated gitblit ({0} @ {1})",
               registration.name, registration.url), e);
      }

      try {
         // Pull SCRIPTS
         Map<String, String> scripts = FederationUtils.getScripts(registration);
         if (scripts != null && scripts.size() > 0) {
            for (Map.Entry<String, String> script : scripts.entrySet()) {
               String scriptName = script.getKey();
               if (scriptName.endsWith(".groovy")) {
                  scriptName = scriptName.substring(0, scriptName.indexOf(".groovy"));
               }
               File file = new File(registrationFolderFile, registration.name + "_"
                     + scriptName + ".groovy");
               FileUtils.writeContent(file, script.getValue());
            }
         }
      } catch (ForbiddenException e) {
         // ignore forbidden exceptions
      } catch (IOException e) {
         logger.warn(MessageFormat.format(
               "Failed to retrieve SCRIPTS from federated gitblit ({0} @ {1})",
               registration.name, registration.url), e);
      }
   }

   /**
    * Sends a status acknowledgment to the origin Gitblit instance. This
    * includes the results of the federated pull.
    *
    * @param registration
    * @throws Exception
    */
   private void sendStatusAcknowledgment(FederationModel registration) throws Exception {
      if (!registration.sendStatus) {
         // skip status acknowledgment
         return;
      }
      InetAddress addr = InetAddress.getLocalHost();
      String federationName = gitblit.getSettings().getString(Keys.federation.name, null);
      if (StringUtils.isEmpty(federationName)) {
         federationName = addr.getHostName();
      }
      FederationUtils.acknowledgeStatus(addr.getHostAddress(), registration);
      logger.info(MessageFormat.format("Pull status sent to {0}", registration.url));
   }
}