From eecaad8b8e2c447429c31a01d49260ddd6b4ee03 Mon Sep 17 00:00:00 2001
From: Paul Martin <paul@paulsputer.com>
Date: Sat, 16 Apr 2016 17:35:32 -0400
Subject: [PATCH] Proof of concept #1026
---
src/main/java/com/gitblit/utils/CompressionUtils.java | 134 ++++++++++++++++++++++++++++++++------------
1 files changed, 97 insertions(+), 37 deletions(-)
diff --git a/src/main/java/com/gitblit/utils/CompressionUtils.java b/src/main/java/com/gitblit/utils/CompressionUtils.java
index a8dcdd8..1b8e6fc 100644
--- a/src/main/java/com/gitblit/utils/CompressionUtils.java
+++ b/src/main/java/com/gitblit/utils/CompressionUtils.java
@@ -15,6 +15,10 @@
*/
package com.gitblit.utils;
+import java.io.ByteArrayOutputStream;
+import java.io.EOFException;
+import java.io.File;
+import java.io.FileInputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.text.MessageFormat;
@@ -27,10 +31,11 @@
import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream;
import org.apache.commons.compress.compressors.CompressorException;
import org.apache.commons.compress.compressors.CompressorStreamFactory;
-import org.apache.wicket.util.io.ByteArrayOutputStream;
+import org.apache.commons.io.IOUtils;
import org.eclipse.jgit.lib.Constants;
import org.eclipse.jgit.lib.FileMode;
import org.eclipse.jgit.lib.MutableObjectId;
+import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.ObjectLoader;
import org.eclipse.jgit.lib.ObjectReader;
import org.eclipse.jgit.lib.Repository;
@@ -41,11 +46,16 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.gitblit.GitBlit;
+import com.gitblit.manager.IFilestoreManager;
+import com.gitblit.models.FilestoreModel;
+import com.gitblit.models.FilestoreModel.Status;
+
/**
* Collection of static methods for retrieving information from a repository.
- *
+ *
* @author James Moger
- *
+ *
*/
public class CompressionUtils {
@@ -53,7 +63,7 @@
/**
* Log an error message and exception.
- *
+ *
* @param t
* @param repository
* if repository is not null it MUST be the {0} parameter in the
@@ -77,7 +87,7 @@
/**
* Zips the contents of the tree at the (optionally) specified revision and
* the (optionally) specified basepath to the supplied outputstream.
- *
+ *
* @param repository
* @param basePath
* if unspecified, entire repository is assumed.
@@ -87,7 +97,7 @@
* @return true if repository was successfully zipped to supplied output
* stream
*/
- public static boolean zip(Repository repository, String basePath, String objectId,
+ public static boolean zip(Repository repository, IFilestoreManager filestoreManager, String basePath, String objectId,
OutputStream os) {
RevCommit commit = JGitUtils.getCommit(repository, objectId);
if (commit == null) {
@@ -115,16 +125,40 @@
continue;
}
tw.getObjectId(id, 0);
-
+
+ ObjectLoader loader = repository.open(id);
+
ZipArchiveEntry entry = new ZipArchiveEntry(tw.getPathString());
- entry.setSize(reader.getObjectSize(id, Constants.OBJ_BLOB));
+
+ FilestoreModel filestoreItem = null;
+
+ if (JGitUtils.isPossibleFilestoreItem(loader.getSize())) {
+ filestoreItem = JGitUtils.getFilestoreItem(tw.getObjectReader().open(id));
+ }
+
+ final long size = (filestoreItem == null) ? loader.getSize() : filestoreItem.getSize();
+
+ entry.setSize(size);
entry.setComment(commit.getName());
entry.setUnixMode(mode.getBits());
entry.setTime(modified);
zos.putArchiveEntry(entry);
+
+ if (filestoreItem == null) {
+ //Copy repository stored file
+ loader.copyTo(zos);
+ } else {
+ //Copy filestore file
+ try (FileInputStream streamIn = new FileInputStream(filestoreManager.getStoragePath(filestoreItem.oid))) {
+ IOUtils.copyLarge(streamIn, zos);
+ } catch (Throwable e) {
+ LOGGER.error(MessageFormat.format("Failed to archive filestore item {0}", filestoreItem.oid), e);
- ObjectLoader ldr = repository.open(id);
- ldr.copyTo(zos);
+ //Handle as per other errors
+ throw e;
+ }
+ }
+
zos.closeArchiveEntry();
}
zos.finish();
@@ -132,16 +166,16 @@
} catch (IOException e) {
error(e, repository, "{0} failed to zip files from commit {1}", commit.getName());
} finally {
- tw.release();
+ tw.close();
rw.dispose();
}
return success;
}
-
+
/**
* tar the contents of the tree at the (optionally) specified revision and
* the (optionally) specified basepath to the supplied outputstream.
- *
+ *
* @param repository
* @param basePath
* if unspecified, entire repository is assumed.
@@ -151,15 +185,15 @@
* @return true if repository was successfully zipped to supplied output
* stream
*/
- public static boolean tar(Repository repository, String basePath, String objectId,
+ public static boolean tar(Repository repository, IFilestoreManager filestoreManager, String basePath, String objectId,
OutputStream os) {
- return tar(null, repository, basePath, objectId, os);
+ return tar(null, repository, filestoreManager, basePath, objectId, os);
}
-
+
/**
* tar.gz the contents of the tree at the (optionally) specified revision and
* the (optionally) specified basepath to the supplied outputstream.
- *
+ *
* @param repository
* @param basePath
* if unspecified, entire repository is assumed.
@@ -169,15 +203,15 @@
* @return true if repository was successfully zipped to supplied output
* stream
*/
- public static boolean gz(Repository repository, String basePath, String objectId,
+ public static boolean gz(Repository repository, IFilestoreManager filestoreManager, String basePath, String objectId,
OutputStream os) {
- return tar(CompressorStreamFactory.GZIP, repository, basePath, objectId, os);
+ return tar(CompressorStreamFactory.GZIP, repository, filestoreManager, basePath, objectId, os);
}
-
+
/**
* tar.xz the contents of the tree at the (optionally) specified revision and
* the (optionally) specified basepath to the supplied outputstream.
- *
+ *
* @param repository
* @param basePath
* if unspecified, entire repository is assumed.
@@ -187,15 +221,15 @@
* @return true if repository was successfully zipped to supplied output
* stream
*/
- public static boolean xz(Repository repository, String basePath, String objectId,
+ public static boolean xz(Repository repository, IFilestoreManager filestoreManager, String basePath, String objectId,
OutputStream os) {
- return tar(CompressorStreamFactory.XZ, repository, basePath, objectId, os);
+ return tar(CompressorStreamFactory.XZ, repository, filestoreManager, basePath, objectId, os);
}
-
+
/**
* tar.bzip2 the contents of the tree at the (optionally) specified revision and
* the (optionally) specified basepath to the supplied outputstream.
- *
+ *
* @param repository
* @param basePath
* if unspecified, entire repository is assumed.
@@ -205,17 +239,17 @@
* @return true if repository was successfully zipped to supplied output
* stream
*/
- public static boolean bzip2(Repository repository, String basePath, String objectId,
+ public static boolean bzip2(Repository repository, IFilestoreManager filestoreManager, String basePath, String objectId,
OutputStream os) {
-
- return tar(CompressorStreamFactory.BZIP2, repository, basePath, objectId, os);
+
+ return tar(CompressorStreamFactory.BZIP2, repository, filestoreManager, basePath, objectId, os);
}
-
+
/**
* Compresses/archives the contents of the tree at the (optionally)
* specified revision and the (optionally) specified basepath to the
* supplied outputstream.
- *
+ *
* @param algorithm
* compression algorithm for tar (optional)
* @param repository
@@ -227,13 +261,13 @@
* @return true if repository was successfully zipped to supplied output
* stream
*/
- private static boolean tar(String algorithm, Repository repository, String basePath, String objectId,
+ private static boolean tar(String algorithm, Repository repository, IFilestoreManager filestoreManager, String basePath, String objectId,
OutputStream os) {
RevCommit commit = JGitUtils.getCommit(repository, objectId);
if (commit == null) {
return false;
}
-
+
OutputStream cos = os;
if (!StringUtils.isEmpty(algorithm)) {
try {
@@ -263,8 +297,9 @@
if (mode == FileMode.GITLINK || mode == FileMode.TREE) {
continue;
}
- tw.getObjectId(id, 0);
+ tw.getObjectId(id, 0);
+
ObjectLoader loader = repository.open(id);
if (FileMode.SYMLINK == mode) {
TarArchiveEntry entry = new TarArchiveEntry(tw.getPathString(),TarArchiveEntry.LF_SYMLINK);
@@ -278,9 +313,34 @@
TarArchiveEntry entry = new TarArchiveEntry(tw.getPathString());
entry.setMode(mode.getBits());
entry.setModTime(modified);
- entry.setSize(loader.getSize());
- tos.putArchiveEntry(entry);
- loader.copyTo(tos);
+
+ FilestoreModel filestoreItem = null;
+
+ if (JGitUtils.isPossibleFilestoreItem(loader.getSize())) {
+ filestoreItem = JGitUtils.getFilestoreItem(tw.getObjectReader().open(id));
+ }
+
+ final long size = (filestoreItem == null) ? loader.getSize() : filestoreItem.getSize();
+
+ entry.setSize(size);
+ tos.putArchiveEntry(entry);
+
+ if (filestoreItem == null) {
+ //Copy repository stored file
+ loader.copyTo(tos);
+ } else {
+ //Copy filestore file
+ try (FileInputStream streamIn = new FileInputStream(filestoreManager.getStoragePath(filestoreItem.oid))) {
+
+ IOUtils.copyLarge(streamIn, tos);
+ } catch (Throwable e) {
+ LOGGER.error(MessageFormat.format("Failed to archive filestore item {0}", filestoreItem.oid), e);
+
+ //Handle as per other errors
+ throw e;
+ }
+ }
+
tos.closeArchiveEntry();
}
}
@@ -291,7 +351,7 @@
} catch (IOException e) {
error(e, repository, "{0} failed to {1} stream files from commit {2}", algorithm, commit.getName());
} finally {
- tw.release();
+ tw.close();
rw.dispose();
}
return success;
--
Gitblit v1.9.1