Browse Source
Adds a JGit built-in implementation of the "git lfs clean" filter. This filter should do the same as the one described in [1]. But since this filter is written in Java and can be called by JGit without forking new processes it should be much faster [1] https://github.com/github/git-lfs/blob/master/docs/man/git-lfs-clean.1.ronn Change-Id: If60e387e97870245b4bd765eda6717eb84cffb1dstable-4.6
Christian Halstrick
8 years ago
committed by
Matthias Sohn
12 changed files with 616 additions and 7 deletions
@ -0,0 +1,176 @@
|
||||
/* |
||||
* Copyright (C) 2016, Christian Halstrick <christian.halstrick@sap.com> |
||||
* and other copyright owners as documented in the project's IP log. |
||||
* |
||||
* This program and the accompanying materials are made available |
||||
* under the terms of the Eclipse Distribution License v1.0 which |
||||
* accompanies this distribution, is reproduced below, and is |
||||
* available at http://www.eclipse.org/org/documents/edl-v10.php
|
||||
* |
||||
* All rights reserved. |
||||
* |
||||
* Redistribution and use in source and binary forms, with or |
||||
* without modification, are permitted provided that the following |
||||
* conditions are met: |
||||
* |
||||
* - Redistributions of source code must retain the above copyright |
||||
* notice, this list of conditions and the following disclaimer. |
||||
* |
||||
* - Redistributions in binary form must reproduce the above |
||||
* copyright notice, this list of conditions and the following |
||||
* disclaimer in the documentation and/or other materials provided |
||||
* with the distribution. |
||||
* |
||||
* - Neither the name of the Eclipse Foundation, Inc. nor the |
||||
* names of its contributors may be used to endorse or promote |
||||
* products derived from this software without specific prior |
||||
* written permission. |
||||
* |
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND |
||||
* CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, |
||||
* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES |
||||
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE |
||||
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR |
||||
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
||||
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT |
||||
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; |
||||
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER |
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, |
||||
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) |
||||
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF |
||||
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
||||
*/ |
||||
package org.eclipse.jgit.lfs; |
||||
|
||||
import java.io.IOException; |
||||
import java.io.InputStream; |
||||
import java.io.OutputStream; |
||||
import java.nio.file.Files; |
||||
import java.nio.file.Path; |
||||
import java.nio.file.StandardOpenOption; |
||||
import java.security.DigestOutputStream; |
||||
|
||||
import org.eclipse.jgit.attributes.FilterCommand; |
||||
import org.eclipse.jgit.attributes.FilterCommandFactory; |
||||
import org.eclipse.jgit.attributes.FilterCommandRegistry; |
||||
import org.eclipse.jgit.lfs.errors.CorruptMediaFile; |
||||
import org.eclipse.jgit.lfs.lib.Constants; |
||||
import org.eclipse.jgit.lfs.lib.LongObjectId; |
||||
import org.eclipse.jgit.lib.Repository; |
||||
import org.eclipse.jgit.util.FileUtils; |
||||
|
||||
/** |
||||
* Built-in LFS clean filter |
||||
* |
||||
* When new content is about to be added to the git repository and this filter |
||||
* is configured for that content, then this filter will replace the original |
||||
* content with content of a so-called LFS pointer file. The pointer file |
||||
* content will then be added to the git repository. Additionally this filter |
||||
* writes the original content in a so-called 'media file' to '.git/lfs/objects/ |
||||
* <first-two-characters-of-contentid>/<rest-of-contentid>' |
||||
* |
||||
* @see <a href="https://github.com/github/git-lfs/blob/master/docs/spec.md">Git |
||||
* LFS Specification</a> |
||||
* @since 4.6 |
||||
*/ |
||||
public class CleanFilter extends FilterCommand { |
||||
/** |
||||
* The factory is responsible for creating instances of {@link CleanFilter} |
||||
*/ |
||||
public final static FilterCommandFactory FACTORY = new FilterCommandFactory() { |
||||
|
||||
@Override |
||||
public FilterCommand create(Repository db, InputStream in, |
||||
OutputStream out) throws IOException { |
||||
return new CleanFilter(db, in, out); |
||||
} |
||||
}; |
||||
|
||||
/** |
||||
* Registers this filter by calling |
||||
* {@link FilterCommandRegistry#register(String, FilterCommandFactory)} |
||||
*/ |
||||
public final static void register() { |
||||
FilterCommandRegistry.register( |
||||
org.eclipse.jgit.lib.Constants.BUILTIN_FILTER_PREFIX |
||||
+ "lfs/clean", //$NON-NLS-1$
|
||||
FACTORY); |
||||
} |
||||
|
||||
// The OutputStream to a temporary file which will be renamed to mediafile
|
||||
// when the operation succeeds
|
||||
private OutputStream tmpOut; |
||||
|
||||
// Used to compute the hash for the original content
|
||||
private DigestOutputStream dOut; |
||||
|
||||
private Lfs lfsUtil; |
||||
|
||||
// the size of the original content
|
||||
private long size; |
||||
|
||||
// a temporary file into which the original content is written. When no
|
||||
// errors occur this file will be renamed to the mediafile
|
||||
private Path tmpFile; |
||||
|
||||
/** |
||||
* @param db |
||||
* the repository |
||||
* @param in |
||||
* an {@link InputStream} providing the original content |
||||
* @param out |
||||
* the {@link OutputStream} into which the content of the pointer |
||||
* file should be written. That's the content which will be added |
||||
* to the git repository |
||||
* @throws IOException |
||||
* when the creation of the temporary file fails or when no |
||||
* {@link OutputStream} for this file can be created |
||||
*/ |
||||
public CleanFilter(Repository db, InputStream in, OutputStream out) |
||||
throws IOException { |
||||
super(in, out); |
||||
lfsUtil = new Lfs(db.getDirectory().toPath().resolve("lfs")); //$NON-NLS-1$
|
||||
Files.createDirectories(lfsUtil.getLfsTmpDir()); |
||||
tmpFile = lfsUtil.createTmpFile(); |
||||
tmpOut = Files.newOutputStream(tmpFile, |
||||
StandardOpenOption.CREATE); |
||||
this.dOut = new DigestOutputStream( |
||||
tmpOut, |
||||
Constants.newMessageDigest()); |
||||
} |
||||
|
||||
public int run() throws IOException { |
||||
try { |
||||
int b = in.read(); |
||||
if (b != -1) { |
||||
dOut.write(b); |
||||
size++; |
||||
return 1; |
||||
} else { |
||||
dOut.close(); |
||||
tmpOut.close(); |
||||
LongObjectId loid = LongObjectId |
||||
.fromRaw(dOut.getMessageDigest().digest()); |
||||
Path mediaFile = lfsUtil.getMediaFile(loid); |
||||
if (Files.isRegularFile(mediaFile)) { |
||||
long fsSize = Files.size(mediaFile); |
||||
if (fsSize != size) { |
||||
throw new CorruptMediaFile(mediaFile, size, fsSize); |
||||
} |
||||
} else { |
||||
FileUtils.mkdirs(mediaFile.getParent().toFile(), true); |
||||
FileUtils.rename(tmpFile.toFile(), mediaFile.toFile()); |
||||
} |
||||
LfsPointer lfsPointer = new LfsPointer(loid, size); |
||||
lfsPointer.encode(out); |
||||
out.close(); |
||||
return -1; |
||||
} |
||||
} catch (IOException e) { |
||||
out.close(); |
||||
dOut.close(); |
||||
tmpOut.close(); |
||||
throw e; |
||||
} |
||||
} |
||||
} |
||||
@ -0,0 +1,124 @@
|
||||
/* |
||||
* Copyright (C) 2016, Christian Halstrick <christian.halstrick@sap.com> |
||||
* and other copyright owners as documented in the project's IP log. |
||||
* |
||||
* This program and the accompanying materials are made available |
||||
* under the terms of the Eclipse Distribution License v1.0 which |
||||
* accompanies this distribution, is reproduced below, and is |
||||
* available at http://www.eclipse.org/org/documents/edl-v10.php
|
||||
* |
||||
* All rights reserved. |
||||
* |
||||
* Redistribution and use in source and binary forms, with or |
||||
* without modification, are permitted provided that the following |
||||
* conditions are met: |
||||
* |
||||
* - Redistributions of source code must retain the above copyright |
||||
* notice, this list of conditions and the following disclaimer. |
||||
* |
||||
* - Redistributions in binary form must reproduce the above |
||||
* copyright notice, this list of conditions and the following |
||||
* disclaimer in the documentation and/or other materials provided |
||||
* with the distribution. |
||||
* |
||||
* - Neither the name of the Eclipse Foundation, Inc. nor the |
||||
* names of its contributors may be used to endorse or promote |
||||
* products derived from this software without specific prior |
||||
* written permission. |
||||
* |
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND |
||||
* CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, |
||||
* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES |
||||
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE |
||||
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR |
||||
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
||||
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT |
||||
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; |
||||
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER |
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, |
||||
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) |
||||
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF |
||||
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
||||
*/ |
||||
package org.eclipse.jgit.lfs; |
||||
|
||||
import java.io.IOException; |
||||
import java.nio.file.Files; |
||||
import java.nio.file.Path; |
||||
|
||||
import org.eclipse.jgit.lfs.lib.LongObjectId; |
||||
|
||||
/** |
||||
* Class which represents the lfs folder hierarchy inside a .git folder |
||||
* |
||||
* @since 4.6 |
||||
*/ |
||||
public class Lfs { |
||||
private Path root; |
||||
|
||||
private Path objDir; |
||||
|
||||
private Path tmpDir; |
||||
|
||||
/** |
||||
* @param root |
||||
* the path to the LFS media directory. Will be "<repo>/.git/lfs" |
||||
*/ |
||||
public Lfs(Path root) { |
||||
this.root = root; |
||||
} |
||||
|
||||
/** |
||||
* @return the path to the LFS directory |
||||
*/ |
||||
public Path getLfsRoot() { |
||||
return root; |
||||
} |
||||
|
||||
/** |
||||
* @return the path to the temp directory used by LFS. Will be |
||||
* "<repo>/.git/lfs/tmp" |
||||
*/ |
||||
public Path getLfsTmpDir() { |
||||
if (tmpDir == null) { |
||||
tmpDir = root.resolve("tmp"); //$NON-NLS-1$
|
||||
} |
||||
return tmpDir; |
||||
} |
||||
|
||||
/** |
||||
* @return the path to the object directory used by LFS. Will be |
||||
* "<repo>/.git/lfs/objects" |
||||
*/ |
||||
public Path getLfsObjDir() { |
||||
if (objDir == null) { |
||||
objDir = root.resolve("objects"); //$NON-NLS-1$
|
||||
} |
||||
return objDir; |
||||
} |
||||
|
||||
/** |
||||
* @param id |
||||
* the id of the mediafile |
||||
* @return the file which stores the original content. This will be files |
||||
* underneath |
||||
* "<repo>/.git/lfs/objects/<firstTwoLettersOfID>/<remainingLettersOfID>" |
||||
*/ |
||||
public Path getMediaFile(LongObjectId id) { |
||||
String idStr = LongObjectId.toString(id); |
||||
return getLfsObjDir().resolve(idStr.substring(0, 2)) |
||||
.resolve(idStr.substring(2)); |
||||
} |
||||
|
||||
/** |
||||
* Create a new temp file in the LFS directory |
||||
* |
||||
* @return a new temporary file in the LFS directory |
||||
* @throws IOException |
||||
* when the temp file could not be created |
||||
*/ |
||||
public Path createTmpFile() throws IOException { |
||||
return Files.createTempFile(getLfsTmpDir(), null, null); |
||||
} |
||||
|
||||
} |
||||
@ -0,0 +1,121 @@
|
||||
/* |
||||
* Copyright (C) 2016, Christian Halstrick <christian.halstrick@sap.com> |
||||
* and other copyright owners as documented in the project's IP log. |
||||
* |
||||
* This program and the accompanying materials are made available |
||||
* under the terms of the Eclipse Distribution License v1.0 which |
||||
* accompanies this distribution, is reproduced below, and is |
||||
* available at http://www.eclipse.org/org/documents/edl-v10.php
|
||||
* |
||||
* All rights reserved. |
||||
* |
||||
* Redistribution and use in source and binary forms, with or |
||||
* without modification, are permitted provided that the following |
||||
* conditions are met: |
||||
* |
||||
* - Redistributions of source code must retain the above copyright |
||||
* notice, this list of conditions and the following disclaimer. |
||||
* |
||||
* - Redistributions in binary form must reproduce the above |
||||
* copyright notice, this list of conditions and the following |
||||
* disclaimer in the documentation and/or other materials provided |
||||
* with the distribution. |
||||
* |
||||
* - Neither the name of the Eclipse Foundation, Inc. nor the |
||||
* names of its contributors may be used to endorse or promote |
||||
* products derived from this software without specific prior |
||||
* written permission. |
||||
* |
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND |
||||
* CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, |
||||
* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES |
||||
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE |
||||
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR |
||||
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
||||
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT |
||||
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; |
||||
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER |
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, |
||||
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) |
||||
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF |
||||
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
||||
*/ |
||||
package org.eclipse.jgit.lfs; |
||||
|
||||
import java.io.OutputStream; |
||||
import java.io.PrintStream; |
||||
|
||||
import org.eclipse.jgit.lfs.lib.Constants; |
||||
import org.eclipse.jgit.lfs.lib.LongObjectId; |
||||
|
||||
/** |
||||
* Represents an LFS pointer file |
||||
* |
||||
* @since 4.6 |
||||
*/ |
||||
public class LfsPointer { |
||||
/** |
||||
* The version of the LfsPointer file format |
||||
*/ |
||||
public static final String VERSION = "https://git-lfs.github.com/spec/v1"; //$NON-NLS-1$
|
||||
|
||||
/** |
||||
* The name of the hash function as used in the pointer files. This will |
||||
* evaluate to "sha256" |
||||
*/ |
||||
public static final String HASH_FUNCTION_NAME = Constants.LONG_HASH_FUNCTION |
||||
.toLowerCase().replace("-", ""); //$NON-NLS-1$ //$NON-NLS-2$
|
||||
|
||||
private LongObjectId oid; |
||||
|
||||
private long size; |
||||
|
||||
/** |
||||
* @param oid |
||||
* the id of the content |
||||
* @param size |
||||
* the size of the content |
||||
*/ |
||||
public LfsPointer(LongObjectId oid, long size) { |
||||
this.oid = oid; |
||||
this.size = size; |
||||
} |
||||
|
||||
/** |
||||
* @return the id of the content |
||||
*/ |
||||
public LongObjectId getOid() { |
||||
return oid; |
||||
} |
||||
|
||||
/** |
||||
* @return the size of the content |
||||
*/ |
||||
public long getSize() { |
||||
return size; |
||||
} |
||||
|
||||
/** |
||||
* Encode this object into the LFS format defined by {@link #VERSION} |
||||
* |
||||
* @param out |
||||
* the {@link OutputStream} into which the encoded data should be |
||||
* written |
||||
*/ |
||||
public void encode(OutputStream out) { |
||||
try (PrintStream ps = new PrintStream(out)) { |
||||
ps.print("version "); //$NON-NLS-1$
|
||||
ps.println(VERSION); |
||||
ps.print("oid " + HASH_FUNCTION_NAME + ":"); //$NON-NLS-1$ //$NON-NLS-2$
|
||||
ps.println(LongObjectId.toString(oid)); |
||||
ps.print("size "); //$NON-NLS-1$
|
||||
ps.println(size); |
||||
} |
||||
} |
||||
|
||||
@Override |
||||
public String toString() { |
||||
return "LfsPointer: oid=" + LongObjectId.toString(oid) + ", size=" //$NON-NLS-1$ //$NON-NLS-2$
|
||||
+ size; |
||||
} |
||||
} |
||||
@ -0,0 +1,100 @@
|
||||
/* |
||||
* Copyright (C) 2016, Christian Halstrick <christian.halstrick@sap.com> |
||||
* and other copyright owners as documented in the project's IP log. |
||||
* |
||||
* This program and the accompanying materials are made available |
||||
* under the terms of the Eclipse Distribution License v1.0 which |
||||
* accompanies this distribution, is reproduced below, and is |
||||
* available at http://www.eclipse.org/org/documents/edl-v10.php
|
||||
* |
||||
* All rights reserved. |
||||
* |
||||
* Redistribution and use in source and binary forms, with or |
||||
* without modification, are permitted provided that the following |
||||
* conditions are met: |
||||
* |
||||
* - Redistributions of source code must retain the above copyright |
||||
* notice, this list of conditions and the following disclaimer. |
||||
* |
||||
* - Redistributions in binary form must reproduce the above |
||||
* copyright notice, this list of conditions and the following |
||||
* disclaimer in the documentation and/or other materials provided |
||||
* with the distribution. |
||||
* |
||||
* - Neither the name of the Eclipse Foundation, Inc. nor the |
||||
* names of its contributors may be used to endorse or promote |
||||
* products derived from this software without specific prior |
||||
* written permission. |
||||
* |
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND |
||||
* CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, |
||||
* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES |
||||
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE |
||||
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR |
||||
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
||||
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT |
||||
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; |
||||
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER |
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, |
||||
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) |
||||
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF |
||||
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
||||
*/ |
||||
package org.eclipse.jgit.lfs.errors; |
||||
|
||||
import java.io.IOException; |
||||
import java.nio.file.Path; |
||||
import java.text.MessageFormat; |
||||
|
||||
import org.eclipse.jgit.lfs.internal.LfsText; |
||||
|
||||
/** |
||||
* Thrown when a LFS mediafile is found which doesn't have the expected size |
||||
* |
||||
* @since 4.6 |
||||
*/ |
||||
public class CorruptMediaFile extends IOException { |
||||
private static final long serialVersionUID = 1L; |
||||
|
||||
private Path mediaFile; |
||||
|
||||
private long expectedSize; |
||||
|
||||
private long size; |
||||
|
||||
/** |
||||
* @param mediaFile |
||||
* @param expectedSize |
||||
* @param size |
||||
*/ |
||||
@SuppressWarnings("boxing") |
||||
public CorruptMediaFile(Path mediaFile, long expectedSize, |
||||
long size) { |
||||
super(MessageFormat.format(LfsText.get().inconsistentMediafileLength, |
||||
mediaFile, expectedSize, size)); |
||||
this.mediaFile = mediaFile; |
||||
this.expectedSize = expectedSize; |
||||
this.size = size; |
||||
} |
||||
|
||||
/** |
||||
* @return the media file which seems to be corrupt |
||||
*/ |
||||
public Path getMediaFile() { |
||||
return mediaFile; |
||||
} |
||||
|
||||
/** |
||||
* @return the expected size of the media file |
||||
*/ |
||||
public long getExpectedSize() { |
||||
return expectedSize; |
||||
} |
||||
|
||||
/** |
||||
* @return the actual size of the media file in the file system |
||||
*/ |
||||
public long getSize() { |
||||
return size; |
||||
} |
||||
} |
||||
Loading…
Reference in new issue