From 8fd500e20c96ee250f7e1573f09bd097be11bc41 Mon Sep 17 00:00:00 2001 From: Hongkai Liu Date: Mon, 23 Jan 2017 11:33:40 -0500 Subject: [PATCH] Clean up orphan files in GC An orphan file is either a bitmap or an idx file in pack folder, and its corresponding pack file is missing. Change-Id: I3c4cb1f7aa99dd7b398bdb8d513f528d7761edff Signed-off-by: Hongkai Liu Signed-off-by: Matthias Sohn --- .../storage/file/GcOrphanFilesTest.java | 119 ++++++++++++++++++ .../jgit/internal/storage/file/GC.java | 53 ++++++++ 2 files changed, 172 insertions(+) create mode 100644 org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcOrphanFilesTest.java diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcOrphanFilesTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcOrphanFilesTest.java new file mode 100644 index 000000000..d9317b89a --- /dev/null +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcOrphanFilesTest.java @@ -0,0 +1,119 @@ +/* + * Copyright (C) 2017 Ericsson + * and other copyright owners as documented in the project's IP log. + * + * This program and the accompanying materials are made available + * under the terms of the Eclipse Distribution License v1.0 which + * accompanies this distribution, is reproduced below, and is + * available at http://www.eclipse.org/org/documents/edl-v10.php + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * - Neither the name of the Eclipse Foundation, Inc. nor the + * names of its contributors may be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package org.eclipse.jgit.internal.storage.file; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.io.File; +import java.io.IOException; + +import org.junit.Before; +import org.junit.Test; + +public class GcOrphanFilesTest extends GcTestCase { + private final static String PACK = "pack"; + + private final static String BITMAP_File_1 = PACK + "-1.bitmap"; + + private final static String IDX_File_2 = PACK + "-2.idx"; + + private final static String IDX_File_malformed = PACK + "-1234idx"; + + private final static String PACK_File_2 = PACK + "-2.pack"; + + private final static String PACK_File_3 = PACK + "-3.pack"; + + private File packDir; + + @Before + public void setUp() throws Exception { + super.setUp(); + packDir = new File(repo.getObjectsDirectory(), PACK); + } + + @Test + public void bitmapAndIdxDeletedButPackNot() throws Exception { + createFileInPackFolder(BITMAP_File_1); + createFileInPackFolder(IDX_File_2); + createFileInPackFolder(PACK_File_3); + gc.gc(); + assertFalse(new File(packDir, BITMAP_File_1).exists()); + assertFalse(new File(packDir, IDX_File_2).exists()); + assertTrue(new File(packDir, PACK_File_3).exists()); + } + + @Test + public void bitmapDeletedButIdxAndPackNot() throws Exception { + createFileInPackFolder(BITMAP_File_1); + createFileInPackFolder(IDX_File_2); + createFileInPackFolder(PACK_File_2); + createFileInPackFolder(PACK_File_3); + gc.gc(); + assertFalse(new File(packDir, BITMAP_File_1).exists()); + assertTrue(new File(packDir, IDX_File_2).exists()); + assertTrue(new File(packDir, PACK_File_2).exists()); + assertTrue(new File(packDir, PACK_File_3).exists()); + } + + @Test + public void malformedIdxNotDeleted() throws Exception { + createFileInPackFolder(IDX_File_malformed); + gc.gc(); + assertTrue(new File(packDir, IDX_File_malformed).exists()); + } + + private void createFileInPackFolder(String fileName) throws IOException { + if (!packDir.exists() || !packDir.isDirectory()) { + assertTrue(packDir.mkdirs()); + } + assertTrue(new File(packDir, fileName).createNewFile()); + } + + @Test + public void noSuchPackFolder() throws Exception { + assertTrue(packDir.delete()); + gc.gc(); + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java index 147e54dce..e3e73e25f 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java @@ -55,6 +55,7 @@ import java.nio.channels.FileChannel; import java.nio.file.DirectoryStream; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; import java.nio.file.StandardCopyOption; import java.text.MessageFormat; import java.text.ParseException; @@ -73,6 +74,8 @@ import java.util.Objects; import java.util.Set; import java.util.TreeMap; import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.stream.Stream; import org.eclipse.jgit.annotations.NonNull; import org.eclipse.jgit.dircache.DirCacheIterator; @@ -126,6 +129,13 @@ public class GC { private static final Pattern PATTERN_LOOSE_OBJECT = Pattern .compile("[0-9a-fA-F]{38}"); //$NON-NLS-1$ + private static final String PACK_EXT = "." + PackExt.PACK.getExtension();//$NON-NLS-1$ + + private static final String BITMAP_EXT = "." //$NON-NLS-1$ + + PackExt.BITMAP_INDEX.getExtension(); + + private static final String INDEX_EXT = "." + PackExt.INDEX.getExtension(); //$NON-NLS-1$ + private static final int DEFAULT_AUTOPACKLIMIT = 50; private static final int DEFAULT_AUTOLIMIT = 6700; @@ -654,12 +664,55 @@ public class GC { throw new IOException(e); } prunePacked(); + deleteOrphans(); lastPackedRefs = refsBefore; lastRepackTime = time; return ret; } + /** + * Deletes orphans + *

+ * A file is considered an orphan if it is either a "bitmap" or an index + * file, and its corresponding pack file is missing in the list. + *

+ */ + private void deleteOrphans() { + Path packDir = Paths.get(repo.getObjectsDirectory().getAbsolutePath(), + "pack"); //$NON-NLS-1$ + List fileNames = null; + try (Stream files = Files.list(packDir)) { + fileNames = files.map(path -> path.getFileName().toString()) + .filter(name -> { + return (name.endsWith(PACK_EXT) + || name.endsWith(BITMAP_EXT) + || name.endsWith(INDEX_EXT)); + }).sorted(Collections.reverseOrder()) + .collect(Collectors.toList()); + } catch (IOException e1) { + // ignore + } + if (fileNames == null) { + return; + } + + String base = null; + for (String n : fileNames) { + if (n.endsWith(PACK_EXT)) { + base = n.substring(0, n.lastIndexOf('.')); + } else { + if (base == null || !n.startsWith(base)) { + try { + Files.delete(new File(packDir.toFile(), n).toPath()); + } catch (IOException e) { + LOG.error(e.getMessage(), e); + } + } + } + } + } + /** * @param ref * the ref which log should be inspected