From bac4d32d39abc6fb9af61628adbed139ea442ee9 Mon Sep 17 00:00:00 2001 From: Hector Caballero Date: Fri, 3 Nov 2017 14:39:37 -0400 Subject: [PATCH] GC: Delete stale temporary packs and indexes When a GC operation is interrupted, temporary packs and indexes can be left on the pack folder. In big, busy repositories this can lead to significant amounts of wasted disk space if this interruption is done with a certain frequency. Remove stale temporary packs and indexes at the end of the GC process so they do not accumulate. To avoid interfering with a possible concurrent JGit GC process in the same repository, only delete temporary files that are older than one day. Change-Id: If9b6c1e57fac8a6a0ecc0a703089634caba4caae Signed-off-by: Hector Caballero --- .../storage/file/GcTemporaryFilesTest.java | 106 ++++++++++++++++++ .../jgit/internal/storage/file/GC.java | 25 +++++ 2 files changed, 131 insertions(+) create mode 100644 org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcTemporaryFilesTest.java diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcTemporaryFilesTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcTemporaryFilesTest.java new file mode 100644 index 000000000..59d544e63 --- /dev/null +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcTemporaryFilesTest.java @@ -0,0 +1,106 @@ +/* + * Copyright (C) 2017 Ericsson + * and other copyright owners as documented in the project's IP log. + * + * This program and the accompanying materials are made available + * under the terms of the Eclipse Distribution License v1.0 which + * accompanies this distribution, is reproduced below, and is + * available at http://www.eclipse.org/org/documents/edl-v10.php + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * - Neither the name of the Eclipse Foundation, Inc. nor the + * names of its contributors may be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package org.eclipse.jgit.internal.storage.file; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.io.File; +import java.nio.file.Paths; +import java.time.Instant; + +import org.junit.Before; +import org.junit.Test; + +public class GcTemporaryFilesTest extends GcTestCase { + private static final String TEMP_IDX = "gc_1234567890.idx_tmp"; + + private static final String TEMP_PACK = "gc_1234567890.pack_tmp"; + + private File packDir; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + packDir = Paths.get(repo.getObjectsDirectory().getAbsolutePath(), + "pack").toFile(); //$NON-NLS-1$ + } + + @Test + public void oldTempPacksAndIdxAreDeleted() throws Exception { + File tempIndex = new File(packDir, TEMP_IDX); + File tempPack = new File(packDir, TEMP_PACK); + if (!packDir.exists() || !packDir.isDirectory()) { + assertTrue(packDir.mkdirs()); + } + assertTrue(tempPack.createNewFile()); + assertTrue(tempIndex.createNewFile()); + assertTrue(tempIndex.exists()); + assertTrue(tempPack.exists()); + long _24HoursBefore = Instant.now().toEpochMilli() + - 24 * 60 * 62 * 1000; + tempIndex.setLastModified(_24HoursBefore); + tempPack.setLastModified(_24HoursBefore); + gc.gc(); + assertFalse(tempIndex.exists()); + assertFalse(tempPack.exists()); + } + + @Test + public void recentTempPacksAndIdxAreNotDeleted() throws Exception { + File tempIndex = new File(packDir, TEMP_IDX); + File tempPack = new File(packDir, TEMP_PACK); + if (!packDir.exists() || !packDir.isDirectory()) { + assertTrue(packDir.mkdirs()); + } + assertTrue(tempPack.createNewFile()); + assertTrue(tempIndex.createNewFile()); + assertTrue(tempIndex.exists()); + assertTrue(tempPack.exists()); + gc.gc(); + assertTrue(tempIndex.exists()); + assertTrue(tempPack.exists()); + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java index f992a3340..928e52d6b 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java @@ -61,6 +61,8 @@ import java.nio.file.Paths; import java.nio.file.StandardCopyOption; import java.text.MessageFormat; import java.text.ParseException; +import java.time.Instant; +import java.time.temporal.ChronoUnit; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -903,6 +905,7 @@ public class GC { } prunePacked(); deleteOrphans(); + deleteTempPacksIdx(); lastPackedRefs = refsBefore; lastRepackTime = time; @@ -959,6 +962,28 @@ public class GC { } } + private void deleteTempPacksIdx() { + Path packDir = Paths.get(repo.getObjectsDirectory().getAbsolutePath(), + "pack"); //$NON-NLS-1$ + Instant threshold = Instant.now().minus(1, ChronoUnit.DAYS); + try { + Files.newDirectoryStream(packDir, "gc_*_tmp") //$NON-NLS-1$ + .forEach(t -> { + try { + Instant lastModified = Files.getLastModifiedTime(t) + .toInstant(); + if (lastModified.isBefore(threshold)) { + Files.deleteIfExists(t); + } + } catch (IOException e) { + LOG.error(e.getMessage(), e); + } + }); + } catch (IOException e) { + LOG.error(e.getMessage(), e); + } + } + /** * @param ref * the ref which log should be inspected