Browse Source

reftable: debug tools

Simple debug programs to experiment with the reftable file format:

  debug-read-reftable
  debug-write-reftable
  debug-verify-reftable
  debug-benchmark-reftable

Change-Id: I79db351d86900f1e58b17e922e195dff06ee71f1
stable-4.9
Shawn Pearce 7 years ago
parent
commit
0398f3dd6e
  1. 3
      org.eclipse.jgit.pgm/META-INF/MANIFEST.MF
  2. 7
      org.eclipse.jgit.pgm/META-INF/services/org.eclipse.jgit.pgm.TextBuiltin
  3. 315
      org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/debug/BenchmarkReftable.java
  4. 97
      org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/debug/ReadReftable.java
  5. 211
      org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/debug/VerifyReftable.java
  6. 301
      org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/debug/WriteReftable.java
  7. 1
      org.eclipse.jgit/META-INF/MANIFEST.MF

3
org.eclipse.jgit.pgm/META-INF/MANIFEST.MF

@ -37,8 +37,11 @@ Import-Package: javax.servlet;version="[3.1.0,4.0.0)",
org.eclipse.jgit.errors;version="[4.9.0,4.10.0)", org.eclipse.jgit.errors;version="[4.9.0,4.10.0)",
org.eclipse.jgit.gitrepo;version="[4.9.0,4.10.0)", org.eclipse.jgit.gitrepo;version="[4.9.0,4.10.0)",
org.eclipse.jgit.internal.ketch;version="[4.9.0,4.10.0)", org.eclipse.jgit.internal.ketch;version="[4.9.0,4.10.0)",
org.eclipse.jgit.internal.storage.dfs;version="[4.9.0,4.10.0)",
org.eclipse.jgit.internal.storage.file;version="[4.9.0,4.10.0)", org.eclipse.jgit.internal.storage.file;version="[4.9.0,4.10.0)",
org.eclipse.jgit.internal.storage.io;version="[4.9.0,4.10.0)",
org.eclipse.jgit.internal.storage.pack;version="[4.9.0,4.10.0)", org.eclipse.jgit.internal.storage.pack;version="[4.9.0,4.10.0)",
org.eclipse.jgit.internal.storage.reftable;version="[4.9.0,4.10.0)",
org.eclipse.jgit.internal.storage.reftree;version="[4.9.0,4.10.0)", org.eclipse.jgit.internal.storage.reftree;version="[4.9.0,4.10.0)",
org.eclipse.jgit.lfs;version="[4.9.0,4.10.0)", org.eclipse.jgit.lfs;version="[4.9.0,4.10.0)",
org.eclipse.jgit.lfs.lib;version="[4.9.0,4.10.0)", org.eclipse.jgit.lfs.lib;version="[4.9.0,4.10.0)",

7
org.eclipse.jgit.pgm/META-INF/services/org.eclipse.jgit.pgm.TextBuiltin

@ -38,10 +38,12 @@ org.eclipse.jgit.pgm.Tag
org.eclipse.jgit.pgm.UploadPack org.eclipse.jgit.pgm.UploadPack
org.eclipse.jgit.pgm.Version org.eclipse.jgit.pgm.Version
org.eclipse.jgit.pgm.debug.BenchmarkReftable
org.eclipse.jgit.pgm.debug.DiffAlgorithms org.eclipse.jgit.pgm.debug.DiffAlgorithms
org.eclipse.jgit.pgm.debug.LfsStore org.eclipse.jgit.pgm.debug.LfsStore
org.eclipse.jgit.pgm.debug.MakeCacheTree org.eclipse.jgit.pgm.debug.MakeCacheTree
org.eclipse.jgit.pgm.debug.ReadDirCache org.eclipse.jgit.pgm.debug.ReadDirCache
org.eclipse.jgit.pgm.debug.ReadReftable
org.eclipse.jgit.pgm.debug.RebuildCommitGraph org.eclipse.jgit.pgm.debug.RebuildCommitGraph
org.eclipse.jgit.pgm.debug.RebuildRefTree org.eclipse.jgit.pgm.debug.RebuildRefTree
org.eclipse.jgit.pgm.debug.ShowCacheTree org.eclipse.jgit.pgm.debug.ShowCacheTree
@ -49,5 +51,6 @@ org.eclipse.jgit.pgm.debug.ShowCommands
org.eclipse.jgit.pgm.debug.ShowDirCache org.eclipse.jgit.pgm.debug.ShowDirCache
org.eclipse.jgit.pgm.debug.ShowPackDelta org.eclipse.jgit.pgm.debug.ShowPackDelta
org.eclipse.jgit.pgm.debug.TextHashFunctions org.eclipse.jgit.pgm.debug.TextHashFunctions
org.eclipse.jgit.pgm.debug.WriteDirCache org.eclipse.jgit.pgm.debug.VerifyReftable
org.eclipse.jgit.pgm.debug.WriteReftable
org.eclipse.jgit.pgm.debug.WriteReftable

315
org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/debug/BenchmarkReftable.java

@ -0,0 +1,315 @@
/*
* Copyright (C) 2017, Google Inc.
* and other copyright owners as documented in the project's IP log.
*
* This program and the accompanying materials are made available
* under the terms of the Eclipse Distribution License v1.0 which
* accompanies this distribution, is reproduced below, and is
* available at http://www.eclipse.org/org/documents/edl-v10.php
*
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials provided
* with the distribution.
*
* - Neither the name of the Eclipse Foundation, Inc. nor the
* names of its contributors may be used to endorse or promote
* products derived from this software without specific prior
* written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
* CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.eclipse.jgit.pgm.debug;
import static java.nio.charset.StandardCharsets.UTF_8;
import static org.eclipse.jgit.lib.Constants.HEAD;
import static org.eclipse.jgit.lib.Constants.MASTER;
import static org.eclipse.jgit.lib.Constants.R_HEADS;
import static org.eclipse.jgit.lib.Ref.Storage.NEW;
import static org.eclipse.jgit.lib.Ref.Storage.PACKED;
import java.io.BufferedReader;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStreamReader;
import org.eclipse.jgit.internal.storage.io.BlockSource;
import org.eclipse.jgit.internal.storage.reftable.RefCursor;
import org.eclipse.jgit.internal.storage.reftable.ReftableReader;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.ObjectIdRef;
import org.eclipse.jgit.lib.Ref;
import org.eclipse.jgit.lib.SymbolicRef;
import org.eclipse.jgit.pgm.Command;
import org.eclipse.jgit.pgm.TextBuiltin;
import org.eclipse.jgit.util.RefList;
import org.kohsuke.args4j.Argument;
import org.kohsuke.args4j.Option;
@Command
class BenchmarkReftable extends TextBuiltin {
enum Test {
SCAN,
SEEK_COLD, SEEK_HOT,
BY_ID_COLD, BY_ID_HOT;
}
@Option(name = "--tries")
private int tries = 10;
@Option(name = "--test")
private Test test = Test.SCAN;
@Option(name = "--ref")
private String ref;
@Option(name = "--object-id")
private String objectId;
@Argument(index = 0)
private String lsRemotePath;
@Argument(index = 1)
private String reftablePath;
@Override
protected void run() throws Exception {
switch (test) {
case SCAN:
scan();
break;
case SEEK_COLD:
seekCold(ref);
break;
case SEEK_HOT:
seekHot(ref);
break;
case BY_ID_COLD:
byIdCold(ObjectId.fromString(objectId));
break;
case BY_ID_HOT:
byIdHot(ObjectId.fromString(objectId));
break;
}
}
private void printf(String fmt, Object... args) throws IOException {
errw.println(String.format(fmt, args));
}
@SuppressWarnings({ "nls", "boxing" })
private void scan() throws Exception {
long start, tot;
start = System.currentTimeMillis();
for (int i = 0; i < tries; i++) {
readLsRemote();
}
tot = System.currentTimeMillis() - start;
printf("%12s %10d ms %6d ms/run", "packed-refs", tot, tot / tries);
start = System.currentTimeMillis();
for (int i = 0; i < tries; i++) {
try (FileInputStream in = new FileInputStream(reftablePath);
BlockSource src = BlockSource.from(in);
ReftableReader reader = new ReftableReader(src)) {
try (RefCursor rc = reader.allRefs()) {
while (rc.next()) {
rc.getRef();
}
}
}
}
tot = System.currentTimeMillis() - start;
printf("%12s %10d ms %6d ms/run", "reftable", tot, tot / tries);
}
private RefList<Ref> readLsRemote()
throws IOException, FileNotFoundException {
RefList.Builder<Ref> list = new RefList.Builder<>();
try (BufferedReader br = new BufferedReader(new InputStreamReader(
new FileInputStream(lsRemotePath), UTF_8))) {
Ref last = null;
String line;
while ((line = br.readLine()) != null) {
ObjectId id = ObjectId.fromString(line.substring(0, 40));
String name = line.substring(41, line.length());
if (last != null && name.endsWith("^{}")) { //$NON-NLS-1$
last = new ObjectIdRef.PeeledTag(PACKED, last.getName(),
last.getObjectId(), id);
list.set(list.size() - 1, last);
continue;
}
if (name.equals(HEAD)) {
last = new SymbolicRef(name, new ObjectIdRef.Unpeeled(NEW,
R_HEADS + MASTER, null));
} else {
last = new ObjectIdRef.PeeledNonTag(PACKED, name, id);
}
list.add(last);
}
}
list.sort();
return list.toRefList();
}
@SuppressWarnings({ "nls", "boxing" })
private void seekCold(String refName) throws Exception {
long start, tot;
int lsTries = Math.min(tries, 64);
start = System.nanoTime();
for (int i = 0; i < lsTries; i++) {
readLsRemote().get(refName);
}
tot = System.nanoTime() - start;
printf("%12s %10d usec %9.1f usec/run %5d runs", "packed-refs",
tot / 1000,
(((double) tot) / lsTries) / 1000,
lsTries);
start = System.nanoTime();
for (int i = 0; i < tries; i++) {
try (FileInputStream in = new FileInputStream(reftablePath);
BlockSource src = BlockSource.from(in);
ReftableReader reader = new ReftableReader(src)) {
try (RefCursor rc = reader.seekRef(refName)) {
while (rc.next()) {
rc.getRef();
}
}
}
}
tot = System.nanoTime() - start;
printf("%12s %10d usec %9.1f usec/run %5d runs", "reftable",
tot / 1000,
(((double) tot) / tries) / 1000,
tries);
}
@SuppressWarnings({ "nls", "boxing" })
private void seekHot(String refName) throws Exception {
long start, tot;
int lsTries = Math.min(tries, 64);
start = System.nanoTime();
RefList<Ref> lsRemote = readLsRemote();
for (int i = 0; i < lsTries; i++) {
lsRemote.get(refName);
}
tot = System.nanoTime() - start;
printf("%12s %10d usec %9.1f usec/run %5d runs", "packed-refs",
tot / 1000, (((double) tot) / lsTries) / 1000, lsTries);
start = System.nanoTime();
try (FileInputStream in = new FileInputStream(reftablePath);
BlockSource src = BlockSource.from(in);
ReftableReader reader = new ReftableReader(src)) {
for (int i = 0; i < tries; i++) {
try (RefCursor rc = reader.seekRef(refName)) {
while (rc.next()) {
rc.getRef();
}
}
}
}
tot = System.nanoTime() - start;
printf("%12s %10d usec %9.1f usec/run %5d runs", "reftable",
tot / 1000, (((double) tot) / tries) / 1000, tries);
}
@SuppressWarnings({ "nls", "boxing" })
private void byIdCold(ObjectId id) throws Exception {
long start, tot;
int lsTries = Math.min(tries, 64);
start = System.nanoTime();
for (int i = 0; i < lsTries; i++) {
for (Ref r : readLsRemote()) {
if (id.equals(r.getObjectId())) {
continue;
}
}
}
tot = System.nanoTime() - start;
printf("%12s %10d usec %9.1f usec/run %5d runs", "packed-refs",
tot / 1000, (((double) tot) / lsTries) / 1000, lsTries);
start = System.nanoTime();
for (int i = 0; i < tries; i++) {
try (FileInputStream in = new FileInputStream(reftablePath);
BlockSource src = BlockSource.from(in);
ReftableReader reader = new ReftableReader(src)) {
try (RefCursor rc = reader.byObjectId(id)) {
while (rc.next()) {
rc.getRef();
}
}
}
}
tot = System.nanoTime() - start;
printf("%12s %10d usec %9.1f usec/run %5d runs", "reftable",
tot / 1000, (((double) tot) / tries) / 1000, tries);
}
@SuppressWarnings({ "nls", "boxing" })
private void byIdHot(ObjectId id) throws Exception {
long start, tot;
int lsTries = Math.min(tries, 64);
start = System.nanoTime();
RefList<Ref> lsRemote = readLsRemote();
for (int i = 0; i < lsTries; i++) {
for (Ref r : lsRemote) {
if (id.equals(r.getObjectId())) {
continue;
}
}
}
tot = System.nanoTime() - start;
printf("%12s %10d usec %9.1f usec/run %5d runs", "packed-refs",
tot / 1000, (((double) tot) / lsTries) / 1000, lsTries);
start = System.nanoTime();
try (FileInputStream in = new FileInputStream(reftablePath);
BlockSource src = BlockSource.from(in);
ReftableReader reader = new ReftableReader(src)) {
for (int i = 0; i < tries; i++) {
try (RefCursor rc = reader.byObjectId(id)) {
while (rc.next()) {
rc.getRef();
}
}
}
}
tot = System.nanoTime() - start;
printf("%12s %10d usec %9.1f usec/run %5d runs", "reftable",
tot / 1000, (((double) tot) / tries) / 1000, tries);
}
}

97
org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/debug/ReadReftable.java

@ -0,0 +1,97 @@
/*
* Copyright (C) 2017, Google Inc.
* and other copyright owners as documented in the project's IP log.
*
* This program and the accompanying materials are made available
* under the terms of the Eclipse Distribution License v1.0 which
* accompanies this distribution, is reproduced below, and is
* available at http://www.eclipse.org/org/documents/edl-v10.php
*
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials provided
* with the distribution.
*
* - Neither the name of the Eclipse Foundation, Inc. nor the
* names of its contributors may be used to endorse or promote
* products derived from this software without specific prior
* written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
* CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.eclipse.jgit.pgm.debug;
import java.io.FileInputStream;
import java.io.IOException;
import org.eclipse.jgit.internal.storage.io.BlockSource;
import org.eclipse.jgit.internal.storage.reftable.RefCursor;
import org.eclipse.jgit.internal.storage.reftable.ReftableReader;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.Ref;
import org.eclipse.jgit.pgm.Command;
import org.eclipse.jgit.pgm.TextBuiltin;
import org.kohsuke.args4j.Argument;
@Command
class ReadReftable extends TextBuiltin {
@Argument(index = 0)
private String input;
@Argument(index = 1, required = false)
private String ref;
@Override
protected void run() throws Exception {
try (FileInputStream in = new FileInputStream(input);
BlockSource src = BlockSource.from(in);
ReftableReader reader = new ReftableReader(src)) {
try (RefCursor rc = ref != null
? reader.seekRef(ref)
: reader.allRefs()) {
while (rc.next()) {
write(rc.getRef());
}
}
}
}
private void write(Ref r) throws IOException {
if (r.isSymbolic()) {
outw.println(r.getTarget().getName() + '\t' + r.getName());
return;
}
ObjectId id1 = r.getObjectId();
if (id1 != null) {
outw.println(id1.name() + '\t' + r.getName());
}
ObjectId id2 = r.getPeeledObjectId();
if (id2 != null) {
outw.println('^' + id2.name());
}
}
}

211
org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/debug/VerifyReftable.java

@ -0,0 +1,211 @@
/*
* Copyright (C) 2017, Google Inc.
* and other copyright owners as documented in the project's IP log.
*
* This program and the accompanying materials are made available
* under the terms of the Eclipse Distribution License v1.0 which
* accompanies this distribution, is reproduced below, and is
* available at http://www.eclipse.org/org/documents/edl-v10.php
*
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials provided
* with the distribution.
*
* - Neither the name of the Eclipse Foundation, Inc. nor the
* names of its contributors may be used to endorse or promote
* products derived from this software without specific prior
* written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
* CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.eclipse.jgit.pgm.debug;
import java.io.FileInputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Random;
import org.eclipse.jgit.internal.storage.io.BlockSource;
import org.eclipse.jgit.internal.storage.reftable.RefCursor;
import org.eclipse.jgit.internal.storage.reftable.ReftableReader;
import org.eclipse.jgit.lib.AnyObjectId;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.Ref;
import org.eclipse.jgit.lib.RefComparator;
import org.eclipse.jgit.lib.TextProgressMonitor;
import org.eclipse.jgit.pgm.Command;
import org.eclipse.jgit.pgm.TextBuiltin;
import org.kohsuke.args4j.Argument;
@Command
class VerifyReftable extends TextBuiltin {
private static final long SEED1 = 0xaba8bb4de4caf86cL;
private static final long SEED2 = 0x28bb5c25ad43ecb5L;
@Argument(index = 0)
private String lsRemotePath;
@Argument(index = 1)
private String reftablePath;
@Override
protected void run() throws Exception {
List<Ref> refs = WriteReftable.readRefs(lsRemotePath);
try (FileInputStream in = new FileInputStream(reftablePath);
BlockSource src = BlockSource.from(in);
ReftableReader reader = new ReftableReader(src)) {
scan(refs, reader);
seek(refs, reader);
byId(refs, reader);
}
}
@SuppressWarnings("nls")
private void scan(List<Ref> refs, ReftableReader reader)
throws IOException {
errw.print(String.format("%-20s", "sequential scan..."));
errw.flush();
try (RefCursor rc = reader.allRefs()) {
for (Ref exp : refs) {
verify(exp, rc);
}
if (rc.next()) {
throw die("expected end of table");
}
}
errw.println(" OK");
}
@SuppressWarnings("nls")
private void seek(List<Ref> refs, ReftableReader reader)
throws IOException {
List<Ref> rnd = new ArrayList<>(refs);
Collections.shuffle(rnd, new Random(SEED1));
TextProgressMonitor pm = new TextProgressMonitor(errw);
pm.beginTask("random seek", rnd.size());
for (Ref exp : rnd) {
try (RefCursor rc = reader.seekRef(exp.getName())) {
verify(exp, rc);
if (rc.next()) {
throw die("should not have ref after " + exp.getName());
}
}
pm.update(1);
}
pm.endTask();
}
@SuppressWarnings("nls")
private void byId(List<Ref> refs, ReftableReader reader)
throws IOException {
Map<ObjectId, List<Ref>> want = groupById(refs);
List<List<Ref>> rnd = new ArrayList<>(want.values());
Collections.shuffle(rnd, new Random(SEED2));
TextProgressMonitor pm = new TextProgressMonitor(errw);
pm.beginTask("byObjectId", rnd.size());
for (List<Ref> exp : rnd) {
Collections.sort(exp, RefComparator.INSTANCE);
ObjectId id = exp.get(0).getObjectId();
try (RefCursor rc = reader.byObjectId(id)) {
for (Ref r : exp) {
verify(r, rc);
}
}
pm.update(1);
}
pm.endTask();
}
private static Map<ObjectId, List<Ref>> groupById(List<Ref> refs) {
Map<ObjectId, List<Ref>> m = new HashMap<>();
for (Ref r : refs) {
ObjectId id = r.getObjectId();
if (id != null) {
List<Ref> c = m.get(id);
if (c == null) {
c = new ArrayList<>(2);
m.put(id, c);
}
c.add(r);
}
}
return m;
}
@SuppressWarnings("nls")
private void verify(Ref exp, RefCursor rc) throws IOException {
if (!rc.next()) {
throw die("ended before " + exp.getName());
}
Ref act = rc.getRef();
if (!exp.getName().equals(act.getName())) {
throw die(String.format("expected %s, found %s",
exp.getName(),
act.getName()));
}
if (exp.isSymbolic()) {
if (!act.isSymbolic()) {
throw die("expected " + act.getName() + " to be symbolic");
}
if (!exp.getTarget().getName().equals(act.getTarget().getName())) {
throw die(String.format("expected %s to be %s, found %s",
exp.getName(),
exp.getLeaf().getName(),
act.getLeaf().getName()));
}
return;
}
if (!AnyObjectId.equals(exp.getObjectId(), act.getObjectId())) {
throw die(String.format("expected %s to be %s, found %s",
exp.getName(),
id(exp.getObjectId()),
id(act.getObjectId())));
}
if (exp.getPeeledObjectId() != null
&& !AnyObjectId.equals(exp.getPeeledObjectId(), act.getPeeledObjectId())) {
throw die(String.format("expected %s to be %s, found %s",
exp.getName(),
id(exp.getPeeledObjectId()),
id(act.getPeeledObjectId())));
}
}
@SuppressWarnings("nls")
private static String id(ObjectId id) {
return id != null ? id.name() : "<null>";
}
}

301
org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/debug/WriteReftable.java

@ -0,0 +1,301 @@
/*
* Copyright (C) 2017, Google Inc.
* and other copyright owners as documented in the project's IP log.
*
* This program and the accompanying materials are made available
* under the terms of the Eclipse Distribution License v1.0 which
* accompanies this distribution, is reproduced below, and is
* available at http://www.eclipse.org/org/documents/edl-v10.php
*
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials provided
* with the distribution.
*
* - Neither the name of the Eclipse Foundation, Inc. nor the
* names of its contributors may be used to endorse or promote
* products derived from this software without specific prior
* written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
* CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package org.eclipse.jgit.pgm.debug;
import static java.nio.charset.StandardCharsets.UTF_8;
import static org.eclipse.jgit.lib.Constants.HEAD;
import static org.eclipse.jgit.lib.Constants.MASTER;
import static org.eclipse.jgit.lib.Constants.R_HEADS;
import static org.eclipse.jgit.lib.Ref.Storage.NEW;
import static org.eclipse.jgit.lib.Ref.Storage.PACKED;
import java.io.BufferedReader;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.eclipse.jgit.internal.storage.reftable.ReftableConfig;
import org.eclipse.jgit.internal.storage.reftable.ReftableWriter;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.ObjectIdRef;
import org.eclipse.jgit.lib.PersonIdent;
import org.eclipse.jgit.lib.Ref;
import org.eclipse.jgit.lib.SymbolicRef;
import org.eclipse.jgit.pgm.Command;
import org.eclipse.jgit.pgm.TextBuiltin;
import org.kohsuke.args4j.Argument;
import org.kohsuke.args4j.Option;
@Command
class WriteReftable extends TextBuiltin {
private static final int KIB = 1 << 10;
private static final int MIB = 1 << 20;
@Option(name = "--block-size")
private int refBlockSize;
@Option(name = "--log-block-size")
private int logBlockSize;
@Option(name = "--restart-interval")
private int restartInterval;
@Option(name = "--index-levels")
private int indexLevels;
@Option(name = "--reflog-in")
private String reflogIn;
@Option(name = "--no-index-objects")
private boolean noIndexObjects;
@Argument(index = 0)
private String in;
@Argument(index = 1)
private String out;
@SuppressWarnings({ "nls", "boxing" })
@Override
protected void run() throws Exception {
List<Ref> refs = readRefs(in);
List<LogEntry> logs = readLog(reflogIn);
ReftableWriter.Stats stats;
try (OutputStream os = new FileOutputStream(out)) {
ReftableConfig cfg = new ReftableConfig();
cfg.setIndexObjects(!noIndexObjects);
if (refBlockSize > 0) {
cfg.setRefBlockSize(refBlockSize);
}
if (logBlockSize > 0) {
cfg.setLogBlockSize(logBlockSize);
}
if (restartInterval > 0) {
cfg.setRestartInterval(restartInterval);
}
if (indexLevels > 0) {
cfg.setMaxIndexLevels(indexLevels);
}
ReftableWriter w = new ReftableWriter(cfg);
w.setMinUpdateIndex(min(logs)).setMaxUpdateIndex(max(logs));
w.begin(os);
w.sortAndWriteRefs(refs);
for (LogEntry e : logs) {
w.writeLog(e.ref, e.updateIndex, e.who,
e.oldId, e.newId, e.message);
}
stats = w.finish().getStats();
}
double fileMiB = ((double) stats.totalBytes()) / MIB;
printf("Summary:");
printf(" file sz : %.1f MiB (%d bytes)", fileMiB, stats.totalBytes());
printf(" padding : %d KiB", stats.paddingBytes() / KIB);
errw.println();
printf("Refs:");
printf(" ref blk : %d", stats.refBlockSize());
printf(" restarts: %d", stats.restartInterval());
printf(" refs : %d", stats.refCount());
if (stats.refIndexLevels() > 0) {
int idxSize = (int) Math.round(((double) stats.refIndexSize()) / KIB);
printf(" idx sz : %d KiB", idxSize);
printf(" idx lvl : %d", stats.refIndexLevels());
}
printf(" avg ref : %d bytes", stats.refBytes() / refs.size());
errw.println();
if (stats.objCount() > 0) {
int objMiB = (int) Math.round(((double) stats.objBytes()) / MIB);
int idLen = stats.objIdLength();
printf("Objects:");
printf(" obj blk : %d", stats.refBlockSize());
printf(" restarts: %d", stats.restartInterval());
printf(" objects : %d", stats.objCount());
printf(" obj sz : %d MiB (%d bytes)", objMiB, stats.objBytes());
if (stats.objIndexSize() > 0) {
int s = (int) Math.round(((double) stats.objIndexSize()) / KIB);
printf(" idx sz : %d KiB", s);
printf(" idx lvl : %d", stats.objIndexLevels());
}
printf(" id len : %d bytes (%d hex digits)", idLen, 2 * idLen);
printf(" avg obj : %d bytes", stats.objBytes() / stats.objCount());
errw.println();
}
if (stats.logCount() > 0) {
int logMiB = (int) Math.round(((double) stats.logBytes()) / MIB);
printf("Log:");
printf(" log blk : %d", stats.logBlockSize());
printf(" logs : %d", stats.logCount());
printf(" log sz : %d MiB (%d bytes)", logMiB, stats.logBytes());
printf(" avg log : %d bytes", stats.logBytes() / logs.size());
errw.println();
}
}
private void printf(String fmt, Object... args) throws IOException {
errw.println(String.format(fmt, args));
}
static List<Ref> readRefs(String inputFile) throws IOException {
List<Ref> refs = new ArrayList<>();
try (BufferedReader br = new BufferedReader(
new InputStreamReader(new FileInputStream(inputFile), UTF_8))) {
String line;
while ((line = br.readLine()) != null) {
ObjectId id = ObjectId.fromString(line.substring(0, 40));
String name = line.substring(41, line.length());
if (name.endsWith("^{}")) { //$NON-NLS-1$
int lastIdx = refs.size() - 1;
Ref last = refs.get(lastIdx);
refs.set(lastIdx, new ObjectIdRef.PeeledTag(PACKED,
last.getName(), last.getObjectId(), id));
continue;
}
Ref ref;
if (name.equals(HEAD)) {
ref = new SymbolicRef(name, new ObjectIdRef.Unpeeled(NEW,
R_HEADS + MASTER, null));
} else {
ref = new ObjectIdRef.PeeledNonTag(PACKED, name, id);
}
refs.add(ref);
}
}
Collections.sort(refs, (a, b) -> a.getName().compareTo(b.getName()));
return refs;
}
private static List<LogEntry> readLog(String logPath)
throws FileNotFoundException, IOException {
if (logPath == null) {
return Collections.emptyList();
}
List<LogEntry> log = new ArrayList<>();
try (BufferedReader br = new BufferedReader(
new InputStreamReader(new FileInputStream(logPath), UTF_8))) {
@SuppressWarnings("nls")
Pattern pattern = Pattern.compile("([^,]+)" // 1: ref
+ ",([0-9]+(?:[.][0-9]+)?)" // 2: time
+ ",([^,]+)" // 3: who
+ ",([^,]+)" // 4: old
+ ",([^,]+)" // 5: new
+ ",(.*)"); // 6: msg
String line;
while ((line = br.readLine()) != null) {
Matcher m = pattern.matcher(line);
if (!m.matches()) {
throw new IOException("unparsed line: " + line); //$NON-NLS-1$
}
String ref = m.group(1);
double t = Double.parseDouble(m.group(2));
long time = ((long) t) * 1000L;
long index = (long) (t * 1e6);
String user = m.group(3);
ObjectId oldId = parseId(m.group(4));
ObjectId newId = parseId(m.group(5));
String msg = m.group(6);
String email = user + "@gerrit"; //$NON-NLS-1$
PersonIdent who = new PersonIdent(user, email, time, -480);
log.add(new LogEntry(ref, index, who, oldId, newId, msg));
}
}
Collections.sort(log, LogEntry::compare);
return log;
}
private static long min(List<LogEntry> log) {
return log.stream().mapToLong(e -> e.updateIndex).min().orElse(0);
}
private static long max(List<LogEntry> log) {
return log.stream().mapToLong(e -> e.updateIndex).max().orElse(0);
}
private static ObjectId parseId(String s) {
if ("NULL".equals(s)) { //$NON-NLS-1$
return ObjectId.zeroId();
}
return ObjectId.fromString(s);
}
private static class LogEntry {
static int compare(LogEntry a, LogEntry b) {
int cmp = a.ref.compareTo(b.ref);
if (cmp == 0) {
cmp = Long.signum(b.updateIndex - a.updateIndex);
}
return cmp;
}
final String ref;
final long updateIndex;
final PersonIdent who;
final ObjectId oldId;
final ObjectId newId;
final String message;
LogEntry(String ref, long updateIndex, PersonIdent who,
ObjectId oldId, ObjectId newId, String message) {
this.ref = ref;
this.updateIndex = updateIndex;
this.who = who;
this.oldId = oldId;
this.newId = newId;
this.message = message;
}
}
}

1
org.eclipse.jgit/META-INF/MANIFEST.MF

@ -74,6 +74,7 @@ Export-Package: org.eclipse.jgit.annotations;version="4.9.0",
org.eclipse.jgit.lfs, org.eclipse.jgit.lfs,
org.eclipse.jgit.pgm, org.eclipse.jgit.pgm,
org.eclipse.jgit.pgm.test", org.eclipse.jgit.pgm.test",
org.eclipse.jgit.internal.storage.io;version="4.9.0";x-friends:="org.eclipse.jgit.junit,org.eclipse.jgit.test,org.eclipse.jgit.pgm",
org.eclipse.jgit.internal.storage.pack;version="4.9.0";x-friends:="org.eclipse.jgit.junit,org.eclipse.jgit.test,org.eclipse.jgit.pgm", org.eclipse.jgit.internal.storage.pack;version="4.9.0";x-friends:="org.eclipse.jgit.junit,org.eclipse.jgit.test,org.eclipse.jgit.pgm",
org.eclipse.jgit.internal.storage.reftable;version="4.9.0";x-friends:="org.eclipse.jgit.junit,org.eclipse.jgit.test,org.eclipse.jgit.pgm", org.eclipse.jgit.internal.storage.reftable;version="4.9.0";x-friends:="org.eclipse.jgit.junit,org.eclipse.jgit.test,org.eclipse.jgit.pgm",
org.eclipse.jgit.internal.storage.reftree;version="4.9.0";x-friends:="org.eclipse.jgit.junit,org.eclipse.jgit.test,org.eclipse.jgit.pgm", org.eclipse.jgit.internal.storage.reftree;version="4.9.0";x-friends:="org.eclipse.jgit.junit,org.eclipse.jgit.test,org.eclipse.jgit.pgm",

Loading…
Cancel
Save