Add support for indexers and alternate odb packfiles

This allows for implementations of git servers written in Go.

5 files changed, 356 insertions, 1 deletions

diff --git a/indexer.go b/indexer.go
new file mode 100644
index 0000000..d1b9372
--- /dev/null
+++ b/indexer.go
@@ -0,0 +1,99 @@
+package git
+
+/*
+#include <git2.h>
+
+extern const git_oid * git_indexer_hash(const git_indexer *idx);
+extern int git_indexer_append(git_indexer *idx, const void *data, size_t size, git_transfer_progress *stats);
+extern int git_indexer_commit(git_indexer *idx, git_transfer_progress *stats);
+extern int _go_git_indexer_new(git_indexer **out, const char *path, unsigned int mode, git_odb *odb, void *progress_cb_payload);
+extern void git_indexer_free(git_indexer *idx);
+*/
+import "C"
+import (
+	"reflect"
+	"runtime"
+	"unsafe"
+)
+
+// Indexer can post-process packfiles and create an .idx file for efficient
+// lookup.
+type Indexer struct {
+	ptr             *C.git_indexer
+	stats           C.git_transfer_progress
+	callbacks       RemoteCallbacks
+	callbacksHandle unsafe.Pointer
+}
+
+// NewIndexer creates a new indexer instance.
+func NewIndexer(packfilePath string, odb *Odb, callback TransferProgressCallback) (indexer *Indexer, err error) {
+	indexer = new(Indexer)
+
+	runtime.LockOSThread()
+	defer runtime.UnlockOSThread()
+
+	var odbPtr *C.git_odb = nil
+	if odb != nil {
+		odbPtr = odb.ptr
+	}
+
+	indexer.callbacks.TransferProgressCallback = callback
+	indexer.callbacksHandle = pointerHandles.Track(&indexer.callbacks)
+
+	cstr := C.CString(packfilePath)
+	defer C.free(unsafe.Pointer(cstr))
+
+	ret := C._go_git_indexer_new(&indexer.ptr, cstr, 0, odbPtr, indexer.callbacksHandle)
+	runtime.KeepAlive(odb)
+	if ret < 0 {
+		pointerHandles.Untrack(indexer.callbacksHandle)
+		return nil, MakeGitError(ret)
+	}
+
+	runtime.SetFinalizer(indexer, (*Indexer).Free)
+	return indexer, nil
+}
+
+// Write adds data to the indexer.
+func (indexer *Indexer) Write(data []byte) (int, error) {
+	header := (*reflect.SliceHeader)(unsafe.Pointer(&data))
+	ptr := unsafe.Pointer(header.Data)
+	size := C.size_t(header.Len)
+
+	runtime.LockOSThread()
+	defer runtime.UnlockOSThread()
+
+	ret := C.git_indexer_append(indexer.ptr, ptr, size, &indexer.stats)
+	runtime.KeepAlive(indexer)
+	if ret < 0 {
+		return 0, MakeGitError(ret)
+	}
+
+	return len(data), nil
+}
+
+// Commit finalizes the pack and index. It resolves any pending deltas and
+// writes out the index file.
+//
+// It also returns the packfile's hash. A packfile's name is derived from the
+// sorted hashing of all object names.
+func (indexer *Indexer) Commit() (*Oid, error) {
+	runtime.LockOSThread()
+	defer runtime.UnlockOSThread()
+
+	ret := C.git_indexer_commit(indexer.ptr, &indexer.stats)
+	if ret < 0 {
+		return nil, MakeGitError(ret)
+	}
+
+	id := newOidFromC(C.git_indexer_hash(indexer.ptr))
+	runtime.KeepAlive(indexer)
+	return id, nil
+}
+
+// Free frees the indexer and its resources.
+func (indexer *Indexer) Free() {
+	pointerHandles.Untrack(indexer.callbacksHandle)
+	runtime.SetFinalizer(indexer, nil)
+	C.git_indexer_free(indexer.ptr)
+}
diff --git a/indexer_test.go b/indexer_test.go
new file mode 100644
index 0000000..1b65c95
--- /dev/null
+++ b/indexer_test.go
@@ -0,0 +1,93 @@
+package git
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path"
+	"testing"
+)
+
+var (
+	// This is a packfile with three objects. The second is a delta which
+	// depends on the third, which is also a delta.
+	outOfOrderPack = []byte{
+		0x50, 0x41, 0x43, 0x4b, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x03,
+		0x32, 0x78, 0x9c, 0x63, 0x67, 0x00, 0x00, 0x00, 0x10, 0x00, 0x08, 0x76,
+		0xe6, 0x8f, 0xe8, 0x12, 0x9b, 0x54, 0x6b, 0x10, 0x1a, 0xee, 0x95, 0x10,
+		0xc5, 0x32, 0x8e, 0x7f, 0x21, 0xca, 0x1d, 0x18, 0x78, 0x9c, 0x63, 0x62,
+		0x66, 0x4e, 0xcb, 0xcf, 0x07, 0x00, 0x02, 0xac, 0x01, 0x4d, 0x75, 0x01,
+		0xd7, 0x71, 0x36, 0x66, 0xf4, 0xde, 0x82, 0x27, 0x76, 0xc7, 0x62, 0x2c,
+		0x10, 0xf1, 0xb0, 0x7d, 0xe2, 0x80, 0xdc, 0x78, 0x9c, 0x63, 0x62, 0x62,
+		0x62, 0xb7, 0x03, 0x00, 0x00, 0x69, 0x00, 0x4c, 0xde, 0x7d, 0xaa, 0xe4,
+		0x19, 0x87, 0x58, 0x80, 0x61, 0x09, 0x9a, 0x33, 0xca, 0x7a, 0x31, 0x92,
+		0x6f, 0xae, 0x66, 0x75,
+	}
+)
+
+func TestIndexerOutOfOrder(t *testing.T) {
+	t.Parallel()
+
+	tmpPath, err := ioutil.TempDir("", "git2go")
+	checkFatal(t, err)
+	defer os.RemoveAll(tmpPath)
+
+	var finalStats TransferProgress
+	idx, err := NewIndexer(tmpPath, nil, func(stats TransferProgress) ErrorCode {
+		finalStats = stats
+		return ErrOk
+	})
+	checkFatal(t, err)
+	defer idx.Free()
+
+	_, err = idx.Write(outOfOrderPack)
+	checkFatal(t, err)
+	oid, err := idx.Commit()
+	checkFatal(t, err)
+
+	// The packfile contains the hash as the last 20 bytes.
+	expectedOid := NewOidFromBytes(outOfOrderPack[len(outOfOrderPack)-20:])
+	if !expectedOid.Equal(oid) {
+		t.Errorf("mismatched packfile hash, expected %v, got %v", expectedOid, oid)
+	}
+	if finalStats.TotalObjects != 3 {
+		t.Errorf("mismatched transferred objects, expected 3, got %v", finalStats.TotalObjects)
+	}
+	if finalStats.ReceivedObjects != 3 {
+		t.Errorf("mismatched received objects, expected 3, got %v", finalStats.ReceivedObjects)
+	}
+	if finalStats.IndexedObjects != 3 {
+		t.Errorf("mismatched indexed objects, expected 3, got %v", finalStats.IndexedObjects)
+	}
+
+	odb, err := NewOdb()
+	checkFatal(t, err)
+	defer odb.Free()
+
+	backend, err := NewOdbBackendOnePack(path.Join(tmpPath, fmt.Sprintf("pack-%s.idx", oid.String())))
+	checkFatal(t, err)
+	// Transfer the ownership of the backend to the odb, no freeing needed.
+	err = odb.AddBackend(backend, 1)
+	checkFatal(t, err)
+
+	packfileObjects := 0
+	err = odb.ForEach(func(id *Oid) error {
+		packfileObjects += 1
+		return nil
+	})
+	checkFatal(t, err)
+	if packfileObjects != 3 {
+		t.Errorf("mismatched packfile objects, expected 3, got %v", packfileObjects)
+	}
+
+	// Inspect one of the well-known objects in the packfile.
+	obj, err := odb.Read(NewOidFromBytes([]byte{
+		0x19, 0x10, 0x28, 0x15, 0x66, 0x3d, 0x23, 0xf8, 0xb7, 0x5a, 0x47, 0xe7,
+		0xa0, 0x19, 0x65, 0xdc, 0xdc, 0x96, 0x46, 0x8c,
+	}))
+	checkFatal(t, err)
+	defer obj.Free()
+	if "foo" != string(obj.Data()) {
+		t.Errorf("mismatched packfile object contents, expected foo, got %q", string(obj.Data()))
+	}
+}
diff --git a/odb.go b/odb.go
index 6489653..994ff2a 100644
--- a/odb.go
+++ b/odb.go
@@ -3,8 +3,13 @@ package git
 /*
 #include <git2.h>
 
+extern int git_odb_backend_one_pack(git_odb_backend **out, const char *index_file);
 extern int _go_git_odb_foreach(git_odb *db, void *payload);
 extern void _go_git_odb_backend_free(git_odb_backend *backend);
+extern int _go_git_odb_write_pack(git_odb_writepack **out, git_odb *db, void *progress_payload);
+extern int _go_git_odb_writepack_append(git_odb_writepack *writepack, const void *, size_t, git_transfer_progress *);
+extern int _go_git_odb_writepack_commit(git_odb_writepack *writepack, git_transfer_progress *);
+extern void _go_git_odb_writepack_free(git_odb_writepack *writepack);
 */
 import "C"
 import (
@@ -42,8 +47,20 @@ func NewOdbBackendFromC(ptr unsafe.Pointer) (backend *OdbBackend) {
 	return backend
 }
 
-func (v *Odb) AddBackend(backend *OdbBackend, priority int) (err error) {
+func (v *Odb) AddAlternate(backend *OdbBackend, priority int) (err error) {
+	runtime.LockOSThread()
+	defer runtime.UnlockOSThread()
 
+	ret := C.git_odb_add_alternate(v.ptr, backend.ptr, C.int(priority))
+	runtime.KeepAlive(v)
+	if ret < 0 {
+		backend.Free()
+		return MakeGitError(ret)
+	}
+	return nil
+}
+
+func (v *Odb) AddBackend(backend *OdbBackend, priority int) (err error) {
 	runtime.LockOSThread()
 	defer runtime.UnlockOSThread()
 
@@ -56,6 +73,21 @@ func (v *Odb) AddBackend(backend *OdbBackend, priority int) (err error) {
 	return nil
 }
 
+func NewOdbBackendOnePack(packfileIndexPath string) (backend *OdbBackend, err error) {
+	runtime.LockOSThread()
+	defer runtime.UnlockOSThread()
+
+	cstr := C.CString(packfileIndexPath)
+	defer C.free(unsafe.Pointer(cstr))
+
+	var odbOnePack *C.git_odb_backend = nil
+	ret := C.git_odb_backend_one_pack(&odbOnePack, cstr)
+	if ret < 0 {
+		return nil, MakeGitError(ret)
+	}
+	return NewOdbBackendFromC(unsafe.Pointer(odbOnePack)), nil
+}
+
 func (v *Odb) ReadHeader(oid *Oid) (uint64, ObjectType, error) {
 	runtime.LockOSThread()
 	defer runtime.UnlockOSThread()
@@ -231,6 +263,31 @@ func (v *Odb) NewWriteStream(size int64, otype ObjectType) (*OdbWriteStream, err
 	return stream, nil
 }
 
+// NewWritePack opens a stream for writing a pack file to the ODB. If the ODB
+// layer understands pack files, then the given packfile will likely be
+// streamed directly to disk (and a corresponding index created). If the ODB
+// layer does not understand pack files, the objects will be stored in whatever
+// format the ODB layer uses.
+func (v *Odb) NewWritePack(callback TransferProgressCallback) (*OdbWritepack, error) {
+	writepack := new(OdbWritepack)
+
+	runtime.LockOSThread()
+	defer runtime.UnlockOSThread()
+
+	writepack.callbacks.TransferProgressCallback = callback
+	writepack.callbacksHandle = pointerHandles.Track(&writepack.callbacks)
+
+	ret := C._go_git_odb_write_pack(&writepack.ptr, v.ptr, writepack.callbacksHandle)
+	runtime.KeepAlive(v)
+	if ret < 0 {
+		pointerHandles.Untrack(writepack.callbacksHandle)
+		return nil, MakeGitError(ret)
+	}
+
+	runtime.SetFinalizer(writepack, (*OdbWritepack).Free)
+	return writepack, nil
+}
+
 func (v *OdbBackend) Free() {
 	C._go_git_odb_backend_free(v.ptr)
 }
@@ -360,3 +417,47 @@ func (stream *OdbWriteStream) Free() {
 	runtime.SetFinalizer(stream, nil)
 	C.git_odb_stream_free(stream.ptr)
 }
+
+// OdbWritepack is a stream to write a packfile to the ODB.
+type OdbWritepack struct {
+	ptr             *C.git_odb_writepack
+	stats           C.git_transfer_progress
+	callbacks       RemoteCallbacks
+	callbacksHandle unsafe.Pointer
+}
+
+func (writepack *OdbWritepack) Write(data []byte) (int, error) {
+	header := (*reflect.SliceHeader)(unsafe.Pointer(&data))
+	ptr := unsafe.Pointer(header.Data)
+	size := C.size_t(header.Len)
+
+	runtime.LockOSThread()
+	defer runtime.UnlockOSThread()
+
+	ret := C._go_git_odb_writepack_append(writepack.ptr, ptr, size, &writepack.stats)
+	runtime.KeepAlive(writepack)
+	if ret < 0 {
+		return 0, MakeGitError(ret)
+	}
+
+	return len(data), nil
+}
+
+func (writepack *OdbWritepack) Commit() error {
+	runtime.LockOSThread()
+	defer runtime.UnlockOSThread()
+
+	ret := C._go_git_odb_writepack_commit(writepack.ptr, &writepack.stats)
+	runtime.KeepAlive(writepack)
+	if ret < 0 {
+		return MakeGitError(ret)
+	}
+
+	return nil
+}
+
+func (writepack *OdbWritepack) Free() {
+	pointerHandles.Untrack(writepack.callbacksHandle)
+	runtime.SetFinalizer(writepack, nil)
+	C._go_git_odb_writepack_free(writepack.ptr)
+}
diff --git a/odb_test.go b/odb_test.go
index 46acdba..e44c927 100644
--- a/odb_test.go
+++ b/odb_test.go
@@ -152,3 +152,37 @@ func TestOdbForeach(t *testing.T) {
 		t.Fatalf("Odb.ForEach() did not return the expected error, got %v", err)
 	}
 }
+
+func TestOdbWritepack(t *testing.T) {
+	t.Parallel()
+	repo := createTestRepo(t)
+	defer cleanupTestRepo(t, repo)
+
+	_, _ = seedTestRepo(t, repo)
+
+	odb, err := repo.Odb()
+	checkFatal(t, err)
+
+	var finalStats TransferProgress
+	writepack, err := odb.NewWritePack(func(stats TransferProgress) ErrorCode {
+		finalStats = stats
+		return ErrOk
+	})
+	checkFatal(t, err)
+	defer writepack.Free()
+
+	_, err = writepack.Write(outOfOrderPack)
+	checkFatal(t, err)
+	err = writepack.Commit()
+	checkFatal(t, err)
+
+	if finalStats.TotalObjects != 3 {
+		t.Errorf("mismatched transferred objects, expected 3, got %v", finalStats.TotalObjects)
+	}
+	if finalStats.ReceivedObjects != 3 {
+		t.Errorf("mismatched received objects, expected 3, got %v", finalStats.ReceivedObjects)
+	}
+	if finalStats.IndexedObjects != 3 {
+		t.Errorf("mismatched indexed objects, expected 3, got %v", finalStats.IndexedObjects)
+	}
+}
diff --git a/wrapper.c b/wrapper.c
index 3656773..c4a5ff0 100644
--- a/wrapper.c
+++ b/wrapper.c
@@ -185,4 +185,32 @@ git_credtype_t _go_git_cred_credtype(git_cred *cred) {
 	return cred->credtype;
 }
 
+int _go_git_odb_write_pack(git_odb_writepack **out, git_odb *db, void *progress_payload)
+{
+	return git_odb_write_pack(out, db, (git_transfer_progress_cb)transferProgressCallback, progress_payload);
+}
+
+int _go_git_odb_writepack_append(git_odb_writepack *writepack, const void *data, size_t size, git_transfer_progress *stats)
+{
+	return writepack->append(writepack, data, size, stats);
+}
+
+int _go_git_odb_writepack_commit(git_odb_writepack *writepack, git_transfer_progress *stats)
+{
+	return writepack->commit(writepack, stats);
+}
+
+void _go_git_odb_writepack_free(git_odb_writepack *writepack)
+{
+	writepack->free(writepack);
+}
+
+int _go_git_indexer_new(git_indexer **out, const char *path, unsigned int mode, git_odb *odb, void *progress_cb_payload)
+{
+	git_indexer_options indexer_options = GIT_INDEXER_OPTIONS_INIT;
+	indexer_options.progress_cb = (git_transfer_progress_cb)transferProgressCallback;
+	indexer_options.progress_cb_payload = progress_cb_payload;
+	return git_indexer_new(out, path, mode, odb, &indexer_options);
+}
+
 /* EOF */