From 4a79cccffd6cd20e42271eada567c4b8c0edc2f4 Mon Sep 17 00:00:00 2001 From: Jeff Carr Date: Fri, 29 Nov 2024 18:15:25 -0600 Subject: attempt to use go.wit.com/apps/autogenpb for sort --- Makefile | 28 +++++----- gitTag.proto | 19 +++++++ gitTag.query.go | 134 +++++++++++++++++++++++++++++++++++++++++++++++ gitTag.update.go | 83 +++++++++++++++++++++++++++++ gitTags.proto | 13 ----- gitTags.query.go | 134 ----------------------------------------------- gitTags.sort.go | 149 ----------------------------------------------------- gitTags.update.go | 83 ----------------------------- goDep.helpers.go | 52 +++++++++++++++++++ goDep.proto | 21 ++++++++ goDep.redoGoMod.go | 132 +++++++++++++++++++++++++++++++++++++++++++++++ godep.helpers.go | 57 -------------------- godep.proto | 15 ------ godep.redoGoMod.go | 132 ----------------------------------------------- godep.sort.go | 99 ----------------------------------- refs.sort.go | 149 ----------------------------------------------------- refs.update.go | 55 -------------------- repo.helpers.go | 14 ----- repo.proto | 29 +++++------ 19 files changed, 469 insertions(+), 929 deletions(-) create mode 100644 gitTag.proto create mode 100644 gitTag.query.go create mode 100644 gitTag.update.go delete mode 100644 gitTags.proto delete mode 100644 gitTags.query.go delete mode 100644 gitTags.sort.go delete mode 100644 gitTags.update.go create mode 100644 goDep.helpers.go create mode 100644 goDep.proto create mode 100644 goDep.redoGoMod.go delete mode 100644 godep.helpers.go delete mode 100644 godep.proto delete mode 100644 godep.redoGoMod.go delete mode 100644 godep.sort.go delete mode 100644 refs.sort.go delete mode 100644 refs.update.go diff --git a/Makefile b/Makefile index cefd157..0f39f91 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ # go install -all: refs.pb.go gitTags.pb.go godep.pb.go repo.pb.go +all: gitTag.pb.go goDep.pb.go repo.pb.go make -C scanGoSrc/ vet: lint @@ -29,26 +29,28 @@ clean: -rm -f go.* make -C scanGoSrc clean -refs.pb.go: refs.proto - cd ~/go/src && protoc --go_out=. --proto_path=go.wit.com/lib/protobuf/gitpb \ - --go_opt=Mrefs.proto=go.wit.com/lib/protobuf/gitpb \ - refs.proto +#refs.pb.go: refs.proto +# cd ~/go/src && protoc --go_out=. --proto_path=go.wit.com/lib/protobuf/gitpb \ +# --go_opt=Mrefs.proto=go.wit.com/lib/protobuf/gitpb \ +# refs.proto -gitTags.pb.go: gitTags.proto +gitTag.pb.go: gitTag.proto cd ~/go/src && protoc --go_out=. --proto_path=go.wit.com/lib/protobuf/gitpb \ - --go_opt=MgitTags.proto=go.wit.com/lib/protobuf/gitpb \ - gitTags.proto + --go_opt=MgitTag.proto=go.wit.com/lib/protobuf/gitpb \ + gitTag.proto + autogenpb --proto gitTag.proto --sort "ByName,Refname" --sort "ByHash,Hash" --no-marshal -godep.pb.go: godep.proto +goDep.pb.go: goDep.proto cd ~/go/src && protoc --go_out=. --proto_path=go.wit.com/lib/protobuf/gitpb \ - --go_opt=Mgodep.proto=go.wit.com/lib/protobuf/gitpb \ - godep.proto + --go_opt=MgoDep.proto=go.wit.com/lib/protobuf/gitpb \ + goDep.proto + autogenpb --proto goDep.proto --sort "ByPath,GoPath" --no-marshal repo.pb.go: repo.proto cd ~/go/src && protoc --go_out=. --proto_path=go.wit.com/lib/protobuf/gitpb \ --go_opt=Mrefs.proto=go.wit.com/lib/protobuf/gitpb \ - --go_opt=Mgodep.proto=go.wit.com/lib/protobuf/gitpb \ + --go_opt=MgoDep.proto=go.wit.com/lib/protobuf/gitpb \ --go_opt=Mrepo.proto=go.wit.com/lib/protobuf/gitpb \ - --go_opt=MgitTags.proto=go.wit.com/lib/protobuf/gitpb \ + --go_opt=MgitTag.proto=go.wit.com/lib/protobuf/gitpb \ repo.proto autogenpb --proto repo.proto --sort "ByPath,GoPath" diff --git a/gitTag.proto b/gitTag.proto new file mode 100644 index 0000000..14bf4dc --- /dev/null +++ b/gitTag.proto @@ -0,0 +1,19 @@ +syntax = "proto3"; + +package gitpb; + +import "google/protobuf/timestamp.proto"; // Import the well-known type for Timestamp + +message GitTag { + string refname = 1; // tag name. treated as unique + google.protobuf.Timestamp creatordate = 2; // git creatordate + google.protobuf.Timestamp authordate = 3; // git author date + string hash = 4; // git hash + string subject = 5; // git tag subject +} + +message GitTags { + string uuid = 1; // I guess why not just have this on each file + string version = 2; // maybe can be used for protobuf schema change violations + repeated GitTag gitTags = 3; +} diff --git a/gitTag.query.go b/gitTag.query.go new file mode 100644 index 0000000..8c288e4 --- /dev/null +++ b/gitTag.query.go @@ -0,0 +1,134 @@ +package gitpb + +// runs git, parses output +// types faster than you can + +import ( + "errors" + "path/filepath" + "strings" + + "go.wit.com/log" +) + +func (repo *Repo) GetLastTag() string { + cmd := []string{"git", "rev-list", "--tags", "--max-count=1"} + result := repo.RunQuiet(cmd) + log.Info("getLastTagVersion()", result.Stdout) + + if len(result.Stdout) != 1 { + log.Log(GITPBWARN, "git LastTag() error:", result.Stdout) + return "" + } + + hash := result.Stdout[0] + + cmd = []string{"git", "describe", "--tags", "--always", hash} + result = repo.RunQuiet(cmd) + + if len(result.Stdout) != 1 { + log.Log(GITPBWARN, "git LastTag() error:", result.Stdout) + return "" + } + + return result.Stdout[0] +} + +func (repo *Repo) GitMasterVersion() string { + v, err := repo.gitVersionByName("master") + /* + count := repo.LenGitTags() + log.Info(repo.GoPath, "tag count", count) + repo.UpdateGitTags() + count = repo.LenGitTags() + log.Info(repo.GoPath, "tag count", count) + */ + + if err == nil { + return v + } else { + log.Log(GITPBWARN, "gitpb.GitMasterVersion() error:", err) + return "" + } +} + +func (repo *Repo) GitDevelVersion() string { + v, err := repo.gitVersionByName("devel") + if err == nil { + return v + } else { + log.Log(GITPBWARN, "gitpb.GitDevelVersion() error:", err) + return "" + } +} + +func (repo *Repo) GitUserVersion() string { + v, err := repo.gitVersionByName("jcarr") + if err == nil { + return v + } else { + log.Log(GITPBWARN, "gitpb.GitUserVersion() error:", err) + return "" + } +} + +func (repo *Repo) gitVersionByName(name string) (string, error) { + name = strings.TrimSpace(name) + + if name == "" { + // git will return the current tag + r := repo.RunQuiet([]string{"git", "describe", "--tags", "--always"}) + output := strings.Join(r.Stdout, "\n") + if r.Error != nil { + log.Log(GITPBWARN, "gitDescribeByName() output might have worked anyway:", output) + log.Log(GITPBWARN, "gitDescribeByName() not in a git repo?", r.Error, repo.GoPath) + return "", r.Error + } + return strings.TrimSpace(output), nil + } + if ! repo.IsBranch(name) { + // tag does not exist + log.Log(GITPBWARN, "LocalTagExists()", name, "did not exist") + return "", errors.New("gitDescribeByName() git fatal: Not a valid object name: " + name) + } + cmd := []string{"git", "describe", "--tags", "--always", name} + result := repo.RunQuiet(cmd) + output := strings.Join(result.Stdout, "\n") + if result.Error != nil { + log.Log(GITPBWARN, "cmd =", cmd) + log.Log(GITPBWARN, "err =", result.Error) + log.Log(GITPBWARN, "output (might have worked with error?) =", output) + log.Log(GITPBWARN, "not in a git repo or bad tag?", repo.GoPath) + return "", result.Error + } + + return strings.TrimSpace(output), nil +} + +// find a branch name +// will find "master" or "devel" +// will also find "v0.1.1" +// or will find "patches-from-foo" +// will return *any* match on any git branch because it doesn't +// matter much here yet +// eventually this will be worked out by forge in some future code that hasn't been made yet +func (repo *Repo) IsBranch(findname string) bool { + loop := repo.Tags.All() + for loop.Scan() { + t := loop.Next() + // log.Info("LocalTagExists() tag:", t.Refname) + + tagname := t.Refname + if strings.HasPrefix(tagname, "refs/remotes") { + continue + } + path, filename := filepath.Split(tagname) + log.Log(GITPB, "gitpb.IsBranch() tag:", path, filename, "from", repo.GoPath) + if filename == findname { + log.Log(GITPB, "gitpb.IsBranch() found tag:", path, filename, "from", repo.GoPath) + return true + } + } + log.Log(GITPBWARN, "did not find tag:", findname, "in", repo.GoPath) + return false +} diff --git a/gitTag.update.go b/gitTag.update.go new file mode 100644 index 0000000..163b379 --- /dev/null +++ b/gitTag.update.go @@ -0,0 +1,83 @@ +package gitpb + +import ( + "slices" + "strings" + "time" + + "go.wit.com/lib/gui/shell" + "go.wit.com/log" + timestamppb "google.golang.org/protobuf/types/known/timestamppb" +) + +// Update repo.Refs from .git/ +func (repo *Repo) UpdateGitTags() error { + // delete the old hash + // r.DeleteByHash(hash) + repo.Tags.GitTags = nil + + tags := []string{"%(objectname)", "%(creatordate)", "%(*authordate)", "%(refname)", "%(subject)"} + format := strings.Join(tags, "_,,,_") + cmd := []string{"git", "for-each-ref", "--sort=taggerdate", "--format", format} + // log.Info("RUNNING:", strings.Join(cmd, " ")) + result := shell.PathRunQuiet(repo.FullPath, cmd) + if result.Error != nil { + log.Warn("git for-each-ref error:", result.Error) + return result.Error + } + + lines := result.Stdout + // reverse the git order + slices.Reverse(lines) + + var refname string + var hash string + var subject string + var ctime *timestamppb.Timestamp + var atime *timestamppb.Timestamp + + for i, line := range lines { + var parts []string + parts = make([]string, 0) + parts = strings.Split(line, "_,,,_") + if len(parts) != 5 { + log.Info("tag error:", i, parts) + continue + } + hash = parts[0] + if parts[1] != "" { + tmp := getGitDateStamp(parts[1]) + ctime = timestamppb.New(tmp) + } + if parts[2] != "" { + tmp := getGitDateStamp(parts[2]) + atime = timestamppb.New(tmp) + } + refname = parts[3] + subject = parts[4] + + newr := GitTag{ + Refname: refname, + Hash: hash, + Subject: subject, + Creatordate: ctime, + Authordate: atime, + } + + repo.Tags.Append(&newr) + } + return nil +} + +// converts a git for-each-ref date. "Wed Feb 7 10:13:38 2024 -0600" +func getGitDateStamp(gitdefault string) time.Time { + // now := time.Now().Format("Wed Feb 7 10:13:38 2024 -0600") + const gitLayout = "Mon Jan 2 15:04:05 2006 -0700" + tagTime, err := time.Parse(gitLayout, gitdefault) + if err != nil { + log.Warn("GOT THIS IN PARSE AAA." + gitdefault + ".AAA") + log.Warn(err) + return time.Now() + } + return tagTime +} diff --git a/gitTags.proto b/gitTags.proto deleted file mode 100644 index dd3d77a..0000000 --- a/gitTags.proto +++ /dev/null @@ -1,13 +0,0 @@ -syntax = "proto3"; - -package gitpb; - -import "google/protobuf/timestamp.proto"; // Import the well-known type for Timestamp - -message GitTag { - string refname = 1; // tag name. treated as unique - google.protobuf.Timestamp creatordate = 2; // git creatordate - google.protobuf.Timestamp authordate = 3; // git creatordate - string objectname = 4; // git hash - string subject = 5; // git tag subject -} diff --git a/gitTags.query.go b/gitTags.query.go deleted file mode 100644 index 029bd0b..0000000 --- a/gitTags.query.go +++ /dev/null @@ -1,134 +0,0 @@ -package gitpb - -// runs git, parses output -// types faster than you can - -import ( - "errors" - "path/filepath" - "strings" - - "go.wit.com/log" -) - -func (repo *Repo) GetLastTag() string { - cmd := []string{"git", "rev-list", "--tags", "--max-count=1"} - result := repo.RunQuiet(cmd) - log.Info("getLastTagVersion()", result.Stdout) - - if len(result.Stdout) != 1 { - log.Log(GITPBWARN, "git LastTag() error:", result.Stdout) - return "" - } - - hash := result.Stdout[0] - - cmd = []string{"git", "describe", "--tags", "--always", hash} - result = repo.RunQuiet(cmd) - - if len(result.Stdout) != 1 { - log.Log(GITPBWARN, "git LastTag() error:", result.Stdout) - return "" - } - - return result.Stdout[0] -} - -func (repo *Repo) GitMasterVersion() string { - v, err := repo.gitVersionByName("master") - /* - count := repo.LenGitTags() - log.Info(repo.GoPath, "tag count", count) - repo.UpdateGitTags() - count = repo.LenGitTags() - log.Info(repo.GoPath, "tag count", count) - */ - - if err == nil { - return v - } else { - log.Log(GITPBWARN, "gitpb.GitMasterVersion() error:", err) - return "" - } -} - -func (repo *Repo) GitDevelVersion() string { - v, err := repo.gitVersionByName("devel") - if err == nil { - return v - } else { - log.Log(GITPBWARN, "gitpb.GitDevelVersion() error:", err) - return "" - } -} - -func (repo *Repo) GitUserVersion() string { - v, err := repo.gitVersionByName("jcarr") - if err == nil { - return v - } else { - log.Log(GITPBWARN, "gitpb.GitUserVersion() error:", err) - return "" - } -} - -func (repo *Repo) gitVersionByName(name string) (string, error) { - name = strings.TrimSpace(name) - - if name == "" { - // git will return the current tag - r := repo.RunQuiet([]string{"git", "describe", "--tags", "--always"}) - output := strings.Join(r.Stdout, "\n") - if r.Error != nil { - log.Log(GITPBWARN, "gitDescribeByName() output might have worked anyway:", output) - log.Log(GITPBWARN, "gitDescribeByName() not in a git repo?", r.Error, repo.GoPath) - return "", r.Error - } - return strings.TrimSpace(output), nil - } - if ! repo.IsBranch(name) { - // tag does not exist - log.Log(GITPBWARN, "LocalTagExists()", name, "did not exist") - return "", errors.New("gitDescribeByName() git fatal: Not a valid object name: " + name) - } - cmd := []string{"git", "describe", "--tags", "--always", name} - result := repo.RunQuiet(cmd) - output := strings.Join(result.Stdout, "\n") - if result.Error != nil { - log.Log(GITPBWARN, "cmd =", cmd) - log.Log(GITPBWARN, "err =", result.Error) - log.Log(GITPBWARN, "output (might have worked with error?) =", output) - log.Log(GITPBWARN, "not in a git repo or bad tag?", repo.GoPath) - return "", result.Error - } - - return strings.TrimSpace(output), nil -} - -// find a branch name -// will find "master" or "devel" -// will also find "v0.1.1" -// or will find "patches-from-foo" -// will return *any* match on any git branch because it doesn't -// matter much here yet -// eventually this will be worked out by forge in some future code that hasn't been made yet -func (repo *Repo) IsBranch(findname string) bool { - loop := repo.AllTags() - for loop.Scan() { - t := loop.Next() - // log.Info("LocalTagExists() tag:", t.Refname) - - tagname := t.Refname - if strings.HasPrefix(tagname, "refs/remotes") { - continue - } - path, filename := filepath.Split(tagname) - log.Log(GITPB, "gitpb.IsBranch() tag:", path, filename, "from", repo.GoPath) - if filename == findname { - log.Log(GITPB, "gitpb.IsBranch() found tag:", path, filename, "from", repo.GoPath) - return true - } - } - log.Log(GITPBWARN, "did not find tag:", findname, "in", repo.GoPath) - return false -} diff --git a/gitTags.sort.go b/gitTags.sort.go deleted file mode 100644 index b0079ff..0000000 --- a/gitTags.sort.go +++ /dev/null @@ -1,149 +0,0 @@ -package gitpb - -// this is becoming a standard format -// todo: autogenerate this from the .proto file? - -import ( - "fmt" - "os" - "sort" - sync "sync" - "time" -) - -// bad global lock until I figure out some other plan -var gitTagslock sync.RWMutex - -type GitTagIterator struct { - sync.RWMutex - - packs []*GitTag - index int -} - -// NewGitTagIterator initializes a new iterator. -func NewGitTagIterator(packs []*GitTag) *GitTagIterator { - return &GitTagIterator{packs: packs} -} - -// Scan moves to the next element and returns false if there are no more packs. -func (it *GitTagIterator) Scan() bool { - if it.index >= len(it.packs) { - return false - } - it.index++ - return true -} - -// GitTag returns the current repo. -func (it *GitTagIterator) Next() *GitTag { - if it.packs[it.index-1] == nil { - for i, d := range it.packs { - fmt.Println("i =", i, d) - } - fmt.Println("len =", len(it.packs)) - fmt.Println("repo == nil", it.index, it.index-1) - os.Exit(-1) - } - return it.packs[it.index-1] -} - -// Use Scan() in a loop, similar to a while loop -// -// for iterator.Scan() { -// d := iterator.GitTag() -// fmt.Println("GitTag UUID:", d.Uuid) -// } - -func (r *Repo) AllTags() *GitTagIterator { - repoPointers := r.selectAllGitTags() - - iterator := NewGitTagIterator(repoPointers) - return iterator -} - -func (r *Repo) SortTagsByName() *GitTagIterator { - packs := r.selectAllGitTags() - - sort.Sort(GitTagsByName(packs)) - - iterator := NewGitTagIterator(packs) - return iterator -} - -// enforces no duplicate package names -func (repo *Repo) AppendGitTag(newP *GitTag) bool { - gitTagslock.Lock() - defer gitTagslock.Unlock() - - for _, p := range repo.GitTags { - if p.Refname == newP.Refname { - return false - } - } - - repo.GitTags = append(repo.GitTags, newP) - return true -} - -// returns time.Duration since last Update() -func (r *GitTag) Age(newP *GitTag) time.Duration { - t := time.Since(r.Creatordate.AsTime()) - return t -} - -// find a package by name -func (repo *Repo) FindGitTagByName(name string) *GitTag { - gitTagslock.RLock() - defer gitTagslock.RUnlock() - - for _, p := range repo.GitTags { - if p.Refname == name { - return p - } - } - - return nil -} - -func (repo *Repo) LenGitTags() int { - gitTagslock.RLock() - defer gitTagslock.RUnlock() - - return len(repo.GitTags) -} - -type GitTagsByName []*GitTag - -func (a GitTagsByName) Len() int { return len(a) } -func (a GitTagsByName) Less(i, j int) bool { return a[i].Refname < a[j].Refname } -func (a GitTagsByName) Swap(i, j int) { a[i], a[j] = a[j], a[i] } - -// safely returns a slice of pointers to the GitTag protobufs -func (repo *Repo) selectAllGitTags() []*GitTag { - gitTagslock.RLock() - defer gitTagslock.RUnlock() - - // Create a new slice to hold pointers to eachGitTag - var allGitTags []*GitTag - allGitTags = make([]*GitTag, len(repo.GitTags)) - for i, p := range repo.GitTags { - allGitTags[i] = p // Copy pointers for safe iteration - } - - return allGitTags -} - -func (repo *Repo) DeleteTagByHash(hash string) *GitTag { - gitTagslock.Lock() - defer gitTagslock.Unlock() - - for i, _ := range repo.GitTags { - if repo.GitTags[i].Objectname == hash { - repo.GitTags[i] = repo.GitTags[len(repo.GitTags)-1] - repo.GitTags = repo.GitTags[:len(repo.GitTags)-1] - return nil - } - } - return nil -} diff --git a/gitTags.update.go b/gitTags.update.go deleted file mode 100644 index 41e71ed..0000000 --- a/gitTags.update.go +++ /dev/null @@ -1,83 +0,0 @@ -package gitpb - -import ( - "slices" - "strings" - "time" - - "go.wit.com/lib/gui/shell" - "go.wit.com/log" - timestamppb "google.golang.org/protobuf/types/known/timestamppb" -) - -// Update repo.Refs from .git/ -func (repo *Repo) UpdateGitTags() error { - // delete the old hash - // r.DeleteByHash(hash) - repo.GitTags = nil - - tags := []string{"%(objectname)", "%(creatordate)", "%(*authordate)", "%(refname)", "%(subject)"} - format := strings.Join(tags, "_,,,_") - cmd := []string{"git", "for-each-ref", "--sort=taggerdate", "--format", format} - // log.Info("RUNNING:", strings.Join(cmd, " ")) - result := shell.PathRunQuiet(repo.FullPath, cmd) - if result.Error != nil { - log.Warn("git for-each-ref error:", result.Error) - return result.Error - } - - lines := result.Stdout - // reverse the git order - slices.Reverse(lines) - - var refname string - var hash string - var subject string - var ctime *timestamppb.Timestamp - var atime *timestamppb.Timestamp - - for i, line := range lines { - var parts []string - parts = make([]string, 0) - parts = strings.Split(line, "_,,,_") - if len(parts) != 5 { - log.Info("tag error:", i, parts) - continue - } - hash = parts[0] - if parts[1] != "" { - tmp := getGitDateStamp(parts[1]) - ctime = timestamppb.New(tmp) - } - if parts[2] != "" { - tmp := getGitDateStamp(parts[2]) - atime = timestamppb.New(tmp) - } - refname = parts[3] - subject = parts[4] - - newr := GitTag{ - Refname: refname, - Objectname: hash, - Subject: subject, - Creatordate: ctime, - Authordate: atime, - } - - repo.AppendGitTag(&newr) - } - return nil -} - -// converts a git for-each-ref date. "Wed Feb 7 10:13:38 2024 -0600" -func getGitDateStamp(gitdefault string) time.Time { - // now := time.Now().Format("Wed Feb 7 10:13:38 2024 -0600") - const gitLayout = "Mon Jan 2 15:04:05 2006 -0700" - tagTime, err := time.Parse(gitLayout, gitdefault) - if err != nil { - log.Warn("GOT THIS IN PARSE AAA." + gitdefault + ".AAA") - log.Warn(err) - return time.Now() - } - return tagTime -} diff --git a/goDep.helpers.go b/goDep.helpers.go new file mode 100644 index 0000000..470d750 --- /dev/null +++ b/goDep.helpers.go @@ -0,0 +1,52 @@ +package gitpb + +// this is becoming a standard format +// todo: autogenerate this from the .proto file? + +import ( + "time" +) + +func (repo *Repo) DeleteGoDepByHash(hash string) *GoDep { + /* + refslock.Lock() + defer refslock.Unlock() + + for i, _ := range repo.GoDeps { + if repo.GoDeps[i].Hash == hash { + repo.GoDeps[i] = repo.GoDeps[len(repo.GoDeps)-1] + repo.GoDeps = repo.GoDeps[:len(repo.GoDeps)-1] + return nil + } + } + */ + return nil +} + +// enforces no duplicate package names +func (repo *Repo) AppendGoDep(newP *GoDep) bool { + /* + refslock.Lock() + defer refslock.Unlock() + + for _, p := range repo.GoDeps { + if p.GoPath == newP.GoPath { + return false + } + } + + repo.GoDeps = append(repo.GoDeps, newP) + */ + return repo.GoDeps.AppendUniqueGoPath(newP) +} + +// returns time.Duration since last scan of go.sum & go.mod +func (repo *Repo) AgeGoDep() time.Duration { + t := time.Since(repo.LastGoDep.AsTime()) + return t +} + +// find a dependancy by the go path +func (repo *Repo) FindGoDepByPath(gopath string) *GoDep { + return repo.GoDeps.FindByGoPath(gopath) +} diff --git a/goDep.proto b/goDep.proto new file mode 100644 index 0000000..5128c1c --- /dev/null +++ b/goDep.proto @@ -0,0 +1,21 @@ +syntax = "proto3"; + +// store go dependancies + +package gitpb; + +import "google/protobuf/timestamp.proto"; // Import the well-known type for Timestamp + +message GoDep { + string hash = 1; // md5sum/hash value from the go.sum file + google.protobuf.Timestamp ctime = 2; // get the go date from 'go list' ? + string version = 3; // v1.2.2 + string goPath = 4; // "go.wit.com/lib/foo" + string goVersion = 5; // version of golang the developer used to make this package version +} + +message GoDeps { + string uuid = 1; // I guess why not just have this on each file + string version = 2; // maybe can be used for protobuf schema change violations + repeated GoDep goDeps = 3; +} diff --git a/goDep.redoGoMod.go b/goDep.redoGoMod.go new file mode 100644 index 0000000..6b6f222 --- /dev/null +++ b/goDep.redoGoMod.go @@ -0,0 +1,132 @@ +package gitpb + +// does processing on the go.mod and go.sum files + +import ( + "bufio" + "errors" + "os" + "path/filepath" + "strings" + + "go.wit.com/log" +) + +// poor name perhaps. It's because in most of these +// repos you can also type "make redomod" to do the same thing +// since it's a Makefile task that is also useful to be able to run +// from the command line +func (repo *Repo) MakeRedomod() (bool, error) { + // unset the go development ENV var to generate release files + os.Unsetenv("GO111MODULE") + if ok, err := repo.strictRun([]string{"rm", "-f", "go.mod", "go.sum"}); !ok { + log.Warn("rm go.mod go.sum failed", err) + return ok, err + } + if ok, err := repo.strictRun([]string{"go", "mod", "init", repo.GoPath}); !ok { + log.Warn("go mod init failed", err) + return ok, err + } + if ok, err := repo.strictRun([]string{"go", "mod", "tidy"}); !ok { + log.Warn("go mod tidy failed", err) + return ok, err + } + log.Info("MakeRedomod() worked", repo.GoPath) + + if repo.Exists("go.sum") { + // return the attempt to parse go.mod & go.sum + return repo.parseGoSum() + } + repo.GoDeps = nil + repo.GoPrimitive = false + + ok, err := repo.isPrimativeGoMod() + if err != nil { + // this means this repo does not depend on any other package + log.Info("PRIMATIVE repo error:", repo.GoPath, "err =", err) + return false, err + } + if ok { + // this means the repo is primitive so there is no go.sum + repo.GoPrimitive = true + return true, nil + } + // this should never happen + return false, errors.New("MakeRedomod() logic failed") +} + +// reads and parses the go.sum file +func (repo *Repo) parseGoSum() (bool, error) { + // empty out what was there before + repo.GoDeps = nil + tmp := filepath.Join(repo.FullPath, "go.sum") + gosum, err := os.Open(tmp) + if err != nil { + log.Warn("missing go.sum", repo.FullPath) + return false, err + } + defer gosum.Close() + + scanner := bufio.NewScanner(gosum) + log.Info("gosum:", tmp) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + + parts := strings.Split(line, " ") + if len(parts) == 3 { + godep := strings.TrimSpace(parts[0]) + version := strings.TrimSpace(parts[1]) + if strings.HasSuffix(version, "/go.mod") { + version = strings.TrimSuffix(version, "/go.mod") + } + new1 := GoDep{ + GoPath: godep, + Version: version, + } + repo.GoDeps.AppendUniqueGoPath(&new1) + /* + found := repo.FindGoDepByPath(godep) + if found == nil { + currentversion, ok := deps[godep] + if ok { + // only use the first value found in the file? + // this shouldn't have been possible. this function should + // only be called from MakeRedomod() + // todo: make go things a seperate package so this function + // isn't exported? + if version != currentversion { + log.Warn("\tgo.sum ", godep, "had both", version, currentversion) + } + } else { + deps[godep] = version + log.Info("\t", godep, "=", version) + } + */ + } else { + // I've never seen this happen yet + panic(errors.New("go.sum invalid: " + line)) + // return false, errors.New("go.sum invalid: " + line) + } + } + + if err := scanner.Err(); err != nil { + repo.GoDeps = nil + return false, err + } + return true, nil +} +func (repo *Repo) RepoType() string { + os.Setenv("GO111MODULE", "off") + cmd := []string{"go", "list", "-f", "'{{if eq .Name \"main\"}}binary{{else}}library{{end}}'"} + // cmd := []string{"go", "list", "-f", "'{{.Name}}'"} // probably use this. this just prints out the package name + // cmd := []string{"go", "list", "-f", "'{{.ImportPath}}'"} // returns go.wit.com/lib/protobuf/gitpb + + result := repo.RunQuiet(cmd) + if result.Error != nil { + log.Warn("go list binary detect failed", result.Error) + return "" + } + output := strings.TrimSpace(strings.Join(result.Stdout, "\n")) + output = strings.Trim(output, "'") + return output +} diff --git a/godep.helpers.go b/godep.helpers.go deleted file mode 100644 index 007cc61..0000000 --- a/godep.helpers.go +++ /dev/null @@ -1,57 +0,0 @@ -package gitpb - -// this is becoming a standard format -// todo: autogenerate this from the .proto file? - -import ( - "time" -) - -func (repo *Repo) DeleteGoDepByHash(hash string) *GoDep { - refslock.Lock() - defer refslock.Unlock() - - for i, _ := range repo.GoDeps { - if repo.GoDeps[i].Hash == hash { - repo.GoDeps[i] = repo.GoDeps[len(repo.GoDeps)-1] - repo.GoDeps = repo.GoDeps[:len(repo.GoDeps)-1] - return nil - } - } - return nil -} - -// enforces no duplicate package names -func (repo *Repo) AppendGoDep(newP *GoDep) bool { - refslock.Lock() - defer refslock.Unlock() - - for _, p := range repo.GoDeps { - if p.GoPath == newP.GoPath { - return false - } - } - - repo.GoDeps = append(repo.GoDeps, newP) - return true -} - -// returns time.Duration since last scan of go.sum & go.mod -func (repo *Repo) AgeGoDep() time.Duration { - t := time.Since(repo.LastGoDep.AsTime()) - return t -} - -// find a dependancy by the go path -func (repo *Repo) FindGoDepByPath(gopath string) *GoDep { - refslock.RLock() - defer refslock.RUnlock() - - for _, p := range repo.GoDeps { - if p.GoPath == gopath { - return p - } - } - - return nil -} diff --git a/godep.proto b/godep.proto deleted file mode 100644 index 1789cd3..0000000 --- a/godep.proto +++ /dev/null @@ -1,15 +0,0 @@ -syntax = "proto3"; - -// store go dependancies - -package gitpb; - -import "google/protobuf/timestamp.proto"; // Import the well-known type for Timestamp - -message GoDep { - string hash = 1; // md5sum/hash value from the go.sum file - google.protobuf.Timestamp ctime = 2; // get the go date from 'go list' ? - string version = 3; // v1.2.2 - string goPath = 4; // "go.wit.com/lib/foo" - string goVersion = 5; // version of golang the developer used to make this package version -} diff --git a/godep.redoGoMod.go b/godep.redoGoMod.go deleted file mode 100644 index 79a86f7..0000000 --- a/godep.redoGoMod.go +++ /dev/null @@ -1,132 +0,0 @@ -package gitpb - -// does processing on the go.mod and go.sum files - -import ( - "bufio" - "errors" - "os" - "path/filepath" - "strings" - - "go.wit.com/log" -) - -// poor name perhaps. It's because in most of these -// repos you can also type "make redomod" to do the same thing -// since it's a Makefile task that is also useful to be able to run -// from the command line -func (repo *Repo) MakeRedomod() (bool, error) { - // unset the go development ENV var to generate release files - os.Unsetenv("GO111MODULE") - if ok, err := repo.strictRun([]string{"rm", "-f", "go.mod", "go.sum"}); !ok { - log.Warn("rm go.mod go.sum failed", err) - return ok, err - } - if ok, err := repo.strictRun([]string{"go", "mod", "init", repo.GoPath}); !ok { - log.Warn("go mod init failed", err) - return ok, err - } - if ok, err := repo.strictRun([]string{"go", "mod", "tidy"}); !ok { - log.Warn("go mod tidy failed", err) - return ok, err - } - log.Info("MakeRedomod() worked", repo.GoPath) - - if repo.Exists("go.sum") { - // return the attempt to parse go.mod & go.sum - return repo.parseGoSum() - } - repo.GoDeps = nil - repo.GoPrimitive = false - - ok, err := repo.isPrimativeGoMod() - if err != nil { - // this means this repo does not depend on any other package - log.Info("PRIMATIVE repo error:", repo.GoPath, "err =", err) - return false, err - } - if ok { - // this means the repo is primitive so there is no go.sum - repo.GoPrimitive = true - return true, nil - } - // this should never happen - return false, errors.New("MakeRedomod() logic failed") -} - -// reads and parses the go.sum file -func (repo *Repo) parseGoSum() (bool, error) { - // empty out what was there before - repo.GoDeps = nil - tmp := filepath.Join(repo.FullPath, "go.sum") - gosum, err := os.Open(tmp) - if err != nil { - log.Warn("missing go.sum", repo.FullPath) - return false, err - } - defer gosum.Close() - - scanner := bufio.NewScanner(gosum) - log.Info("gosum:", tmp) - for scanner.Scan() { - line := strings.TrimSpace(scanner.Text()) - - parts := strings.Split(line, " ") - if len(parts) == 3 { - godep := strings.TrimSpace(parts[0]) - version := strings.TrimSpace(parts[1]) - if strings.HasSuffix(version, "/go.mod") { - version = strings.TrimSuffix(version, "/go.mod") - } - new1 := GoDep{ - GoPath: godep, - Version: version, - } - repo.AppendGoDep(&new1) - /* - found := repo.FindGoDepByPath(godep) - if found == nil { - currentversion, ok := deps[godep] - if ok { - // only use the first value found in the file? - // this shouldn't have been possible. this function should - // only be called from MakeRedomod() - // todo: make go things a seperate package so this function - // isn't exported? - if version != currentversion { - log.Warn("\tgo.sum ", godep, "had both", version, currentversion) - } - } else { - deps[godep] = version - log.Info("\t", godep, "=", version) - } - */ - } else { - // I've never seen this happen yet - panic(errors.New("go.sum invalid: " + line)) - // return false, errors.New("go.sum invalid: " + line) - } - } - - if err := scanner.Err(); err != nil { - repo.GoDeps = nil - return false, err - } - return true, nil -} -func (repo *Repo) RepoType() string { - os.Setenv("GO111MODULE", "off") - cmd := []string{"go", "list", "-f", "'{{if eq .Name \"main\"}}binary{{else}}library{{end}}'"} - // cmd := []string{"go", "list", "-f", "'{{.Name}}'"} // probably use this. this just prints out the package name - // cmd := []string{"go", "list", "-f", "'{{.ImportPath}}'"} // returns go.wit.com/lib/protobuf/gitpb - - result := repo.RunQuiet(cmd) - if result.Error != nil { - log.Warn("go list binary detect failed", result.Error) - return "" - } - output := strings.TrimSpace(strings.Join(result.Stdout, "\n")) - output = strings.Trim(output, "'") - return output -} diff --git a/godep.sort.go b/godep.sort.go deleted file mode 100644 index ead6e59..0000000 --- a/godep.sort.go +++ /dev/null @@ -1,99 +0,0 @@ -package gitpb - -// this is becoming a standard format -// todo: autogenerate this from the .proto file? - -import ( - "fmt" - "os" - "sort" - sync "sync" -) - -// bad global lock until I figure out some other plan -var godeplock sync.RWMutex - -type GoDepIterator struct { - sync.RWMutex - - packs []*GoDep - index int -} - -// NewGoDepGoDepIterator initializes a new iterator. -func NewGoDepIterator(packs []*GoDep) *GoDepIterator { - return &GoDepIterator{packs: packs} -} - -// Scan moves to the next element and returns false if there are no more packs. -func (it *GoDepIterator) Scan() bool { - if it.index >= len(it.packs) { - return false - } - it.index++ - return true -} - -// GoDep returns the current repo. -func (it *GoDepIterator) GoDep() *GoDep { - if it.packs[it.index-1] == nil { - for i, d := range it.packs { - fmt.Println("i =", i, d) - } - fmt.Println("len =", len(it.packs)) - fmt.Println("repo == nil", it.index, it.index-1) - os.Exit(-1) - } - return it.packs[it.index-1] -} - -// Use Scan() in a loop, similar to a while loop -// -// for iterator.Scan() { -// d := iterator.GoDep() -// fmt.Println("GoDep UUID:", d.Uuid) -// } - -func (r *Repo) AllGoDeps() *GoDepIterator { - repoPointers := r.selectAllGoDeps() - - iterator := NewGoDepIterator(repoPointers) - return iterator -} - -func (r *Repo) SortGoDepsByName() *GoDepIterator { - packs := r.selectAllGoDeps() - - sort.Sort(GoDepByPath(packs)) - - iterator := NewGoDepIterator(packs) - return iterator -} - -func (repo *Repo) Len() int { - refslock.RLock() - defer refslock.RUnlock() - - return len(repo.GoDeps) -} - -type GoDepByPath []*GoDep - -func (a GoDepByPath) Len() int { return len(a) } -func (a GoDepByPath) Less(i, j int) bool { return a[i].GoPath < a[j].GoPath } -func (a GoDepByPath) Swap(i, j int) { a[i], a[j] = a[j], a[i] } - -// safely returns a slice of pointers to the GoDep protobufs -func (r *Repo) selectAllGoDeps() []*GoDep { - refslock.RLock() - defer refslock.RUnlock() - - // Create a new slice to hold pointers to each GoDep - var allPacks []*GoDep - allPacks = make([]*GoDep, len(r.GoDeps)) - for i, p := range r.GoDeps { - allPacks[i] = p // Copy pointers for safe iteration - } - - return allPacks -} diff --git a/refs.sort.go b/refs.sort.go deleted file mode 100644 index bc8fbaf..0000000 --- a/refs.sort.go +++ /dev/null @@ -1,149 +0,0 @@ -package gitpb - -// this is becoming a standard format -// todo: autogenerate this from the .proto file? - -import ( - "fmt" - "os" - "sort" - sync "sync" - "time" -) - -// bad global lock until I figure out some other plan -var refslock sync.RWMutex - -type RefIterator struct { - sync.RWMutex - - packs []*Ref - index int -} - -// NewRefIterator initializes a new iterator. -func NewRefIterator(packs []*Ref) *RefIterator { - return &RefIterator{packs: packs} -} - -// Scan moves to the next element and returns false if there are no more packs. -func (it *RefIterator) Scan() bool { - if it.index >= len(it.packs) { - return false - } - it.index++ - return true -} - -// Ref returns the current repo. -func (it *RefIterator) Ref() *Ref { - if it.packs[it.index-1] == nil { - for i, d := range it.packs { - fmt.Println("i =", i, d) - } - fmt.Println("len =", len(it.packs)) - fmt.Println("repo == nil", it.index, it.index-1) - os.Exit(-1) - } - return it.packs[it.index-1] -} - -// Use Scan() in a loop, similar to a while loop -// -// for iterator.Scan() { -// d := iterator.Ref() -// fmt.Println("Ref UUID:", d.Uuid) -// } - -func (r *Repo) All() *RefIterator { - repoPointers := r.selectAllRefs() - - iterator := NewRefIterator(repoPointers) - return iterator -} - -func (r *Repo) SortByName() *RefIterator { - packs := r.selectAllRefs() - - sort.Sort(RefsByName(packs)) - - iterator := NewRefIterator(packs) - return iterator -} - -// enforces no duplicate package names -func (repo *Repo) AppendRef(newP *Ref) bool { - refslock.Lock() - defer refslock.Unlock() - - for _, p := range repo.Refs { - if p.RefName == newP.RefName { - return false - } - } - - repo.Refs = append(repo.Refs, newP) - return true -} - -// returns time.Duration since last Update() -func (r *Ref) Age(newP *Ref) time.Duration { - t := time.Since(r.Ctime.AsTime()) - return t -} - -// find a package by name -func (repo *Repo) FindRefByName(name string) *Ref { - refslock.RLock() - defer refslock.RUnlock() - - for _, p := range repo.Refs { - if p.RefName == name { - return p - } - } - - return nil -} - -func (repo *Repo) LenRefs() int { - refslock.RLock() - defer refslock.RUnlock() - - return len(repo.Refs) -} - -type RefsByName []*Ref - -func (a RefsByName) Len() int { return len(a) } -func (a RefsByName) Less(i, j int) bool { return a[i].RefName < a[j].RefName } -func (a RefsByName) Swap(i, j int) { a[i], a[j] = a[j], a[i] } - -// safely returns a slice of pointers to the Ref protobufs -func (repo *Repo) selectAllRefs() []*Ref { - refslock.RLock() - defer refslock.RUnlock() - - // Create a new slice to hold pointers to each Ref - var allRefs []*Ref - allRefs = make([]*Ref, len(repo.Refs)) - for i, p := range repo.Refs { - allRefs[i] = p // Copy pointers for safe iteration - } - - return allRefs -} - -func (repo *Repo) DeleteByHash(hash string) *Ref { - refslock.Lock() - defer refslock.Unlock() - - for i, _ := range repo.Refs { - if repo.Refs[i].Hash == hash { - repo.Refs[i] = repo.Refs[len(repo.Refs)-1] - repo.Refs = repo.Refs[:len(repo.Refs)-1] - return nil - } - } - return nil -} diff --git a/refs.update.go b/refs.update.go deleted file mode 100644 index f103417..0000000 --- a/refs.update.go +++ /dev/null @@ -1,55 +0,0 @@ -package gitpb - -/* -// Update repo.Refs from .git/ -func (repo *Repo) UpdateGit() error { - // delete the old hash - // r.DeleteByHash(hash) - repo.Refs = nil - - tags := []string{"%(objectname)", "%(creatordate)", "%(*authordate)", "%(refname)", "%(subject)"} - format := strings.Join(tags, "_,,,_") - cmd := []string{"git", "for-each-ref", "--sort=taggerdate", "--format", format} - // log.Info("RUNNING:", strings.Join(cmd, " ")) - result := shell.PathRunQuiet(repo.FullPath, cmd) - if result.Error != nil { - log.Warn("git for-each-ref error:", result.Error) - return result.Error - } - - lines := result.Stdout - // reverse the git order - slices.Reverse(lines) - - var refName string - var hash string - var subject string - var ctime time.Time - - for i, line := range lines { - var parts []string - parts = make([]string, 0) - parts = strings.Split(line, "_,,,_") - if len(parts) != 5 { - log.Info("tag error:", i, parts) - continue - } - refName = parts[3] - hash = parts[0] - - ctime = getGitDateStamp(parts[1]) - - subject = parts[4] - } - - newr := Ref{ - Hash: hash, - Subject: subject, - RefName: refName, - Ctime: timestamppb.New(ctime), - } - - repo.AppendRef(&newr) - return nil -} -*/ diff --git a/repo.helpers.go b/repo.helpers.go index 50f3277..3c33583 100644 --- a/repo.helpers.go +++ b/repo.helpers.go @@ -16,20 +16,6 @@ func (all *Repos) DeleteByPath(gopath string) *Repo { return nil } -// find a package by gopath -func (all *Repos) FindByGoPath(gopath string) *Repo { - reposMu.RLock() - defer reposMu.RUnlock() - - for _, p := range all.Repos { - if p.GoPath == gopath { - return p - } - } - - return nil -} - // enforces no duplicate gopath's func (all *Repos) add(newP *Repo) bool { reposMu.Lock() diff --git a/repo.proto b/repo.proto index 57ba63c..f4ca350 100644 --- a/repo.proto +++ b/repo.proto @@ -5,28 +5,25 @@ package gitpb; // stores information about git repos // If the project is in golang, also gets the go language dependacies -import "refs.proto"; -import "godep.proto"; -import "gitTags.proto"; +import "goDep.proto"; +import "gitTag.proto"; import "google/protobuf/timestamp.proto"; // Import the well-known type for Timestamp message Repo { string fullPath = 1; // the actual path to the .git directory: '/home/devel/golang.org/x/tools' - repeated Ref refs = 2; - google.protobuf.Timestamp lastPull = 3; // last time a git pull was done + google.protobuf.Timestamp lastPull = 2; // last time a git pull was done - // things specific to golang projects - string goPath = 4; // the logical path as used by golang: 'go.wit.com/apps/helloworld' - repeated GoDep GoDeps = 6; - google.protobuf.Timestamp lastGoDep = 7; // last time go.sum was processed - bool goLibrary = 8; // if this is a golang library - bool goPrimitive = 9; // if this is a golang primitive - - repeated GitTag gitTags = 10; + string masterBranchName = 3; // git 'main' or 'master' branch name + string develBranchName = 4; // whatever the git 'devel' branch name is + string userBranchName = 5; // whatever your username branch is - string masterBranchName = 11; // git 'main' or 'master' branch name - string develBranchName = 12; // whatever the git 'devel' branch name is - string userBranchName = 13; // whatever your username branch is + // things specific to golang projects + string goPath = 6; // the logical path as used by golang: 'go.wit.com/apps/helloworld' + bool goLibrary = 7; // if this is a golang library + bool goPrimitive = 8; // if this is a golang primitive + GitTags tags = 9; + GoDeps goDeps = 10; + google.protobuf.Timestamp lastGoDep = 11; // last time go.sum was processed } message Repos { -- cgit v1.2.3