// Copyright 2017-2025 WIT.COM Inc. All rights reserved. // Use of this source code is governed by the GPL 3.0 package main import ( "errors" "fmt" "os" "path/filepath" "slices" "strings" "go.wit.com/lib/cobol" "go.wit.com/lib/config" "go.wit.com/lib/env" "go.wit.com/lib/protobuf/argvpb" "go.wit.com/lib/protobuf/gitpb" "go.wit.com/log" "google.golang.org/protobuf/types/known/timestamppb" ) func doStats(cmd *StatsCmd) (string, error) { var allerr error if argv.All { for r := range me.forge.Repos.IterByFullPath() { if r.Stats().Len() == 0 { _, err := doStatsRepo(r) allerr = errors.Join(allerr, err) } } return "verify ran everywhere", nil } repo := workingDirToRepo() if repo == nil { return "no repo", errors.New("working dir isn't a repo I know about") } if cmd.List { s := repo.Stats() s.SortPatchId() // s.SortCtime() footer := s.PrintTableLimit(-1) return "stats table: " + footer, nil } s, err := doStatsRepo(repo) allerr = errors.Join(allerr, err) user, err := refHash(repo, "heads/"+repo.GetUserBranchName()) allerr = errors.Join(allerr, err) master, err := refHash(repo, "remotes/origin/master") allerr = errors.Join(allerr, err) HEAD, err := refHash(repo, "remotes/origin/HEAD") allerr = errors.Join(allerr, err) log.Printf("user=%10.10s master=%10.10s HEAD=%10.10s\n", user, master, HEAD) safeDelete(repo, "refs/heads/"+repo.GetUserBranchName(), "refs/remotes/origin/HEAD") // delete user if safely contained in HEAD safeDelete(repo, user, HEAD) // checkes by hash ID // delete user if safely contained in HEAD return s, allerr } func doStatsRepo(r *gitpb.Repo) (string, error) { var allerr error pb, err := r.LoadStats("origin") if err == nil { log.Info("LoadStats() ok", pb.Filename) } else { log.Info("LoadStats() err", err) } allerr = errors.Join(allerr, err) if hasOrigin(r) { log.Info("todo: detect origin") } // collect the stats counter, err := last100(r, pb) allerr = errors.Join(allerr, err) s := log.Sprintf("found %d new hashes", counter) for stat := range pb.IterAll() { if stat.PatchId == "" { stat.PatchId, err = r.FindPatchIdByHash(stat.Hash) allerr = errors.Join(allerr, err) log.Info("patchid for hash", stat.Hash, "is", stat.PatchId) counter += 1 } } if counter > 0 { pb.SaveVerbose() } return s, nil } func collectStats(r *gitpb.Repo, pb *gitpb.Stats) error { return nil } // git show-ref refs/heads/devel func refHash(r *gitpb.Repo, name string) (string, error) { var hash string refname := "refs/" + name cmd := []string{"git", "show-ref", refname} cmdout := r.Run(cmd) for i, line := range cmdout.Stdout { parts := strings.Fields(line) if env.If("stats") { log.Info(parts[0], "LINE:", i, line) } hash = parts[0] } if len(cmdout.Stdout) != 1 { return "", errors.New("no refname:" + name) } return hash, nil } // git show-ref --verify refs/heads/devel func hasOrigin(r *gitpb.Repo) bool { // git show-ref refs/heads/devel return true } var standardFmts []string = []string{"H", "T", "at", "ct", "f"} var standardSeperator string = "___FORGE___" func makeFmts() string { // fmts := strings.Fields(config.GetPanic("standardFmts")) // fmts := strings.Fields(config.GetPanic("standardSeperator")) var all []string for _, fmtvar := range standardFmts { all = append(all, "%"+fmtvar) } return "--format=" + strings.Join(all, standardSeperator) } // the correct syntax for // git log -n 8 --format=%H%00%ae%00%as%00%s origin/HEAD func last100(r *gitpb.Repo, pb *gitpb.Stats) (int, error) { var allerr error var counter int cmd := []string{"git", "log", "-n", "100", makeFmts(), "origin/" + r.GetMasterBranchName()} // must use 'master' as queried from the git server // cmd := []string{"git", "log", "-n", "100", makeFmts(), "origin/HEAD"} // HEAD is _NOT_ always set if env.If("stats") { log.Info("Run:", cmd) } cmdout := r.Run(cmd) for i, line := range cmdout.Stdout { parts := strings.Split(line, standardSeperator) hash := parts[0] if env.If("stats") { log.Printf("LINE:%8.8s %2d %v\n", hash, i, parts[1:]) } found := pb.FindByHash(hash) if found != nil { // already have this hash continue } counter += 1 astat := new(gitpb.Stat) astat.Hash = hash ctime, err := cobol.GetTime(parts[2]) allerr = errors.Join(allerr, err) astat.Ctime = timestamppb.New(*ctime) // astat.Subject = parts[4] // astat.Type = gitpb.Stat_REMOTE pb.Append(astat) } return counter, allerr } func findPatchIdInStats(pb *gitpb.Stats, patchId string) *gitpb.Stat { for stat := range pb.IterAll() { if stat.PatchId == patchId { return stat } } // log.Info("findPatchId searched in", pb.Len(), "stats") return nil } // delete localRef if it's completely contained in the masterRef func safeDelete(r *gitpb.Repo, deleteHash string, keepHash string) error { // compare the branches hashok, hashbad, cmd1, cmd2, err := r.CompareHashes(keepHash, deleteHash) if err != nil { // things are really really messed up. might be 'branchless' at this point (?) log.Printf("%-13.13s %-55.55s err='%v' %s %v\n", "CMD ERR", r.FullPath, err, "NOT SAFE TO DELETE. Reload()?", cmd1) log.Printf("%-13.13s %-55.55s err='%v' %s %v\n", "CMD ERR", r.FullPath, err, "NOT SAFE TO DELETE. Reload()?", cmd2) return err } // things only in the master branch (safe to ignore) for _, line := range hashok { parts := strings.Split(line, "%00") // git log doesn't actually convert %00 to NULL patchId, err := r.FindPatchIdByHash(parts[0]) _ = err log.Printf("%-13.13s %-55.55s hId %10.10s pId %10.10s %v\n", "OK delete", r.FullPath, parts[0], patchId, parts[2:]) } if len(hashbad) > 0 { log.Printf("%-13.13s %v\n", "BAD cmd", cmd1) } var ACTUALLYOK bool = true // things still only in the local branch (bad to delete) for _, line := range hashbad { parts := strings.Split(line, "%00") // git log doesn't actually convert %00 to NULL patchId, err := r.FindPatchIdByHash(parts[0]) _ = err searchResult := log.Sprintf("NOPE(%d)", r.Stats().Len()) stat := findPatchIdInStats(r.Stats(), patchId) if stat != nil { searchResult = log.Sprintf("FOUND %10.10s %s", stat.PatchId, "todo: []slice") } else { ACTUALLYOK = false } log.Printf("%-13.13s %-55.55s hId %10.10s pId %10.10s %s %v\n", "BAD keep", r.FullPath, parts[0], patchId, searchResult, parts[2:]) } if ACTUALLYOK { // todo: force checkout to local master branch // before doing this cmd := []string{"git", "update-ref", "-d", deleteHash} // log.Printf("%-13.13s %-55.55s %v %s\n", "CMD OK", r.FullPath, cmd1, "") // log.Printf("%-13.13s %-55.55s %v %s\n", "CMD OK", r.FullPath, cmd2, "") if r.GetCurrentBranchName() == r.GetMasterBranchName() { log.Printf("%-55.55s %v %s\n", r.FullPath, cmd, "SAFE TO DELETE add --fix") } else { log.Printf("%-55.55s %v %s\n", r.FullPath, cmd, "SAFE TO DELETE BUT NOT ON MASTER BRANCH add --fix") } if argv.Fix { err := r.RunVerbose(cmd) if err != nil { log.Info(deleteHash, r.FullPath) s := "local user branch could not be deleted" argvpb.BadExit(s, err) } } return ErrorNeedArgvFix } return log.Errorf("NOT SAFE") } // /tmp/go-nsupdate$ git ls-remote // From git@github.com:SpComb/go-nsupdate.git // d65f28d10991354b3af0168001a4beea6b8242f9 HEAD // d65f28d10991354b3af0168001a4beea6b8242f9 refs/heads/master // fb04ebe13a16c01e1a7eb3892a0aca8c6db96fa0 refs/pull/3/head // d774220311b2d44c770e7431ec663c8875488a1e refs/pull/6/head // fefb9ea80745893a2203576bdc2872d33e539064 refs/pull/7/head // c09c90d0a931888862262a6ed3753eed0af4ddca refs/pull/8/head // the correct syntax for // git log -n 8 --format=%H%00%ae%00%as%00%s origin/HEAD func updateStats(r *gitpb.Repo, pb *gitpb.Stats, remoteName string) (int, error) { var allerr error var counter int cmd := []string{"git", "log", "-n", "100", makeFmts(), remoteName} // must use 'master' as queried from the git server // cmd := []string{"git", "log", "-n", "100", makeFmts(), "origin/HEAD"} // HEAD is _NOT_ always set if env.True("stats") { log.Info("Run:", cmd) } cmdout := r.Run(cmd) for i, line := range cmdout.Stdout { parts := strings.Split(line, standardSeperator) hash := parts[0] if len(parts) < 2 { log.Printf("Repo: %s\n", r.FullPath) log.Printf("CMD: %v\n", cmd) log.Printf("LINE:%8.8s %2d %v\n", hash, i, parts[1:]) } if env.True("stats") { log.Printf("LINE:%8.8s %2d %v\n", hash, i, parts[1:]) } found := pb.FindByHash(hash) if found != nil { // already have this hash continue } counter += 1 astat := new(gitpb.Stat) astat.Hash = hash ctime, err := cobol.GetTime(parts[2]) allerr = errors.Join(allerr, err) astat.Ctime = timestamppb.New(*ctime) // astat.Subject = parts[4] // astat.Type = gitpb.Stat_REMOTE pb.Append(astat) } return counter, allerr } /* func lookupRefs(r *gitpb.Repo, remoteRefs *gitpb.Stats) error { rname := remoteRefs.Name() for stat := range remoteRefs.IterAll() { log.Info("remote", rname, stat.Name) } } */ // returns err if anything changes or anything is wrong (todo: should these be different?) func makeRefs(r *gitpb.Repo, remoteName string) error { var counter int // inc if anything changes fullname := filepath.Join(r.FullPath, ".git", remoteName+".refs.pb") stats := gitpb.NewStats() stats.Filename = fullname err := config.ForceCreatePB(stats) if err != nil { return err } var last *gitpb.Stat for stat := range stats.IterAll() { if last == nil { last = stat continue } if strings.Compare(stat.Hash, last.Hash) > 0 { log.Info("Compare worked", stat.Hash, last.Hash, r.FullPath) } else { log.Info("Compare failed", stat.Hash, last.Hash, r.FullPath) os.Remove(stats.Filename) return errors.New("out of order") } if strings.Compare(stat.Hash, last.Hash) == 0 { panic("was match") } last = stat } // cmd := []string{"git", "show-ref"} // must use 'master' as queried from the git server cmd := []string{"git", "ls-remote", remoteName} // must use 'master' as queried from the git server // GO has 250k remote refs // cmd := []string{"git", "log", "-n", "100", makeFmts(), "origin/HEAD"} // HEAD is _NOT_ always set if env.True("stats") { log.Info("STATS VERBOSE Run:", cmd) } cmdout := r.Run(cmd) if len(cmdout.Stdout) == 0 { return errors.New("got nothing back") } var ticker int var done int var allnew []*gitpb.Stat for _, line := range cmdout.Stdout { line = strings.TrimSpace(line) parts := strings.Fields(line) if len(parts) != 2 { log.Printf("Repo: %s\n", r.FullPath) log.Printf("CMD: %v\n", cmd) log.Printf("LINE:%s\n", line) return errors.New(line) } if env.True("stats") { log.Printf("LINE:%v %d %s\n", parts, ticker, r.FullPath) } // refpath := filepath.Join("refs/remote", remoteName) // if strings.HasPrefix(parts[1], refpath) { // this ref is not from the remote // continue // } counter += 1 newstat := new(gitpb.Stat) // newstat.Type = gitpb.Stat_REMOTE newstat.Hash = parts[0] // newstat.Name = parts[1] if stats.Len() == 0 { stats.Append(newstat) continue } n, found := slices.BinarySearchFunc(stats.Stats, newstat, func(a, b *gitpb.Stat) int { return strings.Compare(a.Hash, b.Hash) }) _ = n if n > stats.Len() { log.Info("WTF n,len =", n, stats.Len()) stats.Append(newstat) continue } if n-1 < 0 { log.Info("WTF n,len =", n, stats.Len()) continue } testfind := stats.Stats[n-1] if testfind.Hash == newstat.Hash { done += 1 log.Info(counter, ticker, "N WAS RIGHT", n, found, newstat.Hash, "HASH", testfind.Hash) panic("fucknuts") } if found { done += 1 teststat := stats.FindByHash(newstat.Hash) if teststat == nil { // log.Info(counter, ticker, "FOUND TEST STAT:", n, found, "HASH", newstat) log.Printf("FOUND:%v %d/%d/%d %s %v\n", parts, done, counter, len(cmdout.Stdout), r.FullPath, teststat) panic("fucknuts") } continue } else { teststat := stats.FindByHash(newstat.Hash) if teststat != nil { log.Printf("NOT FOUND:%v %d/%d/%d %s %v\n", parts, done, counter, len(cmdout.Stdout), r.FullPath, teststat) // log.Info(counter, ticker, "NOT FOUND TEST STAT:", n, found, teststat, "HASH", newstat) panic("fucknuts") } } allnew = append(allnew, newstat) ticker += 1 if ticker > 1000 { log.Printf("TICKER:%v %d/%d/%d %s\n", parts, done, counter, len(cmdout.Stdout), r.FullPath) ticker = 0 break /* stats.Stats = append(stats.Stats, allnew...) stats.SortByHash() stats.Save() log.Printf("TICKER:%v %d/%d/%d %s\n", parts, done, counter, len(cmdout.Stdout), r.FullPath) return nil */ } } if counter > 0 { stats.Stats = append(stats.Stats, allnew...) stats.SortByHash() stats.Save() return errors.New(fmt.Sprintf("len(%d), ticker(%d) counter(%d) refs changed", stats.Len(), ticker, counter)) } return nil }