From 340872788e76776574be3ba292e1f7a43cc41a8f Mon Sep 17 00:00:00 2001 From: Jeff Carr Date: Sun, 15 Dec 2024 08:48:06 -0600 Subject: clone() and misc other fixes and improvements --- Makefile | 6 -- clone.go | 223 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ gitCheckout.go | 2 +- goSrcScan.go | 1 + goWork.go | 2 +- repoClone.go | 219 ------------------------------------------------------- 6 files changed, 226 insertions(+), 227 deletions(-) create mode 100644 clone.go delete mode 100644 repoClone.go diff --git a/Makefile b/Makefile index d65515c..e754bad 100644 --- a/Makefile +++ b/Makefile @@ -15,12 +15,6 @@ vet: goimports: goimports -w *.go -redomod: - rm -f go.* - GO111MODULE= go mod init - GO111MODULE= go mod tidy - go mod edit -go=1.20 - clean: rm -f *.pb.go -rm -f go.* diff --git a/clone.go b/clone.go new file mode 100644 index 0000000..514624f --- /dev/null +++ b/clone.go @@ -0,0 +1,223 @@ +package forgepb + +import ( + "errors" + "io/ioutil" + "net/http" + "os" + "path/filepath" + "strings" + + "go.wit.com/lib/gui/shell" + "go.wit.com/lib/protobuf/gitpb" + "go.wit.com/log" +) + +// TODO: make some config file for things like this +// can be used to work around temporary problems +func clonePathHack(dirname string, basedir string, gopath string) (string, error) { + // newdir = helloworld + // basedir = /home/jcarr/go/src/go.wit.com/apps + // giturl = https://gitea.wit.com/gui/helloworld + // func cloneActual(newdir, basedir, giturl string) error { + + switch gopath { + case "golang.org/x/crypto": + return cloneActual(dirname, basedir, "https://"+"go.googlesource.com/crypto") + case "golang.org/x/mod": + return cloneActual(dirname, basedir, "https://"+"go.googlesource.com/mod") + case "golang.org/x/net": + return cloneActual(dirname, basedir, "https://"+"go.googlesource.com/net") + case "golang.org/x/sys": + return cloneActual(dirname, basedir, "https://"+"go.googlesource.com/sys") + case "golang.org/x/sync": + return cloneActual(dirname, basedir, "https://"+"go.googlesource.com/sync") + case "golang.org/x/term": + return cloneActual(dirname, basedir, "https://"+"go.googlesource.com/term") + case "golang.org/x/text": + return cloneActual(dirname, basedir, "https://"+"go.googlesource.com/text") + case "golang.org/x/tools": + return cloneActual(dirname, basedir, "https://"+"go.googlesource.com/tools") + case "golang.org/x/xerrors": + return cloneActual(dirname, basedir, "https://"+"go.googlesource.com/xerrors") + case "google.golang.org/protobuf": + return cloneActual(dirname, basedir, "https://"+"go.googlesource.com/protobuf") + case "google.golang.org/genproto": + return cloneActual(dirname, basedir, "https://"+"go.googlesource.com/genproto") + case "google.golang.org/api": + return cloneActual(dirname, basedir, "https://"+"go.googlesource.com/api") + case "google.golang.org/grpc": + return cloneActual(dirname, basedir, "https://"+"go.googlesource.com/grpc") + case "google.golang.org/appengine": + return cloneActual(dirname, basedir, "https://"+"go.googlesource.com/appengine") + } + if strings.HasPrefix(gopath, "github.com/go-gl/glfw") { + return cloneActual(dirname, basedir, "https://github.com/go-gl/glfw") + } + + return "", errors.New("no gopath override here") +} + +// attempt to git clone if the go path doesn't exist +// does a git clone, if it works, returns true +// workdir = /home/jcarr/go/src/ +// gopath = go.wit.com/apps/helloworld +func (f *Forge) Clone(gopath string) (*gitpb.Repo, error) { + var err error + pb, err := f.Repos.NewGoPath(f.goSrc, gopath, "") + if err == nil { + return pb, err + } + workdir := f.goSrc + fullpath := filepath.Join(workdir, gopath) + dirname := filepath.Base(fullpath) + + basedir := strings.TrimSuffix(fullpath, dirname) + + url := "https://" + gopath + log.Info("trying git clone") + log.Info("gopath =", gopath) + + // try a direct git clone against the gopath + // cloneActual("helloworld", "/home/jcarr/go/src/go.wit.com/apps", "https://go.wit.com/apps/helloworld") + if finalurl, err := cloneActual(dirname, basedir, url); err == nil { + f.Repos.DeleteByGoPath(gopath) + // git clone worked! + return f.Repos.NewGoPath(f.goSrc, gopath, finalurl) + } + log.Info("direct attempt at git clone failed", url) + + // if direct git clone doesn't work, look for a redirect + // go directly to the URL as that is autoritive. If that failes + // try the go package system as maybe the git site no longer exists + if url, err = findGoImport(url); err != nil { + log.Info("findGoImport() DID NOT WORK", url) + log.Info("findGoImport() DID NOT WORK", err) + } else { + if finalurl, err := cloneActual(dirname, basedir, url); err == nil { + f.Repos.DeleteByGoPath(gopath) + // git clone worked! + return f.Repos.NewGoPath(f.goSrc, gopath, finalurl) + } + } + log.Info("git clone from 'go-import' info failed", url) + + // query the golang package system for the last known location + // NOTE: take time to thank the go developers and google for designing this wonderful system + if url, err = runGoList(gopath); err != nil { + log.Info("go list failed", err) + } else { + if finalurl, err := cloneActual(dirname, basedir, url); err == nil { + f.Repos.DeleteByGoPath(gopath) + // git clone worked! + return f.Repos.NewGoPath(f.goSrc, gopath, finalurl) + } + } + log.Info("git clone from 'git list' info failed", url) + + // try to parse a redirect + + if finalurl, err := clonePathHack(dirname, basedir, gopath); err == nil { + f.Repos.DeleteByGoPath(gopath) + // WTF didn't go-import or go list work? + return f.Repos.NewGoPath(f.goSrc, gopath, finalurl) + } + + return nil, errors.New("can not find git sources for gopath " + gopath) +} + +// newdir = helloworld +// basedir = /home/jcarr/go/src/go.wit.com/apps +// giturl = https://gitea.wit.com/gui/helloworld +func cloneActual(newdir, basedir, giturl string) (string, error) { + log.Info("cloneActual() newdir =", newdir) + log.Info("cloneActual() basedir =", basedir) + log.Info("cloneActual() giturl =", giturl) + if !IsDirectory(basedir) { + os.MkdirAll(basedir, 0750) + } + err := os.Chdir(basedir) + if err != nil { + log.Warn("chdir failed", basedir, err) + return giturl, err + } + + cmd := []string{"git", "clone", "--verbose", "--progress", giturl, newdir} + log.Info("Running:", basedir, cmd) + r := shell.PathRunRealtime(basedir, cmd) + if r.Error != nil { + log.Warn("git clone error", r.Error) + return giturl, r.Error + } + + fullpath := filepath.Join(basedir, newdir) + if !IsDirectory(fullpath) { + log.Info("git clone failed", giturl) + return giturl, errors.New("git clone failed " + giturl) + } + gitdir := filepath.Join(fullpath, ".git") + if IsDirectory(gitdir) { + log.Info("git cloned worked to", fullpath) + // also clone notes -- this can store the go.mod and go.sum files + cmd := []string{"git", "fetch", "origin", "refs/notes/*:refs/notes/*"} + shell.PathRunRealtime(fullpath, cmd) + return giturl, nil + } + // git clone didn't really work but did make a directory + log.Info("fullpath is probably empty", fullpath) + return giturl, errors.New("crapnuts. rmdir fullpath here? " + fullpath) +} + +// check the server for the current go path to git url mapping +// for example: +// This will check go.wit.com for "go.wit.com/apps/go-clone" +// and return where the URL to do git clone should be +func findGoImport(url string) (string, error) { + resp, err := http.Get(url) + if err != nil { + return "", err + } + defer resp.Body.Close() + + bodyBytes, err := ioutil.ReadAll(resp.Body) + if err != nil { + return "", err + } + + tmp := string(bodyBytes) + parts := strings.Split(tmp, "go-import") + if len(parts) < 2 { + return "", errors.New("missing go-import") + } + // this is terrible, it doesn't even look for 'content=' + // but again, this is just a hack for my own code to be + // usuable after the removal in go v1.22 of the go get clone behavior that was in v1.21 + parts = strings.Split(parts[1], "\"") + var newurl string + for { + if len(parts) == 0 { + break + } + tmp := strings.TrimSpace(parts[0]) + fields := strings.Split(tmp, " ") + // log.Info("FIELDS:", fields) + if len(fields) == 3 { + newurl = fields[2] + break + } + parts = parts[1:] + } + if newurl == "" { + return "", errors.New("missing git content string") + } + + return newurl, nil +} + +func IsDirectory(path string) bool { + info, err := os.Stat(path) + if err != nil { + return false + } + return info.IsDir() +} diff --git a/gitCheckout.go b/gitCheckout.go index b0ccc18..2952ccf 100644 --- a/gitCheckout.go +++ b/gitCheckout.go @@ -133,7 +133,7 @@ func (f *Forge) CheckoutMaster() bool { } func (f *Forge) CheckoutUser() bool { - log.Log(FORGEPBWARN, "running git checkout master everwhere") + log.Log(FORGEPBWARN, "running git checkout user everwhere") var failed int = 0 var count int = 0 all := f.Repos.SortByGoPath() diff --git a/goSrcScan.go b/goSrcScan.go index c13dce3..a50dbb2 100644 --- a/goSrcScan.go +++ b/goSrcScan.go @@ -65,6 +65,7 @@ func gitDirectoriesNew(srcDir string) ([]string, error) { switch fname { case "repos.pb": case "go.work": + case "go.work.last": default: // todo: figure out a way to do padding for init() log.Info("WARNING: you have an untracked file outside of any .git repository:", path) diff --git a/goWork.go b/goWork.go index afb09da..f8f75d8 100644 --- a/goWork.go +++ b/goWork.go @@ -23,7 +23,7 @@ func (f *Forge) MakeGoWork() error { } defer workf.Close() - fmt.Fprintln(workf, "go 1.20") // fix this + fmt.Fprintln(workf, "go 1.21") // fix this fmt.Fprintln(workf, "") fmt.Fprintln(workf, "use (") diff --git a/repoClone.go b/repoClone.go deleted file mode 100644 index ab1acee..0000000 --- a/repoClone.go +++ /dev/null @@ -1,219 +0,0 @@ -package forgepb - -import ( - "errors" - "io/ioutil" - "net/http" - "os" - "path/filepath" - "strings" - - "go.wit.com/lib/gui/shell" - "go.wit.com/lib/protobuf/gitpb" - "go.wit.com/log" -) - -// TODO: make some config file for things like this -// can be used to work around temporary problems -func clonePathHack(dirname string, basedir string, gopath string) (string, error) { - // newdir = helloworld - // basedir = /home/jcarr/go/src/go.wit.com/apps - // giturl = https://gitea.wit.com/gui/helloworld - // func cloneActual(newdir, basedir, giturl string) error { - - switch gopath { - case "golang.org/x/crypto": - return cloneActual(dirname, basedir, "https://"+"go.googlesource.com/crypto") - case "golang.org/x/mod": - return cloneActual(dirname, basedir, "https://"+"go.googlesource.com/mod") - case "golang.org/x/net": - return cloneActual(dirname, basedir, "https://"+"go.googlesource.com/net") - case "golang.org/x/sys": - return cloneActual(dirname, basedir, "https://"+"go.googlesource.com/sys") - case "golang.org/x/sync": - return cloneActual(dirname, basedir, "https://"+"go.googlesource.com/sync") - case "golang.org/x/term": - return cloneActual(dirname, basedir, "https://"+"go.googlesource.com/term") - case "golang.org/x/text": - return cloneActual(dirname, basedir, "https://"+"go.googlesource.com/text") - case "golang.org/x/tools": - return cloneActual(dirname, basedir, "https://"+"go.googlesource.com/tools") - case "golang.org/x/xerrors": - return cloneActual(dirname, basedir, "https://"+"go.googlesource.com/xerrors") - case "google.golang.org/protobuf": - return cloneActual(dirname, basedir, "https://"+"go.googlesource.com/protobuf") - case "google.golang.org/genproto": - return cloneActual(dirname, basedir, "https://"+"go.googlesource.com/genproto") - case "google.golang.org/api": - return cloneActual(dirname, basedir, "https://"+"go.googlesource.com/api") - case "google.golang.org/grpc": - return cloneActual(dirname, basedir, "https://"+"go.googlesource.com/grpc") - case "google.golang.org/appengine": - return cloneActual(dirname, basedir, "https://"+"go.googlesource.com/appengine") - } - if strings.HasPrefix(gopath, "github.com/go-gl/glfw") { - return cloneActual(dirname, basedir, "https://github.com/go-gl/glfw") - } - - return "", errors.New("no gopath override here") -} - -// attempt to git clone if the go path doesn't exist -// does a git clone, if it works, returns true -// workdir = /home/jcarr/go/src/ -// gopath = go.wit.com/apps/helloworld -func (f *Forge) Clone(gopath string) (*gitpb.Repo, error) { - var err error - pb, err := f.Repos.NewGoPath(f.goSrc, gopath, "") - if err == nil { - return pb, err - } - workdir := f.goSrc - fullpath := filepath.Join(workdir, gopath) - dirname := filepath.Base(fullpath) - - basedir := strings.TrimSuffix(fullpath, dirname) - - url := "https://" + gopath - log.Info("trying git clone") - log.Info("gopath =", gopath) - - // try a direct git clone against the gopath - // cloneActual("helloworld", "/home/jcarr/go/src/go.wit.com/apps", "https://go.wit.com/apps/helloworld") - if finalurl, err := cloneActual(dirname, basedir, url); err == nil { - // git clone worked! - return f.Repos.NewGoPath(f.goSrc, gopath, finalurl) - } - log.Info("direct attempt at git clone failed", url) - - // if direct git clone doesn't work, look for a redirect - // go directly to the URL as that is autoritive. If that failes - // try the go package system as maybe the git site no longer exists - if url, err = findGoImport(url); err != nil { - log.Info("findGoImport() DID NOT WORK", url) - log.Info("findGoImport() DID NOT WORK", err) - } else { - if finalurl, err := cloneActual(dirname, basedir, url); err == nil { - // git clone worked! - return f.Repos.NewGoPath(f.goSrc, gopath, finalurl) - } - } - log.Info("git clone from 'go-import' info failed", url) - - // query the golang package system for the last known location - // NOTE: take time to thank the go developers and google for designing this wonderful system - if url, err = runGoList(gopath); err != nil { - log.Info("go list failed", err) - } else { - if finalurl, err := cloneActual(dirname, basedir, url); err == nil { - // git clone worked! - return f.Repos.NewGoPath(f.goSrc, gopath, finalurl) - } - } - log.Info("git clone from 'git list' info failed", url) - - // try to parse a redirect - - if finalurl, err := clonePathHack(dirname, basedir, gopath); err == nil { - // WTF didn't go-import or go list work? - return f.Repos.NewGoPath(f.goSrc, gopath, finalurl) - } - - return nil, errors.New("can not find git sources for gopath " + gopath) -} - -// newdir = helloworld -// basedir = /home/jcarr/go/src/go.wit.com/apps -// giturl = https://gitea.wit.com/gui/helloworld -func cloneActual(newdir, basedir, giturl string) (string, error) { - log.Info("cloneActual() newdir =", newdir) - log.Info("cloneActual() basedir =", basedir) - log.Info("cloneActual() giturl =", giturl) - if !IsDirectory(basedir) { - os.MkdirAll(basedir, 0750) - } - err := os.Chdir(basedir) - if err != nil { - log.Warn("chdir failed", basedir, err) - return giturl, err - } - - cmd := []string{"git", "clone", "--verbose", "--progress", giturl, newdir} - log.Info("Running:", basedir, cmd) - r := shell.PathRunRealtime(basedir, cmd) - if r.Error != nil { - log.Warn("git clone error", r.Error) - return giturl, r.Error - } - - fullpath := filepath.Join(basedir, newdir) - if !IsDirectory(fullpath) { - log.Info("git clone failed", giturl) - return giturl, errors.New("git clone failed " + giturl) - } - gitdir := filepath.Join(fullpath, ".git") - if IsDirectory(gitdir) { - log.Info("git cloned worked to", fullpath) - // also clone notes -- this can store the go.mod and go.sum files - cmd := []string{"git", "fetch", "origin", "refs/notes/*:refs/notes/*"} - shell.PathRunRealtime(fullpath, cmd) - return giturl, nil - } - // git clone didn't really work but did make a directory - log.Info("fullpath is probably empty", fullpath) - return giturl, errors.New("crapnuts. rmdir fullpath here? " + fullpath) -} - -// check the server for the current go path to git url mapping -// for example: -// This will check go.wit.com for "go.wit.com/apps/go-clone" -// and return where the URL to do git clone should be -func findGoImport(url string) (string, error) { - resp, err := http.Get(url) - if err != nil { - return "", err - } - defer resp.Body.Close() - - bodyBytes, err := ioutil.ReadAll(resp.Body) - if err != nil { - return "", err - } - - tmp := string(bodyBytes) - parts := strings.Split(tmp, "go-import") - if len(parts) < 2 { - return "", errors.New("missing go-import") - } - // this is terrible, it doesn't even look for 'content=' - // but again, this is just a hack for my own code to be - // usuable after the removal in go v1.22 of the go get clone behavior that was in v1.21 - parts = strings.Split(parts[1], "\"") - var newurl string - for { - if len(parts) == 0 { - break - } - tmp := strings.TrimSpace(parts[0]) - fields := strings.Split(tmp, " ") - // log.Info("FIELDS:", fields) - if len(fields) == 3 { - newurl = fields[2] - break - } - parts = parts[1:] - } - if newurl == "" { - return "", errors.New("missing git content string") - } - - return newurl, nil -} - -func IsDirectory(path string) bool { - info, err := os.Stat(path) - if err != nil { - return false - } - return info.IsDir() -} -- cgit v1.2.3