diff options
| -rw-r--r-- | config.go | 2 | ||||
| -rw-r--r-- | configDefault.go (renamed from sampleConfig.go) | 2 | ||||
| -rw-r--r-- | forgeConfig/main.go | 2 | ||||
| -rw-r--r-- | goList.go | 117 | ||||
| -rw-r--r-- | goSrcFind.go (renamed from goSrc.go) | 0 | ||||
| -rw-r--r-- | goSrcScan.go (renamed from scanGoSrc.go) | 2 | ||||
| -rw-r--r-- | init.go | 4 | ||||
| -rw-r--r-- | repoClone.go | 278 | ||||
| -rw-r--r-- | repoNew.go | 7 | ||||
| -rw-r--r-- | repoSettings.go (renamed from settings.go) | 6 |
10 files changed, 412 insertions, 8 deletions
@@ -111,7 +111,7 @@ func (c *ForgeConfigs) ConfigLoad() error { } // first time user. make a template config file - c.SampleConfig() + c.sampleConfig() return nil } diff --git a/sampleConfig.go b/configDefault.go index 52038ab..0f505d5 100644 --- a/sampleConfig.go +++ b/configDefault.go @@ -6,7 +6,7 @@ import ( "go.wit.com/log" ) -func (all *ForgeConfigs) SampleConfig() { +func (all *ForgeConfigs) sampleConfig() { new1 := new(ForgeConfig) new1.GoPath = "go.wit.com" new1.Writable = true diff --git a/forgeConfig/main.go b/forgeConfig/main.go index 3ce6c7d..b1e496c 100644 --- a/forgeConfig/main.go +++ b/forgeConfig/main.go @@ -15,7 +15,7 @@ func main() { if argv.List { f.ConfigPrintTable() - loop := f.SortByPath() // get the list of forge configs + loop := f.Config.SortByPath() // get the list of forge configs for loop.Scan() { r := loop.Next() log.Info("repo:", r.GoPath) diff --git a/goList.go b/goList.go new file mode 100644 index 0000000..583502a --- /dev/null +++ b/goList.go @@ -0,0 +1,117 @@ +package forgepb + +import ( + "encoding/json" + "strings" + "time" + + "go.wit.com/lib/gui/shell" + "go.wit.com/log" +) + +// go list -json -m go.wit.com/apps/go-clone@latest + +// go list -json -m go.wit.com/apps/go-clone@latest +// { +// "Path": "go.wit.com/apps/go-clone", +// "Version": "v0.0.6", +// "Query": "latest", +// "Time": "2024-03-10T04:12:15Z", +// "GoMod": "/home/jcarr/go/pkg/mod/cache/download/go.wit.com/apps/go-clone/@v/v0.0.6.mod", +// "GoVersion": "1.22.0" +// } + +type Module struct { + Path string // module path + Query string // version query corresponding to this version + Version string // module version + Versions []string // available module versions + Replace *Module // replaced by this module + Time *time.Time // time version was created + Update *Module // available update (with -u) + Main bool // is this the main module? + Indirect bool // module is only indirectly needed by main module + Dir string // directory holding local copy of files, if any + GoMod string // path to go.mod file describing module, if any + GoVersion string // go version used in module + Retracted []string // retraction information, if any (with -retracted or -u) + Deprecated string // deprecation message, if any (with -u) + Error *ModuleError // error loading module + Sum string // checksum for path, version (as in go.sum) + GoModSum string // checksum for go.mod (as in go.sum) + Reuse bool // reuse of old module info is safe + Origin Origin +} + +type ModuleError struct { + Err string // the error itself +} + +// An Origin describes the provenance of a given repo method result. +// It can be passed to CheckReuse (usually in a different go command invocation) +// to see whether the result remains up-to-date. +type Origin struct { + VCS string `json:",omitempty"` // "git" etc + URL string `json:",omitempty"` // URL of repository + Subdir string `json:",omitempty"` // subdirectory in repo + + Hash string `json:",omitempty"` // commit hash or ID + + // If TagSum is non-empty, then the resolution of this module version + // depends on the set of tags present in the repo, specifically the tags + // of the form TagPrefix + a valid semver version. + // If the matching repo tags and their commit hashes still hash to TagSum, + // the Origin is still valid (at least as far as the tags are concerned). + // The exact checksum is up to the Repo implementation; see (*gitRepo).Tags. + TagPrefix string `json:",omitempty"` + TagSum string `json:",omitempty"` + + // If Ref is non-empty, then the resolution of this module version + // depends on Ref resolving to the revision identified by Hash. + // If Ref still resolves to Hash, the Origin is still valid (at least as far as Ref is concerned). + // For Git, the Ref is a full ref like "refs/heads/main" or "refs/tags/v1.2.3", + // and the Hash is the Git object hash the ref maps to. + // Other VCS might choose differently, but the idea is that Ref is the name + // with a mutable meaning while Hash is a name with an immutable meaning. + Ref string `json:",omitempty"` + + // If RepoSum is non-empty, then the resolution of this module version + // failed due to the repo being available but the version not being present. + // This depends on the entire state of the repo, which RepoSum summarizes. + // For Git, this is a hash of all the refs and their hashes. + RepoSum string `json:",omitempty"` +} + +// A Tags describes the available tags in a code repository. + +func runGoList(url string) (string, error) { + ver, err := getLatestVersion(url) + if err != nil { + return "", err + } + r := shell.Run([]string{"go", "list", "-json", "-m", url + "@" + ver}) + var modInfo Module + out := strings.Join(r.Stdout, "\n") + err = json.Unmarshal([]byte(out), &modInfo) + if err != nil { + log.Info("runGoList() r.Output =", out) + log.Info("runGoList() json.Unmarshal() error =", err) + return "", err + } + log.Info("runGoList() Output.Origin.URL =", modInfo.Origin.URL) + return modInfo.Origin.URL, err +} + +func getLatestVersion(url string) (string, error) { + r := shell.Run([]string{"go", "list", "-json", "-m", url + "@latest"}) + var modInfo Module + out := strings.Join(r.Stdout, "\n") + err := json.Unmarshal([]byte(out), &modInfo) + if err != nil { + log.Info("runGoList() r.Output =", out) + log.Info("runGoList() json.Unmarshal() error =", err) + return "", err + } + log.Info("runGoList() Output.Version =", modInfo.Version) + return modInfo.Version, err +} diff --git a/scanGoSrc.go b/goSrcScan.go index e0fba71..0f85c33 100644 --- a/scanGoSrc.go +++ b/goSrcScan.go @@ -50,7 +50,7 @@ func (f *Forge) ScanGoSrc() (bool, error) { if newr.IsBranch(uname) { newr.SetUserBranchName(uname) } else { - newr.SetUserBranchName(uname+"FIXME") + newr.SetUserBranchName(uname + "FIXME") } } else { log.Log(FORGEPBWARN, "ScanGoSrc() bad:", dir) @@ -50,7 +50,3 @@ func Init() *Forge { log.Warn("GOT HERE. forge.Init(). f can not be nil") return f } - -func (f *Forge) SortByPath() *ForgeConfigIterator { - return f.Config.SortByPath() -} diff --git a/repoClone.go b/repoClone.go new file mode 100644 index 0000000..8078d36 --- /dev/null +++ b/repoClone.go @@ -0,0 +1,278 @@ +package forgepb + +import ( + "errors" + "io/ioutil" + "net/http" + "os" + "path/filepath" + "strings" + + "go.wit.com/lib/gui/shell" + "go.wit.com/lib/protobuf/gitpb" + "go.wit.com/log" +) + +/* +// guess the paths. returns + +// realpath : the actual path on the filesystem +// goSrcPath : this could be ~/go/src, or where the go.work file is + +// goPath : go.wit.com/lib/gui/repostatus (for example) +// true/false if the repo is a golang repo +func (f *Forge) guessPaths(path string) (string, string, string, bool, error) { + var realpath, goSrcDir string + var isGoLang bool = false + + homeDir, err := os.UserHomeDir() + if err != nil { + log.Log(FORGEPBWARN, "Error getting home directory:", err) + return path, realpath, goSrcDir, false, err + } + goSrcDir = filepath.Join(homeDir, "go/src") + + // allow arbitrary paths, otherwise, assume the repo is in ~/go/src + // unless REPO_WORK_PATH was set. to over-ride ~/go/src + // todo, look for go.work + if os.Getenv("REPO_WORK_PATH") == "" { + os.Setenv("REPO_WORK_PATH", goSrcDir) + } else { + goSrcDir = os.Getenv("REPO_WORK_PATH") + } + + // todo: this is dumb + if strings.HasPrefix(path, "/") { + realpath = path + } else if strings.HasPrefix(path, "~") { + tmp := strings.TrimPrefix(path, "~") + realpath = filepath.Join(homeDir, tmp) + } else { + realpath = filepath.Join(goSrcDir, path) + isGoLang = true + } + + if os.Getenv("REPO_AUTO_CLONE") == "true" { + err := f.Clone(goSrcDir, path) + if err != nil { + // directory doesn't exist. exit with nil and error nil + return path, realpath, goSrcDir, false, errors.New("git clone") + } + } + + if !IsDirectory(realpath) { + log.Log(FORGEPBWARN, "directory doesn't exist", realpath) + // directory doesn't exist. exit with nil and error nil + return path, realpath, goSrcDir, false, errors.New(realpath + " does not exist") + } + + filename := filepath.Join(realpath, ".git/config") + + _, err = os.Open(filename) + if err != nil { + // log.Log(WARN, "Error reading .git/config:", filename, err) + // log.Log(WARN, "TODO: find .git/config in parent directory") + return path, realpath, goSrcDir, false, err + } + return path, realpath, goSrcDir, isGoLang, nil +} +*/ + +// TODO: make some config file for things like this +// can be used to work around temporary problems +func clonePathHack(dirname string, basedir string, gopath string) error { + // newdir = helloworld + // basedir = /home/jcarr/go/src/go.wit.com/apps + // giturl = https://gitea.wit.com/gui/helloworld + // func cloneActual(newdir, basedir, giturl string) error { + + switch gopath { + case "golang.org/x/crypto": + return cloneActual(dirname, basedir, "https://"+"go.googlesource.com/crypto") + case "golang.org/x/mod": + return cloneActual(dirname, basedir, "https://"+"go.googlesource.com/mod") + case "golang.org/x/net": + return cloneActual(dirname, basedir, "https://"+"go.googlesource.com/net") + case "golang.org/x/sys": + return cloneActual(dirname, basedir, "https://"+"go.googlesource.com/sys") + case "golang.org/x/sync": + return cloneActual(dirname, basedir, "https://"+"go.googlesource.com/sync") + case "golang.org/x/term": + return cloneActual(dirname, basedir, "https://"+"go.googlesource.com/term") + case "golang.org/x/text": + return cloneActual(dirname, basedir, "https://"+"go.googlesource.com/text") + case "golang.org/x/tools": + return cloneActual(dirname, basedir, "https://"+"go.googlesource.com/tools") + case "golang.org/x/xerrors": + return cloneActual(dirname, basedir, "https://"+"go.googlesource.com/xerrors") + case "google.golang.org/protobuf": + return cloneActual(dirname, basedir, "https://"+"go.googlesource.com/protobuf") + case "google.golang.org/genproto": + return cloneActual(dirname, basedir, "https://"+"go.googlesource.com/genproto") + case "google.golang.org/api": + return cloneActual(dirname, basedir, "https://"+"go.googlesource.com/api") + case "google.golang.org/grpc": + return cloneActual(dirname, basedir, "https://"+"go.googlesource.com/grpc") + case "google.golang.org/appengine": + return cloneActual(dirname, basedir, "https://"+"go.googlesource.com/appengine") + } + + return errors.New("no gopath override here") +} + +// attempt to git clone if the go path doesn't exist +// does a git clone, if it works, returns true +// workdir = /home/jcarr/go/src/ +// gopath = go.wit.com/apps/helloworld +func (f *Forge) Clone(gopath string) (*gitpb.Repo, error) { + var err error + pb, err := f.Repos.NewGoPath(f.goSrc, gopath) + if err == nil { + return pb, err + } + workdir := f.goSrc + fullpath := filepath.Join(workdir, gopath) + dirname := filepath.Base(fullpath) + + basedir := strings.TrimSuffix(fullpath, dirname) + + url := "https://" + gopath + log.Info("trying git clone") + log.Info("gopath =", gopath) + + // try a direct git clone against the gopath + // cloneActual("helloworld", "/home/jcarr/go/src/go.wit.com/apps", "https://go.wit.com/apps/helloworld") + if err = cloneActual(dirname, basedir, url); err == nil { + // git clone worked! + return f.Repos.NewGoPath(f.goSrc, gopath) + } + log.Info("direct attempt at git clone failed", url) + + // if direct git clone doesn't work, look for a redirect + // go directly to the URL as that is autoritive. If that failes + // try the go package system as maybe the git site no longer exists + if url, err = findGoImport(url); err != nil { + log.Info("findGoImport() DID NOT WORK", url) + log.Info("findGoImport() DID NOT WORK", err) + } else { + if err := cloneActual(dirname, basedir, url); err == nil { + // git clone worked! + return f.Repos.NewGoPath(f.goSrc, gopath) + } + } + log.Info("git clone from 'go-import' info failed", url) + + // query the golang package system for the last known location + // NOTE: take time to thank the go developers and google for designing this wonderful system + if url, err = runGoList(gopath); err != nil { + log.Info("go list failed", err) + } else { + if err := cloneActual(dirname, basedir, url); err == nil { + // git clone worked! + return f.Repos.NewGoPath(f.goSrc, gopath) + } + } + log.Info("git clone from 'git list' info failed", url) + + // try to parse a redirect + + if err = clonePathHack(dirname, basedir, gopath); err == nil { + // WTF didn't go-import or go list work? + return f.Repos.NewGoPath(f.goSrc, gopath) + } + + return nil, errors.New("can not find git sources for gopath " + gopath) +} + +// newdir = helloworld +// basedir = /home/jcarr/go/src/go.wit.com/apps +// giturl = https://gitea.wit.com/gui/helloworld +func cloneActual(newdir, basedir, giturl string) error { + log.Info("cloneActual() newdir =", newdir) + log.Info("cloneActual() basedir =", basedir) + log.Info("cloneActual() giturl =", giturl) + if !IsDirectory(basedir) { + os.MkdirAll(basedir, 0750) + } + err := os.Chdir(basedir) + if err != nil { + log.Warn("chdir failed", basedir, err) + return err + } + + cmd := []string{"git", "clone", "--verbose", "--progress", giturl, newdir} + log.Info("Running:", cmd) + r := shell.PathRunRealtime(basedir, cmd) + if r.Error != nil { + log.Warn("git clone error", r.Error) + return r.Error + } + + fullpath := filepath.Join(basedir, newdir) + if !IsDirectory(fullpath) { + log.Info("git clone failed", giturl) + return errors.New("git clone failed " + giturl) + } + gitdir := filepath.Join(fullpath, ".git") + if IsDirectory(gitdir) { + log.Info("git cloned worked to", fullpath) + return nil + } + // git clone didn't really work but did make a directory + log.Info("fullpath is probably empty", fullpath) + panic("crapnuts. rmdir fullpath here?") +} + +// check the server for the current go path to git url mapping +// for example: +// This will check go.wit.com for "go.wit.com/apps/go-clone" +// and return where the URL to do git clone should be +func findGoImport(url string) (string, error) { + resp, err := http.Get(url) + if err != nil { + return "", err + } + defer resp.Body.Close() + + bodyBytes, err := ioutil.ReadAll(resp.Body) + if err != nil { + return "", err + } + + tmp := string(bodyBytes) + parts := strings.Split(tmp, "go-import") + if len(parts) < 2 { + return "", errors.New("missing go-import") + } + // this is terrible, it doesn't even look for 'content=' + // but again, this is just a hack for my own code to be + // usuable after the removal in go v1.22 of the go get clone behavior that was in v1.21 + parts = strings.Split(parts[1], "\"") + var newurl string + for { + if len(parts) == 0 { + break + } + tmp := strings.TrimSpace(parts[0]) + fields := strings.Split(tmp, " ") + // log.Info("FIELDS:", fields) + if len(fields) == 3 { + newurl = fields[2] + break + } + parts = parts[1:] + } + if newurl == "" { + return "", errors.New("missing git content string") + } + + return newurl, nil +} + +func IsDirectory(path string) bool { + info, err := os.Stat(path) + if err != nil { + return false + } + return info.IsDir() +} diff --git a/repoNew.go b/repoNew.go new file mode 100644 index 0000000..3ee2b38 --- /dev/null +++ b/repoNew.go @@ -0,0 +1,7 @@ +package forgepb + +import "go.wit.com/lib/protobuf/gitpb" + +func (f *Forge) NewGoPath(gopath string) (*gitpb.Repo, error) { + return f.Repos.NewGoPath(f.goSrc, gopath) +} diff --git a/settings.go b/repoSettings.go index c1a7cb8..c44ea58 100644 --- a/settings.go +++ b/repoSettings.go @@ -14,6 +14,12 @@ import ( "strings" ) +/* +func (f *Forge) SortByPath() *ForgeConfigIterator { + return f.Config.SortByPath() +} +*/ + func (all *ForgeConfigs) UpdateGoPath(name string, gopath string) bool { oldr := all.DeleteByGoPath(name) if oldr == nil { |