diff options
| -rw-r--r-- | DEBIAN/control | 9 | ||||
| -rw-r--r-- | Makefile | 3 | ||||
| -rw-r--r-- | doEverything.go | 17 | ||||
| -rw-r--r-- | doVerify.go | 8 | ||||
| -rw-r--r-- | doWalk.go | 106 | ||||
| -rw-r--r-- | main.go | 31 | ||||
| -rw-r--r-- | makePackagesFile.go | 199 | ||||
| -rw-r--r-- | stuff.go | 128 |
8 files changed, 329 insertions, 172 deletions
diff --git a/DEBIAN/control b/DEBIAN/control new file mode 100644 index 0000000..829bc5e --- /dev/null +++ b/DEBIAN/control @@ -0,0 +1,9 @@ +Package: andlabs +Version: 0.22.46-202510071306 +Architecture: amd64 +Maintainer: todo: get from ENV +Packager: Jeff Carr <[email protected]> +URL: https://cgit.grid.wit.com/toolkits/andlabs +Package-Build-Date: 2025/10/07 18:06:16 UTC +Git-Tag-Date: todo: get from repo +Description: go.wit.com/toolkits/andlabs @@ -5,7 +5,8 @@ GUIVERSION = $(shell git describe --tags) BUILDTIME = $(shell date +%s) all: install - mirrors + mirrors walk + mirrors verify everything: install mirrors diff --git a/doEverything.go b/doEverything.go index 66bc887..7418eeb 100644 --- a/doEverything.go +++ b/doEverything.go @@ -73,22 +73,11 @@ func doEverything() (string, error) { } } + // sort the packages & write out the Packages file newest.SortPackage() - var myshit string - for p := range newest.IterAll() { - controlfile, err := p.GetDebianControlFile() - if err != nil { - log.Info("make debInfo file error", err) - panic("deb error") - } - // log.Info(controlfile) - myshit += controlfile - myshit += log.Sprintf("Installed-Size: %s\n", "234234234") - myshit += "\n" - } - + thefile := doMakePackagesFile(newest) fullname := "/home/mirrors/wit/dists/sid/main/binary-amd64/Packages" - if err := os.WriteFile(fullname, []byte(myshit), 0644); err != nil { + if err := os.WriteFile(fullname, []byte(thefile), 0644); err != nil { return fullname, err } diff --git a/doVerify.go b/doVerify.go index 09c8c74..a121b8e 100644 --- a/doVerify.go +++ b/doVerify.go @@ -17,6 +17,7 @@ func doVerify() (string, error) { if err := os.Chdir(me.pb.BaseDir); err != nil { return "no '" + me.pb.BaseDir + "' directory", err } + if !shell.IsDir("pool/") { return "no " + filepath.Join(me.pb.BaseDir, "pool") + " directory", errors.New("mount -a ? missing wit/pool/") } @@ -27,8 +28,13 @@ func doVerify() (string, error) { for p := range me.pb.IterAll() { pdump := log.Sprintf("%v", p) if p.DebInfo == nil { - log.Printf("debinfo == nil %-130.130s\n", pdump) + fullname := filepath.Join(me.pb.BaseDir, p.Filename) + log.Printf("debinfo == nil. need to run dpkg -I %s\n", fullname) + populateDebInfo(p) counter += 1 + if counter > 3 { + break + } continue } if p.Filename != strings.TrimSpace(p.Filename) { @@ -1,95 +1,45 @@ package main import ( + "errors" "os" - "time" + "path/filepath" + "strings" - "go.wit.com/lib/debian" "go.wit.com/lib/protobuf/zoopb" "go.wit.com/log" - "google.golang.org/protobuf/types/known/timestamppb" ) +var errStopWalk = errors.New("walk stopped by user") + func doWalk() (string, error) { os.Chdir(me.pb.BaseDir) - // 2. Scan pool directory for .deb files and gather info - log.Printf("Scanning for .deb files in %s\n", poolDir) - debInfos, err := scanDebs(poolDir, 200) - if err != nil { - log.Printf("Failed to scan .deb files: %v\n", err) - me.sh.BadExit("scan pool scan dir", err) - } - var counter int - for _, deb := range debInfos { - newdeb := new(zoopb.Package) - newdeb.DebInfo = new(zoopb.DebInfo) - newdeb.Filename = deb.Filename - newdeb.DebInfo.Filename = deb.Filename - newdeb.DebInfo.MD5SUM = deb.MD5Sum - // newdeb.DebInfo.SHA1 = deb.SHA1Sum - newdeb.DebInfo.SHA256 = deb.SHA256Sum - - // log.Info("len(CONTROLDATA)", len(deb.ControlData)) - // log.Sprintf("VAR='%s' VAL='%s'\n", varname, varvalue) - // log.Info("%v", deb.ControlData) - for varname, varvalue := range deb.ControlData { - switch varname { - case "Package": - newdeb.Package = varvalue - case "Version": - newdeb.Version = varvalue - case "Architecture": - newdeb.Architecture = varvalue - case "Git-Tag-Date": - if argv.Verbose { - log.Info("CONVERT THIS TO TIME", varname, varvalue) - } - case "Depends": - newdeb.DebInfo.Depends = varvalue - case "Build-Depends": - newdeb.DebInfo.BuildDepends = varvalue - case "Packager": - newdeb.DebInfo.Packager = varvalue - case "Package-Build-Date": - varname = "PackageBuildDate" - const layout = "2006/01/02 15:04:05 MST" - parsedTime, err := time.Parse(layout, varvalue) - if err != nil { - log.Info("CONVERT TO TIME failed", varname, varvalue, err) - } - newdeb.BuildDate = timestamppb.New(parsedTime) - case "URL": - newdeb.DebInfo.Homepage = varvalue - default: - if err := debian.SetDebInfoString(newdeb, varname, varvalue); err == nil { - if argv.Verbose { - log.Printf("Searching for Sugarman WORKED: VAR='%-30s' VAL='%s'\n", varname, varvalue) - } - } else { - log.Printf("Searching for Sugarman (unknwon var): VAR='%-30s' VAL='%s' err=%v\n", varname, varvalue, err) - } - // todo: add to protomap - } + err := filepath.Walk("pool", func(path string, info os.FileInfo, err error) error { + if err != nil { + return err } - counter += 1 - me.pb.AppendByFilename(newdeb) - if newdeb.Filename == "" { - log.Info(deb) - log.Info("file=", deb.Filename) - panic("newdeb.Filename = ''") - } - if argv.Verbose { - newdeb.Print() + if !info.IsDir() && strings.HasSuffix(info.Name(), ".deb") { + found := me.pb.FindByFilename(path) + if found != nil { + return nil + } + counter += 1 + // if counter > 10 { + // return errStopWalk + // } + newdeb := new(zoopb.Package) + newdeb.Filename = path + me.pb.AppendByFilename(newdeb) + log.Info("added new", path) } - // arch := deb.ControlData["Architecture"] - } + return nil + }) - // footer := me.pb.PrintTable() - // log.Info("found so far:", footer) - - me.pb.Save() - log.Printf("add %d new packages. Total packages.Len()=%d\n", counter, me.pb.Len()) - return "mirrors/ has been scanned for new files", nil + if counter > 0 { + me.pb.Save() + } + s := log.Sprintf("add %d new packages. Total packages.Len()=%d", counter, me.pb.Len()) + return s, err } @@ -52,24 +52,27 @@ func main() { me.sh.BadExit("no "+filepath.Join(me.pb.BaseDir, "pool")+" directory", errors.New("mount -a ? missing wit/pool/")) } - var s string - var err error - - // STANDARD START + if me.sh.Cmd == "" { + // STANDARD START + // walk for new .deb files + s, err := doWalk() + if err != nil { + me.sh.BadExit(s, err) + } - // walk for new .deb files - s, err = doWalk() - if err != nil { - me.sh.BadExit(s, err) - } + // move files from inccoming into pooo/ + s, err = doIncoming(me.pb) + if err != nil { + me.sh.BadExit(s, err) + } - // move files from inccoming into pooo/ - s, err = doIncoming(me.pb) - if err != nil { - me.sh.BadExit(s, err) + me.sh.GoodExit(s) + // STANDARD START END } - // STANDARD START END + //// start standard argv subcommand processing here //// + var s string + var err error // handling of argv subcommands if argv.Incoming != nil { diff --git a/makePackagesFile.go b/makePackagesFile.go index 9f28e46..d7067e5 100644 --- a/makePackagesFile.go +++ b/makePackagesFile.go @@ -1,13 +1,150 @@ package main import ( + "bufio" + "errors" + "path/filepath" "strings" "go.wit.com/lib/cobol" + "go.wit.com/lib/gui/shell" "go.wit.com/lib/protobuf/zoopb" "go.wit.com/log" + "google.golang.org/protobuf/types/known/timestamppb" ) +func populateDebInfo(p *zoopb.Package) error { + var verbose bool = false + if p.DebInfo != nil { + // already added p.DebInfo + return nil + } + fullname := filepath.Join(me.pb.BaseDir, p.Filename) + cmd := []string{"dpkg", "-I", fullname} + r := shell.Run(cmd) + if r.Error != nil { + return r.Error + } + if r.Exit != 0 { + return errors.New("dpkg returned -1") + } + p.DebInfo = new(zoopb.DebInfo) + + starting := true + all := strings.Join(r.Stdout, "\n") + scanner := bufio.NewScanner(strings.NewReader(all)) + for scanner.Scan() { + line := scanner.Text() + parts := strings.Fields(line) + if starting { + if parts[0] == "new" { + if verbose { + log.Printf("new: %v\n", parts) + } + // skip the first dpkg -I line + continue + } + if parts[0] == "size" { + if verbose { + log.Printf("size: %v\n", parts) + } + p.DebInfo.InstalledSize = parts[1] + // scan all the entries from size + for scanner.Scan() { + line = scanner.Text() + parts = strings.Fields(line) + if strings.HasPrefix(line, " ") { + if verbose { + log.Printf("sizeline: %v\n", parts) + } + continue + } + starting = false + break + } + } + if starting { + if verbose { + log.Printf("parts: %v\n", parts) + } + continue + } + } + varname := strings.TrimSuffix(parts[0], ":") + // varname = strings.TrimSuffix(varname, ":") // WTF is causing this to be needed + varval := strings.Join(parts[1:], " ") + // log.Printf("varname:%s varval:%s\n", varname, varval) + switch varname { + case "Package": + p.Package = varval + case "Filename": + log.Printf("Filename: varname:%s varval:%s\n", varname, varval) + // p.Package = varval + case "Version": + p.Version = varval + case "Architecture": + p.Architecture = varval + case "GoPath": + p.Namespace = varval + case "Maintainer": + p.DebInfo.Maintainer = varval + case "Packager": + p.DebInfo.Packager = varval + case "Depends": + p.DebInfo.Depends = varval + case "URL": + p.DebInfo.URL = varval + case "Build-Depends": + p.DebInfo.BuildDepends = varval + case "Installed-Size": + p.DebInfo.InstalledSize = varval + case "Homepage": + p.DebInfo.URL = varval + case "Package-Build-Date": + t, err := cobol.GetTime(varval) + if err == nil { + p.BuildDate = timestamppb.New(t) + } else { + if verbose { + log.Info("FIXME: Package-Build-Date", varval, err) + } + } + case "Git-Tag-Date": + if verbose { + log.Info("FIXME: Git-Tag-Date", varval) + } + case "Description": + description := varval + for scanner.Scan() { + line := scanner.Text() + if strings.HasPrefix(line, " ") { + line = strings.TrimSpace(line) + description += line + "\n" + continue + } + break + } + p.DebInfo.Description = description + default: + varname2 := strings.TrimSuffix(varname, ":") + log.Printf("UNKNOWN: varname:%s varval:%s varname2=%s\n", varname, varval, varname2) + panic("fix mirrors populateDebInfo()") + } + } + // log.Info("Ran:", cmd, r, err) + return nil +} + +/* +UNKNOWN: varname:Depends varval:protobuf-compiler, protoc-gen-go | protoc-gen-go-wit +UNKNOWN: varname:Build-Depends varval:golang +UNKNOWN: varname:URL varval:https://go.wit.com/ +UNKNOWN: varname:Package-Build-Date varval:2025/10/05 07:40:30 UTC +UNKNOWN: varname:Git-Tag-Date varval:todo: get from repo +UNKNOWN: varname:Description varval:autogen protobuf marshal and sort files +UNKNOWN: varname:this varval:is needed for go.wit.com/lib/protobuf/ GO packages +*/ + func getDebianControlFile(p *zoopb.Package, varname string) (string, string) { switch varname { case "Package": @@ -75,3 +212,65 @@ func doMakePackagesFile(all *zoopb.Packages) string { } return pfile } + +/* + log.Printf("Scanning for .deb files in %s\n", poolDir) + debInfos, err := scanDebs(poolDir, 200) + if err != nil { + log.Printf("Failed to scan .deb files: %v\n", err) + me.sh.BadExit("scan pool scan dir", err) + } + + var counter int + for _, deb := range debInfos { + newdeb := new(zoopb.Package) + newdeb.DebInfo = new(zoopb.DebInfo) + newdeb.Filename = deb.Filename + newdeb.DebInfo.Filename = deb.Filename + newdeb.DebInfo.MD5SUM = deb.MD5Sum + // newdeb.DebInfo.SHA1 = deb.SHA1Sum + newdeb.DebInfo.SHA256 = deb.SHA256Sum + + // log.Info("len(CONTROLDATA)", len(deb.ControlData)) + // log.Sprintf("VAR='%s' VAL='%s'\n", varname, varvalue) + // log.Info("%v", deb.ControlData) + for varname, varvalue := range deb.ControlData { + switch varname { + case "Package": + newdeb.Package = varvalue + case "Version": + newdeb.Version = varvalue + case "Architecture": + newdeb.Architecture = varvalue + case "Git-Tag-Date": + if argv.Verbose { + log.Info("CONVERT THIS TO TIME", varname, varvalue) + } + case "Depends": + newdeb.DebInfo.Depends = varvalue + case "Build-Depends": + newdeb.DebInfo.BuildDepends = varvalue + case "Packager": + newdeb.DebInfo.Packager = varvalue + case "Package-Build-Date": + varname = "PackageBuildDate" + const layout = "2006/01/02 15:04:05 MST" + parsedTime, err := time.Parse(layout, varvalue) + if err != nil { + log.Info("CONVERT TO TIME failed", varname, varvalue, err) + } + newdeb.BuildDate = timestamppb.New(parsedTime) + case "URL": + newdeb.DebInfo.Homepage = varvalue + default: + if err := debian.SetDebInfoString(newdeb, varname, varvalue); err == nil { + if argv.Verbose { + log.Printf("Searching for Sugarman WORKED: VAR='%-30s' VAL='%s'\n", varname, varvalue) + } + } else { + log.Printf("Searching for Sugarman (unknwon var): VAR='%-30s' VAL='%s' err=%v\n", varname, varvalue, err) + } + // todo: add to protomap + } + } +*/ @@ -18,62 +18,14 @@ import ( "go.wit.com/log" ) -// scanDebs finds all .deb files and extracts their metadata. -func scanDebs(root string, count int) ([]DebInfo, error) { - var debs []DebInfo - var counter int - err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - if !info.IsDir() && strings.HasSuffix(info.Name(), ".deb") { - found := me.pb.FindByFilename(path) - if found != nil { - // log.Info("already processed", path) - return nil - } - if count == -1 { - return nil - } - counter += 1 - if counter > count { - return nil - } - log.Printf(" -> Processing %s\n", path) - - // Get control info - cmd := exec.Command("dpkg-deb", "-I", path) - var out bytes.Buffer - cmd.Stdout = &out - if err := cmd.Run(); err != nil { - return fmt.Errorf("failed to run dpkg-deb on %s: %v", path, err) - } - - controlData := parseControlData(out.String()) - - // Get checksums - md5sum, sha1sum, sha256sum, err := getChecksums(path) - if err != nil { - return err - } - - relativePath, err := filepath.Rel(".", path) - if err != nil { - return err - } - - debs = append(debs, DebInfo{ - ControlData: controlData, - Filename: relativePath, - Size: info.Size(), - MD5Sum: md5sum, - SHA1Sum: sha1sum, - SHA256Sum: sha256sum, - }) - } - return nil - }) - return debs, err +func runCommand(cmd *exec.Cmd) error { + var stderr bytes.Buffer + cmd.Stderr = &stderr + err := cmd.Run() + if err != nil { + return fmt.Errorf("command '%s' failed: %v\nStderr: %s", cmd.String(), err, stderr.String()) + } + return nil } // parseControlData converts the text output of dpkg-deb into a map. @@ -229,12 +181,60 @@ func generateAndSignReleaseFile(distPath string) error { return nil } -func runCommand(cmd *exec.Cmd) error { - var stderr bytes.Buffer - cmd.Stderr = &stderr - err := cmd.Run() - if err != nil { - return fmt.Errorf("command '%s' failed: %v\nStderr: %s", cmd.String(), err, stderr.String()) - } - return nil +// scanDebs finds all .deb files and extracts their metadata. +func scanDebs(root string, count int) ([]DebInfo, error) { + var debs []DebInfo + var counter int + err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if !info.IsDir() && strings.HasSuffix(info.Name(), ".deb") { + found := me.pb.FindByFilename(path) + if found != nil { + // log.Info("already processed", path) + return nil + } + if count == -1 { + return nil + } + counter += 1 + if counter > count { + return nil + } + log.Printf(" -> Processing %s\n", path) + + // Get control info + cmd := exec.Command("dpkg-deb", "-I", path) + var out bytes.Buffer + cmd.Stdout = &out + if err := cmd.Run(); err != nil { + return fmt.Errorf("failed to run dpkg-deb on %s: %v", path, err) + } + + controlData := parseControlData(out.String()) + + // Get checksums + md5sum, sha1sum, sha256sum, err := getChecksums(path) + if err != nil { + return err + } + + relativePath, err := filepath.Rel(".", path) + if err != nil { + return err + } + + debs = append(debs, DebInfo{ + ControlData: controlData, + Filename: relativePath, + Size: info.Size(), + MD5Sum: md5sum, + SHA1Sum: sha1sum, + SHA256Sum: sha256sum, + }) + } + return nil + }) + return debs, err } |