diff options
| author | Jeff Carr <[email protected]> | 2025-10-12 04:58:15 -0500 |
|---|---|---|
| committer | Jeff Carr <[email protected]> | 2025-10-12 04:58:43 -0500 |
| commit | 29bb90b293a8bda889ff30bb56d05ce47af4a9bf (patch) | |
| tree | 240650da2d5f20412060bc118123e9c258383abe | |
| parent | 05363ddbb7c0cd4d868767d32c362f457eb3ae7d (diff) | |
still compilesv0.0.20
| -rw-r--r-- | Makefile | 6 | ||||
| -rw-r--r-- | argv.go | 21 | ||||
| -rw-r--r-- | doEverything.go | 175 | ||||
| -rw-r--r-- | doNewest.go | 145 | ||||
| -rw-r--r-- | main.go | 189 |
5 files changed, 360 insertions, 176 deletions
@@ -6,8 +6,12 @@ BUILDTIME = $(shell date +%s) all: install mirrors + +everything: + mirrors find /home/mirrors/wit/dists - mirrors newest + false + mirrors everything apt update sleep 1 ls -l /var/lib/apt/lists/mirrors.wit.com* @@ -17,16 +17,17 @@ import ( var argv args type args struct { - List *EmptyCmd `arg:"subcommand:list" help:"show the packages"` - Walk *WalkCmd `arg:"subcommand:walk" help:"walk the filesystem for new .deb files"` - Update *EmptyCmd `arg:"subcommand:oldway" help:"the old code to update the apt repo"` - Verify *EmptyCmd `arg:"subcommand:verify" help:"verify the pb is accurate and doesn't have errors"` - Incoming *IncomingCmd `arg:"subcommand:incoming" help:"handle the incoming directory"` - MakeDists *EmptyCmd `arg:"subcommand:makedists" help:"make debian mirrors/dists files for 'apt update'"` - Newest *EmptyCmd `arg:"subcommand:newest" help:"make a list of the newest .deb packages"` - DryRun bool `arg:"--dry-run" help:"only show what would be packaged"` - Verbose bool `arg:"--verbose" help:"be loud about it"` - Force bool `arg:"--force" help:"rebuild everything"` + List *EmptyCmd `arg:"subcommand:list" help:"show the packages"` + Walk *WalkCmd `arg:"subcommand:walk" help:"walk the filesystem for new .deb files"` + Update *EmptyCmd `arg:"subcommand:oldway" help:"the old code to update the apt repo"` + Verify *EmptyCmd `arg:"subcommand:verify" help:"verify the pb is accurate and doesn't have errors"` + Incoming *IncomingCmd `arg:"subcommand:incoming" help:"handle the incoming directory"` + MakeDists *EmptyCmd `arg:"subcommand:makedists" help:"make debian mirrors/dists files for 'apt update'"` + Everything *EmptyCmd `arg:"subcommand:everything" help:"do the whole thing needed. nothing more. just everything."` + Newest *EmptyCmd `arg:"subcommand:newest" help:"make a list of the newest .deb packages"` + DryRun bool `arg:"--dry-run" help:"only show what would be packaged"` + Verbose bool `arg:"--verbose" help:"be loud about it"` + Force bool `arg:"--force" help:"rebuild everything"` } type EmptyCmd struct { diff --git a/doEverything.go b/doEverything.go new file mode 100644 index 0000000..9e24644 --- /dev/null +++ b/doEverything.go @@ -0,0 +1,175 @@ +package main + +import ( + "crypto/sha256" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "time" + + "go.wit.com/lib/cobol" + "go.wit.com/lib/gui/shell" + "go.wit.com/lib/protobuf/zoopb" + "go.wit.com/log" +) + +func doEverything() (string, error) { + // log.Info(r, err2) + shell.RunVerbose([]string{"rm", "-rf", "/home/mirrors/wit/dists"}) + // shell.RunVerbose([]string{"rm", "f", "/home/mirrors/wit/dists/sid/InRelease"}) + // shell.RunVerbose([]string{"rm", "f", "/home/mirrors/wit/dists/sid/Release"}) + // shell.RunVerbose([]string{"rm", "f", "/home/mirrors/wit/dists/sid/Release.gpg"}) + + // dists.working/sid/main/binary-amd64 + if err := os.MkdirAll("/home/mirrors/wit/dists/sid/main/binary-amd64", 0755); err != nil { + log.Info("did not work", err) + } else { + log.Info("did work") + } + + if me.pb.BaseDir != "/home/mirrors/wit" { + me.pb.Filename = "/home/mirrors/wit/mirrors.wit.com.pb" + me.pb.BaseDir = "/home/mirrors/wit" + me.pb.Save() + panic("missing /home/mirrors/wit as BaseDir") + } + + if err := os.Chdir(me.pb.BaseDir); err != nil { + me.sh.BadExit("no '"+me.pb.BaseDir+"' directory", err) + } + + if !shell.IsDir("pool/") { + me.sh.BadExit("no "+filepath.Join(me.pb.BaseDir, "pool")+" directory", errors.New("mount -a ? missing wit/pool/")) + } + + log.Info("Processing dir", filepath.Join(me.pb.BaseDir, "pool")) + + arch := "amd64" + + newest := zoopb.NewPackages() + for p := range me.pb.IterAll() { + if p.Architecture != arch { + continue + } + found := newest.FindByPackage(p.Package) + if found == nil { + // package is new + log.Printf("%-20.20s %-20.20s %-80.80s\n", "new package", p.Package, p.Filename) + newest.Clone(p) + continue + } + curtime := p.Ctime.AsTime() + newtime := found.Ctime.AsTime() + durs := cobol.Since(p.Ctime) + " vs found " + cobol.Since(found.Ctime) + if time.Since(curtime) > time.Since(newtime) { + log.Printf("%-20.20s %-20.20s %-80.80s %s\n", "found is newer", p.Package, p.Filename, durs) + } else { + log.Printf("%-20.20s %-20.20s %-80.80s %s\n", "found is older", p.Package, p.Filename, durs) + newest.Delete(found) + newest.Clone(p) + } + } + + newest.SortPackage() + var myshit string + for p := range newest.IterAll() { + controlfile, err := p.GetDebianControlFile() + if err != nil { + log.Info("make debInfo file error", err) + panic("deb error") + } + log.Info(controlfile) + myshit += controlfile + "\n" + } + + fullname := "/home/mirrors/wit/dists/sid/main/binary-amd64/Packages" + if err := os.WriteFile(fullname, []byte(myshit), 0644); err != nil { + return fullname, err + } + + r, err := shell.RunVerbose([]string{"gzip", "-k", fullname}) + log.Info(r, err) + // r, err := shell.RunVerbose([]string{"gzip", fullname}) + // panic("blah") + // shell.RunVerbose([]string{"bzip2", "-k", fullname}) + // shell.RunVerbose([]string{"bzip2", fullname}) + + allfiles, _ := FindFiles("dists/sid/main") + for i, filename := range allfiles { + log.Info(i, filename) + } + + releasePath := filepath.Join("/home/mirrors/wit/dists/sid", "Release") + + rfile, _ := os.OpenFile(releasePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) + fmt.Fprintf(rfile, "Origin: WIT.COM Debian Sid\n") + fmt.Fprintf(rfile, "Label: WIT.COM Debian Sid\n") + fmt.Fprintf(rfile, "Suite: sid\n") + fmt.Fprintf(rfile, "Codename: sid\n") + fmt.Fprintf(rfile, "Date: %s\n", time.Now().UTC().Format(time.RFC1123Z)) + fmt.Fprintf(rfile, "Architectures: riscv64 amd64 arm64 all\n") + fmt.Fprintf(rfile, "Components: main\n") + fmt.Fprintf(rfile, "Description: Tooling for RiscV, Semiconductor Designs & Private Clouds\n") + + /* + fmt.Fprintf(rfile, "MD5SUM:\n") + for i, filename := range allfiles { + log.Info(i, filename) + fileBytes, _ := os.ReadFile(filename) + sum := fmt.Sprintf("%x", md5.Sum(fileBytes)) // deprecated + newfile := strings.TrimPrefix(filename, "dists/sid/") + fmt.Fprintf(rfile, " %s %d %s\n", sum, len(fileBytes), newfile) + } + fmt.Fprintf(rfile, "SHA1:\n") + for i, filename := range allfiles { + log.Info(i, filename) + fileBytes, _ := os.ReadFile(filename) + sum := fmt.Sprintf("%x", sha1.Sum(fileBytes)) + newfile := strings.TrimPrefix(filename, "dists/sid/") + fmt.Fprintf(rfile, " %s %d %s\n", sum, len(fileBytes), newfile) + } + */ + fmt.Fprintf(rfile, "SHA256:\n") + for i, filename := range allfiles { + log.Info(i, filename) + fileBytes, _ := os.ReadFile(filename) + sum := fmt.Sprintf("%x", sha256.Sum256(fileBytes)) + newfile := strings.TrimPrefix(filename, "dists/sid/") + fmt.Fprintf(rfile, " %s %d %s\n", sum, len(fileBytes), newfile) + } + // fmt.Fprintf(rfile, "SHA1:\n") + // fmt.Fprintf(rfile, "SHA256:\n") + rfile.Close() + + // Sign the file + log.Println("Signing with GPG key:", gpgKeyID) + + distPath := "/home/mirrors/wit/dists/sid" + // Create InRelease + cmdClearSign := exec.Command("gpg", "--default-key", gpgKeyID, "--clearsign", "-o", filepath.Join(distPath, "InRelease"), releasePath) + if err := runCommand(cmdClearSign); err != nil { + log.Printf("failed to create InRelease: %v\n", err) + } + + // Create Release.gpg + cmdDetachedSign := exec.Command("gpg", "--default-key", gpgKeyID, "-abs", "-o", filepath.Join(distPath, "Release.gpg"), releasePath) + if err := runCommand(cmdDetachedSign); err != nil { + log.Printf("failed to create Release.gpg: %v\n", err) + } + /* + var sum string + switch name { + case "MD5Sum": + sum = fmt.Sprintf("%x", md5.Sum(fileBytes)) + case "SHA1": + sum = fmt.Sprintf("%x", sha1.Sum(fileBytes)) + case "SHA256": + // FIX 3: Use the correct sha256.Sum256 function. + sum = fmt.Sprintf("%x", sha256.Sum256(fileBytes)) + */ + + return "doNewest", nil +} diff --git a/doNewest.go b/doNewest.go new file mode 100644 index 0000000..224163f --- /dev/null +++ b/doNewest.go @@ -0,0 +1,145 @@ +package main + +import ( + "crypto/sha256" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "time" + + "go.wit.com/lib/cobol" + "go.wit.com/lib/gui/shell" + "go.wit.com/lib/protobuf/zoopb" + "go.wit.com/log" +) + +// make a list of the newest .deb files +func doNewest(arch string) (string, error) { + log.Info("Processing dir", filepath.Join(me.pb.BaseDir, "pool")) + + newest := zoopb.NewPackages() + for p := range me.pb.IterAll() { + if p.Architecture != arch { + continue + } + found := newest.FindByPackage(p.Package) + if found == nil { + // package is new + log.Printf("%-20.20s %-20.20s %-80.80s\n", "new package", p.Package, p.Filename) + newest.Clone(p) + continue + } + curtime := p.Ctime.AsTime() + newtime := found.Ctime.AsTime() + durs := cobol.Since(p.Ctime) + " vs found " + cobol.Since(found.Ctime) + if time.Since(curtime) > time.Since(newtime) { + log.Printf("%-20.20s %-20.20s %-80.80s %s\n", "found is newer", p.Package, p.Filename, durs) + } else { + log.Printf("%-20.20s %-20.20s %-80.80s %s\n", "found is older", p.Package, p.Filename, durs) + newest.Delete(found) + newest.Clone(p) + } + } + + newest.SortPackage() + var myshit string + for p := range newest.IterAll() { + controlfile, err := p.GetDebianControlFile() + if err != nil { + log.Info("make debInfo file error", err) + panic("deb error") + } + log.Info(controlfile) + myshit += controlfile + "\n" + } + + fullname := "/home/mirrors/wit/dists/sid/main/binary-amd64/Packages" + if err := os.WriteFile(fullname, []byte(myshit), 0644); err != nil { + return fullname, err + } + + r, err := shell.RunVerbose([]string{"gzip", "-k", fullname}) + log.Info(r, err) + // r, err := shell.RunVerbose([]string{"gzip", fullname}) + // panic("blah") + // shell.RunVerbose([]string{"bzip2", "-k", fullname}) + // shell.RunVerbose([]string{"bzip2", fullname}) + + allfiles, _ := FindFiles("dists/sid/main") + for i, filename := range allfiles { + log.Info(i, filename) + } + + releasePath := filepath.Join("/home/mirrors/wit/dists/sid", "Release") + + rfile, _ := os.OpenFile(releasePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) + fmt.Fprintf(rfile, "Origin: WIT.COM Debian Sid\n") + fmt.Fprintf(rfile, "Label: WIT.COM Debian Sid\n") + fmt.Fprintf(rfile, "Suite: sid\n") + fmt.Fprintf(rfile, "Codename: sid\n") + fmt.Fprintf(rfile, "Date: %s\n", time.Now().UTC().Format(time.RFC1123Z)) + fmt.Fprintf(rfile, "Architectures: riscv64 amd64 arm64 all\n") + fmt.Fprintf(rfile, "Components: main\n") + fmt.Fprintf(rfile, "Description: Tooling for RiscV, Semiconductor Designs & Private Clouds\n") + + /* + fmt.Fprintf(rfile, "MD5SUM:\n") + for i, filename := range allfiles { + log.Info(i, filename) + fileBytes, _ := os.ReadFile(filename) + sum := fmt.Sprintf("%x", md5.Sum(fileBytes)) // deprecated + newfile := strings.TrimPrefix(filename, "dists/sid/") + fmt.Fprintf(rfile, " %s %d %s\n", sum, len(fileBytes), newfile) + } + fmt.Fprintf(rfile, "SHA1:\n") + for i, filename := range allfiles { + log.Info(i, filename) + fileBytes, _ := os.ReadFile(filename) + sum := fmt.Sprintf("%x", sha1.Sum(fileBytes)) + newfile := strings.TrimPrefix(filename, "dists/sid/") + fmt.Fprintf(rfile, " %s %d %s\n", sum, len(fileBytes), newfile) + } + */ + fmt.Fprintf(rfile, "SHA256:\n") + for i, filename := range allfiles { + log.Info(i, filename) + fileBytes, _ := os.ReadFile(filename) + sum := fmt.Sprintf("%x", sha256.Sum256(fileBytes)) + newfile := strings.TrimPrefix(filename, "dists/sid/") + fmt.Fprintf(rfile, " %s %d %s\n", sum, len(fileBytes), newfile) + } + // fmt.Fprintf(rfile, "SHA1:\n") + // fmt.Fprintf(rfile, "SHA256:\n") + rfile.Close() + + // Sign the file + log.Println("Signing with GPG key:", gpgKeyID) + + distPath := "/home/mirrors/wit/dists/sid" + // Create InRelease + cmdClearSign := exec.Command("gpg", "--default-key", gpgKeyID, "--clearsign", "-o", filepath.Join(distPath, "InRelease"), releasePath) + if err := runCommand(cmdClearSign); err != nil { + log.Printf("failed to create InRelease: %v\n", err) + } + + // Create Release.gpg + cmdDetachedSign := exec.Command("gpg", "--default-key", gpgKeyID, "-abs", "-o", filepath.Join(distPath, "Release.gpg"), releasePath) + if err := runCommand(cmdDetachedSign); err != nil { + log.Printf("failed to create Release.gpg: %v\n", err) + } + /* + var sum string + switch name { + case "MD5Sum": + sum = fmt.Sprintf("%x", md5.Sum(fileBytes)) + case "SHA1": + sum = fmt.Sprintf("%x", sha1.Sum(fileBytes)) + case "SHA256": + // FIX 3: Use the correct sha256.Sum256 function. + sum = fmt.Sprintf("%x", sha256.Sum256(fileBytes)) + */ + + return "doNewest", nil +} @@ -1,22 +1,15 @@ package main import ( - "crypto/sha256" "embed" "errors" - "fmt" "os" - "os/exec" "path/filepath" - "strings" - "time" - "go.wit.com/lib/cobol" "go.wit.com/lib/config" "go.wit.com/lib/gui/prep" "go.wit.com/lib/gui/shell" "go.wit.com/lib/protobuf/zoopb" - "go.wit.com/log" ) // sent via -ldflags @@ -44,26 +37,6 @@ func main() { } } - // log.Info(r, err2) - shell.RunVerbose([]string{"rm", "-rf", "/home/mirrors/wit/dists"}) - shell.RunVerbose([]string{"rm", "f", "/home/mirrors/wit/dists/sid/InRelease"}) - shell.RunVerbose([]string{"rm", "f", "/home/mirrors/wit/dists/sid/Release"}) - shell.RunVerbose([]string{"rm", "f", "/home/mirrors/wit/dists/sid/Release.gpg"}) - - // dists.working/sid/main/binary-amd64 - if err := os.MkdirAll("/home/mirrors/wit/dists/sid/main/binary-amd64", 0755); err != nil { - log.Info("did not work", err) - } else { - log.Info("did work") - } - - if me.pb.BaseDir != "/home/mirrors/wit" { - me.pb.Filename = "/home/mirrors/wit/mirrors.wit.com.pb" - me.pb.BaseDir = "/home/mirrors/wit" - me.pb.Save() - panic("missing /home/mirrors/wit as BaseDir") - } - if err := os.Chdir(me.pb.BaseDir); err != nil { me.sh.BadExit("no '"+me.pb.BaseDir+"' directory", err) } @@ -72,15 +45,26 @@ func main() { me.sh.BadExit("no "+filepath.Join(me.pb.BaseDir, "pool")+" directory", errors.New("mount -a ? missing wit/pool/")) } - if me.sh.Cmd == "" { - // default behavior when no argv - s := log.Sprintf("You have %d packages in %s", me.pb.Len(), me.pb.BaseDir) - me.sh.GoodExit(s) - } - - // default handling of argv subcommands var s string var err error + + // STANDARD START + + // walk for new .deb files + s, err = doWalk() + if err != nil { + me.sh.BadExit(s, err) + } + + // move files from inccoming into pooo/ + s, err = doIncoming(me.pb) + if err != nil { + me.sh.BadExit(s, err) + } + + // STANDARD START END + + // handling of argv subcommands if argv.Incoming != nil { s, err = doIncoming(me.pb) } @@ -89,6 +73,10 @@ func main() { s, err = doWalk() } + if argv.Newest != nil { + s, err = doNewest("amd64") + } + if argv.List != nil { s, err = doList() } @@ -105,8 +93,8 @@ func main() { s, err = doMakeDists() } - if argv.Newest != nil { - s, err = doNewest("amd64") + if argv.Everything != nil { + s, err = doEverything() } if err != nil { @@ -114,132 +102,3 @@ func main() { } me.sh.GoodExit(s) } - -// make a list of the newest .deb files -func doNewest(arch string) (string, error) { - log.Info("Processing dir", filepath.Join(me.pb.BaseDir, "pool")) - - newest := zoopb.NewPackages() - for p := range me.pb.IterAll() { - if p.Architecture != arch { - continue - } - found := newest.FindByPackage(p.Package) - if found == nil { - // package is new - log.Printf("%-20.20s %-20.20s %-80.80s\n", "new package", p.Package, p.Filename) - newest.Clone(p) - continue - } - curtime := p.Ctime.AsTime() - newtime := found.Ctime.AsTime() - durs := cobol.Since(p.Ctime) + " vs found " + cobol.Since(found.Ctime) - if time.Since(curtime) > time.Since(newtime) { - log.Printf("%-20.20s %-20.20s %-80.80s %s\n", "found is newer", p.Package, p.Filename, durs) - } else { - log.Printf("%-20.20s %-20.20s %-80.80s %s\n", "found is older", p.Package, p.Filename, durs) - newest.Delete(found) - newest.Clone(p) - } - } - - newest.SortPackage() - var myshit string - for p := range newest.IterAll() { - controlfile, err := p.GetDebianControlFile() - if err != nil { - log.Info("make debInfo file error", err) - panic("deb error") - } - log.Info(controlfile) - myshit += controlfile + "\n" - } - - fullname := "/home/mirrors/wit/dists/sid/main/binary-amd64/Packages" - if err := os.WriteFile(fullname, []byte(myshit), 0644); err != nil { - return fullname, err - } - - r, err := shell.RunVerbose([]string{"gzip", "-k", fullname}) - log.Info(r, err) - // r, err := shell.RunVerbose([]string{"gzip", fullname}) - // panic("blah") - // shell.RunVerbose([]string{"bzip2", "-k", fullname}) - // shell.RunVerbose([]string{"bzip2", fullname}) - - allfiles, _ := FindFiles("dists/sid/main") - for i, filename := range allfiles { - log.Info(i, filename) - } - - releasePath := filepath.Join("/home/mirrors/wit/dists/sid", "Release") - - rfile, _ := os.OpenFile(releasePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) - fmt.Fprintf(rfile, "Origin: WIT.COM Debian Sid\n") - fmt.Fprintf(rfile, "Label: WIT.COM Debian Sid\n") - fmt.Fprintf(rfile, "Suite: sid\n") - fmt.Fprintf(rfile, "Codename: sid\n") - fmt.Fprintf(rfile, "Date: %s\n", time.Now().UTC().Format(time.RFC1123Z)) - fmt.Fprintf(rfile, "Architectures: riscv64 amd64 arm64 all\n") - fmt.Fprintf(rfile, "Components: main\n") - fmt.Fprintf(rfile, "Description: Tooling for RiscV, Semiconductor Designs & Private Clouds\n") - - /* - fmt.Fprintf(rfile, "MD5SUM:\n") - for i, filename := range allfiles { - log.Info(i, filename) - fileBytes, _ := os.ReadFile(filename) - sum := fmt.Sprintf("%x", md5.Sum(fileBytes)) // deprecated - newfile := strings.TrimPrefix(filename, "dists/sid/") - fmt.Fprintf(rfile, " %s %d %s\n", sum, len(fileBytes), newfile) - } - fmt.Fprintf(rfile, "SHA1:\n") - for i, filename := range allfiles { - log.Info(i, filename) - fileBytes, _ := os.ReadFile(filename) - sum := fmt.Sprintf("%x", sha1.Sum(fileBytes)) - newfile := strings.TrimPrefix(filename, "dists/sid/") - fmt.Fprintf(rfile, " %s %d %s\n", sum, len(fileBytes), newfile) - } - */ - fmt.Fprintf(rfile, "SHA256:\n") - for i, filename := range allfiles { - log.Info(i, filename) - fileBytes, _ := os.ReadFile(filename) - sum := fmt.Sprintf("%x", sha256.Sum256(fileBytes)) - newfile := strings.TrimPrefix(filename, "dists/sid/") - fmt.Fprintf(rfile, " %s %d %s\n", sum, len(fileBytes), newfile) - } - // fmt.Fprintf(rfile, "SHA1:\n") - // fmt.Fprintf(rfile, "SHA256:\n") - rfile.Close() - - // Sign the file - log.Println("Signing with GPG key:", gpgKeyID) - - distPath := "/home/mirrors/wit/dists/sid" - // Create InRelease - cmdClearSign := exec.Command("gpg", "--default-key", gpgKeyID, "--clearsign", "-o", filepath.Join(distPath, "InRelease"), releasePath) - if err := runCommand(cmdClearSign); err != nil { - log.Printf("failed to create InRelease: %v\n", err) - } - - // Create Release.gpg - cmdDetachedSign := exec.Command("gpg", "--default-key", gpgKeyID, "-abs", "-o", filepath.Join(distPath, "Release.gpg"), releasePath) - if err := runCommand(cmdDetachedSign); err != nil { - log.Printf("failed to create Release.gpg: %v\n", err) - } - /* - var sum string - switch name { - case "MD5Sum": - sum = fmt.Sprintf("%x", md5.Sum(fileBytes)) - case "SHA1": - sum = fmt.Sprintf("%x", sha1.Sum(fileBytes)) - case "SHA256": - // FIX 3: Use the correct sha256.Sum256 function. - sum = fmt.Sprintf("%x", sha256.Sum256(fileBytes)) - */ - - return "doNewest", nil -} |
