diff options
| -rw-r--r-- | rill.go | 115 |
1 files changed, 83 insertions, 32 deletions
@@ -67,8 +67,8 @@ func (f *Forge) updateRepo(repo *gitpb.Repo) error { } */ -var RillX int = 10 -var RillY int = 20 +// var RillX int = 10 +// var RillY int = 20 // x is the size of the queued up pool (shouldn't matter here for this I think) // y is how many simultanous functions will run @@ -92,11 +92,11 @@ func (f *Forge) RillReload() int { var counter int // Read users from the API. // Concurrency = 20 - dirs := rill.Map(ids, RillX, func(repo *gitpb.Repo) (*gitpb.Repo, error) { + dirs := rill.Map(ids, int(f.Config.RillX), func(repo *gitpb.Repo) (*gitpb.Repo, error) { return repo, nil }) - rill.ForEach(dirs, RillY, func(repo *gitpb.Repo) error { + rill.ForEach(dirs, int(f.Config.RillY), func(repo *gitpb.Repo) error { if err := repo.HasChanged(); err != nil { counter += 1 return err @@ -129,6 +129,39 @@ type RillStats struct { var rillMu sync.Mutex +func rillSetError(stats map[string]*RillStats, fullpath string, err error) { + rillMu.Lock() + defer rillMu.Unlock() + if s, ok := stats[fullpath]; ok { + s.Err = err + return + } + log.Info("WHAT THE FUCK STATS ERROR", fullpath) +} + +func rillSetStartTime(stats map[string]*RillStats, fullpath string) { + rillMu.Lock() + defer rillMu.Unlock() + if s, ok := stats[fullpath]; ok { + s.Start = time.Now() + return + } + var s *RillStats + s = new(RillStats) + s.Start = time.Now() + stats[fullpath] = s +} + +func rillSetEndTime(stats map[string]*RillStats, fullpath string) { + rillMu.Lock() + defer rillMu.Unlock() + if s, ok := stats[fullpath]; ok { + s.End = time.Now() + return + } + log.Info("WHAT THE FUCK STATS END TIME", fullpath) +} + // x is the size of the queued up pool (shouldn't matter here for this I think) // y is how many simultanous functions will run // todo: tune and compute x,y by # of CPUs and disk io @@ -183,35 +216,53 @@ func (f *Forge) RillRepos(rillf func(*gitpb.Repo) error) map[string]*RillStats { return stats } -func rillSetError(stats map[string]*RillStats, fullpath string, err error) { - rillMu.Lock() - defer rillMu.Unlock() - if s, ok := stats[fullpath]; ok { - s.Err = err - return - } - log.Info("WHAT THE FUCK STATS ERROR", fullpath) -} +// x is the size of the queued up pool (shouldn't matter here for this I think) +// y is how many simultanous functions will run +// todo: tune and compute x,y by # of CPUs and disk io +// todo: store x,y in forge config ? (or compute them. notsure) +func (f *Forge) RunOnRepos(repos *gitpb.Repos, rillf func(*gitpb.Repo) error) map[string]*RillStats { + var all []*gitpb.Repo -func rillSetStartTime(stats map[string]*RillStats, fullpath string) { - rillMu.Lock() - defer rillMu.Unlock() - if s, ok := stats[fullpath]; ok { - s.Start = time.Now() - return - } - var s *RillStats - s = new(RillStats) - s.Start = time.Now() - stats[fullpath] = s -} + var stats map[string]*RillStats + stats = make(map[string]*RillStats) -func rillSetEndTime(stats map[string]*RillStats, fullpath string) { - rillMu.Lock() - defer rillMu.Unlock() - if s, ok := stats[fullpath]; ok { - s.End = time.Now() - return + for repo := range repos.IterAll() { + if !repo.IsValidDir() { + log.Printf("got an invalid repo in forgepb.RillRepos() %-50s\n", repo.GetFullPath()) + continue + } + all = append(all, repo) } - log.Info("WHAT THE FUCK STATS END TIME", fullpath) + // log.Info("Rill Repos len =", len(all)) + // Convert a slice of user IDs into a channel + ids := rill.FromSlice(all, nil) + + var counter int + var watch int = 10 + + // Read users from the API. + // Concurrency = 20 + dirs := rill.Map(ids, int(f.Config.RillX), func(id *gitpb.Repo) (*gitpb.Repo, error) { + return id, nil + }) + + rill.ForEach(dirs, int(f.Config.RillY), func(repo *gitpb.Repo) error { + // todo: make this a goroutine to show stats to the user + rillMu.Lock() + counter += 1 + if counter > watch { + // log.Info("Processed", watch, "repos") // this doesn't work + watch += 50 + } + rillMu.Unlock() + + rillSetStartTime(stats, repo.GetFullPath()) + if err := rillf(repo); err != nil { + rillSetError(stats, repo.GetFullPath(), err) + } + rillSetEndTime(stats, repo.GetFullPath()) + return nil + }) + + return stats } |