Use goroutinuies for devel updates

The update process was already partly parallel: repo, aur and devel
updates where each given their own goroutine.

This gives two further layers of parallelization. Firstly the
`needsUpdate()` function is now run in parallel for each package checked.
One package may have many vcs sources so `needsUpdate()` also checks all
of its sources in parallel.

This gives an aproxamte 3x speedup for `yay -Su --devel` timing from
when the command starts to when the number menu apears.

unfortunately git://bitbucket.org/volumesoffun/polyvox.git never
resolves on my machine for some reason so it always hits the 5 second
timout period.

It then moves on to http:/bitbucket.org/volumesoffun/polyvox.git/ which
does resolve as expected. I have not looked into it but I fear this
applies to all gitbucket repos. Luckly they are few and far between.
This commit is contained in:
morganamilo 2018-03-08 21:34:12 +00:00
parent f9d4d9bafb
commit 57a8048cb8
No known key found for this signature in database
GPG Key ID: 6FE9E7996B0B082E
3 changed files with 56 additions and 25 deletions

View File

@ -338,21 +338,18 @@ func aurInfo(names []string) ([]rpc.Pkg, error) {
outOfDate := make([]string, 0, len(names)) outOfDate := make([]string, 0, len(names))
makeRequest := func(n, max int) { makeRequest := func(n, max int) {
defer wg.Done()
tempInfo, requestErr := rpc.Info(names[n:max]) tempInfo, requestErr := rpc.Info(names[n:max])
if err != nil { if err != nil {
wg.Done()
return return
} }
if requestErr != nil { if requestErr != nil {
//return info, err
err = requestErr err = requestErr
wg.Done()
return return
} }
mux.Lock() mux.Lock()
info = append(info, tempInfo...) info = append(info, tempInfo...)
mux.Unlock() mux.Unlock()
wg.Done()
} }
for n := 0; n < len(names); n += config.RequestSplitN { for n := 0; n < len(names); n += config.RequestSplitN {

View File

@ -5,6 +5,7 @@ import (
"fmt" "fmt"
"os" "os"
"sort" "sort"
"sync"
"unicode" "unicode"
alpm "github.com/jguer/go-alpm" alpm "github.com/jguer/go-alpm"
@ -128,29 +129,50 @@ loop:
} }
func upDevel(remote []alpm.Package, packageC chan upgrade, done chan bool) { func upDevel(remote []alpm.Package, packageC chan upgrade, done chan bool) {
for vcsName, e := range savedInfo { toUpdate := make([]alpm.Package, 0, 0)
toRemove := make([]string, 0, 0)
var mux1 sync.Mutex
var mux2 sync.Mutex
var wg sync.WaitGroup
checkUpdate := func(vcsName string, e shaInfos) {
defer wg.Done()
if e.needsUpdate() { if e.needsUpdate() {
found := false for _, pkg := range remote {
var pkg alpm.Package if pkg.Name() == vcsName {
for _, r := range remote { mux1.Lock()
if r.Name() == vcsName { toUpdate = append(toUpdate, pkg)
found = true mux1.Unlock()
pkg = r return
} }
} }
if found {
if pkg.ShouldIgnore() { mux2.Lock()
left, right := getVersionDiff(pkg.Version(), "latest-commit") toRemove = append(toRemove, vcsName)
fmt.Print(magenta("Warning: ")) mux2.Unlock()
fmt.Printf("%s ignoring package upgrade (%s => %s)\n", cyan(pkg.Name()), left, right)
} else {
packageC <- upgrade{pkg.Name(), "devel", pkg.Version(), "latest-commit"}
}
} else {
removeVCSPackage([]string{vcsName})
}
} }
} }
for vcsName, e := range savedInfo {
wg.Add(1)
go checkUpdate(vcsName, e)
}
wg.Wait()
for _, pkg := range toUpdate {
if pkg.ShouldIgnore() {
left, right := getVersionDiff(pkg.Version(), "latest-commit")
fmt.Print(magenta("Warning: "))
fmt.Printf("%s ignoring package upgrade (%s => %s)\n", cyan(pkg.Name()), left, right)
} else {
packageC <- upgrade{pkg.Name(), "devel", pkg.Version(), "latest-commit"}
}
}
removeVCSPackage(toRemove)
done <- true done <- true
} }

18
vcs.go
View File

@ -6,6 +6,7 @@ import (
"os" "os"
"os/exec" "os/exec"
"strings" "strings"
"sync"
"time" "time"
) )
@ -142,14 +143,25 @@ func getCommit(url string, branch string, protocols []string) string {
} }
func (infos shaInfos) needsUpdate() bool { func (infos shaInfos) needsUpdate() bool {
for url, info := range infos { var wg sync.WaitGroup
hasUpdate := false
checkHash := func(url string, info shaInfo) {
defer wg.Done()
hash := getCommit(url, info.Brach, info.Protocols) hash := getCommit(url, info.Brach, info.Protocols)
if hash != "" && hash != info.SHA { if hash != "" && hash != info.SHA {
return true hasUpdate = true
} }
} }
return false for url, info := range infos {
wg.Add(1)
go checkHash(url, info)
}
wg.Wait()
return hasUpdate
} }
func inStore(pkgName string) shaInfos { func inStore(pkgName string) shaInfos {