From 80c59a74cc652f28f47d013070da68d63772d419 Mon Sep 17 00:00:00 2001 From: morganamilo Date: Mon, 5 Mar 2018 22:15:34 +0000 Subject: [PATCH] Use git ls-remote to track devel updates Use the command `git ls-remote ` to track devel updates rather than relying on the GitHub API. This allows devel update to work for every git based source and elimantes the rate limiting from GitHub. The yay_vcs.json format has changed to better support packages which multiple vcs sources and to track the protocols each source uses. And track the branch that each source tracks in it's fragment. --- clean.go | 15 ++-- cmd.go | 4 - config.go | 5 +- install.go | 25 ++---- upgrade.go | 8 +- vcs.go | 236 +++++++++++++++++++++++++---------------------------- 6 files changed, 134 insertions(+), 159 deletions(-) diff --git a/clean.go b/clean.go index 0a1ce223..3fb23355 100644 --- a/clean.go +++ b/clean.go @@ -4,16 +4,19 @@ package main // RemovePackage removes package from VCS information func removeVCSPackage(pkgs []string) { + updated := false + for _, pkgName := range pkgs { - for i, e := range savedInfo { - if e.Package == pkgName { - savedInfo[i] = savedInfo[len(savedInfo)-1] - savedInfo = savedInfo[:len(savedInfo)-1] - } + _, ok := savedInfo[pkgName] + if ok { + delete(savedInfo, pkgName) + updated = true } } - _ = saveVCSInfo() + if updated { + saveVCSInfo() + } } // CleanDependencies removes all dangling dependencies in system diff --git a/cmd.go b/cmd.go index 15e9dfac..49a6ba08 100644 --- a/cmd.go +++ b/cmd.go @@ -420,10 +420,6 @@ func handleYay() (err error) { if err != nil { return } - err = saveVCSInfo() - if err != nil { - return - } } else if cmdArgs.existsArg("c", "clean") { err = cleanDependencies() } else if len(cmdArgs.targets) > 0 { diff --git a/config.go b/config.go index b5101890..a08c0a75 100644 --- a/config.go +++ b/config.go @@ -46,7 +46,8 @@ var version = "3.373" // baseURL givers the AUR default address. const baseURL string = "https://aur.archlinux.org" -var savedInfo infos +// savedInfo holds the current vcs info +var savedInfo vcsInfo // configfile holds yay config file path. var configFile string @@ -54,7 +55,7 @@ var configFile string // vcsfile holds yay vcs info file path. var vcsFile string -//completion file +// completion file var completionFile string // changedConfig holds whether or not the config has changed diff --git a/install.go b/install.go index 775ed773..d3497d56 100644 --- a/install.go +++ b/install.go @@ -339,20 +339,6 @@ func askEditPkgBuilds(pkgs []*rpc.Pkg, bases map[string][]*rpc.Pkg) error { return nil } -func updateVSCdb(pkgs []*rpc.Pkg, pkgbuild *gopkg.PKGBUILD) { - for _, pkgsource := range pkgbuild.Source { - owner, repo := parseSource(pkgsource) - if owner != "" && repo != "" { - for _, pkg := range pkgs { - err := branchInfo(pkg.Name, owner, repo) - if err != nil { - fmt.Println(err) - } - } - } - } -} - func parsesrcinfosFile(pkgs []*rpc.Pkg, srcinfos map[string]*gopkg.PKGBUILD, bases map[string][]*rpc.Pkg) error { for k, pkg := range pkgs { dir := config.BuildDir + pkg.PackageBase + "/" @@ -366,7 +352,11 @@ func parsesrcinfosFile(pkgs []*rpc.Pkg, srcinfos map[string]*gopkg.PKGBUILD, bas } srcinfos[pkg.PackageBase] = pkgbuild - updateVSCdb(bases[pkg.PackageBase], pkgbuild) + + for _, pkg := range bases[pkg.PackageBase] { + updateVCSData(pkg.Name, pkgbuild.Source) + } + } return nil @@ -493,7 +483,10 @@ func buildInstallPkgBuilds(pkgs []*rpc.Pkg, srcinfos map[string]*gopkg.PKGBUILD, return err } - updateVSCdb(bases[pkg.PackageBase], srcinfo) + for _, pkg := range bases[pkg.PackageBase] { + updateVCSData(pkg.Name, srcinfo.Source) + } + if len(depArguments.targets) > 0 { _, stderr, err := passToPacmanCapture(depArguments) if err != nil { diff --git a/upgrade.go b/upgrade.go index 95d05f6e..e4006327 100644 --- a/upgrade.go +++ b/upgrade.go @@ -144,12 +144,12 @@ loop: } func upDevel(remote []alpm.Package, packageC chan upgrade, done chan bool) { - for _, e := range savedInfo { + for vcsName, e := range savedInfo { if e.needsUpdate() { found := false var pkg alpm.Package for _, r := range remote { - if r.Name() == e.Package { + if r.Name() == vcsName { found = true pkg = r } @@ -159,10 +159,10 @@ func upDevel(remote []alpm.Package, packageC chan upgrade, done chan bool) { fmt.Print(magenta("Warning: ")) fmt.Printf("%s ignoring package upgrade (%s => %s)\n", cyan(pkg.Name()), pkg.Version(), "git") } else { - packageC <- upgrade{e.Package, "devel", pkg.Version(), "commit-" + e.SHA[0:6]} + packageC <- upgrade{pkg.Name(), "devel", pkg.Version(), "latest-commit"} } } else { - removeVCSPackage([]string{e.Package}) + removeVCSPackage([]string{vcsName}) } } } diff --git a/vcs.go b/vcs.go index 5403fbb4..fa95578e 100644 --- a/vcs.go +++ b/vcs.go @@ -1,38 +1,21 @@ package main import ( + "bytes" "encoding/json" - "fmt" - "io/ioutil" - "net/http" "os" + "os/exec" "strings" + "time" ) -// branch contains the information of a repository branch -type branch struct { - Name string `json:"name"` - Commit struct { - SHA string `json:"sha"` - } `json:"commit"` -} - -type branches []branch - // Info contains the last commit sha of a repo -type Info struct { - Package string `json:"pkgname"` - URL string `json:"url"` - SHA string `json:"sha"` -} - -type infos []Info - -// Repo contains information about the repository -type repo struct { - Name string `json:"name"` - FullName string `json:"full_name"` - DefaultBranch string `json:"default_branch"` +type vcsInfo map[string]shaInfos +type shaInfos map[string]shaInfo +type shaInfo struct { + Protocols []string `json:"protocols"` + Brach string `json:"branch"` + SHA string `json:"sha"` } // createDevelDB forces yay to create a DB of the existing development packages @@ -50,128 +33,127 @@ func createDevelDB() error { return err } -// parseSource returns owner and repo from source -func parseSource(source string) (owner string, repo string) { +// parseSource returns the git url and efault branch +func parseSource(source string) (url string, branch string, protocols []string) { if !(strings.Contains(source, "git://") || strings.Contains(source, ".git") || strings.Contains(source, "git+https://")) { return } - split := strings.Split(source, "github.com/") - if len(split) > 1 { - secondSplit := strings.Split(split[1], "/") - if len(secondSplit) > 1 { - owner = secondSplit[0] - thirdSplit := strings.Split(secondSplit[1], ".git") - if len(thirdSplit) > 0 { - repo = thirdSplit[0] - } - } + split := strings.Split(source, "::") + source = split[len(split)-1] + split = strings.SplitN(source, "://", 2) + + if len(split) != 2 { + return } + + protocols = strings.Split(split[0], "+") + split = strings.SplitN(split[1], "#", 2) + if len(split) == 2 { + secondSplit := strings.SplitN(split[1], "=", 2) + if secondSplit[0] != "branch" { + //source has #commit= or #tag= which makes them not vcs + //packages because they reference a specific point + return + } + + if len(secondSplit) == 2 { + url = split[0] + branch = secondSplit[1] + } + } else { + url = split[0] + branch = "HEAD" + } + return } -func (info *Info) needsUpdate() bool { - var newRepo repo - var newBranches branches - if strings.HasSuffix(info.URL, "/branches") { - info.URL = info.URL[:len(info.URL)-9] - } - infoResp, infoErr := http.Get(info.URL) - if infoErr != nil { - fmt.Println(infoErr) - return false - } - defer infoResp.Body.Close() - - infoBody, _ := ioutil.ReadAll(infoResp.Body) - var err = json.Unmarshal(infoBody, &newRepo) - if err != nil { - fmt.Printf("Cannot update '%v'\nError: %v\nStatus code: %v\nBody: %v\n", - info.Package, err, infoResp.StatusCode, string(infoBody)) - return false +func updateVCSData(pkgName string, sources []string) { + if savedInfo == nil { + savedInfo = make(vcsInfo) } - defaultBranch := newRepo.DefaultBranch - branchesURL := info.URL + "/branches" + info := make(shaInfos) - branchResp, branchErr := http.Get(branchesURL) - if branchErr != nil { - fmt.Println(branchErr) - return false + for _, source := range sources { + url, branch, protocols := parseSource(source) + if url == "" || branch == "" { + continue + } + + commit := getCommit(url, branch, protocols) + if commit == "" { + continue + } + + info[url] = shaInfo{ + protocols, + branch, + commit, + } + + savedInfo[pkgName] = info + saveVCSInfo() } - defer branchResp.Body.Close() +} - branchBody, _ := ioutil.ReadAll(branchResp.Body) - err = json.Unmarshal(branchBody, &newBranches) - if err != nil { - fmt.Printf("Cannot update '%v'\nError: %v\nStatus code: %v\nBody: %v\n", - info.Package, err, branchResp.StatusCode, string(branchBody)) - return false +func getCommit(url string, branch string, protocols []string) string { + for _, protocol := range protocols { + var outbuf bytes.Buffer + + cmd := exec.Command("git", "ls-remote", protocol+"://"+url, branch) + cmd.Stdout = &outbuf + + err := cmd.Start() + if err != nil { + continue + } + + //for some reason + //git://bitbucket.org/volumesoffun/polyvox.git` hangs on my + //machine but using http:// instead of git does not hang. + //Introduce a time out so this can not hang + timer := time.AfterFunc(5*time.Second, func() { + cmd.Process.Kill() + }) + + err = cmd.Wait() + timer.Stop() + + if err != nil { + continue + } + err = cmd.Run() + + stdout := outbuf.String() + split := strings.Fields(stdout) + + if len(split) < 2 { + continue + } + + commit := split[0] + return commit } - for _, e := range newBranches { - if e.Name == defaultBranch { - return e.Commit.SHA != info.SHA + return "" +} + +func (infos shaInfos) needsUpdate() bool { + for url, info := range infos { + hash := getCommit(url, info.Brach, info.Protocols) + if hash != info.SHA { + return true } } + return false } -func inStore(pkgName string) *Info { - for i, e := range savedInfo { - if pkgName == e.Package { - return &savedInfo[i] - } - } - return nil -} - -// branchInfo updates saved information -func branchInfo(pkgName string, owner string, repoName string) (err error) { - updated := false - var newRepo repo - var newBranches branches - url := "https://api.github.com/repos/" + owner + "/" + repoName - repoResp, err := http.Get(url) - if err != nil { - return - } - defer repoResp.Body.Close() - - _ = json.NewDecoder(repoResp.Body).Decode(&newRepo) - defaultBranch := newRepo.DefaultBranch - branchesURL := url + "/branches" - - branchResp, err := http.Get(branchesURL) - if err != nil { - return - } - defer branchResp.Body.Close() - - _ = json.NewDecoder(branchResp.Body).Decode(&newBranches) - - packinfo := inStore(pkgName) - - for _, e := range newBranches { - if e.Name == defaultBranch { - updated = true - - if packinfo != nil { - packinfo.Package = pkgName - packinfo.URL = url - packinfo.SHA = e.Commit.SHA - } else { - savedInfo = append(savedInfo, Info{Package: pkgName, URL: url, SHA: e.Commit.SHA}) - } - } - } - - if updated { - saveVCSInfo() - } - - return +func inStore(pkgName string) shaInfos { + return savedInfo[pkgName] } func saveVCSInfo() error {