From f0e22844a344ddeff1fe1f3d3e13865cc3351f99 Mon Sep 17 00:00:00 2001 From: Maciej Wereski Date: Tue, 7 Mar 2017 11:27:38 +0100 Subject: [PATCH] Add support for ignoring blacklisted repositories This change allows user to use blacklist file (default provided). Such file is a list of regular expressions that describe git repositories (one per line). Default blacklist file makes snapsync ignore profile repositories. Change-Id: I9c82ae82d7b17a4aeeccb18d1c3379a0a08cb37b Signed-off-by: Maciej Wereski --- blacklist | 1 + config/config.go | 33 ++++++++++++++++++++++++++++++--- manifests.go | 25 ++++++++++++++++++++----- snapsync.conf | 1 + snapsync.go | 27 +++++++++++++++++---------- 5 files changed, 69 insertions(+), 18 deletions(-) create mode 100644 blacklist diff --git a/blacklist b/blacklist new file mode 100644 index 0000000..f5f944c --- /dev/null +++ b/blacklist @@ -0,0 +1 @@ +^profile/* diff --git a/config/config.go b/config/config.go index 577eeb3..5495d8e 100644 --- a/config/config.go +++ b/config/config.go @@ -14,11 +14,14 @@ * limitations under the License */ -// Package config finds and parses snapsync config file. +// Package config finds and parses snapsync config and blacklist files. package config import ( + "bufio" + "bytes" "fmt" + "io/ioutil" "net/url" "os" "path" @@ -30,8 +33,9 @@ import ( // General represents a General section in config file. // It contains all keys that don't belong to other sections. type General struct { - Workers int - Timeout string + Workers int + Timeout string + BlacklistFile string `ini:"Blacklist"` } // Repos represents Repos section in config file. @@ -151,3 +155,26 @@ func FindFPath(file, fallbackName string) (string, error) { } return file, nil } + +// GetBlacklist returns slice with regular expressions +// that should match to repositories paths. +func GetBlacklist(path string) ([]*regexp.Regexp, error) { + if path == "" { + return nil, nil + } + blacklist, err := ioutil.ReadFile(path) + if err != nil { + return nil, err + } + + var ret []*regexp.Regexp + scanner := bufio.NewScanner(bytes.NewReader(blacklist)) + for scanner.Scan() { + line, err := regexp.Compile(scanner.Text()) + if err != nil { + return nil, err + } + ret = append(ret, line) + } + return ret, nil +} diff --git a/manifests.go b/manifests.go index d0ea6b5..d8ec446 100644 --- a/manifests.go +++ b/manifests.go @@ -20,7 +20,9 @@ import ( "encoding/xml" "fmt" "io/ioutil" + "log" "net/http" + "regexp" "sync" ) @@ -72,9 +74,9 @@ type RepoXML struct { Repos []RepoElem `xml:"project"` } -// parseReposXMLs takes snapshot XMLs as slice of byte slices (one XML per slice). +// parseReposXMLs takes snapshot XMLs as slice of byte slices (one XML per slice) and blacklist as slice of regular expressions. // It returns map of commits in git repositories and error. -func parseReposXMLs(contents [][]byte) (map[string]string, error) { +func parseReposXMLs(contents [][]byte, blacklist []*regexp.Regexp) (map[string]string, error) { var repoxml RepoXML result := make(map[string]string) errFmt := "Repository %s contains 2 different commits: %s and %s" @@ -86,6 +88,19 @@ func parseReposXMLs(contents [][]byte) (map[string]string, error) { } for _, v := range repoxml.Repos { + if blacklist != nil { + ignore := false + for _, re := range blacklist { + if re.MatchString(v.Path) { + log.Printf("Ignoring %s (matched pattern: %s)\n", v.Path, re) + ignore = true + break + } + } + if ignore { + continue + } + } rev, ok := result[v.Path] if ok { // sanity check @@ -122,9 +137,9 @@ func getBody(url string) ([]byte, error) { return body, nil } -// getSnapshotInfo takes URL to root folder of snapshot. It downloads and parses +// getSnapshotInfo takes URL to root folder of snapshot and a blacklist. It downloads and parses // all snapshot manifest XMLs. -func getSnapshotInfo(url string) (map[string]string, error) { +func getSnapshotInfo(url string, blacklist []*regexp.Regexp) (map[string]string, error) { var wg sync.WaitGroup // download snapshot manifest XML xmlURL := url + "/build.xml" @@ -166,5 +181,5 @@ func getSnapshotInfo(url string) (map[string]string, error) { } // parse & return - return parseReposXMLs(manifests) + return parseReposXMLs(manifests, blacklist) } diff --git a/snapsync.conf b/snapsync.conf index 8ccce2e..558d6f6 100644 --- a/snapsync.conf +++ b/snapsync.conf @@ -1,6 +1,7 @@ [General] #workers = 4 timeout = 3s +blacklist = /usr/share/snapsync/blacklist [Repos] sourceURL = http://download.tizen.org/snapshots/tizen/ diff --git a/snapsync.go b/snapsync.go index d515036..92c0cf4 100644 --- a/snapsync.go +++ b/snapsync.go @@ -57,6 +57,8 @@ var ( // path to configuration path configFile string + // path to file with list of ignored repositories (regexp) + blacklistFile string // number of Git Job workers numWorkers int @@ -66,6 +68,7 @@ var ( func init() { flag.StringVar(&configFile, "config", "", "configuration file") + flag.StringVar(&blacklistFile, "blacklist", "", "file with repositories which should be omitted") flag.StringVar(&target, "target", "", "profile to be synchronized") flag.StringVar(&source, "source", "", "profile from which repositories will be synchronized") flag.StringVar(&sourceSnapshot, "snapshot", "latest", "source snapshot") @@ -89,7 +92,7 @@ func getValue(val, def string, lowerCase bool) string { return ret } -// setup parses commandline options and checks if everything was properly set. +// setup parses commandline options, configuration file and blacklist. // As a result all required variables are set (otherwise non-nil error is returned). func setup() error { var err error @@ -116,6 +119,12 @@ func setup() error { if err != nil { return fmt.Errorf("Parsing timeout failed: %s", err) } + blacklistFile = getValue(blacklistFile, settings.G.BlacklistFile, false) + // if blacklist option wasn't set then check for blacklist in standard location + blacklistFile, err = config.FindFPath(blacklistFile, "blacklist") + if err != nil { + log.Println("Unable to read blacklist file: ", err) + } // Repos target = getValue(target, settings.R.Target, true) @@ -157,13 +166,6 @@ func setup() error { log.Println("Target wasn't set - probably new profile/project.") } - log.Println("GENERAL:") - log.Printf("workers: %d\ntimeout: %s\n", numWorkers, timeout) - log.Println("REPOS:") - log.Printf("source: %s\nsource URL: %s\ntarget: %s\ntarget URL: %s\n", - source, sourceURL, target, targetURL) - log.Printf("SSH URL: %s\nLocal mirror: %s\n", sshURL, localMirror) - return nil } @@ -173,17 +175,22 @@ func main() { log.Fatalln("setup failed: ", err) } + blacklisted, err := config.GetBlacklist(blacklistFile) + if err != nil { + log.Fatalln("Reading blacklist failed: ", err) + } + var submitted map[string]string if target != "" { // get & parse target latest snapshot manifests - submitted, err = getSnapshotInfo(targetURL + target + "/latest") + submitted, err = getSnapshotInfo(targetURL+target+"/latest", blacklisted) if err != nil { log.Fatalln("Couldn't get list of target repositories: ", err) } } // get & parse source snapshot manifests - repoList, err := getSnapshotInfo(sourceURL + source + "/" + sourceSnapshot) + repoList, err := getSnapshotInfo(sourceURL+source+"/"+sourceSnapshot, blacklisted) if err != nil { log.Fatalln("Couldn't get list of source repositories: ", err) } -- 2.7.4