From: Katarzyna Gorska Date: Wed, 13 Dec 2017 14:10:46 +0000 (+0100) Subject: Add downloader to Storage, implement PushArtifact X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=90a168555839eec7a910f54e870d453f2e259900;p=tools%2Fweles.git Add downloader to Storage, implement PushArtifact Change-Id: If757c4745089eb308fca53c732790d3615c5dcb9 --- diff --git a/artifacts/artifact_test.go b/artifacts/artifact_test.go index f76762c..53a616e 100644 --- a/artifacts/artifact_test.go +++ b/artifacts/artifact_test.go @@ -19,7 +19,10 @@ package artifacts import ( "database/sql" + "fmt" "io/ioutil" + "net/http" + "net/http/httptest" "os" "path/filepath" "strconv" @@ -28,6 +31,7 @@ import ( _ "github.com/mattn/go-sqlite3" . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/extensions/table" . "github.com/onsi/gomega" ) @@ -73,8 +77,13 @@ var ( var _ = Describe("ArtifactManager", func() { - BeforeEach(func() { + var ( + silverKangaroo weles.ArtifactManager + validURL weles.ArtifactURI = "validURL" + invalidURL weles.ArtifactURI = "invalidURL" + ) + BeforeEach(func() { testDir, err = ioutil.TempDir("", "test-weles-") Expect(err).ToNot(HaveOccurred()) dbPath = filepath.Join(testDir, "test.db") @@ -89,7 +98,30 @@ var _ = Describe("ArtifactManager", func() { Expect(err).ToNot(HaveOccurred()) }) + checkPathInDb := func(path weles.ArtifactPath) bool { + silverKangaroo.Close() + db, err := sql.Open("sqlite3", dbPath) + Expect(err).ToNot(HaveOccurred()) + var n int + err = db.QueryRow("select count (*) from artifacts where path = ?", path).Scan(&n) + Expect(err).ToNot(HaveOccurred()) + return (n > 0) + } + + prepareServer := func(url weles.ArtifactURI) *httptest.Server { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if url == validURL { + w.WriteHeader(http.StatusOK) + fmt.Fprint(w, poem) + } else { + w.WriteHeader(http.StatusNotFound) + } + })) + return ts + } + It("should create new temp directory for artifacts", func() { + var path, pathSame, pathType weles.ArtifactPath jobDir := filepath.Join(testDir, strconv.Itoa(int(description.JobID))) @@ -147,10 +179,67 @@ var _ = Describe("ArtifactManager", func() { By("Check if it's possible to GetFileInfo", func() { for _, p := range paths { - ai, err := silverKangaroo.GetArtifactInfo(p) - Expect(err).ToNot(HaveOccurred()) - Expect(ai.Path).To(Equal(p)) + Expect(checkPathInDb(p)).To(BeTrue()) + } + }) }) + + Describe("PushArtifact", func() { + + var ( + ad weles.ArtifactDescription = weles.ArtifactDescription{ + job, + weles.AM_IMAGEFILE, + "somealias", + validURL, + } + + adInvalid weles.ArtifactDescription = weles.ArtifactDescription{ + job, + weles.AM_IMAGEFILE, + "somealias", + invalidURL, + } + ) + DescribeTable("Push artifact", + func(ad weles.ArtifactDescription, finalStatus weles.ArtifactStatus) { + + ts := prepareServer(ad.URI) + defer ts.Close() + ad.URI = weles.ArtifactURI(ts.URL) + + ch := make(chan weles.ArtifactStatusChange, 20) + path, err := silverKangaroo.PushArtifact(ad, ch) + + Expect(err).ToNot(HaveOccurred()) + + Eventually(ch).Should(Receive(Equal(weles.ArtifactStatusChange{path, weles.AM_PENDING}))) + Eventually(ch).Should(Receive(Equal(weles.ArtifactStatusChange{path, weles.AM_DOWNLOADING}))) + Eventually(ch).Should(Receive(Equal(weles.ArtifactStatusChange{path, finalStatus}))) + + if finalStatus != weles.AM_FAILED { + By("Check if file exists and has proper content") + + content, err := ioutil.ReadFile(string(path)) + Expect(err).ToNot(HaveOccurred()) + Expect(string(content)).To(BeIdenticalTo(poem)) + + } else { + By("Check if file exists") + Expect(string(path)).NotTo(BeAnExistingFile()) + } + + ai, err := silverKangaroo.GetArtifactInfo(path) + Expect(err).ToNot(HaveOccurred()) + Expect(ai.Status).To(Equal(finalStatus)) + + By("Check if artifact is in ArtifactDB") + Expect(checkPathInDb(path)).To(BeTrue()) + }, + Entry("push artifact to db and download file", ad, weles.AM_READY), + Entry("do not push an invalid artifact", adInvalid, weles.AM_FAILED), + ) + }) }) diff --git a/artifacts/artifacts.go b/artifacts/artifacts.go index ae36d3a..b8fb111 100644 --- a/artifacts/artifacts.go +++ b/artifacts/artifacts.go @@ -26,6 +26,7 @@ import ( . "git.tizen.org/tools/weles" . "git.tizen.org/tools/weles/artifacts/database" + . "git.tizen.org/tools/weles/artifacts/downloader" ) // ArtifactDownloader downloads requested file if there is need to. @@ -45,8 +46,10 @@ type ArtifactDownloader interface { // Storage implements ArtifactManager interface. type Storage struct { ArtifactManager - db ArtifactDB - dir string + db ArtifactDB + dir string + downloader *Downloader + notifier chan ArtifactStatusChange } var ( @@ -54,11 +57,22 @@ var ( defaultDb = "weles.db" // defaultDir is default directory for ArtifactManager storage. defaultDir = "/tmp/weles/" + // notifierCap is default notifier channel capacity. + notifierCap = 100 + // workersCount is default number of workers. + workersCount = 5 ) func newArtifactManager(db, dir string) (ArtifactManager, error) { - am := Storage{dir: dir} + notifier := make(chan ArtifactStatusChange, notifierCap) + + am := Storage{ + dir: dir, + downloader: NewDownloader(notifier, workersCount), + notifier: notifier, + } err := am.db.Open(db) + go am.listenToChanges() return &am, err } @@ -74,7 +88,19 @@ func (s *Storage) ListArtifact(filter ArtifactFilter) ([]ArtifactInfo, error) { // PushArtifact is part of implementation of ArtifactManager interface. func (s *Storage) PushArtifact(artifact ArtifactDescription, ch chan ArtifactStatusChange) (ArtifactPath, error) { - return "", ErrNotImplemented + // Create artifact. + path, err := s.CreateArtifact(artifact) + if err != nil { + return "", err + } + + // Download artifact. + err = s.downloader.Download(artifact.URI, path, ch) + if err != nil { + s.db.SetStatus(ArtifactStatusChange{path, AM_FAILED}) + return "", err + } + return path, nil } // CreateArtifact is part of implementation of ArtifactManager interface. @@ -124,3 +150,15 @@ func (s *Storage) getNewPath(ad ArtifactDescription) (ArtifactPath, error) { defer f.Close() return ArtifactPath(f.Name()), err } + +// listenToChanges updates artifact's status in db everytime Storyge is notified +// about status change. +func (s *Storage) listenToChanges() { + empty := ArtifactStatusChange{} + for change := range s.notifier { + // TODO handle errors returned by SetStatus + if change != empty { + s.db.SetStatus(change) + } + } +} diff --git a/artifacts/artifacts_suite_test.go b/artifacts/artifacts_suite_test.go index 65f699c..98cb290 100644 --- a/artifacts/artifacts_suite_test.go +++ b/artifacts/artifacts_suite_test.go @@ -18,13 +18,25 @@ package artifacts import ( + "testing" + . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - - "testing" ) func TestArtifacts(t *testing.T) { RegisterFailHandler(Fail) RunSpecs(t, "Artifacts Suite") } + +var poem = `How doth the little crocodile +Improve his shining tail, +And pour the waters of the Nile +On every golden scale! + +How cheerfully he seems to grin +How neatly spreads his claws, +And welcomes little fishes in, +With gently smiling jaws! + +-Lewis Carroll`