Add downloader to Storage, implement PushArtifact

Change-Id: If757c4745089eb308fca53c732790d3615c5dcb9

diff --git a/artifacts/artifact_test.go b/artifacts/artifact_test.go
index f76762c..53a616e 100644 (file)
--- a/artifacts/artifact_test.go
+++ b/artifacts/artifact_test.go
@@ -19,7 +19,10 @@ package artifacts
 
 import (
        "database/sql"
+       "fmt"
        "io/ioutil"
+       "net/http"
+       "net/http/httptest"
        "os"
        "path/filepath"
        "strconv"
@@ -28,6 +31,7 @@ import (
 
        _ "github.com/mattn/go-sqlite3"
        . "github.com/onsi/ginkgo"
+       . "github.com/onsi/ginkgo/extensions/table"
        . "github.com/onsi/gomega"
 )
 
@@ -73,8 +77,13 @@ var (
 
 var _ = Describe("ArtifactManager", func() {
 
-       BeforeEach(func() {
+       var (
+               silverKangaroo weles.ArtifactManager
+               validURL       weles.ArtifactURI = "validURL"
+               invalidURL     weles.ArtifactURI = "invalidURL"
+       )
 
+       BeforeEach(func() {
                testDir, err = ioutil.TempDir("", "test-weles-")
                Expect(err).ToNot(HaveOccurred())
                dbPath = filepath.Join(testDir, "test.db")
@@ -89,7 +98,30 @@ var _ = Describe("ArtifactManager", func() {
                Expect(err).ToNot(HaveOccurred())
        })
 
+       checkPathInDb := func(path weles.ArtifactPath) bool {
+               silverKangaroo.Close()
+               db, err := sql.Open("sqlite3", dbPath)
+               Expect(err).ToNot(HaveOccurred())
+               var n int
+               err = db.QueryRow("select count (*) from artifacts where path = ?", path).Scan(&n)
+               Expect(err).ToNot(HaveOccurred())
+               return (n > 0)
+       }
+
+       prepareServer := func(url weles.ArtifactURI) *httptest.Server {
+               ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+                       if url == validURL {
+                               w.WriteHeader(http.StatusOK)
+                               fmt.Fprint(w, poem)
+                       } else {
+                               w.WriteHeader(http.StatusNotFound)
+                       }
+               }))
+               return ts
+       }
+
        It("should create new temp directory for artifacts", func() {
+
                var path, pathSame, pathType weles.ArtifactPath
 
                jobDir := filepath.Join(testDir, strconv.Itoa(int(description.JobID)))
@@ -147,10 +179,67 @@ var _ = Describe("ArtifactManager", func() {
 
                By("Check if it's possible to GetFileInfo", func() {
                        for _, p := range paths {
-                               ai, err := silverKangaroo.GetArtifactInfo(p)
-                               Expect(err).ToNot(HaveOccurred())
-                               Expect(ai.Path).To(Equal(p))
+                               Expect(checkPathInDb(p)).To(BeTrue())
+
                        }
+
                })
        })
+
+       Describe("PushArtifact", func() {
+
+               var (
+                       ad weles.ArtifactDescription = weles.ArtifactDescription{
+                               job,
+                               weles.AM_IMAGEFILE,
+                               "somealias",
+                               validURL,
+                       }
+
+                       adInvalid weles.ArtifactDescription = weles.ArtifactDescription{
+                               job,
+                               weles.AM_IMAGEFILE,
+                               "somealias",
+                               invalidURL,
+                       }
+               )
+               DescribeTable("Push artifact",
+                       func(ad weles.ArtifactDescription, finalStatus weles.ArtifactStatus) {
+
+                               ts := prepareServer(ad.URI)
+                               defer ts.Close()
+                               ad.URI = weles.ArtifactURI(ts.URL)
+
+                               ch := make(chan weles.ArtifactStatusChange, 20)
+                               path, err := silverKangaroo.PushArtifact(ad, ch)
+
+                               Expect(err).ToNot(HaveOccurred())
+
+                               Eventually(ch).Should(Receive(Equal(weles.ArtifactStatusChange{path, weles.AM_PENDING})))
+                               Eventually(ch).Should(Receive(Equal(weles.ArtifactStatusChange{path, weles.AM_DOWNLOADING})))
+                               Eventually(ch).Should(Receive(Equal(weles.ArtifactStatusChange{path, finalStatus})))
+
+                               if finalStatus != weles.AM_FAILED {
+                                       By("Check if file exists and has proper content")
+
+                                       content, err := ioutil.ReadFile(string(path))
+                                       Expect(err).ToNot(HaveOccurred())
+                                       Expect(string(content)).To(BeIdenticalTo(poem))
+
+                               } else {
+                                       By("Check if file exists")
+                                       Expect(string(path)).NotTo(BeAnExistingFile())
+                               }
+
+                               ai, err := silverKangaroo.GetArtifactInfo(path)
+                               Expect(err).ToNot(HaveOccurred())
+                               Expect(ai.Status).To(Equal(finalStatus))
+
+                               By("Check if artifact is in ArtifactDB")
+                               Expect(checkPathInDb(path)).To(BeTrue())
+                       },
+                       Entry("push artifact to db and download file", ad, weles.AM_READY),
+                       Entry("do not push an invalid artifact", adInvalid, weles.AM_FAILED),
+               )
+       })
 })

diff --git a/artifacts/artifacts.go b/artifacts/artifacts.go
index ae36d3a..b8fb111 100644 (file)
--- a/artifacts/artifacts.go
+++ b/artifacts/artifacts.go
@@ -26,6 +26,7 @@ import (
 
        . "git.tizen.org/tools/weles"
        . "git.tizen.org/tools/weles/artifacts/database"
+       . "git.tizen.org/tools/weles/artifacts/downloader"
 )
 
 // ArtifactDownloader downloads requested file if there is need to.
@@ -45,8 +46,10 @@ type ArtifactDownloader interface {
 // Storage implements ArtifactManager interface.
 type Storage struct {
        ArtifactManager
-       db  ArtifactDB
-       dir string
+       db         ArtifactDB
+       dir        string
+       downloader *Downloader
+       notifier   chan ArtifactStatusChange
 }
 
 var (
@@ -54,11 +57,22 @@ var (
        defaultDb = "weles.db"
        // defaultDir is default directory for ArtifactManager storage.
        defaultDir = "/tmp/weles/"
+       // notifierCap is default notifier channel capacity.
+       notifierCap = 100
+       // workersCount is default number of workers.
+       workersCount = 5
 )
 
 func newArtifactManager(db, dir string) (ArtifactManager, error) {
-       am := Storage{dir: dir}
+       notifier := make(chan ArtifactStatusChange, notifierCap)
+
+       am := Storage{
+               dir:        dir,
+               downloader: NewDownloader(notifier, workersCount),
+               notifier:   notifier,
+       }
        err := am.db.Open(db)
+       go am.listenToChanges()
        return &am, err
 }
 
@@ -74,7 +88,19 @@ func (s *Storage) ListArtifact(filter ArtifactFilter) ([]ArtifactInfo, error) {
 
 // PushArtifact is part of implementation of ArtifactManager interface.
 func (s *Storage) PushArtifact(artifact ArtifactDescription, ch chan ArtifactStatusChange) (ArtifactPath, error) {
-       return "", ErrNotImplemented
+       // Create artifact.
+       path, err := s.CreateArtifact(artifact)
+       if err != nil {
+               return "", err
+       }
+
+       // Download artifact.
+       err = s.downloader.Download(artifact.URI, path, ch)
+       if err != nil {
+               s.db.SetStatus(ArtifactStatusChange{path, AM_FAILED})
+               return "", err
+       }
+       return path, nil
 }
 
 // CreateArtifact is part of implementation of ArtifactManager interface.
@@ -124,3 +150,15 @@ func (s *Storage) getNewPath(ad ArtifactDescription) (ArtifactPath, error) {
        defer f.Close()
        return ArtifactPath(f.Name()), err
 }
+
+// listenToChanges updates artifact's status in db everytime Storyge is notified
+// about status change.
+func (s *Storage) listenToChanges() {
+       empty := ArtifactStatusChange{}
+       for change := range s.notifier {
+               // TODO handle errors returned by SetStatus
+               if change != empty {
+                       s.db.SetStatus(change)
+               }
+       }
+}

diff --git a/artifacts/artifacts_suite_test.go b/artifacts/artifacts_suite_test.go
index 65f699c..98cb290 100644 (file)
--- a/artifacts/artifacts_suite_test.go
+++ b/artifacts/artifacts_suite_test.go
@@ -18,13 +18,25 @@
 package artifacts
 
 import (
+       "testing"
+
        . "github.com/onsi/ginkgo"
        . "github.com/onsi/gomega"
-
-       "testing"
 )
 
 func TestArtifacts(t *testing.T) {
        RegisterFailHandler(Fail)
        RunSpecs(t, "Artifacts Suite")
 }
+
+var poem = `How doth the little crocodile
+Improve his shining tail,
+And pour the waters of the Nile
+On every golden scale!
+
+How cheerfully he seems to grin
+How neatly spreads his claws,
+And welcomes little fishes in,
+With gently smiling jaws!
+
+-Lewis Carroll`