From: Katarzyna Gorska Date: Tue, 28 Nov 2017 13:25:32 +0000 (+0100) Subject: Add downloader base functions X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=f05c6d7fc8a06fd8338490b067c85bab9f2a7f1c;p=tools%2Fweles.git Add downloader base functions Downloader implements ArtifactDownloader interface. It schedules download jobs in queue and notifies about artifacts' status changes. Change-Id: I8ff33c46406dd305c62d3874bd6e013fc6606393 Signed-off-by: Katarzyna Gorska --- diff --git a/artifacts/downloader/downloader.go b/artifacts/downloader/downloader.go new file mode 100644 index 0000000..0dbb5fb --- /dev/null +++ b/artifacts/downloader/downloader.go @@ -0,0 +1,129 @@ +/* + * Copyright (c) 2017 Samsung Electronics Co., Ltd All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License + */ + +// Package downloader is responsible for Weles system's job artifact downloading. +package downloader + +import ( + "fmt" + "io" + "net/http" + "os" + + . "git.tizen.org/tools/weles" +) + +// Downloader implements ArtifactDownloader interface. +type Downloader struct { + notification chan ArtifactStatusChange // can be used to monitor ArtifactStatusChanges. + queue chan downloadJob +} + +// downloadJob provides necessary info for download to be done. +type downloadJob struct { +} + +// queueCap is the default length of download queue. +const queueCap = 100 + +// newDownloader returns initilized Downloader. +func newDownloader(notification chan ArtifactStatusChange, workerCount int, queueSize int) *Downloader { + + return &Downloader{ + notification: notification, + queue: make(chan downloadJob, queueSize), + } +} + +// NewDownloader returns Downloader initialized with default queue length +func NewDownloader(notification chan ArtifactStatusChange, workerCount int) *Downloader { + return newDownloader(notification, workerCount, queueCap) +} + +// Close is part of implementation of ArtifactDownloader interface. +// It closes used channels. +func (d *Downloader) Close() { + close(d.queue) +} + +// getData downloads file from provided location and saves it in a prepared path. +func (d *Downloader) getData(URI ArtifactURI, path ArtifactPath) error { + + resp, err := http.Get(string(URI)) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("server error %v %v", URI, resp.Status) + } + + file, err := os.Create(string(path)) + if err != nil { + return err + } + defer file.Close() + + _, err = io.Copy(file, resp.Body) + return err +} + +// download downloads artifact from provided URI and saves it to specified path. +// It sends notification about status changes to two channels - Downloader's notification +// channel, and other one, that can be specified passed as an argument. +func (d *Downloader) download(URI ArtifactURI, path ArtifactPath, ch chan ArtifactStatusChange) { + if path == "" { + return + } + + change := ArtifactStatusChange{ + Path: path, + NewStatus: AM_DOWNLOADING, + } + channels := []chan ArtifactStatusChange{ch, d.notification} + notify(change, channels) + + err := d.getData(URI, path) + if err != nil { + os.Remove(string(path)) + change.NewStatus = AM_FAILED + } else { + change.NewStatus = AM_READY + } + + notify(change, channels) +} + +// Download is part of implementation of ArtifactDownloader interface. +// TODO implement. +func (d *Downloader) Download(URI ArtifactURI, path ArtifactPath, ch chan ArtifactStatusChange) error { + return ErrNotImplemented + +} + +// CheckInCache is part of implementation of ArtifactDownloader interface. +// TODO implement. +func (d *Downloader) CheckInCache(URI ArtifactURI) (ArtifactInfo, error) { + return ArtifactInfo{}, ErrNotImplemented +} + +// notify sends ArtifactStatusChange to all specified channels. +func notify(change ArtifactStatusChange, channels []chan ArtifactStatusChange) { + for _, ch := range channels { + ch <- change + } +} diff --git a/artifacts/downloader/downloader_suite_test.go b/artifacts/downloader/downloader_suite_test.go new file mode 100644 index 0000000..d866229 --- /dev/null +++ b/artifacts/downloader/downloader_suite_test.go @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2017 Samsung Electronics Co., Ltd All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License + */ + +package downloader + +import ( + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestDownloader(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Downloader Suite") +} diff --git a/artifacts/downloader/downloader_test.go b/artifacts/downloader/downloader_test.go new file mode 100644 index 0000000..4956243 --- /dev/null +++ b/artifacts/downloader/downloader_test.go @@ -0,0 +1,168 @@ +/* + * Copyright (c) 2017 Samsung Electronics Co., Ltd All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License + */ + +package downloader + +import ( + "fmt" + "io/ioutil" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + + "git.tizen.org/tools/weles" + . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/extensions/table" + . "github.com/onsi/gomega" +) + +var _ = Describe("Downloader", func() { + + const pigs = `The pig, if I am not mistaken, +Supplies us sausage, ham, and bacon. +Let others say his heart is big -- +I call it stupid of the pig. + +-Ogden Nash` + + var platinumKoala *Downloader + + var ( + tmpDir string + validDir string + invalidDir string + validURL weles.ArtifactURI = "validURL" + invalidURL weles.ArtifactURI = "invalidURL" + ) + var ( + notifyCap int = 100 // notitication channel capacity. + notification chan weles.ArtifactStatusChange + workersCount = 8 + ) + + checkChannels := func(ch1, ch2 chan weles.ArtifactStatusChange, change weles.ArtifactStatusChange) { + Eventually(ch1).Should(Receive(Equal(change))) + Eventually(ch2).Should(Receive(Equal(change))) + } + + BeforeEach(func() { + + var err error + // prepare Downloader. + notification = make(chan weles.ArtifactStatusChange, notifyCap) + platinumKoala = NewDownloader(notification, workersCount) + + // prepare temporary directories. + tmpDir, err = ioutil.TempDir("", "weles-") + Expect(err).ToNot(HaveOccurred()) + validDir = filepath.Join(tmpDir, "valid") + err = os.MkdirAll(validDir, os.ModePerm) + Expect(err).ToNot(HaveOccurred()) + // directory is not created therefore path will be invalid. + invalidDir = filepath.Join(tmpDir, "invalid") + }) + + AfterEach(func() { + platinumKoala.Close() + err := os.RemoveAll(tmpDir) + Expect(err).ToNot(HaveOccurred()) + + }) + + prepareServer := func(url weles.ArtifactURI) *httptest.Server { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if url == validURL { + w.WriteHeader(http.StatusOK) + fmt.Fprint(w, pigs) + } else { + w.WriteHeader(http.StatusNotFound) + } + })) + return ts + } + + DescribeTable("getData(): Notify channels and save data to file", + func(url weles.ArtifactURI, valid bool, finalResult weles.ArtifactStatus) { + + ts := prepareServer(url) + defer ts.Close() + + dir := validDir + if !valid { + dir = invalidDir + } + filename := weles.ArtifactPath(filepath.Join(dir, "test")) + + err := platinumKoala.getData(weles.ArtifactURI(ts.URL), weles.ArtifactPath(filename)) + + if valid && url != invalidURL { + Expect(err).ToNot(HaveOccurred()) + content, err := ioutil.ReadFile(string(filename)) + Expect(err).ToNot(HaveOccurred()) + Expect(string(content)).To(Equal(pigs)) + } else { + Expect(string(filename)).NotTo(BeAnExistingFile()) + _, err := ioutil.ReadFile(string(filename)) + Expect(err).To(HaveOccurred()) + } + + }, + Entry("download valid file to valid path", validURL, true, weles.AM_READY), + Entry("fail when url is invalid", invalidURL, true, weles.AM_FAILED), + Entry("fail when path is invalid", validURL, false, weles.AM_FAILED), + Entry("fail when url and path are invalid", invalidURL, false, weles.AM_FAILED), + ) + + DescribeTable("download(): Notify channels and save data to file", + func(url weles.ArtifactURI, valid bool, finalResult weles.ArtifactStatus) { + + ts := prepareServer(url) + defer ts.Close() + + ch := make(chan weles.ArtifactStatusChange, 5) + + dir := validDir + if !valid { + dir = invalidDir + } + filename := weles.ArtifactPath(filepath.Join(dir, "test")) + + status := weles.ArtifactStatusChange{filename, weles.AM_DOWNLOADING} + + platinumKoala.download(weles.ArtifactURI(ts.URL), weles.ArtifactPath(filename), ch) + + status.NewStatus = weles.AM_DOWNLOADING + checkChannels(ch, platinumKoala.notification, status) + + status.NewStatus = finalResult + checkChannels(ch, platinumKoala.notification, status) + + if valid && url != invalidURL { + content, err := ioutil.ReadFile(string(filename)) + Expect(err).ToNot(HaveOccurred()) + Expect(string(content)).To(Equal(pigs)) + } else { + Expect(string(filename)).NotTo(BeAnExistingFile()) + } + + }, + Entry("download valid file to valid path", validURL, true, weles.AM_READY), + Entry("fail when url is invalid", invalidURL, true, weles.AM_FAILED), + Entry("fail when path is invalid", validURL, false, weles.AM_FAILED), + Entry("fail when url and path are invalid", invalidURL, false, weles.AM_FAILED), + ) +})