From 740069b6acb37eb557440b034d4a72b7076735a1 Mon Sep 17 00:00:00 2001 From: Katarzyna Gorska Date: Wed, 15 Nov 2017 17:52:46 +0100 Subject: [PATCH] Add ArtifactDB to storage ArtifactDB is designed to store information about downloaded artifacts. As for now inserting ArtifactRecord is supported. More operations (updates, deletes and selects) will be implemented soon. Close was added to ArtifactManager interface. Change-Id: I17ac260691d11518bcec5385eaf2689e389feb7f Signed-off-by: Katarzyna Gorska --- artifactmanager.go | 3 + artifacts/artifacts.go | 83 ++++++++++++++++- artifacts/artifacts_suite_test.go | 30 +++++++ artifacts/artifacts_test.go | 182 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 296 insertions(+), 2 deletions(-) create mode 100644 artifacts/artifacts_suite_test.go create mode 100644 artifacts/artifacts_test.go diff --git a/artifactmanager.go b/artifactmanager.go index 41e2ff5..cc9058c 100644 --- a/artifactmanager.go +++ b/artifactmanager.go @@ -103,4 +103,7 @@ type ArtifactManager interface { // GetFileInfo retrieves information about an artifact from ArtifactDB. GetArtifactInfo(path ArtifactPath) (ArtifactInfo, error) + + // Close gracefully closes ArtifactManager. + Close() error } diff --git a/artifacts/artifacts.go b/artifacts/artifacts.go index 40b398f..cf5c6bb 100644 --- a/artifacts/artifacts.go +++ b/artifacts/artifacts.go @@ -18,7 +18,14 @@ package artifacts import ( + "io/ioutil" + "os" + "path/filepath" + "strconv" + "time" + . "git.tizen.org/tools/weles" + . "git.tizen.org/tools/weles/artifacts/database" ) // ArtifactDownloader downloads requested file if there is need to. @@ -38,6 +45,41 @@ type ArtifactDownloader interface { // Storage implements ArtifactManager interface. type Storage struct { ArtifactManager + db ArtifactDB + dir string +} + +const ( + // defaultDb is default ArtifactDB name. + defaultDb = "weles.db" + // defaultDir is default directory for ArtifactManager storage. + defaultDir = "/tmp/weles/" +) + +func newArtifactManager(db, dir string) (ArtifactManager, error) { + err := os.MkdirAll(dir, os.ModePerm) + if err != nil { + return nil, err + } + am := Storage{dir: dir} + err = am.db.Open(db) + + if err != nil { + return nil, err + } + return &am, nil +} + +// NewArtifactManager returns initialized Storage implementing ArtifactManager interface. +// If db or dir is empy, default value will be used. +func NewArtifactManager(db, dir string) (ArtifactManager, error) { + if db == "" { + db = defaultDb + } + if dir == "" { + dir = defaultDir + } + return newArtifactManager(filepath.Join(dir, db), dir) } // ListArtifact is part of implementation of ArtifactManager interface. @@ -52,10 +94,47 @@ func (s *Storage) PushArtifact(artifact ArtifactDescription, ch chan ArtifactSta // CreateArtifact is part of implementation of ArtifactManager interface. func (s *Storage) CreateArtifact(artifact ArtifactDescription) (ArtifactPath, error) { - return "", ErrNotImplemented + path, err := s.getNewPath(artifact) + if err != nil { + return "", err + } + + err = s.db.InsertArtifactInfo(&ArtifactInfo{artifact, path, "", time.Now().UTC()}) + if err != nil { + return "", err + } + return path, nil } // GetArtifactInfo is part of implementation of ArtifactManager interface. func (s *Storage) GetArtifactInfo(path ArtifactPath) (ArtifactInfo, error) { - return ArtifactInfo{}, ErrNotImplemented + return s.db.SelectPath(path) +} + +// Close closes Storage's ArtifactDB. +func (s *Storage) Close() error { + return s.db.Close() +} + +// getNewPath prepares new path for artifact. +func (s *Storage) getNewPath(ad ArtifactDescription) (ArtifactPath, error) { + var ( + jobDir = filepath.Join(s.dir, strconv.FormatUint(uint64(ad.JobID), 10)) + typeDir = filepath.Join(jobDir, string(ad.Type)) + err error + ) + + // Organize by filetypes + err = os.MkdirAll(typeDir, os.ModePerm) + if err != nil { + return "", err + } + + // Add human readable prefix + f, err := ioutil.TempFile(typeDir, string(ad.Alias)) + if err != nil { + return "", err + } + defer f.Close() + return ArtifactPath(f.Name()), err } diff --git a/artifacts/artifacts_suite_test.go b/artifacts/artifacts_suite_test.go new file mode 100644 index 0000000..f11caa6 --- /dev/null +++ b/artifacts/artifacts_suite_test.go @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2017 Samsung Electronics Co., Ltd All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License + */ + +// Package artifacts is responsible for Weles system's job artifact management. +package artifacts + +import ( + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestArtifacts(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Artifacts Suite") +} diff --git a/artifacts/artifacts_test.go b/artifacts/artifacts_test.go new file mode 100644 index 0000000..63e48aa --- /dev/null +++ b/artifacts/artifacts_test.go @@ -0,0 +1,182 @@ +/* + * Copyright (c) 2017 Samsung Electronics Co., Ltd All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License + */ + +// Package artifacts is responsible for Weles system's job artifact management. +package artifacts + +import ( + "database/sql" + "io/ioutil" + "os" + "path/filepath" + "strconv" + + "git.tizen.org/tools/weles" + + _ "github.com/mattn/go-sqlite3" + . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/extensions/table" + . "github.com/onsi/gomega" +) + +var _ = Describe("ArtifactManager", func() { + + var ( + silverKangaroo weles.ArtifactManager + testDir string + dbPath string + err error + job weles.JobID = 58008 + ) + + var ( + description = weles.ArtifactDescription{ + job, + weles.AM_IMAGEFILE, + "alias", + "uri", + } + + dSameJobNType = weles.ArtifactDescription{ + job, + weles.AM_IMAGEFILE, + "other alias", + "other uri", + } + + dSameJobOtherType = weles.ArtifactDescription{ + job, + weles.AM_YAMLFILE, + "another alias", + "another uri", + } + ) + + BeforeEach(func() { + testDir, err = ioutil.TempDir("", "test-weles-") + Expect(err).ToNot(HaveOccurred()) + dbPath = filepath.Join(testDir, "test.db") + + silverKangaroo, err = newArtifactManager(dbPath, testDir) + Expect(err).ToNot(HaveOccurred()) + }) + + AfterEach(func() { + err := os.RemoveAll(testDir) + Expect(err).ToNot(HaveOccurred()) + err = silverKangaroo.Close() + Expect(err).ToNot(HaveOccurred()) + }) + + It("should create new temp directory for artifacts", func() { + var path, pathSame, pathType weles.ArtifactPath + + jobDir := filepath.Join(testDir, strconv.Itoa(int(description.JobID))) + typeDir := filepath.Join(jobDir, string(description.Type)) + newTypeDir := filepath.Join(jobDir, string(dSameJobOtherType.Type)) + + Expect(jobDir).ToNot(BeADirectory()) + + By("CreateArtifact", func() { + path, err = silverKangaroo.CreateArtifact(description) + Expect(err).ToNot(HaveOccurred()) + Expect(path).NotTo(BeNil()) + }) + + By("Check if all subdirs, and new file exists", func() { + Expect(jobDir).To(BeADirectory()) + Expect(typeDir).To(BeADirectory()) + Expect(string(path)).To(BeAnExistingFile()) + Expect(string(path)).To(ContainSubstring(string(description.Alias))) + }) + + By("Add new artifact for the same JobID", func() { + pathSame, err = silverKangaroo.CreateArtifact(dSameJobNType) + Expect(err).ToNot(HaveOccurred()) + + Expect(jobDir).To(BeADirectory()) + Expect(typeDir).To(BeADirectory()) + + Expect(string(pathSame)).To(BeAnExistingFile()) + Expect(string(pathSame)).To(ContainSubstring(string(dSameJobNType.Alias))) + }) + + By("Add artifact with other type for the same JobID", func() { + pathType, err = silverKangaroo.CreateArtifact(dSameJobOtherType) + + Expect(err).ToNot(HaveOccurred()) + Expect(jobDir).To(BeADirectory()) + Expect(newTypeDir).To(BeADirectory()) + + Expect(string(pathType)).To(BeAnExistingFile()) + Expect(string(pathType)).To(ContainSubstring(string(dSameJobOtherType.Alias))) + }) + + paths := []weles.ArtifactPath{path, pathSame, pathType} + By("Check if artifact with path is in ArtifactDB", func() { + db, err := sql.Open("sqlite3", dbPath) + Expect(err).ToNot(HaveOccurred()) + var n int + for _, p := range paths { + err = db.QueryRow("select count (*) from artifacts where path = ?", p).Scan(&n) + Expect(err).ToNot(HaveOccurred()) + Expect(n).NotTo(BeZero()) + } + }) + + By("Check if it's possible to GetFileInfo", func() { + for _, p := range paths { + ai, err := silverKangaroo.GetArtifactInfo(p) + Expect(err).ToNot(HaveOccurred()) + Expect(ai.Path).To(Equal(p)) + } + }) + }) + + Describe("Public initializer", func() { + var ( + defaultDb = "weles.db" + defaultDir = "/tmp/weles/" + customDb = "nawia.db" + customDir = "/tmp/weles-custom/" + ) + + DescribeTable("NewArtifactManager()", func(db, dir string) { + copperPanda, err := NewArtifactManager(db, dir) + Expect(err).ToNot(HaveOccurred()) + + if db == "" { + db = defaultDb + } + if dir == "" { + dir = defaultDir + } + + Expect(dir).To(BeADirectory()) + Expect(filepath.Join(dir, db)).To(BeAnExistingFile()) + + err = copperPanda.Close() + Expect(err).ToNot(HaveOccurred()) + + err = os.RemoveAll(dir) + Expect(err).ToNot(HaveOccurred()) + }, + Entry("create database in default directory", defaultDb, defaultDir), + Entry("create database in default directory, when arguments are empty", "", ""), + Entry("create database in custom directory", customDb, customDir), + ) + }) +}) -- 2.7.4