From 20511f752a15f160353e27872a25c72fe3c6d1a0 Mon Sep 17 00:00:00 2001 From: Katarzyna Gorska Date: Wed, 15 Nov 2017 17:52:46 +0100 Subject: [PATCH] Add ArtifactDB to storage ArtifactDB is designed to store information about downloaded artifacts. As for now inserting ArtifactRecord is supported. More operations (updates, deletes and selects) will be implemented soon. Close was added to ArtifactManager interface. Change-Id: I17ac260691d11518bcec5385eaf2689e389feb7f Signed-off-by: Katarzyna Gorska --- artifactmanager.go | 3 + artifacts/artifact_test.go | 156 ++++++++++++++++++++++++++++++++++++++ artifacts/artifacts.go | 69 ++++++++++++++++- artifacts/artifacts_suite_test.go | 30 ++++++++ 4 files changed, 256 insertions(+), 2 deletions(-) create mode 100644 artifacts/artifact_test.go create mode 100644 artifacts/artifacts_suite_test.go diff --git a/artifactmanager.go b/artifactmanager.go index 41e2ff5..8500759 100644 --- a/artifactmanager.go +++ b/artifactmanager.go @@ -103,4 +103,7 @@ type ArtifactManager interface { // GetFileInfo retrieves information about an artifact from ArtifactDB. GetArtifactInfo(path ArtifactPath) (ArtifactInfo, error) + + // Close gracefully closes ArtifactManager + Close() error } diff --git a/artifacts/artifact_test.go b/artifacts/artifact_test.go new file mode 100644 index 0000000..f76762c --- /dev/null +++ b/artifacts/artifact_test.go @@ -0,0 +1,156 @@ +/* + * Copyright (c) 2017 Samsung Electronics Co., Ltd All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License + */ + +// Package artifacts is responsible for Weles system's job artifact management. +package artifacts + +import ( + "database/sql" + "io/ioutil" + "os" + "path/filepath" + "strconv" + + "git.tizen.org/tools/weles" + + _ "github.com/mattn/go-sqlite3" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +var ( + silverKangaroo weles.ArtifactManager + testDir string + dbPath string + err error + job weles.JobID = 58008 +) + +var ( + description = weles.ArtifactDescription{ + job, + weles.AM_IMAGEFILE, + "alias", + "uri", + } + + dSameJobNType = weles.ArtifactDescription{ + job, + weles.AM_IMAGEFILE, + "other alias", + "other uri", + } + + dSameJobOtherType = weles.ArtifactDescription{ + job, + weles.AM_YAMLFILE, + "another alias", + "another uri", + } + + dOtherJob = weles.ArtifactDescription{ + job + 1, + weles.AM_YAMLFILE, + "another alias", + "another uri", + } + + testDescriptions = []weles.ArtifactDescription{description, dSameJobNType, dSameJobOtherType, dOtherJob} +) + +var _ = Describe("ArtifactManager", func() { + + BeforeEach(func() { + + testDir, err = ioutil.TempDir("", "test-weles-") + Expect(err).ToNot(HaveOccurred()) + dbPath = filepath.Join(testDir, "test.db") + + silverKangaroo, err = newArtifactManager(dbPath, testDir) + Expect(err).ToNot(HaveOccurred()) + }) + + AfterEach(func() { + os.RemoveAll(testDir) + err := silverKangaroo.Close() + Expect(err).ToNot(HaveOccurred()) + }) + + It("should create new temp directory for artifacts", func() { + var path, pathSame, pathType weles.ArtifactPath + + jobDir := filepath.Join(testDir, strconv.Itoa(int(description.JobID))) + typeDir := filepath.Join(jobDir, string(description.Type)) + newTypeDir := filepath.Join(jobDir, string(dSameJobOtherType.Type)) + + Expect(jobDir).ToNot(BeADirectory()) + + By("CreateArtifact", func() { + path, err = silverKangaroo.CreateArtifact(description) + Expect(err).ToNot(HaveOccurred()) + Expect(path).NotTo(BeNil()) + }) + + By("Check if all subdirs, and new file exists", func() { + Expect(jobDir).To(BeADirectory()) + Expect(typeDir).To(BeADirectory()) + Expect(string(path)).To(BeAnExistingFile()) + Expect(string(path)).To(ContainSubstring(string(description.Alias))) + }) + + By("Add new artifact for the same JobID", func() { + pathSame, err = silverKangaroo.CreateArtifact(dSameJobNType) + Expect(err).ToNot(HaveOccurred()) + + Expect(jobDir).To(BeADirectory()) + Expect(typeDir).To(BeADirectory()) + + Expect(string(pathSame)).To(BeAnExistingFile()) + Expect(string(pathSame)).To(ContainSubstring(string(dSameJobNType.Alias))) + }) + + By("Add artifact with other type for the same JobID", func() { + pathType, err = silverKangaroo.CreateArtifact(dSameJobOtherType) + + Expect(err).ToNot(HaveOccurred()) + Expect(jobDir).To(BeADirectory()) + Expect(newTypeDir).To(BeADirectory()) + + Expect(string(pathType)).To(BeAnExistingFile()) + Expect(string(pathType)).To(ContainSubstring(string(dSameJobOtherType.Alias))) + }) + + paths := []weles.ArtifactPath{path, pathSame, pathType} + By("Check if artifact with path is in ArtifactDB", func() { + db, err := sql.Open("sqlite3", dbPath) + Expect(err).ToNot(HaveOccurred()) + var n int + for _, p := range paths { + err = db.QueryRow("select count (*) from artifacts where path = ?", p).Scan(&n) + Expect(err).ToNot(HaveOccurred()) + Expect(n).NotTo(BeZero()) + } + }) + + By("Check if it's possible to GetFileInfo", func() { + for _, p := range paths { + ai, err := silverKangaroo.GetArtifactInfo(p) + Expect(err).ToNot(HaveOccurred()) + Expect(ai.Path).To(Equal(p)) + } + }) + }) +}) diff --git a/artifacts/artifacts.go b/artifacts/artifacts.go index 40b398f..378a226 100644 --- a/artifacts/artifacts.go +++ b/artifacts/artifacts.go @@ -18,7 +18,14 @@ package artifacts import ( + "io/ioutil" + "os" + "path/filepath" + "strconv" + "time" + . "git.tizen.org/tools/weles" + . "git.tizen.org/tools/weles/artifacts/database" ) // ArtifactDownloader downloads requested file if there is need to. @@ -38,6 +45,26 @@ type ArtifactDownloader interface { // Storage implements ArtifactManager interface. type Storage struct { ArtifactManager + db ArtifactDB + dir string +} + +var ( + // defaultDb is default ArtifactDB name. + defaultDb = "weles.db" + // defaultDir is default directory for ArtifactManager storage. + defaultDir = "/tmp/weles/" +) + +func newArtifactManager(db, dir string) (ArtifactManager, error) { + am := Storage{dir: dir} + err := am.db.Open(db) + return &am, err +} + +// NewArtifactManager returns initialized Storage implementing ArtifactManager interface. +func NewArtifactManager() (ArtifactManager, error) { + return newArtifactManager(filepath.Join(defaultDir, defaultDb), defaultDir) } // ListArtifact is part of implementation of ArtifactManager interface. @@ -52,10 +79,48 @@ func (s *Storage) PushArtifact(artifact ArtifactDescription, ch chan ArtifactSta // CreateArtifact is part of implementation of ArtifactManager interface. func (s *Storage) CreateArtifact(artifact ArtifactDescription) (ArtifactPath, error) { - return "", ErrNotImplemented + path, err := s.getNewPath(artifact) + if err != nil { + return "", err + } + + err = s.db.InsertArtifactInfo(&ArtifactInfo{artifact, path, "", time.Now().UTC()}) + if err != nil { + return "", err + } + return path, nil } // GetArtifactInfo is part of implementation of ArtifactManager interface. func (s *Storage) GetArtifactInfo(path ArtifactPath) (ArtifactInfo, error) { - return ArtifactInfo{}, ErrNotImplemented + return s.db.SelectPath(path) +} + +// Close closes Storage's ArtifactDB. +func (s *Storage) Close() error { + return s.db.Close() +} + +// getNewPath prepares new path for artifact. +func (s *Storage) getNewPath(ad ArtifactDescription) (ArtifactPath, error) { + var ( + jobDir = filepath.Join(s.dir, strconv.Itoa(int(ad.JobID))) + typeDir = filepath.Join(jobDir, string(ad.Type)) + err error + ) + + // Organize by filetypes + err = os.MkdirAll(typeDir, os.ModePerm) + if err != nil { + return "", err + } + + // Add human readable prefix + // TODO sanitize alias - spaces, special signs etc. + f, err := ioutil.TempFile(typeDir, string(ad.Alias)) + if err != nil { + return "", err + } + defer f.Close() + return ArtifactPath(f.Name()), err } diff --git a/artifacts/artifacts_suite_test.go b/artifacts/artifacts_suite_test.go new file mode 100644 index 0000000..65f699c --- /dev/null +++ b/artifacts/artifacts_suite_test.go @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2017 Samsung Electronics Co., Ltd All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License + */ + +// Package artifacts is responsible for Weles system's job artifact management. +package artifacts + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "testing" +) + +func TestArtifacts(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Artifacts Suite") +} -- 2.7.4