Add "--tree" argument to tsp/scripts/crawler.py

Add tsp/tree.example

Change-Id: Ic72e4e80812a56b18d606fc36e57096a84e205bd
Signed-off-by: Aleksander Mistewicz <a.mistewicz@samsung.com>

diff --git a/tsp/jobs/trigger_downloads.sh b/tsp/jobs/trigger_downloads.sh
index 648e89a..1684492 100755 (executable)
--- a/tsp/jobs/trigger_downloads.sh
+++ b/tsp/jobs/trigger_downloads.sh
@@ -33,8 +33,11 @@ cd "${WS_WATCHER}"
 touch dispatched_urls
 touch "${TIMESTAMP}"
 
+TREE="${TSP_DIR}/tree"
+test -f "${TREE}" || TREE="${TSP_DIR}/tree.example"
+
 # Crawl given url
-timeout 240 "${TSP_DIR}/scripts/crawler.py" --timestamp "${TIMESTAMP}" --log DEBUG "${URL}" || die "Crawl failed" "$?"
+timeout 240 "${TSP_DIR}/scripts/crawler.py" --timestamp "${TIMESTAMP}" --tree "${TREE}" --log DEBUG "${URL}" || die "Crawl failed" "$?"
 
 # Read next buid_nr
 touch next_dwn

diff --git a/tsp/scripts/crawler.py b/tsp/scripts/crawler.py
index 9ea25fd..86724e3 100755 (executable)
--- a/tsp/scripts/crawler.py
+++ b/tsp/scripts/crawler.py
@@ -68,9 +68,14 @@ def get_urls2check(session, md5sums, url, level=2):
     if level == 0:
         md5sums.add(url + "MD5SUMS")
 
-def get_modified_paths(discovered, timestamp):
+def get_modified_paths(discovered, timestamp, tree):
     logging.info("get_modified_paths")
     ret = set()
+    if tree:
+        logging.debug("Read tree file")
+        with open(tree, 'r') as f:
+            tree_urls = f.read().splitlines()
+            logging.debug(tree_urls)
     str_time = time.strftime('%a, %d %b %Y %H:%M:%S GMT', time.gmtime(time.time()))
     logging.info("Next timestamp: %s", str_time)
     if os.path.exists(dispatched_urls):
@@ -83,8 +88,12 @@ def get_modified_paths(discovered, timestamp):
     s.headers.update({"If-Modified-Since": stamp})
     for url in discovered:
         logging.debug("Check for MD5SUMS change: %s", url)
-        md5sums_urls = set()
-        get_urls2check(s, md5sums_urls, url + "images/")
+        if not tree:
+            logging.debug("Obtain files to check by crawl")
+            md5sums_urls = set()
+            get_urls2check(s, md5sums_urls, url + "images/")
+        else:
+            md5sums_urls = [url + e for e in tree_urls]
         change = False
         for md5sums_url in md5sums_urls:
             r = s.get(md5sums_url)
@@ -124,6 +133,9 @@ def parse_arguments():
     parser.add_argument("-t", "--timestamp", default="timestamp",
             help="File to read a reference timestamp from")
 
+    parser.add_argument("-r", "--tree",
+            help="Path to a file with list of urls to check")
+
     args = parser.parse_args()
 
     return args
@@ -138,6 +150,7 @@ if '__main__' == __name__:
     logging.debug("Begin")
     snapshots = crawl(args.url)
     timestamp_file = args.timestamp
+    tree_file = args.tree
 
     if "snapshots" in args.url:
         discovered = snapshots
@@ -151,7 +164,7 @@ if '__main__' == __name__:
             dispatched = set([url.rstrip() for url in f.readlines()])
 
     # save discovered URLs for dispatching download requests
-    modified = get_modified_paths(discovered, timestamp_file)
+    modified = get_modified_paths(discovered, timestamp_file, tree_file)
     with open(discovered_urls, 'w') as f:
         f.write('\n'.join(modified) + '\n')
 

diff --git a/tsp/tree.example b/tsp/tree.example
new file mode 100644 (file)
index 0000000..edce685
--- /dev/null
+++ b/tsp/tree.example
@@ -0,0 +1,18 @@
+images/arm-wayland/common-headless-3parts-armv7l-artik/MD5SUMS
+images/ia32-wayland/common-wayland-efi-i586/MD5SUMS
+images/emulator32-wayland/common-emulator-wayland-3parts-i586/MD5SUMS
+images/arm64-wayland/common-wayland-3parts-arm64/MD5SUMS
+images/ia32-wayland/common-wayland-mbr-i586/MD5SUMS
+images/arm-wayland/common-boot-armv7l-odroidxu3/MD5SUMS
+images/x86_64-wayland/common-wayland-mbr-x86_64/MD5SUMS
+images/x86_64-wayland/common-minimal-mbr-x86_64/MD5SUMS
+images/arm-wayland/common-wayland-2parts-armv7l/MD5SUMS
+images/arm-wayland/common-wayland-3parts-armv7l-artik/MD5SUMS
+images/emulator32-wayland/common-emulator-wayland-mbr-i586/MD5SUMS
+images/x86_64-wayland/common-installer-efi-x86_64/MD5SUMS
+images/arm-wayland/common-boot-armv7l-artik10/MD5SUMS
+images/arm-wayland/common-wayland-3parts-armv7l-odroidu3/MD5SUMS
+images/arm-wayland/common-boot-armv7l-artik5/MD5SUMS
+images/arm-wayland/common-boot-armv7l-odroidu3/MD5SUMS
+images/ia32-wayland/common-installer-mbr-i586/MD5SUMS
+images/x86_64-wayland/common-wayland-efi-x86_64/MD5SUMS