Extract elapsed seconds since the start of solving from training log

diff --git a/scripts/extract_seconds.py b/scripts/extract_seconds.py
new file mode 100755 (executable)
index 0000000..9cef9f8
--- /dev/null
+++ b/scripts/extract_seconds.py
@@ -0,0 +1,43 @@
+#!/usr/bin/env python
+import datetime
+import os
+import sys
+
+def extract_datetime_from_line(line, year):
+    # Expected format: I0210 13:39:22.381027 25210 solver.cpp:204] Iteration 100, lr = 0.00992565
+    line = line.strip().split()
+    month = int(line[0][1:3])
+    day = int(line[0][3:])
+    timestamp = line[1]
+    pos = timestamp.rfind('.')
+    ts = [int(x) for x in timestamp[:pos].split(':')]
+    hour = ts[0]
+    minute = ts[1]
+    second = ts[2]
+    microsecond = int(timestamp[pos + 1:])
+    dt = datetime.datetime(year, month, day, hour, minute, second, microsecond)
+    return dt
+
+def extract_seconds(input_file, output_file):
+    with open(input_file, 'r') as f:
+        lines = f.readlines()
+    log_created_time = os.path.getctime(input_file)
+    log_created_year = datetime.datetime.fromtimestamp(log_created_time).year
+    start_time_found = False
+    out = open(output_file, 'w')
+    for line in lines:
+        line = line.strip()
+        if not start_time_found and line.find('Solving') != -1:
+            start_time_found = True
+            start_datetime = extract_datetime_from_line(line, log_created_year)
+        if line.find(', loss = ') != -1:
+            dt = extract_datetime_from_line(line, log_created_year)
+            elapsed_seconds = (dt - start_datetime).total_seconds()
+            out.write('%f\n' % elapsed_seconds)
+    out.close()
+
+if __name__ == '__main__':
+    if len(sys.argv) < 3:
+        print('Usage: ./extract_seconds input_file output_file')
+        exit(1)
+    extract_seconds(sys.argv[1], sys.argv[2])
\ No newline at end of file

diff --git a/scripts/parselog.sh b/scripts/parselog.sh
index 7691f2d..76084eb 100755 (executable)
--- a/scripts/parselog.sh
+++ b/scripts/parselog.sh
@@ -9,18 +9,24 @@ echo "Usage parselog.sh /path/to/your.log"
 exit
 fi
 LOG=`basename $1`
-grep -B 2 'Test ' $1 > aux.txt
+# For extraction of time since this line constains the start time
+grep '] Solving ' $1 > aux.txt
+grep -B 2 'Test ' $1 >> aux.txt
 grep 'Iteration ' aux.txt | sed  's/.*Iteration \([[:digit:]]*\).*/\1/g' > aux0.txt
 grep 'Test score #0' aux.txt | awk '{print $8}' > aux1.txt
 grep 'Test score #1' aux.txt | awk '{print $8}' > aux2.txt
-echo '# Iters TestAccuracy TestLoss'> $LOG.test
-paste aux0.txt aux1.txt aux2.txt | column -t >> $LOG.test
-rm aux.txt aux0.txt aux1.txt aux2.txt
+./extract_seconds.py aux.txt aux3.txt
+echo '# Iters Seconds TestAccuracy TestLoss'> $LOG.test
+paste aux0.txt aux3.txt aux1.txt aux2.txt | column -t >> $LOG.test
+rm aux.txt aux0.txt aux1.txt aux2.txt aux3.txt
 
-grep ', loss = ' $1 > aux.txt
+# For extraction of time since this line constains the start time
+grep '] Solving ' $1 > aux.txt
+grep ', loss = ' $1 >> aux.txt
 grep 'Iteration ' aux.txt | sed  's/.*Iteration \([[:digit:]]*\).*/\1/g' > aux0.txt
 grep ', loss = ' $1 | awk '{print $9}' > aux1.txt
 grep ', lr = ' $1 | awk '{print $9}' > aux2.txt
-echo '# Iters TrainingLoss LearningRate'> $LOG.train
-paste aux0.txt aux1.txt aux2.txt | column -t >> $LOG.train
-rm aux.txt aux0.txt aux1.txt aux2.txt
+./extract_seconds.py aux.txt aux3.txt
+echo '# Iters Seconds TrainingLoss LearningRate'> $LOG.train
+paste aux0.txt aux3.txt aux1.txt aux2.txt | column -t >> $LOG.train
+rm aux.txt aux0.txt aux1.txt aux2.txt  aux3.txt