From a420b8dc36b44017eaf98e82485628cdc70a1fc8 Mon Sep 17 00:00:00 2001 From: "jijoong.moon" Date: Mon, 13 Jul 2020 15:52:44 +0900 Subject: [PATCH] [ Application ] mnist with tensorflow This PR includes: . MNIST Tensorflow Example to compare with nntrainer. **Self evaluation:** 1. Build test: [X]Passed [ ]Failed [ ]Skipped 2. Run test: [X]Passed [ ]Failed [ ]Skipped Signed-off-by: jijoong.moon --- Applications/mnist/Tensorflow/Training_Keras.py | 147 ++++++++++++++++++++++++ Applications/mnist/Tensorflow/dataset.py | 63 ++++++++++ Applications/mnist/jni/main.cpp | 1 - Applications/mnist/res/mnist.ini | 16 +-- 4 files changed, 214 insertions(+), 13 deletions(-) create mode 100644 Applications/mnist/Tensorflow/Training_Keras.py create mode 100644 Applications/mnist/Tensorflow/dataset.py diff --git a/Applications/mnist/Tensorflow/Training_Keras.py b/Applications/mnist/Tensorflow/Training_Keras.py new file mode 100644 index 0000000..074d1d3 --- /dev/null +++ b/Applications/mnist/Tensorflow/Training_Keras.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python +# +# Copyright (C) 2020 Jijoong Moon +# +# SPDX-License-Identifier: Apache-2.0-only +# +# @file mnist_Keras.py +# @date 13 July 2020 +# @brief This is Simple Classification Example using Keras +# @see https://github.com/nnstreamer/nntrainer +# @author Jijoong Moon +# @bug No known bugs except for NYI items +# +# mnist example +# inputlayer -> conv2d (5x5) 6 filters -> pooling2d 2x2 (valid) -> conv2d (5x5) 12 filters -> +# pooling2d 2x2 (valid) -> flatten -> fully connected layer -> 10 class +# + +import random +import struct +import os +os.environ['TF_CUDNN_DETERMINISTIC'] = '1' + +import tensorflow as tf +import numpy as np +import dataset as dataset +from tensorflow.keras.layers import Input, Dense, Conv2D, Flatten, AveragePooling2D +from tensorflow.keras import models, layers, optimizers +from tensorflow.keras.callbacks import Callback +from tensorflow.keras import initializers + +SEED=1 +tf.compat.v1.reset_default_graph() +random.seed(SEED) +tf.compat.v1.set_random_seed(SEED) +np.random.seed(SEED) + +batch_size =32 +Learning = True +Test = False +num_epoch = 1500 + +## +# @brief Callback to get the data from model during trining +# @param[in] Callback Keras Callback object +class History_LAW(Callback): + def on_train_begin(self, logs={}): + self.epoch=[] + self.weights =[] + self.history ={} + self.weights.append(self.model.layers[0].get_weights()) + + def on_epoch_end(self, epoch, logs={}): + self.weights.append(self.model.layers[0].get_weights()) + +## +# @brief input data generater with batch_size +# ( 32 x 28 x 28 x 1 :training & 32 x 10 : labels ) +# @param[in] x_data : total input data +# @param[in] y_data : total label data +# @param[in] batch_size : batch_size +# @return (x_batch, y_batch) +def datagen( x_data, y_data, batch_size): + size=len(x_data) + while True: + for i in range(size // batch_size): + x_batch = x_data[i*batch_size: (i+1)*batch_size] + x_batch=np.reshape(x_batch, (batch_size, 1, 28,28)) + x_batch=np.transpose(x_batch, [0,2,3,1]) + y_batch = y_data[i*batch_size: (i+1)*batch_size] + + yield x_batch, y_batch + +## +# @brief training loop +# - epoches : 1500 +# - Optimizer : Adam +# - Activation : softmax +# - loss : cross entropy +# +def train_nntrainer(): + train_data_size, val_data_size, label_size, feature_size = dataset.get_data_info() + InVec, InLabel, ValVec, ValLabel = dataset.load_data() + + print('reading is done') + inputs = tf.placeholder(tf.float32, [None, feature_size], name="input_X") + labels = tf.placeholder(tf.float32,[None, label_size], name = "label") + + model = models.Sequential() + model.add(Conv2D(6, (5,5), padding='valid', activation='sigmoid', input_shape=(28,28,1), kernel_initializer=initializers.Zeros(), bias_initializer=initializers.Zeros())) + model.add(AveragePooling2D(pool_size=(2,2))) + model.add(Conv2D(12, (5,5), padding='valid', activation='sigmoid', kernel_initializer=initializers.Zeros(), bias_initializer=initializers.Zeros())) + model.add(AveragePooling2D(pool_size=(2,2))) + model.add(Flatten()) + model.add(layers.Dense(10,activation='softmax',kernel_initializer=initializers.Zeros(), bias_initializer=initializers.Zeros())) + + model.compile(optimizer = optimizers.Adam(lr=1e-4), + loss=tf.keras.losses.CategoricalCrossentropy(), + metrics=['accuracy']) + + model.summary() + + model_Hist = History_LAW() + + history = model.fit(datagen(InVec, InLabel, batch_size), epochs=num_epoch, steps_per_epoch=len(InVec)//batch_size, validation_data=datagen(ValVec, ValLabel, batch_size), validation_steps=len(ValVec)//batch_size, callbacks=[model_Hist]) + + count =0 + + if not os.path.exists('./nntrainer_tfmodel'): + os.mkdir('./nntrainer_tfmodel') + model.save('./nntrainer_tfmodel/nntrainer_keras.h5') + + ValVec = np.reshape(ValVec, (label_size*val_data_size, 1,28,28)) + ValVec = np.transpose(ValVec, [0,2,3,1]) + score = model.evaluate(ValVec, ValLabel, verbose=1) + print("%s: %.5f%%" % (model.metrics_names[1], score[1]*100)) + +## +# @brief validation loop +def validation(): + ValVector = np.zeros((label_size*val_data_size,feature_size),dtype=np.float) + ValLabel = np.zeros((label_size*val_data_size, label_size),dtype=np.float) + + fin = open('mnist_trainingSet.dat','rb') + for i in range(label_size*val_data_size): + for j in range(feature_size): + data_str = fin.read(4) + ValVector[i,j] = struct.unpack('f',data_str)[0] + for j in range(label_size): + data_str = fin.read(4) + ValLabel[i,j] = struct.unpack('f',data_str)[0] + fin.close() + saved_model = tf.keras.models.load_model('./nntrainer_tfmodel/nntrainer_keras.h5') + saved_model.summary() + + score = saved_model.evaluate(ValVector, ValLabel, verbose=0) + print("%s: %.5f%%" % (saved_model.metrics_names[1], score[1]*100)) + +## +# @brief main loop + +if __name__ == "__main__": + if Learning: + train_nntrainer() + if Test: + validation() + diff --git a/Applications/mnist/Tensorflow/dataset.py b/Applications/mnist/Tensorflow/dataset.py new file mode 100644 index 0000000..305f33c --- /dev/null +++ b/Applications/mnist/Tensorflow/dataset.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python +# +# SPDX-License-Identifier: Apache-2.0-only +# +# Copyright (C) 2020 Jijoong Moon +# +# @file dataset.py +# @date 15 July 2020 +# @brief This is for mnist input generation +# @see https://github.com/nnstreamer/nntrainer +# @author Jijoong Moon +# @bug No known bugs except for NYI items +# +# + +import struct +import os +import numpy as np + +TOTAL_TRAIN_DATA_SIZE=100 +TOTAL_LABEL_SIZE=10 +TOTAL_VAL_DATA_SIZE=100 +FEATURE_SIZE=784 + +def get_data_info(): + return TOTAL_TRAIN_DATA_SIZE, TOTAL_VAL_DATA_SIZE, TOTAL_LABEL_SIZE, FEATURE_SIZE + +## +# @brief load input data from file +# @return (InputVector, InputLabel, Validation Vector, ValidationLabel) +def load_data(): + data_size = TOTAL_TRAIN_DATA_SIZE; + InputVector = np.zeros((TOTAL_LABEL_SIZE*data_size, FEATURE_SIZE),dtype=np.float32) + InputLabel = np.zeros((TOTAL_LABEL_SIZE*data_size, TOTAL_LABEL_SIZE),dtype=np.float32) + + ValVector = np.zeros((TOTAL_LABEL_SIZE*TOTAL_VAL_DATA_SIZE,FEATURE_SIZE),dtype=np.float32) + ValLabel = np.zeros((TOTAL_LABEL_SIZE*TOTAL_VAL_DATA_SIZE, TOTAL_LABEL_SIZE),dtype=np.float32) + + #read Input & Label + + fin = open('mnist_trainingSet.dat','rb') + for i in range(TOTAL_LABEL_SIZE*TOTAL_VAL_DATA_SIZE): + for j in range(FEATURE_SIZE): + data_str = fin.read(4) + ValVector[i,j] = struct.unpack('f',data_str)[0] + for j in range(TOTAL_LABEL_SIZE): + data_str = fin.read(4) + ValLabel[i,j] = struct.unpack('f',data_str)[0] + fin.close() + + # we are using same training data for validation to check how internal implementation is working + fin=open('mnist_trainingSet.dat','rb') + for i in range(TOTAL_LABEL_SIZE*data_size): + for j in range(FEATURE_SIZE): + data_str = fin.read(4) + InputVector[i,j] = struct.unpack('f',data_str)[0] + + for j in range(TOTAL_LABEL_SIZE): + data_str = fin.read(4) + InputLabel[i,j] = struct.unpack('f',data_str)[0] + fin.close() + + return InputVector, InputLabel, ValVector, ValLabel diff --git a/Applications/mnist/jni/main.cpp b/Applications/mnist/jni/main.cpp index ae6215e..2cf9e1f 100644 --- a/Applications/mnist/jni/main.cpp +++ b/Applications/mnist/jni/main.cpp @@ -1,4 +1,3 @@ - /** * Copyright (C) 2020 Jijoong Moon * diff --git a/Applications/mnist/res/mnist.ini b/Applications/mnist/res/mnist.ini index b3fcce8..71a98c7 100644 --- a/Applications/mnist/res/mnist.ini +++ b/Applications/mnist/res/mnist.ini @@ -1,10 +1,8 @@ # Network Section : Network [Network] Type = NeuralNetwork # Network Type : Regression, KNN, NeuralNetwork -Learning_rate = 0.001 # Learning Rate -Decay_rate = 0.96 # for the decay_rate for the decayed learning rate -Decay_steps = 1000 # decay step for the exponential decayed learning rate -Epoch = 1000 # Epoch +Learning_rate = 1e-4 # Learning Rate +Epoch = 1500 # Epoch Optimizer = adam # Optimizer : sgd (stochastic gradien decent), # adam (Adamtive Moment Estimation) Cost = cross # Cost(loss) function : msr (mean square root error) @@ -12,14 +10,13 @@ Cost = cross # Cost(loss) function : msr (mean square root error) Model = "model.bin" # model path to save / read minibatch = 32 # mini batch size beta1 = 0.9 # beta 1 for adam -beta2 = 0.9999 # beta 2 for adam +beta2 = 0.999 # beta 2 for adam epsilon = 1e-7 # epsilon for adam # Layer Section : Name [inputlayer] Type = input Input_Shape = 32:1:28:28 -normalization =true # Layer Section : Name [conv2d_c1_layer] @@ -27,8 +24,6 @@ Type = conv2d kernel_size = 5,5 bias_zero=true Activation=sigmoid -weight_decay=l2norm -weight_decay_lambda = 0.005 wieght_ini = xavier_uniform filter = 6 stride = 1,1 @@ -46,8 +41,6 @@ Type = conv2d kernel_size = 5,5 bias_zero=true Activation=sigmoid -weight_decay=l2norm -weight_decay_lambda = 0.005 wieght_ini = xavier_uniform filter = 12 stride = 1,1 @@ -66,7 +59,6 @@ Type=flatten [outputlayer] Type = fully_connected Unit = 10 # Output Layer Dimension ( = Weight Width ) +wieght_ini = xavier_uniform Bias_zero = true Activation = softmax # activation : sigmoid, softmax -Weight_Decay = l2norm -weight_Decay_Lambda = 0.005 -- 2.7.4