{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"from __future__ import print_function\n",
"from tensorflow.examples.tutorials.mnist import input_data\n",
"import tensorflow as tf\n",
"import numpy as np\n",
"import os"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"######################################\n",
"######### Necessary Flags ############\n",
"######################################\n",
"\n",
"num_classes = 10\n",
"batch_size = 128\n",
"num_epochs = 10"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"##########################################\n",
"######## Learning rate flags #############\n",
"##########################################\n",
"initial_learning_rate = 0.001\n",
"learning_rate_decay_factor = 0.95\n",
"num_epochs_per_decay = 1"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"#########################################\n",
"########## status flags #################\n",
"#########################################\n",
"is_training = False\n",
"fine_tuning = False\n",
"online_test = True\n",
"allow_soft_placement = True\n",
"log_device_placement = False"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {
"collapsed": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Extracting MNIST_data/train-images-idx3-ubyte.gz\n",
"Extracting MNIST_data/train-labels-idx1-ubyte.gz\n",
"Extracting MNIST_data/t10k-images-idx3-ubyte.gz\n",
"Extracting MNIST_data/t10k-labels-idx1-ubyte.gz\n"
]
}
],
"source": [
"##########################################\n",
"####### Load and Organize Data ###########\n",
"##########################################\n",
"'''\n",
"In this part the input must be prepared.\n",
"\n",
" 1 - The MNIST data will be downloaded.\n",
" 2 - The images and labels for both training and testing will be extracted.\n",
" 3 - The prepared data format(?,784) is different by the appropriate image shape(?,28,28,1) which needs\n",
" to be fed to the CNN architecture. So it needs to be reshaped.\n",
"\n",
"'''\n",
"\n",
"# Download and get MNIST dataset(available in tensorflow.contrib.learn.python.learn.datasets.mnist)\n",
"# It checks and download MNIST if it's not already downloaded then extract it.\n",
"# The 'reshape' is True by default to extract feature vectors but we set it to false to we get the original images.\n",
"mnist = input_data.read_data_sets(\"MNIST_data/\", reshape=True, one_hot=True)\n",
"train_data = mnist.train.images\n",
"train_label = mnist.train.labels\n",
"test_data = mnist.test.images\n",
"test_label = mnist.test.labels\n",
"\n",
"# # The 'input.provide_data' is provided to organize any custom dataset which has specific characteristics.\n",
"# data = input.provide_data(mnist)\n",
"\n",
"# Dimentionality of train\n",
"dimensionality_train = train_data.shape\n",
"\n",
"# Dimensions\n",
"num_train_samples = dimensionality_train[0]\n",
"num_features = dimensionality_train[1]"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {
"collapsed": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Epoch #1, Train Loss=0.248\n",
"Test Accuracy= 0.8565\n",
"Epoch #2, Train Loss=0.242\n",
"Test Accuracy= 0.8610\n",
"Epoch #3, Train Loss=0.240\n",
"Test Accuracy= 0.8676\n",
"Epoch #4, Train Loss=0.236\n",
"Test Accuracy= 0.8737\n",
"Epoch #5, Train Loss=0.235\n",
"Test Accuracy= 0.8740\n",
"Epoch #6, Train Loss=0.234\n",
"Test Accuracy= 0.8753\n",
"Epoch #7, Train Loss=0.234\n",
"Test Accuracy= 0.8756\n",
"Epoch #8, Train Loss=0.244\n",
"Test Accuracy= 0.8766\n",
"Epoch #9, Train Loss=0.234\n",
"Test Accuracy= 0.8765\n",
"Epoch #10, Train Loss=0.236\n",
"Test Accuracy= 0.8786\n",
"Final Test Accuracy is 0.88\n"
]
}
],
"source": [
"#######################################\n",
"########## Defining Graph ############\n",
"#######################################\n",
"\n",
"graph = tf.Graph()\n",
"with graph.as_default():\n",
" ###################################\n",
" ########### Parameters ############\n",
" ###################################\n",
"\n",
" # global step\n",
" global_step = tf.Variable(0, name=\"global_step\", trainable=False)\n",
"\n",
" # learning rate policy\n",
" decay_steps = int(num_train_samples / batch_size *\n",
" num_epochs_per_decay)\n",
" learning_rate = tf.train.exponential_decay(initial_learning_rate,\n",
" global_step,\n",
" decay_steps,\n",
" learning_rate_decay_factor,\n",
" staircase=True,\n",
" name='exponential_decay_learning_rate')\n",
"\n",
" ###############################################\n",
" ########### Defining place holders ############\n",
" ###############################################\n",
" image_place = tf.placeholder(tf.float32, shape=([None, num_features]), name='image')\n",
" label_place = tf.placeholder(tf.float32, shape=([None, num_classes]), name='gt')\n",
" dropout_param = tf.placeholder(tf.float32)\n",
"\n",
" ##################################################\n",
" ########### Model + Loss + Accuracy ##############\n",
" ##################################################\n",
"\n",
" # MODEL(MPL with two hidden layer)\n",
"\n",
" # LAYER-1\n",
" net = tf.contrib.layers.fully_connected(inputs=image_place, num_outputs=250, scope='fc-1')\n",
"\n",
" # LAYER-2\n",
" net = tf.contrib.layers.fully_connected(inputs=net, num_outputs=250, scope='fc-2')\n",
"\n",
" # SOFTMAX\n",
" logits_pre_softmax = tf.contrib.layers.fully_connected(inputs=net, num_outputs=num_classes, scope='fc-3')\n",
"\n",
" # Define loss\n",
" softmax_loss = tf.reduce_mean(\n",
" tf.nn.softmax_cross_entropy_with_logits(logits=logits_pre_softmax, labels=label_place))\n",
"\n",
" # Accuracy\n",
" accuracy = tf.reduce_mean(\n",
" tf.cast(tf.equal(tf.argmax(logits_pre_softmax, 1), tf.argmax(label_place, 1)), tf.float32))\n",
"\n",
" #############################################\n",
" ########### training operation ##############\n",
" #############################################\n",
"\n",
" # Define optimizer by its default values\n",
" optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)\n",
"\n",
" # 'train_op' is a operation that is run for gradient update on parameters.\n",
" # Each execution of 'train_op' is a training step.\n",
" # By passing 'global_step' to the optimizer, each time that the 'train_op' is run, Tensorflow\n",
" # update the 'global_step' and increment it by one!\n",
"\n",
" # gradient update.\n",
" with tf.name_scope('train_scope'):\n",
" grads = optimizer.compute_gradients(softmax_loss)\n",
" train_op = optimizer.apply_gradients(grads, global_step=global_step)\n",
"\n",
" ###############################################\n",
" ############ Define Sammaries #################\n",
" ###############################################\n",
"\n",
" # Summaries for loss and accuracy\n",
" tf.summary.scalar(\"loss\", softmax_loss, collections=['train', 'test'])\n",
" tf.summary.scalar(\"accuracy\", accuracy, collections=['train', 'test'])\n",
" tf.summary.scalar(\"global_step\", global_step, collections=['train'])\n",
" tf.summary.scalar(\"learning_rate\", learning_rate, collections=['train'])\n",
"\n",
" # Merge all summaries together.\n",
" summary_train_op = tf.summary.merge_all('train')\n",
" summary_test_op = tf.summary.merge_all('test')\n",
"\n",
" ############################################\n",
" ############ Run the Session ###############\n",
" ############################################\n",
" session_conf = tf.ConfigProto(\n",
" allow_soft_placement=allow_soft_placement,\n",
" log_device_placement=log_device_placement)\n",
" sess = tf.Session(graph=graph, config=session_conf)\n",
"\n",
" with sess.as_default():\n",
"\n",
" # Initialize all variables\n",
" sess.run(tf.global_variables_initializer())\n",
"\n",
" ###################################################################\n",
" ########## Run the training and loop over the batches #############\n",
" ###################################################################\n",
" for epoch in range(num_epochs):\n",
" total_batch_training = int(train_data.shape[0] / batch_size)\n",
"\n",
" # go through the batches\n",
" for batch_num in range(total_batch_training):\n",
" #################################################\n",
" ########## Get the training batches #############\n",
" #################################################\n",
"\n",
" start_idx = batch_num * batch_size\n",
" end_idx = (batch_num + 1) * batch_size\n",
"\n",
" # Fit training using batch data\n",
" train_batch_data, train_batch_label = train_data[start_idx:end_idx], train_label[\n",
" start_idx:end_idx]\n",
"\n",
" ########################################\n",
" ########## Run the session #############\n",
" ########################################\n",
"\n",
" # Run optimization op (backprop) and Calculate batch loss and accuracy\n",
" # When the tensor tensors['global_step'] is evaluated, it will be incremented by one.\n",
" batch_loss, _, training_step = sess.run(\n",
" [softmax_loss, train_op, global_step],\n",
" feed_dict={image_place: train_batch_data,\n",
" label_place: train_batch_label,\n",
" dropout_param: 0.5})\n",
"\n",
"\n",
" #################################################\n",
" ########## Plot the progressive bar #############\n",
" #################################################\n",
"\n",
" print(\"Epoch #\" + str(epoch + 1) + \", Train Loss=\" + \\\n",
" \"{:.3f}\".format(batch_loss))\n",
"\n",
" #####################################################\n",
" ########## Evaluation on the test data #############\n",
" #####################################################\n",
"\n",
" if online_test:\n",
" # WARNING: In this evaluation the whole test data is fed. In case the test data is huge this implementation\n",
" # may lead to memory error. In presense of large testing samples, batch evaluation on testing is\n",
" # recommended as in the training phase.\n",
" test_accuracy_epoch, test_summaries = sess.run(\n",
" [accuracy, summary_test_op],\n",
" feed_dict={image_place: test_data,\n",
" label_place: test_label,\n",
" dropout_param: 1.})\n",
" print(\"Test Accuracy= \" + \\\n",
" \"{:.4f}\".format(test_accuracy_epoch))\n",
"\n",
" ###########################################################\n",
" ########## Write the summaries for test phase #############\n",
" ###########################################################\n",
"\n",
" # Returning the value of global_step if necessary\n",
" current_step = tf.train.global_step(sess, global_step)\n",
"\n",
"\n",
" # Evaluation of the model\n",
" total_test_accuracy = sess.run(accuracy, feed_dict={\n",
" image_place: test_data,\n",
" label_place: test_label,\n",
" dropout_param: 1.})\n",
"\n",
" print(\"Final Test Accuracy is %.2f\" % total_test_accuracy)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 2",
"language": "python",
"name": "python2"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 2
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython2",
"version": "2.7.13"
}
},
"nbformat": 4,
"nbformat_minor": 2
}