mnist_noise_labels.py 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347
  1. # Copyright 2015 The TensorFlow Authors. All Rights Reserved.
  2. #
  3. # Licensed under the Apache License, Version 2.0 (the "License");
  4. # you may not use this file except in compliance with the License.
  5. # You may obtain a copy of the License at
  6. #
  7. # http://www.apache.org/licenses/LICENSE-2.0
  8. #
  9. # Unless required by applicable law or agreed to in writing, software
  10. # distributed under the License is distributed on an "AS IS" BASIS,
  11. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. # See the License for the specific language governing permissions and
  13. # limitations under the License.
  14. # ==============================================================================
  15. """Simple, end-to-end, LeNet-5-like convolutional MNIST model example.
  16. This should achieve a test error of 0.7%. Please keep this model as simple and
  17. linear as possible, it is meant as a tutorial for simple convolutional models.
  18. Run with --self_test on the command line to execute a short self-test.
  19. """
  20. from __future__ import absolute_import
  21. from __future__ import division
  22. from __future__ import print_function
  23. import gzip
  24. import os
  25. import sys
  26. import time
  27. from six.moves import urllib # @UnresolvedImport
  28. import tensorflow as tf
  29. import numpy as np
  30. from six.moves import xrange
  31. SOURCE_URL = 'http://yann.lecun.com/exdb/mnist/'
  32. WORK_DIRECTORY = 'data'
  33. IMAGE_SIZE = 28
  34. NUM_CHANNELS = 1
  35. PIXEL_DEPTH = 255
  36. NUM_LABELS = 10
  37. VALIDATION_SIZE = 5000 # Size of the validation set.
  38. SEED = 66478 # Set to None for random seed.
  39. BATCH_SIZE = 64
  40. NUM_EPOCHS = 10
  41. EVAL_BATCH_SIZE = 64
  42. EVAL_FREQUENCY = 100 # Number of steps between evaluations.
  43. NOISE_FACTOR = 0.2
  44. BETA = 0.8
  45. tf.app.flags.DEFINE_boolean("self_test", False, "True if running a self test.")
  46. tf.app.flags.DEFINE_boolean('use_fp16', False,
  47. "Use half floats instead of full floats if True.")
  48. FLAGS = tf.app.flags.FLAGS
  49. def data_type():
  50. """Return the type of the activations, weights, and placeholder variables."""
  51. if FLAGS.use_fp16:
  52. return tf.float16
  53. else:
  54. return tf.float32
  55. def maybe_download(filename):
  56. """Download the data from Yann's website, unless it's already here."""
  57. if not tf.gfile.Exists(WORK_DIRECTORY):
  58. tf.gfile.MakeDirs(WORK_DIRECTORY)
  59. filepath = os.path.join(WORK_DIRECTORY, filename)
  60. if not tf.gfile.Exists(filepath):
  61. filepath, _ = urllib.request.urlretrieve(SOURCE_URL + filename, filepath)
  62. with tf.gfile.GFile(filepath) as f:
  63. size = f.size()
  64. print('Successfully downloaded', filename, size, 'bytes.')
  65. return filepath
  66. def extract_data(filename, num_images):
  67. """Extract the images into a 4D tensor [image index, y, x, channels].
  68. Values are rescaled from [0, 255] down to [-0.5, 0.5].
  69. """
  70. print('Extracting', filename)
  71. with gzip.open(filename) as bytestream:
  72. bytestream.read(16)
  73. buf = bytestream.read(IMAGE_SIZE * IMAGE_SIZE * num_images * NUM_CHANNELS)
  74. data = np.frombuffer(buf, dtype=np.uint8).astype(np.float32)
  75. data = (data - (PIXEL_DEPTH / 2.0)) / PIXEL_DEPTH
  76. data = data.reshape(num_images, IMAGE_SIZE, IMAGE_SIZE, NUM_CHANNELS)
  77. return data
  78. def extract_labels(filename, num_images):
  79. """Extract the labels into a vector of int64 label IDs."""
  80. print('Extracting', filename)
  81. with gzip.open(filename) as bytestream:
  82. bytestream.read(8)
  83. buf = bytestream.read(1 * num_images)
  84. labels = np.frombuffer(buf, dtype=np.uint8).astype(np.int64)
  85. return labels
  86. def fake_data(num_images):
  87. """Generate a fake dataset that matches the dimensions of MNIST."""
  88. data = np.ndarray(
  89. shape=(num_images, IMAGE_SIZE, IMAGE_SIZE, NUM_CHANNELS),
  90. dtype=np.float32)
  91. labels = np.zeros(shape=(num_images,), dtype=np.int64)
  92. for image in range(num_images):
  93. label = image % 2
  94. data[image, :, :, 0] = label - 0.5
  95. labels[image] = label
  96. return data, labels
  97. def error_rate(predictions, labels):
  98. """Return the error rate based on dense predictions and sparse labels."""
  99. return 100.0 - (
  100. 100.0 *
  101. np.sum(np.argmax(predictions, 1) == labels) /
  102. predictions.shape[0])
  103. def main(argv=None): # pylint: disable=unused-argument
  104. if FLAGS.self_test:
  105. print('Running self-test.')
  106. train_data, train_labels = fake_data(256)
  107. validation_data, validation_labels = fake_data(EVAL_BATCH_SIZE)
  108. test_data, test_labels = fake_data(EVAL_BATCH_SIZE)
  109. num_epochs = 1
  110. else:
  111. # Get the data.
  112. train_data_filename = maybe_download('train-images-idx3-ubyte.gz')
  113. train_labels_filename = maybe_download('train-labels-idx1-ubyte.gz')
  114. test_data_filename = maybe_download('t10k-images-idx3-ubyte.gz')
  115. test_labels_filename = maybe_download('t10k-labels-idx1-ubyte.gz')
  116. # Extract it into numpy arrays.
  117. train_data = extract_data(train_data_filename, 60000)
  118. train_labels = extract_labels(train_labels_filename, 60000)
  119. test_data = extract_data(test_data_filename, 10000)
  120. test_labels = extract_labels(test_labels_filename, 10000)
  121. # Generate a validation set.
  122. validation_data = train_data[:VALIDATION_SIZE, ...]
  123. validation_labels = train_labels[:VALIDATION_SIZE]
  124. train_data = train_data[VALIDATION_SIZE:, ...]
  125. train_labels = train_labels[VALIDATION_SIZE:]
  126. nrof_training_examples = train_labels.shape[0]
  127. nrof_changed_labels = int(nrof_training_examples*NOISE_FACTOR)
  128. shuf = np.arange(0,nrof_training_examples)
  129. np.random.shuffle(shuf)
  130. change_idx = shuf[0:nrof_changed_labels]
  131. train_labels[change_idx] = (train_labels[change_idx] + np.random.randint(1,9,size=(nrof_changed_labels,))) % NUM_LABELS
  132. num_epochs = NUM_EPOCHS
  133. train_size = train_labels.shape[0]
  134. # This is where training samples and labels are fed to the graph.
  135. # These placeholder nodes will be fed a batch of training data at each
  136. # training step using the {feed_dict} argument to the Run() call below.
  137. train_data_node = tf.placeholder(
  138. data_type(),
  139. shape=(BATCH_SIZE, IMAGE_SIZE, IMAGE_SIZE, NUM_CHANNELS))
  140. train_labels_node = tf.placeholder(tf.int64, shape=(BATCH_SIZE,))
  141. eval_data = tf.placeholder(
  142. data_type(),
  143. shape=(EVAL_BATCH_SIZE, IMAGE_SIZE, IMAGE_SIZE, NUM_CHANNELS))
  144. # The variables below hold all the trainable weights. They are passed an
  145. # initial value which will be assigned when we call:
  146. # {tf.global_variables_initializer().run()}
  147. conv1_weights = tf.Variable(
  148. tf.truncated_normal([5, 5, NUM_CHANNELS, 32], # 5x5 filter, depth 32.
  149. stddev=0.1,
  150. seed=SEED, dtype=data_type()))
  151. conv1_biases = tf.Variable(tf.zeros([32], dtype=data_type()))
  152. conv2_weights = tf.Variable(tf.truncated_normal(
  153. [5, 5, 32, 64], stddev=0.1,
  154. seed=SEED, dtype=data_type()))
  155. conv2_biases = tf.Variable(tf.constant(0.1, shape=[64], dtype=data_type()))
  156. fc1_weights = tf.Variable( # fully connected, depth 512.
  157. tf.truncated_normal([IMAGE_SIZE // 4 * IMAGE_SIZE // 4 * 64, 512],
  158. stddev=0.1,
  159. seed=SEED,
  160. dtype=data_type()))
  161. fc1_biases = tf.Variable(tf.constant(0.1, shape=[512], dtype=data_type()))
  162. fc2_weights = tf.Variable(tf.truncated_normal([512, NUM_LABELS],
  163. stddev=0.1,
  164. seed=SEED,
  165. dtype=data_type()))
  166. fc2_biases = tf.Variable(tf.constant(
  167. 0.1, shape=[NUM_LABELS], dtype=data_type()))
  168. # We will replicate the model structure for the training subgraph, as well
  169. # as the evaluation subgraphs, while sharing the trainable parameters.
  170. def model(data, train=False):
  171. """The Model definition."""
  172. # 2D convolution, with 'SAME' padding (i.e. the output feature map has
  173. # the same size as the input). Note that {strides} is a 4D array whose
  174. # shape matches the data layout: [image index, y, x, depth].
  175. conv = tf.nn.conv2d(data,
  176. conv1_weights,
  177. strides=[1, 1, 1, 1],
  178. padding='SAME')
  179. # Bias and rectified linear non-linearity.
  180. relu = tf.nn.relu(tf.nn.bias_add(conv, conv1_biases))
  181. # Max pooling. The kernel size spec {ksize} also follows the layout of
  182. # the data. Here we have a pooling window of 2, and a stride of 2.
  183. pool = tf.nn.max_pool(relu,
  184. ksize=[1, 2, 2, 1],
  185. strides=[1, 2, 2, 1],
  186. padding='SAME')
  187. conv = tf.nn.conv2d(pool,
  188. conv2_weights,
  189. strides=[1, 1, 1, 1],
  190. padding='SAME')
  191. relu = tf.nn.relu(tf.nn.bias_add(conv, conv2_biases))
  192. pool = tf.nn.max_pool(relu,
  193. ksize=[1, 2, 2, 1],
  194. strides=[1, 2, 2, 1],
  195. padding='SAME')
  196. # Reshape the feature map cuboid into a 2D matrix to feed it to the
  197. # fully connected layers.
  198. pool_shape = pool.get_shape().as_list() #pylint: disable=no-member
  199. reshape = tf.reshape(
  200. pool,
  201. [pool_shape[0], pool_shape[1] * pool_shape[2] * pool_shape[3]])
  202. # Fully connected layer. Note that the '+' operation automatically
  203. # broadcasts the biases.
  204. hidden = tf.nn.relu(tf.matmul(reshape, fc1_weights) + fc1_biases)
  205. # Add a 50% dropout during training only. Dropout also scales
  206. # activations such that no rescaling is needed at evaluation time.
  207. if train:
  208. hidden = tf.nn.dropout(hidden, 0.5, seed=SEED)
  209. return tf.matmul(hidden, fc2_weights) + fc2_biases
  210. # Training computation: logits + cross-entropy loss.
  211. logits = model(train_data_node, True)
  212. # t: observed noisy labels
  213. # q: estimated class probabilities (output from softmax)
  214. # z: argmax of q
  215. t = tf.one_hot(train_labels_node, NUM_LABELS)
  216. q = tf.nn.softmax(logits)
  217. qqq = tf.arg_max(q, dimension=1)
  218. z = tf.one_hot(qqq, NUM_LABELS)
  219. #cross_entropy = -tf.reduce_sum(t*tf.log(q),reduction_indices=1)
  220. cross_entropy = -tf.reduce_sum((BETA*t+(1-BETA)*z)*tf.log(q),reduction_indices=1)
  221. loss = tf.reduce_mean(cross_entropy)
  222. # loss = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(
  223. # logits, train_labels_node))
  224. # L2 regularization for the fully connected parameters.
  225. regularizers = (tf.nn.l2_loss(fc1_weights) + tf.nn.l2_loss(fc1_biases) +
  226. tf.nn.l2_loss(fc2_weights) + tf.nn.l2_loss(fc2_biases))
  227. # Add the regularization term to the loss.
  228. loss += 5e-4 * regularizers
  229. # Optimizer: set up a variable that's incremented once per batch and
  230. # controls the learning rate decay.
  231. batch = tf.Variable(0, dtype=data_type())
  232. # Decay once per epoch, using an exponential schedule starting at 0.01.
  233. learning_rate = tf.train.exponential_decay(
  234. 0.01, # Base learning rate.
  235. batch * BATCH_SIZE, # Current index into the dataset.
  236. train_size, # Decay step.
  237. 0.95, # Decay rate.
  238. staircase=True)
  239. # Use simple momentum for the optimization.
  240. optimizer = tf.train.MomentumOptimizer(learning_rate,
  241. 0.9).minimize(loss,
  242. global_step=batch)
  243. # Predictions for the current training minibatch.
  244. train_prediction = tf.nn.softmax(logits)
  245. # Predictions for the test and validation, which we'll compute less often.
  246. eval_prediction = tf.nn.softmax(model(eval_data))
  247. # Small utility function to evaluate a dataset by feeding batches of data to
  248. # {eval_data} and pulling the results from {eval_predictions}.
  249. # Saves memory and enables this to run on smaller GPUs.
  250. def eval_in_batches(data, sess):
  251. """Get all predictions for a dataset by running it in small batches."""
  252. size = data.shape[0]
  253. if size < EVAL_BATCH_SIZE:
  254. raise ValueError("batch size for evals larger than dataset: %d" % size)
  255. predictions = np.ndarray(shape=(size, NUM_LABELS), dtype=np.float32)
  256. for begin in xrange(0, size, EVAL_BATCH_SIZE):
  257. end = begin + EVAL_BATCH_SIZE
  258. if end <= size:
  259. predictions[begin:end, :] = sess.run(
  260. eval_prediction,
  261. feed_dict={eval_data: data[begin:end, ...]})
  262. else:
  263. batch_predictions = sess.run(
  264. eval_prediction,
  265. feed_dict={eval_data: data[-EVAL_BATCH_SIZE:, ...]})
  266. predictions[begin:, :] = batch_predictions[begin - size:, :]
  267. return predictions
  268. # Create a local session to run the training.
  269. start_time = time.time()
  270. with tf.Session() as sess:
  271. # Run all the initializers to prepare the trainable parameters.
  272. tf.global_variables_initializer().run() #pylint: disable=no-member
  273. print('Initialized!')
  274. # Loop through training steps.
  275. for step in xrange(int(num_epochs * train_size) // BATCH_SIZE):
  276. # Compute the offset of the current minibatch in the data.
  277. # Note that we could use better randomization across epochs.
  278. offset = (step * BATCH_SIZE) % (train_size - BATCH_SIZE)
  279. batch_data = train_data[offset:(offset + BATCH_SIZE), ...]
  280. batch_labels = train_labels[offset:(offset + BATCH_SIZE)]
  281. # This dictionary maps the batch data (as a numpy array) to the
  282. # node in the graph it should be fed to.
  283. feed_dict = {train_data_node: batch_data,
  284. train_labels_node: batch_labels}
  285. # Run the graph and fetch some of the nodes.
  286. _, l, lr, predictions = sess.run(
  287. [optimizer, loss, learning_rate, train_prediction],
  288. feed_dict=feed_dict)
  289. if step % EVAL_FREQUENCY == 0:
  290. elapsed_time = time.time() - start_time
  291. start_time = time.time()
  292. print('Step %d (epoch %.2f), %.1f ms' %
  293. (step, float(step) * BATCH_SIZE / train_size,
  294. 1000 * elapsed_time / EVAL_FREQUENCY))
  295. print('Minibatch loss: %.3f, learning rate: %.6f' % (l, lr))
  296. print('Minibatch error: %.1f%%' % error_rate(predictions, batch_labels))
  297. print('Validation error: %.1f%%' % error_rate(
  298. eval_in_batches(validation_data, sess), validation_labels))
  299. sys.stdout.flush()
  300. # Finally print the result!
  301. test_error = error_rate(eval_in_batches(test_data, sess), test_labels)
  302. print('Test error: %.1f%%' % test_error)
  303. if FLAGS.self_test:
  304. print('test_error', test_error)
  305. assert test_error == 0.0, 'expected 0.0 test_error, got %.2f' % (
  306. test_error,)
  307. if __name__ == '__main__':
  308. tf.app.run()