mnist_center_loss.py 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404
  1. # Copyright 2015 The TensorFlow Authors. All Rights Reserved.
  2. #
  3. # Licensed under the Apache License, Version 2.0 (the "License");
  4. # you may not use this file except in compliance with the License.
  5. # You may obtain a copy of the License at
  6. #
  7. # http://www.apache.org/licenses/LICENSE-2.0
  8. #
  9. # Unless required by applicable law or agreed to in writing, software
  10. # distributed under the License is distributed on an "AS IS" BASIS,
  11. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. # See the License for the specific language governing permissions and
  13. # limitations under the License.
  14. # ==============================================================================
  15. """Simple, end-to-end, LeNet-5-like convolutional MNIST model example.
  16. This should achieve a test error of 0.7%. Please keep this model as simple and
  17. linear as possible, it is meant as a tutorial for simple convolutional models.
  18. Run with --self_test on the command line to execute a short self-test.
  19. """
  20. from __future__ import absolute_import
  21. from __future__ import division
  22. from __future__ import print_function
  23. import gzip
  24. import os
  25. import sys
  26. import time
  27. from six.moves import urllib # @UnresolvedImport
  28. import tensorflow as tf
  29. import numpy as np
  30. import matplotlib.pyplot as plt
  31. from tensorflow.python.ops import control_flow_ops
  32. import facenet
  33. from six.moves import xrange
  34. SOURCE_URL = 'http://yann.lecun.com/exdb/mnist/'
  35. WORK_DIRECTORY = 'data'
  36. IMAGE_SIZE = 28
  37. NUM_CHANNELS = 1
  38. PIXEL_DEPTH = 255
  39. NUM_LABELS = 10
  40. VALIDATION_SIZE = 5000 # Size of the validation set.
  41. SEED = 66478 # Set to None for random seed.
  42. BATCH_SIZE = 64
  43. NUM_EPOCHS = 10
  44. EVAL_BATCH_SIZE = 64
  45. EVAL_FREQUENCY = 100 # Number of steps between evaluations.
  46. tf.app.flags.DEFINE_boolean("self_test", False, "True if running a self test.")
  47. tf.app.flags.DEFINE_boolean('use_fp16', False,
  48. "Use half floats instead of full floats if True.")
  49. FLAGS = tf.app.flags.FLAGS
  50. def data_type():
  51. """Return the type of the activations, weights, and placeholder variables."""
  52. if FLAGS.use_fp16:
  53. return tf.float16
  54. else:
  55. return tf.float32
  56. def maybe_download(filename):
  57. """Download the data from Yann's website, unless it's already here."""
  58. if not tf.gfile.Exists(WORK_DIRECTORY):
  59. tf.gfile.MakeDirs(WORK_DIRECTORY)
  60. filepath = os.path.join(WORK_DIRECTORY, filename)
  61. if not tf.gfile.Exists(filepath):
  62. filepath, _ = urllib.request.urlretrieve(SOURCE_URL + filename, filepath)
  63. with tf.gfile.GFile(filepath) as f:
  64. size = f.size()
  65. print('Successfully downloaded', filename, size, 'bytes.')
  66. return filepath
  67. def extract_data(filename, num_images):
  68. """Extract the images into a 4D tensor [image index, y, x, channels].
  69. Values are rescaled from [0, 255] down to [-0.5, 0.5].
  70. """
  71. print('Extracting', filename)
  72. with gzip.open(filename) as bytestream:
  73. bytestream.read(16)
  74. buf = bytestream.read(IMAGE_SIZE * IMAGE_SIZE * num_images * NUM_CHANNELS)
  75. data = np.frombuffer(buf, dtype=np.uint8).astype(np.float32)
  76. data = (data - (PIXEL_DEPTH / 2.0)) / PIXEL_DEPTH
  77. data = data.reshape(num_images, IMAGE_SIZE, IMAGE_SIZE, NUM_CHANNELS)
  78. return data
  79. def extract_labels(filename, num_images):
  80. """Extract the labels into a vector of int64 label IDs."""
  81. print('Extracting', filename)
  82. with gzip.open(filename) as bytestream:
  83. bytestream.read(8)
  84. buf = bytestream.read(1 * num_images)
  85. labels = np.frombuffer(buf, dtype=np.uint8).astype(np.int64)
  86. return labels
  87. def fake_data(num_images):
  88. """Generate a fake dataset that matches the dimensions of MNIST."""
  89. data = np.ndarray(
  90. shape=(num_images, IMAGE_SIZE, IMAGE_SIZE, NUM_CHANNELS),
  91. dtype=np.float32)
  92. labels = np.zeros(shape=(num_images,), dtype=np.int64)
  93. for image in range(num_images):
  94. label = image % 2
  95. data[image, :, :, 0] = label - 0.5
  96. labels[image] = label
  97. return data, labels
  98. def error_rate(predictions, labels):
  99. """Return the error rate based on dense predictions and sparse labels."""
  100. return 100.0 - (
  101. 100.0 *
  102. np.sum(np.argmax(predictions, 1) == labels) /
  103. predictions.shape[0])
  104. def main(argv=None): # pylint: disable=unused-argument
  105. if FLAGS.self_test:
  106. print('Running self-test.')
  107. train_data, train_labels = fake_data(256)
  108. validation_data, validation_labels = fake_data(EVAL_BATCH_SIZE)
  109. test_data, test_labels = fake_data(EVAL_BATCH_SIZE)
  110. num_epochs = 1
  111. else:
  112. # Get the data.
  113. train_data_filename = maybe_download('train-images-idx3-ubyte.gz')
  114. train_labels_filename = maybe_download('train-labels-idx1-ubyte.gz')
  115. test_data_filename = maybe_download('t10k-images-idx3-ubyte.gz')
  116. test_labels_filename = maybe_download('t10k-labels-idx1-ubyte.gz')
  117. # Extract it into numpy arrays.
  118. train_data = extract_data(train_data_filename, 60000)
  119. train_labels = extract_labels(train_labels_filename, 60000)
  120. test_data = extract_data(test_data_filename, 10000)
  121. test_labels = extract_labels(test_labels_filename, 10000)
  122. # Generate a validation set.
  123. validation_data = train_data[:VALIDATION_SIZE, ...]
  124. validation_labels = train_labels[:VALIDATION_SIZE]
  125. train_data = train_data[VALIDATION_SIZE:, ...]
  126. train_labels = train_labels[VALIDATION_SIZE:]
  127. num_epochs = NUM_EPOCHS
  128. train_size = train_labels.shape[0]
  129. # This is where training samples and labels are fed to the graph.
  130. # These placeholder nodes will be fed a batch of training data at each
  131. # training step using the {feed_dict} argument to the Run() call below.
  132. train_data_node = tf.placeholder(
  133. data_type(),
  134. shape=(BATCH_SIZE, IMAGE_SIZE, IMAGE_SIZE, NUM_CHANNELS))
  135. train_labels_node = tf.placeholder(tf.int64, shape=(BATCH_SIZE,))
  136. eval_data = tf.placeholder(
  137. data_type(),
  138. shape=(EVAL_BATCH_SIZE, IMAGE_SIZE, IMAGE_SIZE, NUM_CHANNELS))
  139. # The variables below hold all the trainable weights. They are passed an
  140. # initial value which will be assigned when we call:
  141. # {tf.global_variables_initializer().run()}
  142. conv1_weights = tf.Variable(
  143. tf.truncated_normal([5, 5, NUM_CHANNELS, 32], # 5x5 filter, depth 32.
  144. stddev=0.1,
  145. seed=SEED, dtype=data_type()))
  146. conv1_biases = tf.Variable(tf.zeros([32], dtype=data_type()))
  147. conv2_weights = tf.Variable(tf.truncated_normal(
  148. [5, 5, 32, 64], stddev=0.1,
  149. seed=SEED, dtype=data_type()))
  150. conv2_biases = tf.Variable(tf.constant(0.1, shape=[64], dtype=data_type()))
  151. fc1_weights = tf.Variable( # fully connected, depth 512.
  152. tf.truncated_normal([IMAGE_SIZE // 4 * IMAGE_SIZE // 4 * 64, 512],
  153. stddev=0.1,
  154. seed=SEED,
  155. dtype=data_type()))
  156. fc1_biases = tf.Variable(tf.constant(0.1, shape=[512], dtype=data_type()))
  157. fc1p_weights = tf.Variable( # fully connected, depth 512.
  158. tf.truncated_normal([512, 2],
  159. stddev=0.1,
  160. seed=SEED,
  161. dtype=data_type()))
  162. fc1p_biases = tf.Variable(tf.constant(0.1, shape=[2], dtype=data_type()))
  163. fc2_weights = tf.Variable(tf.truncated_normal([2, NUM_LABELS],
  164. stddev=0.1,
  165. seed=SEED,
  166. dtype=data_type()))
  167. fc2_biases = tf.Variable(tf.constant(
  168. 0.1, shape=[NUM_LABELS], dtype=data_type()))
  169. def batch_norm(x, phase_train): #pylint: disable=unused-variable
  170. """
  171. Batch normalization on convolutional maps.
  172. Args:
  173. x: Tensor, 4D BHWD input maps
  174. n_out: integer, depth of input maps
  175. phase_train: boolean tf.Variable, true indicates training phase
  176. scope: string, variable scope
  177. affn: whether to affn-transform outputs
  178. Return:
  179. normed: batch-normalized maps
  180. Ref: http://stackoverflow.com/questions/33949786/how-could-i-use-batch-normalization-in-tensorflow/33950177
  181. """
  182. name = 'batch_norm'
  183. with tf.variable_scope(name):
  184. phase_train = tf.convert_to_tensor(phase_train, dtype=tf.bool)
  185. n_out = int(x.get_shape()[-1])
  186. beta = tf.Variable(tf.constant(0.0, shape=[n_out], dtype=x.dtype),
  187. name=name+'/beta', trainable=True, dtype=x.dtype)
  188. gamma = tf.Variable(tf.constant(1.0, shape=[n_out], dtype=x.dtype),
  189. name=name+'/gamma', trainable=True, dtype=x.dtype)
  190. batch_mean, batch_var = tf.nn.moments(x, [0], name='moments')
  191. ema = tf.train.ExponentialMovingAverage(decay=0.9)
  192. def mean_var_with_update():
  193. ema_apply_op = ema.apply([batch_mean, batch_var])
  194. with tf.control_dependencies([ema_apply_op]):
  195. return tf.identity(batch_mean), tf.identity(batch_var)
  196. mean, var = control_flow_ops.cond(phase_train,
  197. mean_var_with_update,
  198. lambda: (ema.average(batch_mean), ema.average(batch_var)))
  199. normed = tf.nn.batch_normalization(x, mean, var, beta, gamma, 1e-3)
  200. return normed
  201. # We will replicate the model structure for the training subgraph, as well
  202. # as the evaluation subgraphs, while sharing the trainable parameters.
  203. def model(data, train=False):
  204. """The Model definition."""
  205. # 2D convolution, with 'SAME' padding (i.e. the output feature map has
  206. # the same size as the input). Note that {strides} is a 4D array whose
  207. # shape matches the data layout: [image index, y, x, depth].
  208. conv = tf.nn.conv2d(data,
  209. conv1_weights,
  210. strides=[1, 1, 1, 1],
  211. padding='SAME')
  212. # Bias and rectified linear non-linearity.
  213. relu = tf.nn.relu(tf.nn.bias_add(conv, conv1_biases))
  214. # Max pooling. The kernel size spec {ksize} also follows the layout of
  215. # the data. Here we have a pooling window of 2, and a stride of 2.
  216. pool = tf.nn.max_pool(relu,
  217. ksize=[1, 2, 2, 1],
  218. strides=[1, 2, 2, 1],
  219. padding='SAME')
  220. conv = tf.nn.conv2d(pool,
  221. conv2_weights,
  222. strides=[1, 1, 1, 1],
  223. padding='SAME')
  224. relu = tf.nn.relu(tf.nn.bias_add(conv, conv2_biases))
  225. pool = tf.nn.max_pool(relu,
  226. ksize=[1, 2, 2, 1],
  227. strides=[1, 2, 2, 1],
  228. padding='SAME')
  229. # Reshape the feature map cuboid into a 2D matrix to feed it to the
  230. # fully connected layers.
  231. pool_shape = pool.get_shape().as_list() #pylint: disable=no-member
  232. reshape = tf.reshape(
  233. pool,
  234. [pool_shape[0], pool_shape[1] * pool_shape[2] * pool_shape[3]])
  235. # Fully connected layer. Note that the '+' operation automatically
  236. # broadcasts the biases.
  237. hidden = tf.nn.relu(tf.matmul(reshape, fc1_weights) + fc1_biases)
  238. # Add a 50% dropout during training only. Dropout also scales
  239. # activations such that no rescaling is needed at evaluation time.
  240. if train:
  241. hidden = tf.nn.dropout(hidden, 0.5, seed=SEED)
  242. hidden = tf.matmul(hidden, fc1p_weights) + fc1p_biases
  243. return tf.nn.relu(tf.matmul(hidden, fc2_weights) + fc2_biases), hidden
  244. # Training computation: logits + cross-entropy loss.
  245. logits, hidden = model(train_data_node, True)
  246. #logits = batch_norm(logits, True)
  247. xent_loss = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(
  248. logits, train_labels_node))
  249. beta = 1e-3
  250. #center_loss, update_centers = center_loss_op(hidden, train_labels_node)
  251. center_loss, _ = facenet.center_loss(hidden, train_labels_node, 0.95, NUM_LABELS)
  252. loss = xent_loss + beta * center_loss
  253. # L2 regularization for the fully connected parameters.
  254. regularizers = (tf.nn.l2_loss(fc1_weights) + tf.nn.l2_loss(fc1_biases) +
  255. tf.nn.l2_loss(fc2_weights) + tf.nn.l2_loss(fc2_biases))
  256. # Add the regularization term to the loss.
  257. loss += 5e-4 * regularizers
  258. # Optimizer: set up a variable that's incremented once per batch and
  259. # controls the learning rate decay.
  260. batch = tf.Variable(0, dtype=data_type())
  261. # Decay once per epoch, using an exponential schedule starting at 0.01.
  262. learning_rate = tf.train.exponential_decay(
  263. 0.01, # Base learning rate.
  264. batch * BATCH_SIZE, # Current index into the dataset.
  265. train_size, # Decay step.
  266. 0.95, # Decay rate.
  267. staircase=True)
  268. # Use simple momentum for the optimization.
  269. optimizer = tf.train.MomentumOptimizer(learning_rate,
  270. 0.9).minimize(loss,
  271. global_step=batch)
  272. # Predictions for the current training minibatch.
  273. train_prediction = tf.nn.softmax(logits)
  274. # Predictions for the test and validation, which we'll compute less often.
  275. eval_logits, eval_embeddings = model(eval_data)
  276. eval_prediction = tf.nn.softmax(eval_logits)
  277. # Small utility function to evaluate a dataset by feeding batches of data to
  278. # {eval_data} and pulling the results from {eval_predictions}.
  279. # Saves memory and enables this to run on smaller GPUs.
  280. def eval_in_batches(data, sess):
  281. """Get all predictions for a dataset by running it in small batches."""
  282. size = data.shape[0]
  283. if size < EVAL_BATCH_SIZE:
  284. raise ValueError("batch size for evals larger than dataset: %d" % size)
  285. predictions = np.ndarray(shape=(size, NUM_LABELS), dtype=np.float32)
  286. for begin in xrange(0, size, EVAL_BATCH_SIZE):
  287. end = begin + EVAL_BATCH_SIZE
  288. if end <= size:
  289. predictions[begin:end, :] = sess.run(
  290. eval_prediction,
  291. feed_dict={eval_data: data[begin:end, ...]})
  292. else:
  293. batch_predictions = sess.run(
  294. eval_prediction,
  295. feed_dict={eval_data: data[-EVAL_BATCH_SIZE:, ...]})
  296. predictions[begin:, :] = batch_predictions[begin - size:, :]
  297. return predictions
  298. def calculate_embeddings(data, sess):
  299. """Get all predictions for a dataset by running it in small batches."""
  300. size = data.shape[0]
  301. if size < EVAL_BATCH_SIZE:
  302. raise ValueError("batch size for evals larger than dataset: %d" % size)
  303. predictions = np.ndarray(shape=(size, 2), dtype=np.float32)
  304. for begin in xrange(0, size, EVAL_BATCH_SIZE):
  305. end = begin + EVAL_BATCH_SIZE
  306. if end <= size:
  307. predictions[begin:end, :] = sess.run(
  308. eval_embeddings,
  309. feed_dict={eval_data: data[begin:end, ...]})
  310. else:
  311. batch_predictions = sess.run(
  312. eval_embeddings,
  313. feed_dict={eval_data: data[-EVAL_BATCH_SIZE:, ...]})
  314. predictions[begin:, :] = batch_predictions[begin - size:, :]
  315. return predictions
  316. # Create a local session to run the training.
  317. start_time = time.time()
  318. with tf.Session() as sess:
  319. # Run all the initializers to prepare the trainable parameters.
  320. tf.global_variables_initializer().run() #pylint: disable=no-member
  321. print('Initialized!')
  322. # Loop through training steps.
  323. for step in xrange(int(num_epochs * train_size) // BATCH_SIZE):
  324. # Compute the offset of the current minibatch in the data.
  325. # Note that we could use better randomization across epochs.
  326. offset = (step * BATCH_SIZE) % (train_size - BATCH_SIZE)
  327. batch_data = train_data[offset:(offset + BATCH_SIZE), ...]
  328. batch_labels = train_labels[offset:(offset + BATCH_SIZE)]
  329. # This dictionary maps the batch data (as a numpy array) to the
  330. # node in the graph it should be fed to.
  331. feed_dict = {train_data_node: batch_data,
  332. train_labels_node: batch_labels}
  333. # Run the graph and fetch some of the nodes.
  334. #_, l, lr, predictions = sess.run([optimizer, loss, learning_rate, train_prediction], feed_dict=feed_dict)
  335. _, cl, l, lr, predictions = sess.run([optimizer, center_loss, loss, learning_rate, train_prediction], feed_dict=feed_dict)
  336. if step % EVAL_FREQUENCY == 0:
  337. elapsed_time = time.time() - start_time
  338. start_time = time.time()
  339. print('Step %d (epoch %.2f), %.1f ms' %
  340. (step, float(step) * BATCH_SIZE / train_size,
  341. 1000 * elapsed_time / EVAL_FREQUENCY))
  342. print('Minibatch loss: %.3f %.3f, learning rate: %.6f' % (l, cl*beta, lr))
  343. print('Minibatch error: %.1f%%' % error_rate(predictions, batch_labels))
  344. print('Validation error: %.1f%%' % error_rate(
  345. eval_in_batches(validation_data, sess), validation_labels))
  346. sys.stdout.flush()
  347. # Finally print the result!
  348. test_error = error_rate(eval_in_batches(test_data, sess), test_labels)
  349. print('Test error: %.1f%%' % test_error)
  350. if FLAGS.self_test:
  351. print('test_error', test_error)
  352. assert test_error == 0.0, 'expected 0.0 test_error, got %.2f' % (
  353. test_error,)
  354. train_embeddings = calculate_embeddings(train_data, sess)
  355. color_list = ['b', 'g', 'r', 'c', 'm', 'y', 'k', 'b', 'g', 'r', 'c' ]
  356. plt.figure(1)
  357. for n in range(0,10):
  358. idx = np.where(train_labels[0:10000]==n)
  359. plt.plot(train_embeddings[idx,0], train_embeddings[idx,1], color_list[n]+'.')
  360. plt.show()
  361. if __name__ == '__main__':
  362. tf.app.run()