diff --git a/bindings/py/cpp_src/bindings/algorithms/py_SpatialPooler.cpp b/bindings/py/cpp_src/bindings/algorithms/py_SpatialPooler.cpp index a8225b2311..ed4a4aae56 100644 --- a/bindings/py/cpp_src/bindings/algorithms/py_SpatialPooler.cpp +++ b/bindings/py/cpp_src/bindings/algorithms/py_SpatialPooler.cpp @@ -58,7 +58,7 @@ using namespace sdr; , Real , bool , Real - , UInt + , Int , UInt , Real , Real diff --git a/src/examples/mnist_nni/MNIST_data/t10k-images-idx3-ubyte.gz b/src/examples/mnist_nni/MNIST_data/t10k-images-idx3-ubyte.gz new file mode 100644 index 0000000000..5ace8ea93f Binary files /dev/null and b/src/examples/mnist_nni/MNIST_data/t10k-images-idx3-ubyte.gz differ diff --git a/src/examples/mnist_nni/MNIST_data/t10k-labels-idx1-ubyte.gz b/src/examples/mnist_nni/MNIST_data/t10k-labels-idx1-ubyte.gz new file mode 100644 index 0000000000..a7e141541c Binary files /dev/null and b/src/examples/mnist_nni/MNIST_data/t10k-labels-idx1-ubyte.gz differ diff --git a/src/examples/mnist_nni/MNIST_data/train-images-idx3-ubyte.gz b/src/examples/mnist_nni/MNIST_data/train-images-idx3-ubyte.gz new file mode 100644 index 0000000000..b50e4b6bcc Binary files /dev/null and b/src/examples/mnist_nni/MNIST_data/train-images-idx3-ubyte.gz differ diff --git a/src/examples/mnist_nni/MNIST_data/train-labels-idx1-ubyte.gz b/src/examples/mnist_nni/MNIST_data/train-labels-idx1-ubyte.gz new file mode 100644 index 0000000000..707a576bb5 Binary files /dev/null and b/src/examples/mnist_nni/MNIST_data/train-labels-idx1-ubyte.gz differ diff --git a/src/examples/mnist_nni/config.yml b/src/examples/mnist_nni/config.yml new file mode 100644 index 0000000000..2b8eb66741 --- /dev/null +++ b/src/examples/mnist_nni/config.yml @@ -0,0 +1,16 @@ +authorName: default +experimentName: example_mnist +trialConcurrency: 6 +trainingServicePlatform: local +searchSpacePath: search_space.json +useAnnotation: false +tuner: + #choice: TPE, Random, Anneal, Evolution, BatchTuner, MetisTuner + #SMAC (SMAC should be installed through nnictl) + builtinTunerName: TPE + classArgs: + optimize_mode: maximize +trial: + command: python3 mnist.py + codeDir: . + gpuNum: 0 diff --git a/src/examples/mnist_nni/config_assessor.yml b/src/examples/mnist_nni/config_assessor.yml new file mode 100644 index 0000000000..e55db64946 --- /dev/null +++ b/src/examples/mnist_nni/config_assessor.yml @@ -0,0 +1,29 @@ +authorName: default +experimentName: example_mnist +trialConcurrency: 1 +maxExecDuration: 1h +maxTrialNum: 50 +#choice: local, remote +trainingServicePlatform: local +searchSpacePath: search_space.json +#choice: true, false +useAnnotation: false +tuner: + #choice: TPE, Random, Anneal, Evolution, BatchTuner, MetisTuner + #SMAC (SMAC should be installed through nnictl) + builtinTunerName: TPE + classArgs: + #choice: maximize, minimize + optimize_mode: maximize +assessor: + #choice: Medianstop, Curvefitting + builtinAssessorName: Curvefitting + classArgs: + #choice: maximize, minimize + optimize_mode: maximize + epoch_num: 20 + threshold: 0.9 +trial: + command: python3 mnist.py + codeDir: . + gpuNum: 0 diff --git a/src/examples/mnist_nni/config_frameworkcontroller.yml b/src/examples/mnist_nni/config_frameworkcontroller.yml new file mode 100644 index 0000000000..9d166dcf4f --- /dev/null +++ b/src/examples/mnist_nni/config_frameworkcontroller.yml @@ -0,0 +1,41 @@ +authorName: default +experimentName: example_mnist +trialConcurrency: 1 +maxExecDuration: 1h +maxTrialNum: 10 +#choice: local, remote, pai, kubeflow +trainingServicePlatform: frameworkcontroller +searchSpacePath: search_space.json +#choice: true, false +useAnnotation: false +tuner: + #choice: TPE, Random, Anneal, Evolution, BatchTuner, MetisTuner + builtinTunerName: TPE + classArgs: + #choice: maximize, minimize + optimize_mode: maximize +assessor: + builtinAssessorName: Medianstop + classArgs: + optimize_mode: maximize + gpuNum: 0 +trial: + codeDir: . + taskRoles: + - name: worker + taskNum: 1 + command: python3 mnist.py + gpuNum: 1 + cpuNum: 1 + memoryMB: 8192 + image: msranni/nni:latest + frameworkAttemptCompletionPolicy: + minFailedTaskCount: 1 + minSucceededTaskCount: 1 +frameworkcontrollerConfig: + storage: nfs + nfs: + # Your NFS server IP, like 10.10.10.10 + server: {your_nfs_server_ip} + # Your NFS server export path, like /var/nfs/nni + path: {your_nfs_server_export_path} \ No newline at end of file diff --git a/src/examples/mnist_nni/config_kubeflow.yml b/src/examples/mnist_nni/config_kubeflow.yml new file mode 100644 index 0000000000..8e942c5f33 --- /dev/null +++ b/src/examples/mnist_nni/config_kubeflow.yml @@ -0,0 +1,32 @@ +authorName: default +experimentName: example_dist +trialConcurrency: 1 +maxExecDuration: 1h +maxTrialNum: 1 +#choice: local, remote, pai, kubeflow +trainingServicePlatform: kubeflow +searchSpacePath: search_space.json +#choice: true, false +useAnnotation: false +tuner: + #choice: TPE, Random, Anneal, Evolution, BatchTuner, MetisTuner + builtinTunerName: TPE + classArgs: + #choice: maximize, minimize + optimize_mode: maximize +trial: + codeDir: . + worker: + replicas: 1 + command: python3 mnist.py + gpuNum: 0 + cpuNum: 1 + memoryMB: 8192 + image: msranni/nni:latest +kubeflowConfig: + operator: tf-operator + apiVersion: v1alpha2 + storage: nfs + nfs: + server: 10.10.10.10 + path: /var/nfs/general \ No newline at end of file diff --git a/src/examples/mnist_nni/config_pai.yml b/src/examples/mnist_nni/config_pai.yml new file mode 100644 index 0000000000..1aed04b694 --- /dev/null +++ b/src/examples/mnist_nni/config_pai.yml @@ -0,0 +1,36 @@ +authorName: default +experimentName: example_mnist +trialConcurrency: 1 +maxExecDuration: 1h +maxTrialNum: 10 +#choice: local, remote, pai +trainingServicePlatform: pai +searchSpacePath: search_space.json +#choice: true, false +useAnnotation: false +tuner: + #choice: TPE, Random, Anneal, Evolution, BatchTuner, MetisTuner + #SMAC (SMAC should be installed through nnictl) + builtinTunerName: TPE + classArgs: + #choice: maximize, minimize + optimize_mode: maximize +trial: + command: python3 mnist.py + codeDir: . + gpuNum: 0 + cpuNum: 1 + memoryMB: 8196 + #The docker image to run nni job on pai + image: msranni/nni:latest + #The hdfs directory to store data on pai, format 'hdfs://host:port/directory' + dataDir: hdfs://10.10.10.10:9000/username/nni + #The hdfs directory to store output data generated by nni, format 'hdfs://host:port/directory' + outputDir: hdfs://10.10.10.10:9000/username/nni +paiConfig: + #The username to login pai + userName: username + #The password to login pai + passWord: password + #The host of restful server of pai + host: 10.10.10.10 \ No newline at end of file diff --git a/src/examples/mnist_nni/config_windows.yml b/src/examples/mnist_nni/config_windows.yml new file mode 100644 index 0000000000..2ace0ced24 --- /dev/null +++ b/src/examples/mnist_nni/config_windows.yml @@ -0,0 +1,21 @@ +authorName: default +experimentName: example_mnist +trialConcurrency: 1 +maxExecDuration: 1h +maxTrialNum: 10 +#choice: local, remote, pai +trainingServicePlatform: local +searchSpacePath: search_space.json +#choice: true, false +useAnnotation: false +tuner: + #choice: TPE, Random, Anneal, Evolution, BatchTuner, MetisTuner + #SMAC (SMAC should be installed through nnictl) + builtinTunerName: TPE + classArgs: + #choice: maximize, minimize + optimize_mode: maximize +trial: + command: python mnist.py + codeDir: . + gpuNum: 0 diff --git a/src/examples/mnist_nni/mnist.py b/src/examples/mnist_nni/mnist.py new file mode 100644 index 0000000000..168d21926d --- /dev/null +++ b/src/examples/mnist_nni/mnist.py @@ -0,0 +1,163 @@ +""" An MNIST classifier using Spatial Pooler.""" + +import argparse +import logging +import math +import random +import gzip +import numpy as np +import os +import threading +from pprint import pprint + +from nupic.bindings.algorithms import SpatialPooler, Classifier +from nupic.bindings.sdr import SDR, Metrics + +import nni + + +def load_mnist(path): + """See: http://yann.lecun.com/exdb/mnist/ for MNIST download and binary file format spec.""" + def int32(b): + i = 0 + for char in b: + i *= 256 + # i += ord(char) # python2 + i += char + return i + + def load_labels(file_name): + with gzip.open(file_name, 'rb') as f: + raw = f.read() + assert(int32(raw[0:4]) == 2049) # Magic number + labels = [] + for char in raw[8:]: + # labels.append(ord(char)) # python2 + labels.append(char) + return labels + + def load_images(file_name): + with gzip.open(file_name, 'rb') as f: + raw = f.read() + assert(int32(raw[0:4]) == 2051) # Magic number + num_imgs = int32(raw[4:8]) + rows = int32(raw[8:12]) + cols = int32(raw[12:16]) + assert(rows == 28) + assert(cols == 28) + img_size = rows*cols + data_start = 4*4 + imgs = [] + for img_index in range(num_imgs): + vec = raw[data_start + img_index*img_size : data_start + (img_index+1)*img_size] + # vec = [ord(c) for c in vec] # python2 + vec = list(vec) + vec = np.array(vec, dtype=np.uint8) + buf = np.reshape(vec, (rows, cols, 1)) + imgs.append(buf) + assert(len(raw) == data_start + img_size * num_imgs) # All data should be used. + return imgs + + train_labels = load_labels(os.path.join(path, 'train-labels-idx1-ubyte.gz')) + train_images = load_images(os.path.join(path, 'train-images-idx3-ubyte.gz')) + test_labels = load_labels(os.path.join(path, 't10k-labels-idx1-ubyte.gz')) + test_images = load_images(os.path.join(path, 't10k-images-idx3-ubyte.gz')) + + return train_labels, train_images, test_labels, test_images + + +class BWImageEncoder: + """Simple grey scale image encoder for MNIST.""" + def __init__(self, input_space): + self.output = SDR(tuple(input_space)) + + def encode(self, image): + self.output.dense = image >= np.mean(image) + return self.output + + +def main(args): + # Load data. + train_labels, train_images, test_labels, test_images = load_mnist(args['data_dir']) + training_data = list(zip(train_images, train_labels)) + test_data = list(zip(test_images, test_labels)) + random.shuffle(training_data) + random.shuffle(test_data) + + # Setup the AI. + enc = BWImageEncoder(train_images[0].shape[:2]) + sp = SpatialPooler( + inputDimensions = enc.output.dimensions, + columnDimensions = [int(args['columnDimensions']), 1], + potentialRadius = 99999999, + potentialPct = args['potentialPct'], + globalInhibition = True, + localAreaDensity = args['localAreaDensity'], + numActiveColumnsPerInhArea = -1, + stimulusThreshold = int(round(args['stimulusThreshold'])), + synPermInactiveDec = args['synPermInactiveDec'], + synPermActiveInc = args['synPermActiveInc'], + synPermConnected = args['synPermConnected'], + minPctOverlapDutyCycle = args['minPctOverlapDutyCycle'], + dutyCyclePeriod = int(round(args['dutyCyclePeriod'])), + boostStrength = args['boostStrength'], + seed = 42, + spVerbosity = 99, + wrapAround = False) + columns = SDR( sp.getColumnDimensions() ) + columns_stats = Metrics( columns, 99999999 ) + sdrc = Classifier() + + # Training Loop + for i in range(len(train_images)): + img, lbl = random.choice(training_data) + enc.encode(np.squeeze(img)) + sp.compute( enc.output, True, columns ) + sdrc.learn( columns, lbl ) + + print(str(sp)) + print(str(columns_stats)) + + # Testing Loop + score = 0 + for img, lbl in test_data: + enc.encode(np.squeeze(img)) + sp.compute( enc.output, False, columns ) + if lbl == np.argmax( sdrc.infer( columns ) ): + score += 1 + + print('Score:', 100 * score / len(test_data), '%') + nni.report_final_result( score / len(test_data) ) + + +def get_params(): + ''' Get parameters from command line ''' + parser = argparse.ArgumentParser() + parser.add_argument("--data_dir", type=str, default="./MNIST_data") + + parser.add_argument("--columnDimensions", type=int, default = 10000) + parser.add_argument("--potentialPct", type=float, default = 0.5) + parser.add_argument("--localAreaDensity", type=float, default = .015) + parser.add_argument("--stimulusThreshold", type=int, default = 6) + parser.add_argument("--synPermActiveInc", type=float, default = 0.01) + parser.add_argument("--synPermInactiveDec", type=float, default = 0.005) + parser.add_argument("--synPermConnected", type=float, default = 0.4) + parser.add_argument("--minPctOverlapDutyCycle", type=float, default = 0.001) + parser.add_argument("--dutyCyclePeriod", type=int, default = 1000) + parser.add_argument("--boostStrength", type=float, default = 2.5) + + args, _ = parser.parse_known_args() + return args + +if __name__ == '__main__': + params = vars(get_params()) + tuner_params = nni.get_next_parameter() + if tuner_params is not None: + params.update(tuner_params) + pprint(params) + + timer = threading.Timer( 30 * 60, lambda: os._exit(1)) + timer.daemon = True + timer.start() + main(params) + timer.cancel() diff --git a/src/examples/mnist_nni/mnist_before.py b/src/examples/mnist_nni/mnist_before.py new file mode 100644 index 0000000000..563ea25831 --- /dev/null +++ b/src/examples/mnist_nni/mnist_before.py @@ -0,0 +1,232 @@ +"""A deep MNIST classifier using convolutional layers.""" +import argparse +import logging +import math +import tempfile +import time + +import tensorflow as tf +from tensorflow.examples.tutorials.mnist import input_data + +FLAGS = None + +logger = logging.getLogger('mnist_AutoML') + + +class MnistNetwork(object): + ''' + MnistNetwork is for initializing and building basic network for mnist. + ''' + + def __init__(self, + channel_1_num, + channel_2_num, + conv_size, + hidden_size, + pool_size, + learning_rate, + x_dim=784, + y_dim=10): + self.channel_1_num = channel_1_num + self.channel_2_num = channel_2_num + self.conv_size = conv_size + self.hidden_size = hidden_size + self.pool_size = pool_size + self.learning_rate = learning_rate + self.x_dim = x_dim + self.y_dim = y_dim + + self.images = tf.placeholder( + tf.float32, [None, self.x_dim], name='input_x') + self.labels = tf.placeholder( + tf.float32, [None, self.y_dim], name='input_y') + self.keep_prob = tf.placeholder(tf.float32, name='keep_prob') + + self.train_step = None + self.accuracy = None + + def build_network(self): + ''' + Building network for mnist + ''' + + # Reshape to use within a convolutional neural net. + # Last dimension is for "features" - there is only one here, since images are + # grayscale -- it would be 3 for an RGB image, 4 for RGBA, etc. + with tf.name_scope('reshape'): + try: + input_dim = int(math.sqrt(self.x_dim)) + except: + print( + 'input dim cannot be sqrt and reshape. input dim: ' + str(self.x_dim)) + logger.debug( + 'input dim cannot be sqrt and reshape. input dim: %s', str(self.x_dim)) + raise + x_image = tf.reshape(self.images, [-1, input_dim, input_dim, 1]) + + # First convolutional layer - maps one grayscale image to 32 feature maps. + with tf.name_scope('conv1'): + w_conv1 = weight_variable( + [self.conv_size, self.conv_size, 1, self.channel_1_num]) + b_conv1 = bias_variable([self.channel_1_num]) + h_conv1 = tf.nn.relu(conv2d(x_image, w_conv1) + b_conv1) + + # Pooling layer - downsamples by 2X. + with tf.name_scope('pool1'): + h_pool1 = max_pool(h_conv1, self.pool_size) + + # Second convolutional layer -- maps 32 feature maps to 64. + with tf.name_scope('conv2'): + w_conv2 = weight_variable([self.conv_size, self.conv_size, + self.channel_1_num, self.channel_2_num]) + b_conv2 = bias_variable([self.channel_2_num]) + h_conv2 = tf.nn.relu(conv2d(h_pool1, w_conv2) + b_conv2) + + # Second pooling layer. + with tf.name_scope('pool2'): + h_pool2 = max_pool(h_conv2, self.pool_size) + + # Fully connected layer 1 -- after 2 round of downsampling, our 28x28 image + # is down to 7x7x64 feature maps -- maps this to 1024 features. + last_dim = int(input_dim / (self.pool_size * self.pool_size)) + with tf.name_scope('fc1'): + w_fc1 = weight_variable( + [last_dim * last_dim * self.channel_2_num, self.hidden_size]) + b_fc1 = bias_variable([self.hidden_size]) + + h_pool2_flat = tf.reshape( + h_pool2, [-1, last_dim * last_dim * self.channel_2_num]) + h_fc1 = tf.nn.relu(tf.matmul(h_pool2_flat, w_fc1) + b_fc1) + + # Dropout - controls the complexity of the model, prevents co-adaptation of features. + with tf.name_scope('dropout'): + h_fc1_drop = tf.nn.dropout(h_fc1, self.keep_prob) + + # Map the 1024 features to 10 classes, one for each digit + with tf.name_scope('fc2'): + w_fc2 = weight_variable([self.hidden_size, self.y_dim]) + b_fc2 = bias_variable([self.y_dim]) + y_conv = tf.matmul(h_fc1_drop, w_fc2) + b_fc2 + + with tf.name_scope('loss'): + cross_entropy = tf.reduce_mean( + tf.nn.softmax_cross_entropy_with_logits(labels=self.labels, logits=y_conv)) + with tf.name_scope('adam_optimizer'): + self.train_step = tf.train.AdamOptimizer( + self.learning_rate).minimize(cross_entropy) + + with tf.name_scope('accuracy'): + correct_prediction = tf.equal( + tf.argmax(y_conv, 1), tf.argmax(self.labels, 1)) + self.accuracy = tf.reduce_mean( + tf.cast(correct_prediction, tf.float32)) + + +def conv2d(x_input, w_matrix): + """conv2d returns a 2d convolution layer with full stride.""" + return tf.nn.conv2d(x_input, w_matrix, strides=[1, 1, 1, 1], padding='SAME') + + +def max_pool(x_input, pool_size): + """max_pool downsamples a feature map by 2X.""" + return tf.nn.max_pool(x_input, ksize=[1, pool_size, pool_size, 1], + strides=[1, pool_size, pool_size, 1], padding='SAME') + + +def weight_variable(shape): + """weight_variable generates a weight variable of a given shape.""" + initial = tf.truncated_normal(shape, stddev=0.1) + return tf.Variable(initial) + + +def bias_variable(shape): + """bias_variable generates a bias variable of a given shape.""" + initial = tf.constant(0.1, shape=shape) + return tf.Variable(initial) + +def download_mnist_retry(data_dir, max_num_retries=20): + """Try to download mnist dataset and avoid errors""" + for _ in range(max_num_retries): + try: + return input_data.read_data_sets(data_dir, one_hot=True) + except tf.errors.AlreadyExistsError: + time.sleep(1) + raise Exception("Failed to download MNIST.") + +def main(params): + ''' + Main function, build mnist network, run and send result to NNI. + ''' + # Import data + mnist = download_mnist_retry(params['data_dir']) + print('Mnist download data done.') + logger.debug('Mnist download data done.') + + # Create the model + # Build the graph for the deep net + mnist_network = MnistNetwork(channel_1_num=params['channel_1_num'], + channel_2_num=params['channel_2_num'], + conv_size=params['conv_size'], + hidden_size=params['hidden_size'], + pool_size=params['pool_size'], + learning_rate=params['learning_rate']) + mnist_network.build_network() + logger.debug('Mnist build network done.') + + # Write log + graph_location = tempfile.mkdtemp() + logger.debug('Saving graph to: %s', graph_location) + train_writer = tf.summary.FileWriter(graph_location) + train_writer.add_graph(tf.get_default_graph()) + + test_acc = 0.0 + with tf.Session() as sess: + sess.run(tf.global_variables_initializer()) + for i in range(params['batch_num']): + batch = mnist.train.next_batch(params['batch_size']) + mnist_network.train_step.run(feed_dict={mnist_network.images: batch[0], + mnist_network.labels: batch[1], + mnist_network.keep_prob: 1 - params['dropout_rate']} + ) + + if i % 100 == 0: + test_acc = mnist_network.accuracy.eval( + feed_dict={mnist_network.images: mnist.test.images, + mnist_network.labels: mnist.test.labels, + mnist_network.keep_prob: 1.0}) + + logger.debug('test accuracy %g', test_acc) + logger.debug('Pipe send intermediate result done.') + + test_acc = mnist_network.accuracy.eval( + feed_dict={mnist_network.images: mnist.test.images, + mnist_network.labels: mnist.test.labels, + mnist_network.keep_prob: 1.0}) + + logger.debug('Final result is %g', test_acc) + logger.debug('Send final result done.') + +def get_params(): + ''' Get parameters from command line ''' + parser = argparse.ArgumentParser() + parser.add_argument("--data_dir", type=str, default='/tmp/tensorflow/mnist/input_data', help="data directory") + parser.add_argument("--dropout_rate", type=float, default=0.5, help="dropout rate") + parser.add_argument("--channel_1_num", type=int, default=32) + parser.add_argument("--channel_2_num", type=int, default=64) + parser.add_argument("--conv_size", type=int, default=5) + parser.add_argument("--pool_size", type=int, default=2) + parser.add_argument("--hidden_size", type=int, default=1024) + parser.add_argument("--learning_rate", type=float, default=1e-4) + parser.add_argument("--batch_num", type=int, default=2000) + parser.add_argument("--batch_size", type=int, default=32) + + args, _ = parser.parse_known_args() + return args + +if __name__ == '__main__': + try: + params = vars(get_params()) + main(params) + except Exception as exception: + logger.exception(exception) + raise diff --git a/src/examples/mnist_nni/search_space.json b/src/examples/mnist_nni/search_space.json new file mode 100644 index 0000000000..325f77d109 --- /dev/null +++ b/src/examples/mnist_nni/search_space.json @@ -0,0 +1,12 @@ +{ + "columnDimensions":{ "_type":"uniform", "_value":[0, 20000]}, + "potentialPct":{ "_type":"uniform", "_value":[0, 1]}, + "localAreaDensity":{ "_type":"uniform", "_value":[0, 0.5]}, + "stimulusThreshold":{ "_type":"uniform", "_value":[0, 1000]}, + "synPermInactiveDec":{ "_type":"uniform", "_value":[0, 1]}, + "synPermActiveInc":{ "_type":"uniform", "_value":[0, 1]}, + "synPermConnected":{ "_type":"uniform", "_value":[0, 1]}, + "minPctOverlapDutyCycle":{ "_type":"uniform", "_value":[0, 1]}, + "dutyCyclePeriod":{ "_type":"uniform", "_value":[0, 10000]}, + "boostStrength":{ "_type":"uniform", "_value":[0, 1000]} +}