用tensorflow实现RBM
我正在尝试使用tensorflow实现RBM,下面是代码: rbm.py用tensorflow实现RBM,tensorflow,Tensorflow,我正在尝试使用tensorflow实现RBM,下面是代码: rbm.py """ An rbm implementation for TensorFlow, based closely on the one in Theano """ import tensorflow as tf import math def sample_prob(probs): return tf.nn.relu( tf.sign( probs - tf.random_un
""" An rbm implementation for TensorFlow, based closely on the one in Theano """
import tensorflow as tf
import math
def sample_prob(probs):
return tf.nn.relu(
tf.sign(
probs - tf.random_uniform(probs.get_shape())))
class RBM(object):
def __init__(self, name, input_size, output_size):
with tf.name_scope("rbm_" + name):
self.weights = tf.Variable(
tf.truncated_normal([input_size, output_size],
stddev=1.0 / math.sqrt(float(input_size))), name="weights")
self.v_bias = tf.Variable(tf.zeros([input_size]), name="v_bias")
self.h_bias = tf.Variable(tf.zeros([output_size]), name="h_bias")
def propup(self, visible):
return tf.nn.sigmoid(tf.matmul(visible, self.weights) + self.h_bias)
def propdown(self, hidden):
return tf.nn.sigmoid(tf.matmul(hidden, tf.transpose(self.weights)) + self.v_bias)
def sample_h_given_v(self, v_sample):
return sample_prob(self.propup(v_sample))
def sample_v_given_h(self, h_sample):
return sample_prob(self.propdown(h_sample))
def gibbs_hvh(self, h0_sample):
v_sample = self.sample_v_given_h(h0_sample)
h_sample = self.sample_h_given_v(v_sample)
return [v_sample, h_sample]
def gibbs_vhv(self, v0_sample):
h_sample = self.sample_h_given_v(v0_sample)
v_sample = self.sample_v_given_h(h_sample)
return [h_sample, v_sample]
def cd1(self, visibles, learning_rate=0.1):
h_start = self.propup(visibles)
v_end = self.propdown(h_start)
h_end = self.propup(v_end)
w_positive_grad = tf.matmul(tf.transpose(visibles), h_start)
w_negative_grad = tf.matmul(tf.transpose(v_end), h_end)
update_w = self.weights.assign_add(learning_rate * (w_positive_grad - w_negative_grad))
update_vb = self.v_bias.assign_add(learning_rate * tf.reduce_mean(visibles - v_end, 0))
update_hb = self.h_bias.assign_add(learning_rate * tf.reduce_mean(h_start - h_end, 0))
return [update_w, update_vb, update_hb]
def reconstruction_error(self, dataset):
err = tf.stop_gradient(dataset - self.gibbs_vhv(dataset)[1])
return tf.reduce_sum(err * err)
import tensorflow as tf
import numpy as np
import rbm
import input_data
def build_model(X, w1, b1, wo, bo):
h1 = tf.nn.sigmoid(tf.matmul(X, w1)+b1)
model = tf.nn.sigmoid(tf.matmul(h1, wo)+bo)
return model
def init_weight(shape):
return tf.Variable(tf.random_normal(shape, mean=0.0, stddev=0.01))
def init_bias(dim):
return tf.Variable(tf.zeros([dim]))
mnist = input_data.read_data_sets("MNIST_data/", one_hot=True)
trX, trY, teX, teY = mnist.train.images, mnist.train.labels, mnist.test.images, mnist.test.labels
X = tf.placeholder("float", [None, 784])
Y = tf.placeholder("float", [None, 10])
rbm_layer = rbm.RBM("mnist", 784, 500)
for i in range(10):
print "RBM CD: ", i
rbm_layer.cd1(trX)
rbm_w, rbm_vb, rbm_hb = rbm_layer.cd1(trX)
wo = init_weight([500,10])
bo = init_bias(10)
py_x = build_model(X, rbm_w, rbm_hb, wo, bo)
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(py_x, Y))
train_op = tf.train.GradientDescentOptimizer(0.05).minimize(cost)
predict_op = tf.argmax(py_x, 1)
sess = tf.Session()
init = tf.initialize_all_variables()
sess.run(init)
for i in range(10):
for start, end in zip(range(0, len(trX), 128), range(128, len(trX), 128)):
sess.run(train_op, feed_dict={X: trX[start:end], Y: trY[start:end]})
print i, np.mean(np.argmax(teY, axis=1) ==
sess.run(predict_op, feed_dict={X: teX, Y: teY}))
rbm\u MNIST\u test.py
""" An rbm implementation for TensorFlow, based closely on the one in Theano """
import tensorflow as tf
import math
def sample_prob(probs):
return tf.nn.relu(
tf.sign(
probs - tf.random_uniform(probs.get_shape())))
class RBM(object):
def __init__(self, name, input_size, output_size):
with tf.name_scope("rbm_" + name):
self.weights = tf.Variable(
tf.truncated_normal([input_size, output_size],
stddev=1.0 / math.sqrt(float(input_size))), name="weights")
self.v_bias = tf.Variable(tf.zeros([input_size]), name="v_bias")
self.h_bias = tf.Variable(tf.zeros([output_size]), name="h_bias")
def propup(self, visible):
return tf.nn.sigmoid(tf.matmul(visible, self.weights) + self.h_bias)
def propdown(self, hidden):
return tf.nn.sigmoid(tf.matmul(hidden, tf.transpose(self.weights)) + self.v_bias)
def sample_h_given_v(self, v_sample):
return sample_prob(self.propup(v_sample))
def sample_v_given_h(self, h_sample):
return sample_prob(self.propdown(h_sample))
def gibbs_hvh(self, h0_sample):
v_sample = self.sample_v_given_h(h0_sample)
h_sample = self.sample_h_given_v(v_sample)
return [v_sample, h_sample]
def gibbs_vhv(self, v0_sample):
h_sample = self.sample_h_given_v(v0_sample)
v_sample = self.sample_v_given_h(h_sample)
return [h_sample, v_sample]
def cd1(self, visibles, learning_rate=0.1):
h_start = self.propup(visibles)
v_end = self.propdown(h_start)
h_end = self.propup(v_end)
w_positive_grad = tf.matmul(tf.transpose(visibles), h_start)
w_negative_grad = tf.matmul(tf.transpose(v_end), h_end)
update_w = self.weights.assign_add(learning_rate * (w_positive_grad - w_negative_grad))
update_vb = self.v_bias.assign_add(learning_rate * tf.reduce_mean(visibles - v_end, 0))
update_hb = self.h_bias.assign_add(learning_rate * tf.reduce_mean(h_start - h_end, 0))
return [update_w, update_vb, update_hb]
def reconstruction_error(self, dataset):
err = tf.stop_gradient(dataset - self.gibbs_vhv(dataset)[1])
return tf.reduce_sum(err * err)
import tensorflow as tf
import numpy as np
import rbm
import input_data
def build_model(X, w1, b1, wo, bo):
h1 = tf.nn.sigmoid(tf.matmul(X, w1)+b1)
model = tf.nn.sigmoid(tf.matmul(h1, wo)+bo)
return model
def init_weight(shape):
return tf.Variable(tf.random_normal(shape, mean=0.0, stddev=0.01))
def init_bias(dim):
return tf.Variable(tf.zeros([dim]))
mnist = input_data.read_data_sets("MNIST_data/", one_hot=True)
trX, trY, teX, teY = mnist.train.images, mnist.train.labels, mnist.test.images, mnist.test.labels
X = tf.placeholder("float", [None, 784])
Y = tf.placeholder("float", [None, 10])
rbm_layer = rbm.RBM("mnist", 784, 500)
for i in range(10):
print "RBM CD: ", i
rbm_layer.cd1(trX)
rbm_w, rbm_vb, rbm_hb = rbm_layer.cd1(trX)
wo = init_weight([500,10])
bo = init_bias(10)
py_x = build_model(X, rbm_w, rbm_hb, wo, bo)
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(py_x, Y))
train_op = tf.train.GradientDescentOptimizer(0.05).minimize(cost)
predict_op = tf.argmax(py_x, 1)
sess = tf.Session()
init = tf.initialize_all_variables()
sess.run(init)
for i in range(10):
for start, end in zip(range(0, len(trX), 128), range(128, len(trX), 128)):
sess.run(train_op, feed_dict={X: trX[start:end], Y: trY[start:end]})
print i, np.mean(np.argmax(teY, axis=1) ==
sess.run(predict_op, feed_dict={X: teX, Y: teY}))
但错误来了: 文件 “/usr/local/lib/python2.7/dist-packages/tensorflow/python/framework/ops.py”, 第1626行,如图所示 提升值错误(“GraphDef不能大于2GB。”)值错误:GraphDef不能大于2GB
有人能帮我解决这个问题吗?TensorFlow在
GraphDef
协议上有2GB的限制,这源于协议缓冲区实现的限制。如果图形中有较大的常数张量,则可以很快达到该极限。特别是,如果多次使用相同的numpy数组,TensorFlow将向图形中添加多个常量张量
在您的例子中,input\u data.read\u set
返回的mnist.train.images
是一个整数浮点数组,其形状(55000784)
,因此大约为164MB
。将该numpy数组传递给rbm_layer.cd1
,在该函数中,每次使用visibles
,都会从numpy数组创建一个TensorFlowConst
节点。您在3个位置使用visibiles
,因此每次调用cd1
都会将图形大小增加约492MB
,因此您很容易超过限制。解决方案是创建一个TensorFlow常量,并将该常量传递给cd1
函数,如下所示:
trX_constant = tf.constant(trX)
for i in range(10):
print "RBM CD: ", i
rbm_layer.cd1(trX_constant)
顺便说一句,我不知道你的意图是在上述循环。请注意,
cd1
函数只是将assign\u add
节点添加到图形中,实际上并不执行赋值。如果你真的希望在你训练的时候发生这些分配,你应该考虑通过控制依赖将这些赋值链接到你的最终<代码> TrimeOP> <代码>节点。 < P> >以完成Kevman的问题,我认为你正在尝试使用这个循环来实现<代码> CD-K/<代码>(对比发散)步骤。
但是我担心代码太不合适了,因为CD-k
是应该在RBM
中自动区分的功能。这意味着成本
和列车运行
不是在RBM
中使用梯度下降的正确方法(这是因为CD-k的特殊作用)。顺便说一下,RBM层应该一个接一个地进行训练,而不是完全连接的层,这不在您的代码中
我是tensorflow的新手,我也想得到实现。我想我最好不要使用tensorflow提供的梯度下降法,因为我需要CD-k
来进行特殊的微分。但愿我能很快找到解决办法
更新:
我为这个实现工作了整整一天。这是目前的情况。我已经实现了一个简单明了的版本,但它只是得到了错误的结果。
请参阅
我只是参考了来自的具体方法。我认为我试图通过tensorflow
实现的过程是可以的,但不知道实际代码出了什么问题
更新2:我修改了代码,现在通过tensorflow实现了最简单的rbm。请参见上面的代码和结果链接