Python Tensorflow:噪声对比估计语言模型
我想将示例中的loss函数更改为Python Tensorflow:噪声对比估计语言模型,python,tensorflow,Python,Tensorflow,我想将示例中的loss函数更改为tf.nn.nce\u loss。查看tf.nn.nce\u损失实现: def nce_loss(weights, biases, inputs, labels, num_sampled, num_classes, num_true=1, sampled_values=None, remove_accidental_hits=False, partition_strategy="mod",
tf.nn.nce\u loss
。查看tf.nn.nce\u损失实现:
def nce_loss(weights, biases, inputs, labels, num_sampled, num_classes,
num_true=1,
sampled_values=None,
remove_accidental_hits=False,
partition_strategy="mod",
name="nce_loss"):
我想
- 第三个参数(输入)是语言模型的logits
- 第四个参数(标签)是语言的下一个单词(self.\u targets)
模型
- num_classes是语音大小
但我不知道前两个参数是什么,权重和偏差。我如何使tf.nn.nce\u loss
适应语言模型?谢谢
########更新
@亚伦:
谢谢,我尝试了以下方法:
loss = tf.reduce_mean(
tf.nn.nce_loss(softmax_w, softmax_b, logits, tf.reshape(self._targets, [-1,1]),
64, vocab_size))
根据以下文件:
- 权重:形状的张量[num_classes,dim]或张量列表
物体
其沿维度0的连接具有形状[num_classes,dim]。类嵌入(可能是分区的)
- 偏差:形状的张量[num_类]。阶级偏见
- 输入:形状张量[批次大小,尺寸]。正向激活
输入网络的一部分
- 标签:int64类型的张量和形状[batch\u size,num\u true]。这个
目标类
- num_sampled:一个整数。每个类随机抽样的数量
一批
- num_classes:一个整数。可能的类数
所以
- 权重是具有形状(隐藏大小,
人声大小)
- 偏差为softmax_b,具有形状(声音大小)
- 输入是logits,它具有形状(批处理大小*num步数,vocab大小)
- 标签为自。_目标,具有形状(批次大小、数量步数),
因此,我们需要重塑它,tf.重塑(自我目标,[-1,1])
我的PTBModel模型看起来像
class PTBModel(object):
def __init__(self, is_training, config):
self.batch_size = batch_size = config.batch_size
self.num_steps = num_steps = config.num_steps
size = config.hidden_size
vocab_size = config.vocab_size
self._input_data = tf.placeholder(tf.int32, [batch_size, num_steps])
self._targets = tf.placeholder(tf.int32, [batch_size, num_steps])
lstm_cell = rnn_cell.BasicLSTMCell(size, forget_bias=0.0)
if is_training and config.keep_prob < 1:
lstm_cell = rnn_cell.DropoutWrapper(lstm_cell, output_keep_prob=config.keep_prob)
cell = rnn_cell.MultiRNNCell([lstm_cell] * config.num_layers)
self._initial_state = cell.zero_state(batch_size, tf.float32)
with tf.device("/cpu:0"):
embedding = tf.get_variable("embedding", [vocab_size, size])
inputs = tf.nn.embedding_lookup(embedding, self._input_data)
if is_training and config.keep_prob < 1:
inputs = tf.nn.dropout(inputs, config.keep_prob)
outputs = []
states = []
state = self._initial_state
with tf.variable_scope("RNN"):
for time_step in range(num_steps):
if time_step > 0: tf.get_variable_scope().reuse_variables()
(cell_output, state) = cell(inputs[:, time_step, :], state)
outputs.append(cell_output)
states.append(state)
output = tf.reshape(tf.concat(1, outputs), [-1, size])
softmax_w = tf.get_variable("softmax_w", [size, vocab_size])
softmax_b = tf.get_variable("softmax_b", [vocab_size])
logits = tf.matmul(output, softmax_w) + softmax_b
'''
#minimize the average negative log probability using sequence_loss_by_example
loss = seq2seq.sequence_loss_by_example([logits],
[tf.reshape(self._targets, [-1])],
[tf.ones([batch_size * num_steps])],
vocab_size)
loss = tf.reduce_mean(
tf.nn.nce_loss(nce_weights, nce_biases, embed, train_labels,
num_sampled, vocabulary_size))
weights: A Tensor of shape [num_classes, dim], or a list of Tensor objects
whose concatenation along dimension 0 has shape [num_classes, dim]. The (possibly-partitioned) class embeddings.
biases: A Tensor of shape [num_classes]. The class biases.
inputs: A Tensor of shape [batch_size, dim]. The forward activations of the input network.
labels: A Tensor of type int64 and shape [batch_size, num_true]. The target classes.
num_sampled: An int. The number of classes to randomly sample per batch.
num_classes: An int. The number of possible classes.
'''
loss = tf.reduce_mean(
tf.nn.nce_loss(softmax_w, softmax_b, logits, tf.reshape(self._targets, [-1,1]),
64, vocab_size))
self._cost = cost = tf.reduce_sum(loss) / batch_size
self._final_state = states[-1]
if not is_training:
return
self._lr = tf.Variable(0.0, trainable=False)
tvars = tf.trainable_variables()
grads, _ = tf.clip_by_global_norm(tf.gradients(cost, tvars),
config.max_grad_norm)
optimizer = tf.train.GradientDescentOptimizer(self.lr)
self._train_op = optimizer.apply_gradients(zip(grads, tvars))
class PTBModel(对象):
定义初始化(自我、is\U培训、配置):
self.batch\u size=batch\u size=config.batch\u size
self.num\u steps=num\u steps=config.num\u steps
size=config.hidden\u size
vocab_size=config.vocab_size
self.\u input\u data=tf.placeholder(tf.int32,[batch\u size,num\u steps])
self.\u targets=tf.placeholder(tf.int32,[batch\u size,num\u steps])
lstm\u单元=rnn\u单元。基本STMCELL(大小,偏差=0.0)
如果是培训和配置保持问题<1:
lstm\u单元=rnn\u单元.dropoutrapper(lstm\u单元,输出保持保持保持保持保持保持配置)
cell=rnn_cell.MultiRNNCell([lstm_cell]*config.num_layers)
self.\u初始\u状态=cell.zero\u状态(批处理大小,tf.float32)
使用tf.device(“/cpu:0”):
嵌入=tf.get_变量(“嵌入”,[vocab_大小,大小])
inputs=tf.nn.embedding\u lookup(嵌入,自输入数据)
如果是培训和配置保持问题<1:
输入=tf.nn.dropout(输入,配置keep_prob)
输出=[]
国家=[]
状态=自身。初始状态
使用tf.variable_scope(“RNN”):
对于范围内的时间步长(num步长):
如果时间步长>0:tf.get_variable_scope().reuse_variables()
(单元格输出,状态)=单元格(输入[:,时间步长,:],状态)
输出。追加(单元格输出)
states.append(state)
输出=tf.reforme(tf.concat(1,输出),[-1,大小])
softmax_w=tf.get_变量(“softmax_w,[size,vocab_size])
softmax_b=tf.get_变量(“softmax_b”,[vocab_size])
logits=tf.matmul(输出,softmax_w)+softmax_b
'''
#通过示例使用序列损失最小化平均负对数概率
损耗=序列损耗按示例([logits],
[tf.重塑(自我目标,[-1]),
[tf.ones([batch_size*num_steps]),
人声大小)
损失=tf.reduce_平均值(
tf.nn.nce_损失(nce_权重、nce_偏差、嵌入、训练标签、,
数量(抽样,词汇(大小))
权重:形状[num_classes,dim]的张量或张量对象列表
其沿维度0的连接具有形状[num_classes,dim]。类嵌入(可能是分区的)。
偏差:形状的张量[num_类]。阶级偏见。
输入:形状张量[批次大小,尺寸]。输入网络的正向激活。
标签:int64类型的张量和形状[batch\u size,num\u true]。目标类。
num_sampled:一个整数。每批随机抽样的类数。
num_classes:一个整数。可能的类数。
'''
损失=tf.reduce_平均值(
tf.nn.nce_损失(softmax_w,softmax_b,logits,tf.Reformation(自我目标,[-1,1]),
64个,B_尺寸)
自身成本=成本=tf.减少总量(损失)/批量
self.\u final\u state=状态[-1]
如果不是,则是_培训:
返回
self._lr=tf.变量(0.0,可训练=False)
tvars=tf.可训练的_变量()
梯度,按全球标准(梯度(成本,tVAR)剪裁,
配置(最大梯度标准)
优化器=tf.train.GradientDescentOptimizer(self.lr)
self.\u train\u op=优化器。应用梯度(zip(梯度,TVAR))
然而,我犯了一个错误
Epoch: 1 Learning rate: 1.000
W tensorflow/core/common_runtime/executor.cc:1102] 0x528c980 Compute status: Invalid argument: Index 9971 at offset 0 in Tindices is out of range
[[Node: model/nce_loss/embedding_lookup = Gather[Tindices=DT_INT64, Tparams=DT_FLOAT, validate_indices=true, _device="/job:localhost/replica:0/task:0/cpu:0"](model/softmax_w/read, model/nce_loss/concat)]]
W tensorflow/core/common_runtime/executor.cc:1102] 0x528c980 Compute status: Invalid argument: Index 9971 at offset 0 in Tindices is out of range
[[Node: model/nce_loss/embedding_lookup = Gather[Tindices=DT_INT64, Tparams=DT_FLOAT, validate_indices=true, _device="/job:localhost/replica:0/task:0/cpu:0"](model/softmax_w/read, model/nce_loss/concat)]]
[[Node: _send_model/RNN/concat_19_0 = _Send[T=DT_FLOAT, client_terminated=true, recv_device="/job:localhost/replica:0/task:0/cpu:0", send_device="/job:localhost/replica:0/task:0/cpu:0", send_device_incarnation=1438650956868917036, tensor_name="model/RNN/concat_19:0", _device="/job:localhost/replica:0/task:0/cpu:0"](model/RNN/concat_19)]]
Traceback (most recent call last):
File "/home/user/works/workspace/python/ptb_word_lm/ptb_word_lm.py", line 235, in <module>
tf.app.run()
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/platform/default/_app.py", line 30, in run
sys.exit(main(sys.argv))
File "/home/user/works/workspace/python/ptb_word_lm/ptb_word_lm.py", line 225, in main
verbose=True)
File "/home/user/works/workspace/python/ptb_word_lm/ptb_word_lm.py", line 189, in run_epoch
m.initial_state: state})
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/client/session.py", line 315, in run
return self._run(None, fetches, feed_dict)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/client/session.py", line 511, in _run
feed_dict_string)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/client/session.py", line 564, in _do_run
target_list)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/client/session.py", line 586, in _do_call
e.code)
tensorflow.python.framework.errors.InvalidArgumentError: Index 9971 at offset 0 in Tindices is out of range
[[Node: model/nce_loss/embedding_lookup = Gather[Tindices=DT_INT64, Tparams=DT_FLOAT, validate_indices=true, _device="/job:localhost/replica:0/task:0/cpu:0"](model/softmax_w/read, model/nce_loss/concat)]]
Caused by op u'model/nce_loss/embedding_lookup', defined at:
File "/home/user/works/workspace/python/ptb_word_lm/ptb_word_lm.py", line 235, in <module>
tf.app.run()
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/platform/default/_app.py", line 30, in run
sys.exit(main(sys.argv))
File "/home/user/works/workspace/python/ptb_word_lm/ptb_word_lm.py", line 214, in main
m = PTBModel(is_training=True, config=config)
File "/home/user/works/workspace/python/ptb_word_lm/ptb_word_lm.py", line 122, in __init__
64, vocab_size))
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/nn.py", line 798, in nce_loss
name=name)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/nn.py", line 660, in _compute_sampled_logits
weights, all_ids, partition_strategy=partition_strategy)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/embedding_ops.py", line 86, in embedding_lookup
validate_indices=validate_indices)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/gen_array_ops.py", line 447, in gather
validate_indices=validate_indices, name=name)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/op_def_library.py", line 655, in apply_op
op_def=op_def)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/framework/ops.py", line 2040, in create_op
original_op=self._default_original_op, op_def=op_def)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/framework/ops.py", line 1087, in __init__
self._traceback = _extract_stack()
Epoch:1学习率:1.000
W tensorflow/core/common_runtime/executor.cc:1102]0x528c980计算状态:无效参数:Tindices中偏移量0处的索引9971超出范围
[[Node:model/nce\u loss/embedding\u lookup=Gather[Tindices=DT\u INT64,Tparams=DT\u FLOAT,validate\u index=true,\u device=“/job:localhost/replica:0/task:0/cpu:0”](model/softmax w/read,model/nce\u loss/concat)]]
W tensorflow/core/common_runtime/executor.cc:1102]0x528c980计算状态:无效参数:Tindices中偏移量0处的索引9971超出范围
[[Node:model/nce\u loss/embedding\u lookup=Gather[Tindices=DT\u INT64,Tpar