Python 如何使用Tensorflow计算RNN和LSTM模型的AUC并生成ROC曲线？

我使用自定义预定义功能运行RNN和LSTM模型

import tensorflow as tf
from tensorflow.contrib.layers import fully_connected
import h5py
import time
from sklearn.utils import shuffle
def trainDNN(path, n_days, n_features, n_neurons, 
            train_sequences, train_lengths, train_y,
            test_sequences, test_y, test_lengths,
            lstm=False, n_epochs=50, batch_size=256,
            learning_rate=0.0003, TRAIN_REC=8, TEST_REC=8):
    # we're doing binary classification
    n_outputs = 2

    # this is the initial learning rate
    # adam optimzer decays the learning rate automatically
#     learning_rate = 0.0001
    #learning rate decay is determined by epsilon
    epsilon = 0.001

    # setup the graph
    tf.reset_default_graph()

    # inputs to the network
    X = tf.placeholder(tf.float32, [None, n_days, n_features])
    y = tf.placeholder(tf.int32, [None])
    seq_length = tf.placeholder(tf.int32, [None])

    # the network itself
    cell = tf.contrib.rnn.BasicLSTMCell(num_units=n_neurons) if lstm else tf.contrib.rnn.BasicRNNCell(num_units=n_neurons)
    outputs, states = tf.nn.dynamic_rnn(cell, X, dtype=tf.float32, sequence_length=seq_length)
    logits = fully_connected(states[-1] if lstm else states, n_outputs)

    # the training process (minimize loss) including the training operatin itself
    xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits)
    loss = tf.reduce_mean(xentropy)
    optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate, epsilon=epsilon)
    training_op = optimizer.minimize(loss)

    # hold onto the accuracy for the logwriter
    correct = tf.nn.in_top_k(logits, y, 1)
    accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))

    # this saves the network for later querying
    # currently only saves after all epochs are complete
    # but we could for example save checkpoints on a
    # regular basis
    saver = tf.train.Saver()

    # this is where we save the log files for tensorboard
    now = int(time.time())
    name = 'lstm' if lstm else 'rnn'
    root_logdir = path+"tensorflow_logs/{}/{}-{}/".format(name.upper(), name, now)
    train_logdir = "{}train".format(root_logdir)
    eval_logdir = "{}eval".format(root_logdir)
    print('train_logdir', train_logdir)
    print('eval_logdir', eval_logdir)

    # scalars that are written to the log files
    loss_summary = tf.summary.scalar('loss', loss)
    acc_summary = tf.summary.scalar('accuracy', accuracy)

    # summary operation and writer for the training data
    train_summary_op = tf.summary.merge([loss_summary, acc_summary])
    train_writer = tf.summary.FileWriter(train_logdir, tf.get_default_graph())
    # summary operation and writer for the validation data
    eval_summary_op = tf.summary.merge([loss_summary, acc_summary])
    eval_writer = tf.summary.FileWriter(eval_logdir, tf.get_default_graph())

    # initialize variables
    init = tf.global_variables_initializer()
    n_batches = len(train_sequences) // batch_size
    print(n_batches, 'batches of size', batch_size, n_epochs, 'epochs,', n_neurons, 'neurons')

    with tf.Session() as sess:
        # actually run the initialization
        init.run()
        start_time = time.time()
        for epoch in range(n_epochs):
            # at the beginning of each epoch, shuffle the training data
            train_sequences, train_y, train_lengths = shuffle(train_sequences, train_y, train_lengths)
            for iteration in range(n_batches):

                # extract the batch of training data for this iteration
                start = iteration*batch_size
                end = start+batch_size
                X_batch = train_sequences[start:end]
                y_batch = train_y[start:end]
                y_batch = y_batch.ravel()
                seq_length_batch = train_lengths[start:end]

                # every TRAIN_REC steps, save a summary of training accuracy & loss
                if iteration % TRAIN_REC == 0:
                    train_summary_str = train_summary_op.eval(
                        feed_dict = {X: X_batch, y: y_batch, seq_length: seq_length_batch}
                    )
                    step = epoch * n_batches + iteration
                    train_writer.add_summary(train_summary_str, step)
                    # without this flush, tensorboard isn't always current
                    train_writer.flush()

                # every TEST_REC steps, save a summary of validation accuracy & loss
                # TODO: this runs all validation data at once. if validation is
                # sufficiently large, this will fail. better would be to either
                # pick a random subset of validation data, or even better, run
                # validation in multiple batches and save the validation accuracy 
                # & loss based on the aggregation of all of the validation batches.
                if iteration % TEST_REC == 0:
                    summary_str = eval_summary_op.eval(
                        feed_dict = {X: test_sequences, y: test_y.ravel(), seq_length: test_lengths}
                    )
                    step = epoch * n_batches + iteration
                    eval_writer.add_summary(summary_str, step)
                    # without this flush, tensorboard isn't always current
                    eval_writer.flush()

                # run training.
                # this is where the network learns.
                sess.run(
                    training_op,
                    feed_dict = {X: X_batch, y: y_batch, seq_length: seq_length_batch}
                )

            # after every epoch, calculate the accuracy of the last seen training batch 
            acc_train = accuracy.eval(
                feed_dict = {X: X_batch, y: y_batch, seq_length: seq_length_batch}
            )
            # after each epoch, calculate the accuracy of the test data
            acc_test = accuracy.eval(
                feed_dict = {X: test_sequences, y: test_y.ravel(), seq_length: test_lengths}
            )

            # print the training & validation accuracy to the console
            print(epoch, time.strftime('%m/%d %H:%M:%S'), "Accuracy train:", acc_train, "test:", acc_test)


        # save the model (for more training or inference) after all
        # training is complete
        save_path = saver.save(sess, root_logdir+"model_final.ckpt")

        # close the writers
        train_writer.close()
        eval_writer.close()    
        log(["{}-{} model score".format(name.upper(), now), percent(acc_test)])

上述函数对时间序列数据上的RNN和LSTM模型进行训练，并输出二进制分类分数。训练和测试分数都是打印出来的，但我正试图找出如何计算AUC并为RNN和LSTM的二元分类生成ROC曲线

更新：

我使用以下脚本评估了logits和预测：

n_epochs = 2
batch_size = 2000
n_batches = len(train_sequences) // batch_size
print(n_batches)
with tf.Session() as sess:
    init.run()
    #sess.run( tf.local_variables_initializer() )
    for epoch in range(n_epochs):
        train_sequences, train_y, train_lengths = shuffle(train_sequences, train_y, train_lengths)
        for iteration in range(n_batches):
            start = iteration*batch_size
            end = start+batch_size
            X_batch = train_sequences[start:end]
            y_batch = train_y[start:end]
            seq_length_batch = train_lengths[start:end]
            if iteration % 20 == 0:
                train_summary_str = train_summary_op.eval(
                    feed_dict = {X: X_batch, y: y_batch, seq_length: seq_length_batch}
                )
                step = epoch * n_batches + iteration
            if iteration % 200 == 0:
                summary_str = eval_summary_op.eval(
                    feed_dict = {X: test_sequences, y: test_y, seq_length: test_lengths}
                )
                step = epoch * n_batches + iteration
            sess.run(
                training_op,
                feed_dict = {X: X_batch, y: y_batch, seq_length: seq_length_batch}
            )

        acc_train = accuracy.eval(
            feed_dict = {X: X_batch, y: y_batch, seq_length: seq_length_batch}
        )
        acc_test = accuracy.eval(
            feed_dict = {X: test_sequences, y: test_y, seq_length: test_lengths}
        )
        probs = logits.eval(feed_dict = {X: test_sequences, y: test_y, seq_length: test_lengths})
        predictions = correct.eval(feed_dict = {logits:probs, y: test_y})
        print(epoch, "Train accuracy:", acc_train, "Test accuracy:", acc_test)# "Manual score:", score)

这将返回me probs，它基本上是一个矩阵，行数等于测试用例数，2列包含2个二进制类中每个类的概率。预测对象包含预测是否正确。

---

我对此表示怀疑，因为ReLU函数概率分数不像S形函数分数那样直观，因为它不再基于正负预测的默认0.5截止值。相反，预测是基于哪个类别的概率更大。真的可以从ReLu输出生成ROC曲线吗？

请注意，下面提到的

tf.metrics.auc（）

已经过时。有一个替代品，但即使是现在也不推荐使用，建议使用。抱歉，没有时间重构下面的代码

---

为此，可以使用

tf.metrics.auc（）

。请注意，您需要一个热编码标签和对此的预测，如果您试图在多个

sess.run（）

命令上累积AUC，还需要运行

update\u op

，它将返回

在您的代码中，您使用创建了

y\u one\u hot

，并且您将所有这些都放在

准确性之后

可能：

y_one_hot = tf.one_hot( y, n_outputs )
auc, auc_update_op = tf.metrics.auc( y_one_hot, logits )

在开始训练循环之前，您还需要初始化auc创建的局部变量，可能就在

init.run（）之后：

然后，在运行精度时，还需要在
sess.run（）
中使用
accurity
运行
auc
，而不是像这样运行
.eval（）
（未测试）：


多批次累积
如果您确实想使用
tf.metrics.auc（）
的累加功能，那么您还需要注意在开始新计算时重置累加。为此，您需要收集创建的局部变量。因此，创建如下auc：

with tf.variable_scope( "AUC" ):
    auc, auc_update_op = tf.metrics.auc( predictions=y_pred, labels=y_true, curve = 'ROC' )
auc_variables = [ v for v in tf.local_variables() if v.name.startswith( "AUC" ) ]
auc_reset_op = tf.initialize_variables( auc_variables )

完成累加后，按如下方式重置auc的内部变量：

session.run( auc_reset_op )

# after every epoch, calculate the accuracy of the last seen training batch 
acc_train, auc_val, _ = sess.run( [ accuracy, auc, auc_update_op ],
    feed_dict = {X: X_batch, y: y_batch, seq_length: seq_length_batch}
)
session.run( auc_reset_op ) # maybe you want to do this here...
# after each epoch, calculate the accuracy of the test data
acc_test, auc_val, _ = sess.run( [ accuracy, auc, auc_update_op ],
    feed_dict = {X: test_sequences, y: test_y.ravel(), seq_length: test_lengths}
)

您还需要确保每次运行
sess.run（）
时都运行
auc\u update\u op
，如下所示：

session.run( auc_reset_op )

# after every epoch, calculate the accuracy of the last seen training batch 
acc_train, auc_val, _ = sess.run( [ accuracy, auc, auc_update_op ],
    feed_dict = {X: X_batch, y: y_batch, seq_length: seq_length_batch}
)
session.run( auc_reset_op ) # maybe you want to do this here...
# after each epoch, calculate the accuracy of the test data
acc_test, auc_val, _ = sess.run( [ accuracy, auc, auc_update_op ],
    feed_dict = {X: test_sequences, y: test_y.ravel(), seq_length: test_lengths}
)

请注意，下面提到的
tf.metrics.auc（）
已过时。有一个替代品，但即使是现在也不推荐使用，建议使用。抱歉，没有时间重构下面的代码


为此，可以使用
tf.metrics.auc（）
。请注意，您需要一个热编码标签和对此的预测，如果您试图在多个
sess.run（）
命令上累积AUC，还需要运行
update\u op
，它将返回

在您的代码中，您使用创建了
y\u one\u hot
，并且您将所有这些都放在
准确性之后
可能：

y_one_hot = tf.one_hot( y, n_outputs )
auc, auc_update_op = tf.metrics.auc( y_one_hot, logits )

在开始训练循环之前，您还需要初始化auc创建的局部变量，可能就在
init.run（）之后：

然后，在运行精度时，还需要在
sess.run（）
中使用
accurity
运行
auc
，而不是像这样运行
.eval（）
（未测试）：


多批次累积
如果您确实想使用
tf.metrics.auc（）
的累加功能，那么您还需要注意在开始新计算时重置累加。为此，您需要收集创建的局部变量。因此，创建如下auc：

with tf.variable_scope( "AUC" ):
    auc, auc_update_op = tf.metrics.auc( predictions=y_pred, labels=y_true, curve = 'ROC' )
auc_variables = [ v for v in tf.local_variables() if v.name.startswith( "AUC" ) ]
auc_reset_op = tf.initialize_variables( auc_variables )

完成累加后，按如下方式重置auc的内部变量：

session.run( auc_reset_op )

# after every epoch, calculate the accuracy of the last seen training batch 
acc_train, auc_val, _ = sess.run( [ accuracy, auc, auc_update_op ],
    feed_dict = {X: X_batch, y: y_batch, seq_length: seq_length_batch}
)
session.run( auc_reset_op ) # maybe you want to do this here...
# after each epoch, calculate the accuracy of the test data
acc_test, auc_val, _ = sess.run( [ accuracy, auc, auc_update_op ],
    feed_dict = {X: test_sequences, y: test_y.ravel(), seq_length: test_lengths}
)

您还需要确保每次运行
sess.run（）
时都运行
auc\u update\u op
，如下所示：

session.run( auc_reset_op )

# after every epoch, calculate the accuracy of the last seen training batch 
acc_train, auc_val, _ = sess.run( [ accuracy, auc, auc_update_op ],
    feed_dict = {X: X_batch, y: y_batch, seq_length: seq_length_batch}
)
session.run( auc_reset_op ) # maybe you want to do this here...
# after each epoch, calculate the accuracy of the test data
acc_test, auc_val, _ = sess.run( [ accuracy, auc, auc_update_op ],
    feed_dict = {X: test_sequences, y: test_y.ravel(), seq_length: test_lengths}
)

为什么要使用
logits
计算AUC？在文章中，it明确指出预测是任意形状的浮点张量，其值在[0,1]范围内，这是logits的缩放版本。嗨，Arat，这是很久以前的事了，我记不清了，但这更多的是关于如何使用
tf.metrics.auc
，而不是网络本身。现在回顾了他的代码，您可能认为变量logits不是正确的使用方法。请注意，在我的代码示例中，我使用y_pred作为变量名，因为我再次关注
tf.metrics.auc
的使用，创建、更新和重置变量。非常感谢您的解释，Peter！是的，你说得对！我错过了。我刚刚看到关于如何计算AUC的信息污染，并想指出它！中的死链接there@jtlz2添加前缀以解释tf.metrics.auc被删除以及使用什么替代。为什么要使用
logits
计算auc？在文章中，it明确指出预测是任意形状的浮点张量，其值在[0,1]范围内，这是logits的缩放版本。嗨，Arat，这是很久以前的事了，我记不清了，但这更多的是关于如何使用
tf.metrics.auc
，而不是网络本身。现在回顾了他的代码，您可能认为变量logits不是正确的使用方法。请注意，在我的代码示例中，我使用y_pred作为变量名，因为我再次关注
tf.metrics.auc
的使用，创建、更新和重置变量。非常感谢您的解释，Peter！是的，你说得对！我错过了。我刚才看到有一条消息