在超参数相同的情况下，Numpy的性能优于Tensorflow和Pytorch

我制作了3个神经网络，分别是Numpy、TensorFlow和Pytorch。使用相同的超参数和1k纪元，我的numpy脚本收敛到0.002损失，但我的pytorch和tensorflow仍然在0.6左右跳跃。有人能帮我弄清楚发生了什么事吗。我不认为谷歌和[Facebook+Nvidia]仅仅为了GPU的提升就做出了比Numpy更糟糕的事情。下面是我的代码

努比 皮托克 张量流

---

不能说我已经检查了你所有的代码（那太多的工作了），但是我注意到你没有在PyTorch中将你的偏差梯度归零，所以这些梯度会不断增长，杀死优化算法。（可能还有其他问题）

更惯用的写作方式是：

optimizer = torch.optim.SGD(net.parameters(), lr)

for _ in range(steps):

    optimizer.zero_grad()

    # ...

一般来说，在这里使用哪个框架并不重要：数字就是数字：如果输入相同的批次，并以相同的方式初始化权重，则应该得到大致相同的梯度

编辑

---

这是另一个很大的区别：在NP中，您计算平均误差（跨批次和通道），而在PT中，您计算它们的总和。这是一个40倍的差异，将影响损失和梯度。

对于TensorFlow，我得到的成本为~1.5，

epochs=10000

，

batches=100

*

学习率=0.001

。你得到了什么？如果我增加历元或改变超参数，我可以减少损失，因为网络仍在学习。但我担心的是，即使使用1000个历元和64个批次，学习率为0.01，我的numpy损失也会减少。这是我的tensorflow损耗-->0.1552949845790863，这是我的Numpy的-->0.00912828950229损耗为0.009的Numpy的精度是多少？@PirateX 0.984375Oh mahn，我没有看到那个错误。谢谢你指出这一点。但这并没有改变我的网络行为。是的，你所说的是我所期待的，我很确定，这是我的错误。那是什么，它与错误/损失计算有关吗？

import numpy as np
import torch as th
from torch.autograd import Variable


input_size = 10
epochs = 1000
batches = 64
lr = 0.01


def binary_enc(num):
    ret = [int(i) for i in '{0:b}'.format(num)]
    return [0] * (input_size - len(ret)) + ret


def binary_dec(array):
    ret = 0
    for i in array:
        ret = ret * 2 + int(i)
    return ret


def training_test_gen(x, y):
    assert len(x) == len(y)
    indices = np.random.permutation(range(len(x)))
    split_size = int(0.9 * len(indices))
    trX = x[indices[:split_size]]
    trY = y[indices[:split_size]]
    teX = x[indices[split_size:]]
    teY = y[indices[split_size:]]
    return trX, trY, teX, teY


def x_y_gen():
    x = []
    y = []
    for i in range(1000):
        x.append(binary_enc(i))
        if i % 15 == 0:
            y.append([1, 0, 0, 0])
        elif i % 5 == 0:
            y.append([0, 1, 0, 0])
        elif i % 3 == 0:
            y.append([0, 0, 1, 0])
        else:
            y.append([0, 0, 0, 1])
    return training_test_gen(np.array(x), np.array(y))


def check_fizbuz(i):
    if i % 15 == 0:
        return 'fizbuz'
    elif i % 5 == 0:
        return 'buz'
    elif i % 3 == 0:
        return 'fiz'
    else:
        return 'number'


trX, trY, teX, teY = x_y_gen()
if th.cuda.is_available():
    dtype = th.cuda.FloatTensor
else:
    dtype = th.FloatTensor
x = Variable(th.from_numpy(trX).type(dtype), requires_grad=False)
y = Variable(th.from_numpy(trY).type(dtype), requires_grad=False)

w1 = Variable(th.randn(10, 100).type(dtype), requires_grad=True)
w2 = Variable(th.randn(100, 4).type(dtype), requires_grad=True)

b1 = Variable(th.zeros(1, 100).type(dtype), requires_grad=True)
b2 = Variable(th.zeros(1, 4).type(dtype), requires_grad=True)

no_of_batches = int(len(trX) / batches)
for epoch in range(epochs):
    for batch in range(no_of_batches):
        start = batch * batches
        end = start + batches
        x_ = x[start:end]
        y_ = y[start:end]

        a2 = x_.mm(w1)
        a2 = a2.add(b1.expand_as(a2))
        h2 = a2.sigmoid()

        a3 = h2.mm(w2)
        a3 = a3.add(b2.expand_as(a3))
        hyp = a3.sigmoid()

        error = hyp - y_
        loss = error.pow(2).sum()
        loss.backward()

        w1.data -= lr * w1.grad.data
        w2.data -= lr * w2.grad.data
        b1.data -= lr * b1.grad.data
        b2.data -= lr * b2.grad.data
        w1.grad.data.zero_()
        w2.grad.data.zero_()
    print(epoch, error.mean().data[0])

import tensorflow as tf
import numpy as np


input_size = 10
epochs = 1000
batches = 64
learning_rate = 0.01


def binary_enc(num):
    ret = [int(i) for i in '{0:b}'.format(num)]
    return [0] * (input_size - len(ret)) + ret


def binary_dec(array):
    ret = 0
    for i in array:
        ret = ret * 2 + int(i)
    return ret


def training_test_gen(x, y):
    assert len(x) == len(y)
    indices = np.random.permutation(range(len(x)))
    split_size = int(0.9 * len(indices))
    trX = x[indices[:split_size]]
    trY = y[indices[:split_size]]
    teX = x[indices[split_size:]]
    teY = y[indices[split_size:]]
    return trX, trY, teX, teY


def x_y_gen():
    x = []
    y = []
    for i in range(1000):
        x.append(binary_enc(i))
        if i % 15 == 0:
            y.append([1, 0, 0, 0])
        elif i % 5 == 0:
            y.append([0, 1, 0, 0])
        elif i % 3 == 0:
            y.append([0, 0, 1, 0])
        else:
            y.append([0, 0, 0, 1])
    return training_test_gen(np.array(x), np.array(y))


def check_fizbuz(i):
    if i % 15 == 0:
        return 'fizbuz'
    elif i % 5 == 0:
        return 'buz'
    elif i % 3 == 0:
        return 'fiz'
    else:
        return 'number'


trX, trY, teX, teY = x_y_gen()

x = tf.placeholder(tf.float32, [None, 10], name='x')
y = tf.placeholder(tf.float32, [None, 4], name='y')

lr = tf.placeholder(tf.float32, [], name='lr')

w1 = tf.Variable(tf.truncated_normal([10, 100]))
w2 = tf.Variable(tf.truncated_normal([100, 4]))

b1 = tf.Variable(tf.zeros(100))
b2 = tf.Variable(tf.zeros(4))


a2 = tf.sigmoid(tf.add(tf.matmul(x, w1), b1))
hyp = tf.sigmoid(tf.add(tf.matmul(a2, w2), b2))

cost = tf.reduce_mean(tf.square(hyp - y))
optmizer = tf.train.GradientDescentOptimizer(lr).minimize(cost)

prediction = tf.argmax(hyp, 1)

no_of_batches = int(len(trX) / batches)
with tf.Session() as sess:
    sess.run(tf.global_variables_initializer())
    for epoch in range(epochs):
        p = np.random.permutation(range(len(trX)))
        trX = trX[p]
        trY = trY[p]
        for batch in range(no_of_batches):
            start = batch * batches
            end = start + batches
            input_batch = trX[start: end]
            target_batch = trY[start: end]
            sess.run(
                optmizer, feed_dict={x: input_batch, y: target_batch, lr: learning_rate})
            if epoch % 100 == 0:
                a = np.argmax(teY, axis=1)
                b = sess.run(prediction, feed_dict={x: teX})
                acc = np.mean(a == b)
                out_cost = sess.run(
                    cost, feed_dict={x: input_batch, y: target_batch, lr: learning_rate})
                print('cost - {} --- accuracy - {}'.format(out_cost.mean(), acc))

optimizer = torch.optim.SGD(net.parameters(), lr)

for _ in range(steps):

    optimizer.zero_grad()

    # ...