Python ResourceExhaustedError（回溯见上文）：使用形状[2880019200]分配张量时的OOM

我发布了一个关于自动编码器（AutoEncoder）的问题

我安装了以下程序，但是现在，当我输入一个160个水平像素乘120个像素的图像时，“ResourceExhausterRor”出现，我无法继续学习。 具体地说，错误发生在第130行。 另一方面，如果您将分辨率设置为宽度80垂直60像素的一半，则EPOC似乎在进步，学习也在进步。 （它通过程序2将图像分割并使其变小。）

我认为图像大小（宽度160 x 120像素）和张数（大约700张）不是特别大，但是为什么你不能教为什么会出现错误以及如何解决它？ 考虑到主存不足可能会受到影响，我制作了128 GB的内存，但同样的错误也发生了

请帮帮我。 多谢各位

环境描述如下

CPU:Xeon E5-1620v4　4芯/8吨

主板：华硕X99-E WS

内存：DDR4-2400 64 GB（8G×8）

GPU:NVIDIA Quadro GP100×2　16GB

操作系统：ubuntu 16.04 LTS

这是源代码

import numpy as np
import tensorflow as tf
from tensorflow.python.framework import ops
import cv2
import os

DATASET_PATH = "/home/densos/workspaces/autoencoder"
DIR_PATH = "input_gray_160*120"
IMAGE_PATH = os.path.join(DATASET_PATH, DIR_PATH)
X_PIXEL, Y_PIXEL = 160, 120
M = 1
N_HIDDENS = np.array(np.array([1.5]) * X_PIXEL * Y_PIXEL // (M*M), dtype = np.int)
TRANCE_FRAME_NUM = 700

ops.reset_default_graph()

def xavier_init(fan_in, fan_out, constant = 1):
    low = -constant * np.sqrt(6.0 / (fan_in + fan_out))
    high = constant * np.sqrt(6.0 / (fan_in + fan_out))
    return tf.random_uniform((fan_in, fan_out), minval = low, maxval = high, dtype = tf.float32)

class AdditiveGaussianNoiseAutoencoder(object):
    def __init__(self, n_input, n_hidden, transfer_function = tf.nn.sigmoid, optimizer = tf.train.AdamOptimizer(), scale = 0.1):
        self.n_input = n_input
        self.n_hidden = n_hidden
        self.transfer = transfer_function
        self.scale = tf.placeholder(tf.float32)
        self.training_scale = scale
        network_weights = self._initialize_weights()
        self.weights = network_weights
        self.sparsity_level = np.repeat([0.05], self.n_hidden).astype(np.float32)
        self.sparse_reg = 0.1

        # model
        self.x = tf.placeholder(tf.float32, [None, self.n_input])
        self.hidden = self.transfer(tf.add(tf.matmul(self.x + scale * tf.random_normal((n_input,)),
                self.weights['w1']),
                self.weights['b1']))
        self.reconstruction = tf.add(tf.matmul(self.hidden, self.weights['w2']), self.weights['b2'])

        # cost
        self.cost = 0.5 * tf.reduce_sum(tf.pow(tf.subtract(self.reconstruction, self.x), 2.0)) + self.sparse_reg \
                        * self.kl_divergence(self.sparsity_level, self.hidden)

        self.optimizer = optimizer.minimize(self.cost)

        init = tf.global_variables_initializer()
        self.sess = tf.Session()
        self.sess.run(init)

    def _initialize_weights(self):
        all_weights = dict()
        all_weights['w1'] = tf.Variable(xavier_init(self.n_input, self.n_hidden))
        all_weights['b1'] = tf.Variable(tf.zeros([self.n_hidden], dtype = tf.float32))
        all_weights['w2'] = tf.Variable(tf.zeros([self.n_hidden, self.n_input], dtype = tf.float32))
        all_weights['b2'] = tf.Variable(tf.zeros([self.n_input], dtype = tf.float32))
        return all_weights

    def partial_fit(self, X):
        cost, opt = self.sess.run((self.cost, self.optimizer), feed_dict = {self.x: X,
                                                                            self.scale: self.training_scale
                                                                            })
        return cost

    def kl_divergence(self, p, p_hat):
        return tf.reduce_mean(p * tf.log(p) - p * tf.log(p_hat) + (1 - p) * tf.log(1 - p) - (1 - p) * tf.log(1 - p_hat))

    def calc_total_cost(self, X):
        return self.sess.run(self.cost, feed_dict = {self.x: X,
                                                     self.scale: self.training_scale
                                                     })

    def transform(self, X):
        return self.sess.run(self.hidden, feed_dict = {self.x: X,
                                                       self.scale: self.training_scale
                                                       })

    def generate(self, hidden = None):
        if hidden is None:
            hidden = np.random.normal(size = self.weights["b1"])
        return self.sess.run(self.reconstruction, feed_dict = {self.hidden: hidden})

    def reconstruct(self, X):
        return self.sess.run(self.reconstruction, feed_dict = {self.x: X,
                                                               self.scale: self.training_scale
                                                               })

    def getWeights(self):
        return self.sess.run(self.weights['w1'])

    def getBiases(self):
        return self.sess.run(self.weights['b1'])

def get_random_block_from_data(data, batch_size):
    start_index = np.random.randint(0, len(data) - batch_size)
    return data[start_index:(start_index + batch_size)]


if __name__ == '__main__':
#get input data lists
    lists = []
    for file in os.listdir(IMAGE_PATH):
        if file.endswith(".jpeg"):
            lists.append(file)
        lists.sort()

#read input data    
    input_images = []
    for image in lists:
        tmp = cv2.imread(os.path.join(IMAGE_PATH, image), cv2.IMREAD_GRAYSCALE)
        tmp = cv2.resize(tmp, (X_PIXEL // M, Y_PIXEL // M))
        tmp = tmp.reshape(tmp.shape[0] * tmp.shape[1])
        input_images.append(tmp)

#preprocess images    
    input_images = np.array(input_images) / 255.

#convert data to float16
    input_images = np.array(input_images, dtype = np.float16)

#set train and test data
    X_train = input_images[:500]
    X_test = input_images[500:]

    n_samples = X_train.shape[0]
    training_epochs = 200
    batch_size = X_train.shape[0] // 4
    display_step = 10

    autoencoder = AdditiveGaussianNoiseAutoencoder(n_input = X_train.shape[1],
                                                   n_hidden = N_HIDDENS[0],
                                                   transfer_function = tf.nn.relu6,
                                                   optimizer = tf.train.AdamOptimizer(learning_rate = 0.001),
                                                   scale = 0.01)

    for epoch in range(training_epochs):
        avg_cost = 0.
        total_batch = int(n_samples / batch_size)
        # Loop over all batches
        for i in range(total_batch):
            batch_xs = get_random_block_from_data(X_train, batch_size)

            # Fit training using batch data
            cost = autoencoder.partial_fit(X_train)
            # Compute average loss
            avg_cost += cost / n_samples * batch_size

        # Display logs per epoch step
        if epoch % display_step == 0:
            print("Epoch:", '%04d' % (epoch + 1), "cost=", avg_cost)

    print("Finish Train")

predicted_imgs = autoencoder.reconstruct(X_test)
predicted_imgs = np.array((predicted_imgs) * 255, dtype = np.uint8)
input_imgs = np.array((X_test) * 255, dtype = np.uint8)

# plot the reconstructed images
for i in range(100):
    im1 = predicted_imgs[i].reshape((Y_PIXEL//M, X_PIXEL//M))
    im2 = input_imgs[i].reshape((Y_PIXEL//M, X_PIXEL//M))

    img_v_union = cv2.vconcat([im1, im2])
    cv2.moveWindow('result.jpg', 100, 200)
    cv2.imshow('result.jpg', img_v_union)

    cv2.waitKey(33)

---

您的

ResourceExhaustedError

不是由超出主内存资源引起的。这是因为您试图在单个GPU中分配超过16GB的可用内存。请注意，

N_HIDDENS

为

28800

，N_输入为

X_像素*Y_像素

，即

19200

。在

\uuuu init\uuuu

中，这些巨大的数字分别作为

n\u hidden

和

n\u input

传递给

\u initialize\u weights（）

。然后使用这些值初始化行

中的权重变量所有权重['w1']=tf.变量（xavier_init（self.n_输入，self.n_隐藏））

。创建了一个大规模的完全连接层，几乎肯定会超过您的GPU内存大小。运行下面的代码以估计该矩阵的大小。如果您的系统没有足够的主内存来存储生成的矩阵，它可能会因内存错误而崩溃

import numpy as np

# Here's a stand in vector - I'm only using it to compute batch_size.
input_images = np.random.rand(1000)
X_train = input_images[:500]
X_test = input_images[500:]
n_samples = X_train.shape[0]
training_epochs = 200
batch_size = X_train.shape[0] // 4
print(batch_size)

# Now, let's compute the number of hidden units
X_PIXEL, Y_PIXEL = 160, 120
M = 1
N_HIDDENS = np.array(np.array([1.5]) * X_PIXEL * Y_PIXEL // (M*M), dtype = np.int)

print(N_HIDDENS[0])

# Now we compute the number of input units.
input_vector_size = X_PIXEL * Y_PIXEL
print(input_vector_size)

# Finally, we make an approximate replica of your first weight matrix.
# Note: THis is huge, and is why you're getting an out of memory error.
your_batch = np.zeros((N_HIDDENS[0], input_vector_size, batch_size), dtype=float)

# If this didn't exceed you main memory allocation, this will print it's size.
print(your_batch.nbytes/1000000000)

您可以看到，在宽度或高度上减小图像大小将以二次方式减少完全连接的层权重矩阵的内存占用。这就是降低图像高度和宽度的原因。请注意，减少批处理大小在这里可能没有帮助！这样做不会改变完全连接层的大小。因此，你应该考虑卷积，而不是完全连接的方法。< /P>

---

希望这个解释对你有所帮助

OOM错误发生在GPU无法为计算矩阵分配足够的内存时，我建议减少批大小值，并给出一个shotThaks供您评论。如果程序不使用GPU，程序能否仅使用CPU学习？　我该怎么写呢？处理时间会更长，但您可以学习使用64 GB的主内存吗？　对不起，有很多问题。有问题没关系！对您可以使用CPU进行训练。这将是不切实际的缓慢，但如果你这样做作为一个概念的证明，这可能是好的。要实现这一点，您需要为正在创建的张量禁用GPU分配。我建议把这个问题作为StackOverflow的另一个单独问题来问。如果这回答了您的问题，请接受答案并使用左上角的蓝色按钮询问新答案。