Python 在InceptionV3网络上运行两次推理会得到完全不同的结果

当我计算《盗梦空间》的分数时，我大部分时间都得到了NaN

试图调查为什么会发生这种情况，我发现在相同的图像上运行两次网络可能会导致某些图像产生完全不同的结果（差异大于0.9，而最大差异可能为1），差异较大的图像会随着运行而变化

我的GPU是2080ti，我使用的是Ubuntu和tensorflow=1.13.1。 我试图改变驱动程序，tensorflow版本，运行docker表单，同样的问题总是发生

我在大学里有另一台服务器，它有相同的GPU（2080ti），当我尝试在那里运行时，问题就消失了

谢谢你的帮助

我的剧本

# Code derived from tensorflow/tensorflow/models/image/imagenet/classify_image.py
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import os.path
import tarfile

import numpy as np
from six.moves import urllib
import tensorflow as tf
import sys
import warnings
from scipy import linalg

MODEL_DIR = '/tmp/imagenet'
DATA_URL = 'http://download.tensorflow.org/models/image/imagenet/inception-2015-12-05.tgz'
softmax = None
pool3 = None

# Call this function with list of images. Each of elements should be a
# numpy array with values ranging from 0 to 255.
def get_features(images):
   assert ((images.shape[3]) == 3)
   assert (np.max(images) > 10)
   assert (np.min(images) >= 0.0)
   images = images.astype(np.float32)
   bs = 100
   sess = tf.get_default_session()
   preds = []
   for inp in np.array_split(images, round(images.shape[0] / bs)):
       sys.stdout.write(".")
       sys.stdout.flush()
       pred = sess.run(softmax, {'InputTensor:0': inp})
       preds.append(pred)
   preds = np.concatenate(preds, 0)
   return preds




# This function is called automatically.
def _init_inception():
   global softmax
   global pool3
   if not os.path.exists(MODEL_DIR):
       os.makedirs(MODEL_DIR)
   filename = DATA_URL.split('/')[-1]
   filepath = os.path.join(MODEL_DIR, filename)
   if not os.path.exists(filepath):
       def _progress(count, block_size, total_size):
           sys.stdout.write('\r>> Downloading %s %.1f%%' % (
               filename, float(count * block_size) / float(total_size) * 100.0))
           sys.stdout.flush()

       filepath, _ = urllib.request.urlretrieve(DATA_URL, filepath, _progress)
       print()
       statinfo = os.stat(filepath)
       print('Succesfully downloaded', filename, statinfo.st_size, 'bytes.')
   tarfile.open(filepath, 'r:gz').extractall(MODEL_DIR)
   with tf.gfile.GFile(os.path.join(
           MODEL_DIR, 'classify_image_graph_def.pb'), 'rb') as f:
       graph_def = tf.GraphDef()
       graph_def.ParseFromString(f.read())
       # Import model with a modification in the input tensor to accept arbitrary
       # batch size.
       input_tensor = tf.placeholder(tf.float32, shape=[None, None, None, 3],
                                     name='InputTensor')
       _ = tf.import_graph_def(graph_def, name='inception_v3',
                               input_map={'ExpandDims:0': input_tensor})
   # Works with an arbitrary minibatch size.
   pool3 = tf.get_default_graph().get_tensor_by_name('inception_v3/pool_3:0')
   ops = pool3.graph.get_operations()
   for op_idx, op in enumerate(ops):
       if 'inception_v3' in op.name:
           for o in op.outputs:
               shape = o.get_shape()
               shape = [s.value for s in shape]
               new_shape = []
               for j, s in enumerate(shape):
                   if s == 1 and j == 0:
                       new_shape.append(None)
                   else:
                       new_shape.append(s)
               o.set_shape(tf.TensorShape(new_shape))
   w = tf.get_default_graph().get_operation_by_name("inception_v3/softmax/logits/MatMul").inputs[1]
   logits = tf.matmul(tf.squeeze(pool3, [1, 2]), w)
   softmax = tf.nn.softmax(logits)

_init_inception()
if __name__ =='__main__':
   (x_train, y_train), (x_test, y_test) = tf.keras.datasets.cifar10.load_data()
   with tf.Session() as sess:
       preds1 = get_features(x_train)
       preds2 = get_features(x_train)
       print(abs(preds1-preds2).max())