Python Tensorflow失败预处理错误，但所有变量都已初始化

编辑：尝试了几件事后，我在代码中添加了以下内容：

with tf.Session(graph=self.graph) as session:
    session.run(tf.initialize_all_variables())
    try:
        session.run(tf.assert_variables_initialized())
    except tf.errors.FailedPreconditionError:
        raise RuntimeError("Not all variables initialized!")

现在，偶尔会失败，即

tf.assert\u variables\u initialized（）

将引发FailedPremissionError，即使在它之前执行了

tf.initialize\u all\u variables（）

。有人知道这是怎么发生的吗

---

原始问题：

背景

我正在使用GradientDescentOptimizer在通过Tensorflow创建的基本神经网络上运行交叉验证（CV）超参数搜索。在看似随机的时刻，对于不同的变量，我得到了一个失败的错误。例如（post末尾的完整堆栈跟踪）：

有些跑步失败得相当快，有些则不然——其中一个跑步已经15个小时没有问题了。我在多个GPU上并行运行这个程序-不是优化本身，而是每个CV折叠

我检查的内容

从和post我了解到，当尝试使用未使用

tf.initialize\u all\u Variables（）

初始化的变量时，会发生此错误。然而，我99%确信我正在这样做（如果没有，我希望它总是失败的）——我将在下面发布代码

报告说

此异常通常在运行以下操作时引发： 在初始化tf.变量之前读取该变量

“最常见”的说法是，它也可以在不同的场景中提出。因此，目前的主要问题是：

问题： 是否存在可能引发此异常的其他场景，它们是什么

代码

MLP等级：

class MLP(object):
    def __init__(self, n_in, hidden_config, n_out, optimizer, f_transfer=tf.nn.tanh, f_loss=mean_squared_error,
                 f_out=tf.identity, seed=None, global_step=None, graph=None, dropout_keep_ratio=1):

        self.graph = tf.Graph() if graph is None else graph           
        # all variables defined below
        with self.graph.as_default():
            self.X = tf.placeholder(tf.float32, shape=(None, n_in))
            self.y = tf.placeholder(tf.float32, shape=(None, n_out))
            self._init_weights(n_in, hidden_config, n_out, seed)
            self._init_computations(f_transfer, f_loss, f_out)
            self._init_optimizer(optimizer, global_step)

     def fit_validate(self, X, y, val_X, val_y, val_f, iters=100, val_step=1):
            [snip]
            with tf.Session(graph=self.graph) as session:
VAR INIT HERE-->tf.initialize_all_variables().run() #<-- VAR INIT HERE
                for i in xrange(iters):
                    [snip: get minibatch here]    
                    _, l = session.run([self.optimizer, self.loss], feed_dict={self.X:X_batch, self.y:y_batch})
                    # validate
                    if i % val_step == 0:
                        val_yhat = self.validation_yhat.eval(feed_dict=val_feed_dict, session=session)

完整示例堆栈跟踪：

FailedPreconditionError: Attempting to use uninitialized value Variable_5
     [[Node: Variable_5/read = Identity[T=DT_FLOAT, _class=["loc:@Variable_5"], _device="/job:localhost/replica:0/task:0/gpu:0"](Variable_5)]]
Caused by op u'Variable_5/read', defined at:
  File "tf_paramsearch.py", line 373, in <module>
    randomized_search_params(int(sys.argv[1]))
  File "tf_paramsearch.py", line 356, in randomized_search_params
    hypersearch.fit()
  File "/home/centos/ODQ/main/python/odq/cv.py", line 430, in fit
    return self._fit(sampled_params)
  File "/home/centos/ODQ/main/python/odq/cv.py", line 190, in _fit
    for train_key, test_key in self.cv)
  File "/home/centos/miniconda2/envs/tensorflow/lib/python2.7/site-packages/sklearn/externals/joblib/parallel.py", line 766, in __call__
    n_jobs = self._initialize_pool()
  File "/home/centos/miniconda2/envs/tensorflow/lib/python2.7/site-packages/sklearn/externals/joblib/parallel.py", line 537, in _initialize_pool
    self._pool = MemmapingPool(n_jobs, **poolargs)
  File "/home/centos/miniconda2/envs/tensorflow/lib/python2.7/site-packages/sklearn/externals/joblib/pool.py", line 580, in __init__
    super(MemmapingPool, self).__init__(**poolargs)
  File "/home/centos/miniconda2/envs/tensorflow/lib/python2.7/site-packages/sklearn/externals/joblib/pool.py", line 418, in __init__
    super(PicklingPool, self).__init__(**poolargs)
  File "/home/centos/miniconda2/envs/tensorflow/lib/python2.7/multiprocessing/pool.py", line 159, in __init__
    self._repopulate_pool()
  File "/home/centos/miniconda2/envs/tensorflow/lib/python2.7/multiprocessing/pool.py", line 223, in _repopulate_pool
    w.start()
  File "/home/centos/miniconda2/envs/tensorflow/lib/python2.7/multiprocessing/process.py", line 130, in start
    self._popen = Popen(self)
  File "/home/centos/miniconda2/envs/tensorflow/lib/python2.7/multiprocessing/forking.py", line 126, in __init__
    code = process_obj._bootstrap()
  File "/home/centos/miniconda2/envs/tensorflow/lib/python2.7/multiprocessing/process.py", line 258, in _bootstrap
    self.run()
  File "/home/centos/miniconda2/envs/tensorflow/lib/python2.7/multiprocessing/process.py", line 114, in run
    self._target(*self._args, **self._kwargs)
  File "/home/centos/miniconda2/envs/tensorflow/lib/python2.7/multiprocessing/pool.py", line 113, in worker
    result = (True, func(*args, **kwds))
  File "/home/centos/miniconda2/envs/tensorflow/lib/python2.7/site-packages/sklearn/externals/joblib/parallel.py", line 130, in __call__
    return self.func(*args, **kwargs)
  File "/home/centos/miniconda2/envs/tensorflow/lib/python2.7/site-packages/sklearn/externals/joblib/parallel.py", line 72, in __call__
    return [func(*args, **kwargs) for func, args, kwargs in self.items]
  File "/home/centos/ODQ/main/python/odq/cv.py", line 131, in _fold_runner
    estimator = estimator_getter(parameters)
  File "tf_paramsearch.py", line 264, in estimator_getter
    net = MLP(config_num_inputs[config_id], hidden, 1, optimizer, seed=params.get('seed',100), global_step=global_step, graph=graph, dropout_keep_ratio=dropout)
  File "tf_paramsearch.py", line 86, in __init__
    self._init_weights(n_in, hidden_config, n_out, seed)
  File "tf_paramsearch.py", line 105, in _init_weights
    self.out_weights = tf.Variable(tf.truncated_normal([hidden_config[-1], n_out], stddev=stdev))
  File "/home/centos/miniconda2/envs/tensorflow/lib/python2.7/site-packages/tensorflow/python/ops/variables.py", line 206, in __init__
    dtype=dtype)
  File "/home/centos/miniconda2/envs/tensorflow/lib/python2.7/site-packages/tensorflow/python/ops/variables.py", line 275, in _init_from_args
    self._snapshot = array_ops.identity(self._variable, name="read")
  File "/home/centos/miniconda2/envs/tensorflow/lib/python2.7/site-packages/tensorflow/python/ops/gen_array_ops.py", line 523, in identity
    return _op_def_lib.apply_op("Identity", input=input, name=name)
  File "/home/centos/miniconda2/envs/tensorflow/lib/python2.7/site-packages/tensorflow/python/ops/op_def_library.py", line 655, in apply_op
    op_def=op_def)
  File "/home/centos/miniconda2/envs/tensorflow/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 2117, in create_op
    original_op=self._default_original_op, op_def=op_def)
  File "/home/centos/miniconda2/envs/tensorflow/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 1128, in __init__
    self._traceback = _extract_stack()

FailedPremissionError:尝试使用未初始化的值变量_5
[[Node:Variable_5/read=Identity[T=DT_FLOAT，[u class=[“loc:@Variable_5”]，[u device=“/job:localhost/replica:0/task:0/gpu:0”]（Variable_5）]]
由op u‘变量_5/读取’引起，定义为：
文件“tf_paramsearch.py”，第373行，在
随机搜索参数（int（sys.argv[1]））
随机搜索参数中第356行的文件“tf_paramsearch.py”
hypersearch.fit（）
文件“/home/centos/ODQ/main/python/ODQ/cv.py”，第430行，合适
返回自拟合（采样参数）
文件“/home/centos/ODQ/main/python/ODQ/cv.py”，第190行，格式为
对于列车钥匙，在self.cv中测试钥匙）
文件“/home/centos/miniconda2/envs/tensorflow/lib/python2.7/site packages/sklearn/externals/joblib/parallel.py”，第766行，in__调用__
n\u jobs=self.\u初始化\u池（）
文件“/home/centos/miniconda2/envs/tensorflow/lib/python2.7/site packages/sklearn/externals/joblib/parallel.py”，第537行，在初始化池中
self._pool=MemmapingPool（n_作业，**poolargs）
文件“/home/centos/miniconda2/envs/tensorflow/lib/python2.7/site packages/sklearn/externals/joblib/pool.py”，第580行，在__
超级（MemmapingPool，self）。\uuuuu初始化（**poolargs）
文件“/home/centos/miniconda2/envs/tensorflow/lib/python2.7/site packages/sklearn/externals/joblib/pool.py”，第418行，在__
超级（PicklingPool，self）。\uuuuuu初始化（**poolargs）
文件“/home/centos/miniconda2/envs/tensorflow/lib/python2.7/multiprocessing/pool.py”，第159行，在__
自我重新填充池（）
文件“/home/centos/miniconda2/envs/tensorflow/lib/python2.7/multiprocessing/pool.py”，第223行，在重新填充池中
w、 开始（）
文件“/home/centos/miniconda2/envs/tensorflow/lib/python2.7/multiprocessing/process.py”，第130行，开始
self.\u popen=popen（self）
文件“/home/centos/miniconda2/envs/tensorflow/lib/python2.7/multiprocessing/forking.py”，第126行，在__
代码=进程\u对象\u引导（）
文件“/home/centos/miniconda2/envs/tensorflow/lib/python2.7/multiprocessing/process.py”，第258行，在引导程序中
self.run（）
文件“/home/centos/miniconda2/envs/tensorflow/lib/python2.7/multiprocessing/process.py”，第114行，正在运行
自我目标（*自我参数，**自我参数）
worker中的文件“/home/centos/miniconda2/envs/tensorflow/lib/python2.7/multiprocessing/pool.py”，第113行
结果=（True，func（*args，**kwds））
文件“/home/centos/miniconda2/envs/tensorflow/lib/python2.7/site packages/sklearn/externals/joblib/parallel.py”，第130行，在调用中__
返回self.func（*args，**kwargs）
文件“/home/centos/miniconda2/envs/tensorflow/lib/python2.7/site packages/sklearn/externals/joblib/parallel.py”，第72行，in__调用__
返回[func（*args，**kwargs），用于self.items中的func、args、kwargs]
文件“/home/centos/ODQ/main/python/ODQ/cv.py”，第131行，在
估计器=估计器（参数）
文件“tf_paramsearch.py”，第264行，在estimator_getter中
net=MLP（config\u num\u inputs[config\u id]，hidden，1，优化器，seed=params.get（'seed'，100），global\u step=global\u step，graph=graph，dropout\u keep\u ratio=dropout）
文件“tf_paramsearch.py”，第86行，在u init中__
self._init_权重（n_in，hidden_config，n_out，seed）
文件“tf_paramsearch.py”，第105行，在_init_权重中
self.out_weights=tf.Variable（tf.truncated_normal（[hidden_config[-1]，n_out]，stddev=stdev））
文件“/home/centos/miniconda2/envs/tensorflow/lib/python2.7/site packages/tensorflow/python/ops/variables.py”，第206行，在__
dtype=dtype）
文件“/home/centos/miniconda2/envs/tensorflow/lib/python2.7/site packages/tensorflow/python/ops/variables.py”，第275行，在参数的_init_中
self.\u snapshot=数组操作标识（self.\u变量，name=“read”）
文件“/home/centos/miniconda2/envs/tensorflow/lib/python2.7/site packages/tensorflow/python/ops/gen_array_ops.py”，第523行，标识
返回_op_def_lib.apply_op（“标识”，输入=输入，名称=名称）
文件“/home/centos/miniconda2/envs/tensorflow/lib/python2.7/site packages/tensorflow/python/ops/op_def_library.py”，第655行，在apply_op
op_def=op_def）
文件“/home/centos/miniconda2/envs/tensorflow/lib/python2.7/site-packages/tensorflow/python/framework
def estimator_getter(params):
    [snip]    
    graph = tf.Graph()
    with graph.as_default():
        global_step = tf.Variable(0, trainable=False)
        learning_rate = tf.train.exponential_decay(params.get('learning_rate',0.1), global_step, decay_steps, decay_rate)
    optimizer = tf.train.GradientDescentOptimizer(learning_rate)
    net = MLP(config_num_inputs[config_id], hidden, 1, optimizer, seed=params.get('seed',100), global_step=global_step, graph=graph, dropout_keep_ratio=dropout)

FailedPreconditionError: Attempting to use uninitialized value Variable_5
     [[Node: Variable_5/read = Identity[T=DT_FLOAT, _class=["loc:@Variable_5"], _device="/job:localhost/replica:0/task:0/gpu:0"](Variable_5)]]
Caused by op u'Variable_5/read', defined at:
  File "tf_paramsearch.py", line 373, in <module>
    randomized_search_params(int(sys.argv[1]))
  File "tf_paramsearch.py", line 356, in randomized_search_params
    hypersearch.fit()
  File "/home/centos/ODQ/main/python/odq/cv.py", line 430, in fit
    return self._fit(sampled_params)
  File "/home/centos/ODQ/main/python/odq/cv.py", line 190, in _fit
    for train_key, test_key in self.cv)
  File "/home/centos/miniconda2/envs/tensorflow/lib/python2.7/site-packages/sklearn/externals/joblib/parallel.py", line 766, in __call__
    n_jobs = self._initialize_pool()
  File "/home/centos/miniconda2/envs/tensorflow/lib/python2.7/site-packages/sklearn/externals/joblib/parallel.py", line 537, in _initialize_pool
    self._pool = MemmapingPool(n_jobs, **poolargs)
  File "/home/centos/miniconda2/envs/tensorflow/lib/python2.7/site-packages/sklearn/externals/joblib/pool.py", line 580, in __init__
    super(MemmapingPool, self).__init__(**poolargs)
  File "/home/centos/miniconda2/envs/tensorflow/lib/python2.7/site-packages/sklearn/externals/joblib/pool.py", line 418, in __init__
    super(PicklingPool, self).__init__(**poolargs)
  File "/home/centos/miniconda2/envs/tensorflow/lib/python2.7/multiprocessing/pool.py", line 159, in __init__
    self._repopulate_pool()
  File "/home/centos/miniconda2/envs/tensorflow/lib/python2.7/multiprocessing/pool.py", line 223, in _repopulate_pool
    w.start()
  File "/home/centos/miniconda2/envs/tensorflow/lib/python2.7/multiprocessing/process.py", line 130, in start
    self._popen = Popen(self)
  File "/home/centos/miniconda2/envs/tensorflow/lib/python2.7/multiprocessing/forking.py", line 126, in __init__
    code = process_obj._bootstrap()
  File "/home/centos/miniconda2/envs/tensorflow/lib/python2.7/multiprocessing/process.py", line 258, in _bootstrap
    self.run()
  File "/home/centos/miniconda2/envs/tensorflow/lib/python2.7/multiprocessing/process.py", line 114, in run
    self._target(*self._args, **self._kwargs)
  File "/home/centos/miniconda2/envs/tensorflow/lib/python2.7/multiprocessing/pool.py", line 113, in worker
    result = (True, func(*args, **kwds))
  File "/home/centos/miniconda2/envs/tensorflow/lib/python2.7/site-packages/sklearn/externals/joblib/parallel.py", line 130, in __call__
    return self.func(*args, **kwargs)
  File "/home/centos/miniconda2/envs/tensorflow/lib/python2.7/site-packages/sklearn/externals/joblib/parallel.py", line 72, in __call__
    return [func(*args, **kwargs) for func, args, kwargs in self.items]
  File "/home/centos/ODQ/main/python/odq/cv.py", line 131, in _fold_runner
    estimator = estimator_getter(parameters)
  File "tf_paramsearch.py", line 264, in estimator_getter
    net = MLP(config_num_inputs[config_id], hidden, 1, optimizer, seed=params.get('seed',100), global_step=global_step, graph=graph, dropout_keep_ratio=dropout)
  File "tf_paramsearch.py", line 86, in __init__
    self._init_weights(n_in, hidden_config, n_out, seed)
  File "tf_paramsearch.py", line 105, in _init_weights
    self.out_weights = tf.Variable(tf.truncated_normal([hidden_config[-1], n_out], stddev=stdev))
  File "/home/centos/miniconda2/envs/tensorflow/lib/python2.7/site-packages/tensorflow/python/ops/variables.py", line 206, in __init__
    dtype=dtype)
  File "/home/centos/miniconda2/envs/tensorflow/lib/python2.7/site-packages/tensorflow/python/ops/variables.py", line 275, in _init_from_args
    self._snapshot = array_ops.identity(self._variable, name="read")
  File "/home/centos/miniconda2/envs/tensorflow/lib/python2.7/site-packages/tensorflow/python/ops/gen_array_ops.py", line 523, in identity
    return _op_def_lib.apply_op("Identity", input=input, name=name)
  File "/home/centos/miniconda2/envs/tensorflow/lib/python2.7/site-packages/tensorflow/python/ops/op_def_library.py", line 655, in apply_op
    op_def=op_def)
  File "/home/centos/miniconda2/envs/tensorflow/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 2117, in create_op
    original_op=self._default_original_op, op_def=op_def)
  File "/home/centos/miniconda2/envs/tensorflow/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 1128, in __init__
    self._traceback = _extract_stack()

tf.reset_default_graph()
a = tf.constant(1)
<add more operations to your graph>
b = tf.Variable(1)
init_op = tf.initialize_all_variables()
tf.get_default_graph().finalize()

sess = tf.InteractiveSession()
sess.run(init_op)
sess.run(compute_op)

with sess.as_default():
    result = compute_fn([seed_input,1])