Python 生成器仅执行12次迭代-无论批大小如何
我有以下数据生成器。它工作并返回预期的数据。除此之外,不管我将epoch或batchsize设置为什么,它只进行12次迭代,然后给出错误(见下文) 我尝试过改变历代的数量和批量大小Python 生成器仅执行12次迭代-无论批大小如何,python,keras,generator,data-generation,Python,Keras,Generator,Data Generation,我有以下数据生成器。它工作并返回预期的数据。除此之外,不管我将epoch或batchsize设置为什么,它只进行12次迭代,然后给出错误(见下文) 我尝试过改变历代的数量和批量大小 # initialize the number of epochs to train for and batch size NUM_EPOCHS = 10 #100 BS = 32 #64 #32 NUM_TRAIN_IMAGES = len(train_uxo_scrap) NUM_TEST_IMAGES = l
# initialize the number of epochs to train for and batch size
NUM_EPOCHS = 10 #100
BS = 32 #64 #32
NUM_TRAIN_IMAGES = len(train_uxo_scrap)
NUM_TEST_IMAGES = len(test_uxo_scrap)
我预计代码将经历10个时代,每个迭代中有32个样本。我每次迭代得到32个样本,但我在第一个历元中只得到12个迭代,然后我得到以下错误。无论设置了什么batchsize或epochs,都会发生这种情况
---------------------------------------------------------------------------
IndexError Traceback (most recent call last)
<ipython-input-83-26f81894773d> in <module>()
5 validation_data=valid_gen,
6 validation_steps=NUM_TEST_IMAGES // BS,
----> 7 epochs=NUM_EPOCHS)
~\AppData\Local\Continuum\anaconda3\envs\dltf1\lib\site-packages\tensorflow\python\keras\engine\training.py in fit_generator(self, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch)
1424 use_multiprocessing=use_multiprocessing,
1425 shuffle=shuffle,
-> 1426 initial_epoch=initial_epoch)
1427
1428 def evaluate_generator(self,
~\AppData\Local\Continuum\anaconda3\envs\dltf1\lib\site-packages\tensorflow\python\keras\engine\training_generator.py in model_iteration(model, data, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch, mode, batch_size, **kwargs)
182 # `batch_size` used for validation data if validation
183 # data is NumPy/EagerTensors.
--> 184 batch_size = int(nest.flatten(batch_data)[0].shape[0])
185
186 # Callbacks batch begin.
IndexError: tuple index out of range
看看这是否有效:
def datagenerator(imgfns, imglabels, batchsize, mode="train", class_mode='binary'):
while True:
start = 0
end = batchsize
while start < len(imgfns):
x = imgfns[start:end]
y = imglabels[start:end]
yield x, y
start += batchsize
end += batchsize
def数据生成器(imgfns、imglabels、batchsize、mode=“train”、class_mode='binary'):
尽管如此:
开始=0
结束=批量大小
启动时
假设
imgfns,imglabel
是numpy数组。稍作修改以满足我的真实代码-这非常有效!谢谢你知道为什么我的代码不起作用吗?乔恩,我没有深入调查。但是,Keras生成器需要一个while True:
,其概念是在该外循环中有一个while
循环,该循环将根据批次大小产生批次。您的内部,而
似乎是一次添加一个图像,并在外部循环中让步。
# train the network
H = model.fit_generator(
train_gen,
steps_per_epoch=NUM_TRAIN_IMAGES // BS,
validation_data=valid_gen,
validation_steps=NUM_TEST_IMAGES // BS,
epochs=NUM_EPOCHS)
---------------------------------------------------------------------------
IndexError Traceback (most recent call last)
<ipython-input-83-26f81894773d> in <module>()
5 validation_data=valid_gen,
6 validation_steps=NUM_TEST_IMAGES // BS,
----> 7 epochs=NUM_EPOCHS)
~\AppData\Local\Continuum\anaconda3\envs\dltf1\lib\site-packages\tensorflow\python\keras\engine\training.py in fit_generator(self, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch)
1424 use_multiprocessing=use_multiprocessing,
1425 shuffle=shuffle,
-> 1426 initial_epoch=initial_epoch)
1427
1428 def evaluate_generator(self,
~\AppData\Local\Continuum\anaconda3\envs\dltf1\lib\site-packages\tensorflow\python\keras\engine\training_generator.py in model_iteration(model, data, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch, mode, batch_size, **kwargs)
182 # `batch_size` used for validation data if validation
183 # data is NumPy/EagerTensors.
--> 184 batch_size = int(nest.flatten(batch_data)[0].shape[0])
185
186 # Callbacks batch begin.
IndexError: tuple index out of range
['C:\\Users\\jfhauris\\Documents\\xtemp\\ML GEO\\MLGeoCode\\FormattedDataStore\\uxo_48-81\\JBCC_Norm_Formatted_48-81_#615.npy', ..., 'C:\\Users\\jfhauris\\Documents\\xtemp\\ML GEO\\MLGeoCode\\FormattedDataStore\\scrap_48-81\\JBCC_Norm_Formatted_48-81_#224.npy']
[1, 0, 1, 1, 0, 0, 0, 1, 0, 0, 1, 1, 0, 1, 0, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, 0, 0, 1, 1, 0, 0, 0]
********** cnt = 352
['C:\\Users\\jfhauris\\Documents\\xtemp\\ML GEO\\MLGeoCode\\FormattedDataStore\\uxo_48-81\\JBCC_Norm_Formatted_48-81_#532.npy', 'C:\\Users\\jfhauris\\Documents\\xtemp\\ML GEO\\MLGeoCode\\FormattedDataStore\\uxo_48-81\\JBCC_Norm_Formatted_48-81_#953.npy',
...
, 'C:\\Users\\jfhauris\\Documents\\xtemp\\ML GEO\\MLGeoCode\\FormattedDataStore\\scrap_48-81\\JBCC_Norm_Formatted_48-81_#1081.npy', 'C:\\Users\\jfhauris\\Documents\\xtemp\\ML GEO\\MLGeoCode\\FormattedDataStore\\scrap_48-81\\JBCC_Norm_Formatted_48-81_#1050.npy']
[1, 1, 0, 1, 1, 1, 1, 0, 1, 1, 0, 0, 1, 1, 1, 0, 1, 0, 0, 0, 1, 1, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0]
********** cnt = 384
def datagenerator(imgfns, imglabels, batchsize, mode="train", class_mode='binary'):
while True:
start = 0
end = batchsize
while start < len(imgfns):
x = imgfns[start:end]
y = imglabels[start:end]
yield x, y
start += batchsize
end += batchsize