Tensorflow 使用model.fit进行培训时，列出索引超出范围
首先，我试图研究其他类似的问题，但它并没有解决我的问题 我有一个项目要求我使用数据扩充来增加样本，并使用VGG16进行迁移学习以提高准确性。我在网上找到了这个Github链接，并尝试将其应用于我的项目，但错误是列表索引超出范围。我已经在数据-->训练或测试-->良性或恶性中加载了图像。我的图像的形状是（360560,3）。我使用的是Tensorflow 2.1.0。这是我的密码Tensorflow 使用model.fit进行培训时，列出索引超出范围,tensorflow,tensorflow2.0,Tensorflow,Tensorflow2.0,首先，我试图研究其他类似的问题，但它并没有解决我的问题 我有一个项目要求我使用数据扩充来增加样本，并使用VGG16进行迁移学习以提高准确性。我在网上找到了这个Github链接，并尝试将其应用于我的项目，但错误是列表索引超出范围。我已经在数据-->训练或测试-->良性或恶性中加载了图像。我的图像的形状是（360560,3）。我使用的是Tensorflow 2.1.0。这是我的密码 import random import numpy as np from numpy.random import r
import random
import numpy as np
from numpy.random import randint as rdi
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from plotly import __version__
import cufflinks as cf
from plotly.offline import download_plotlyjs, init_notebook_mode, plot, iplot
init_notebook_mode(connected=True)
cf.go_offline()
import plotly.graph_objects as go
from plotly.offline import *
import plotly.offline as pyo
import chart_studio.plotly as py
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LinearRegression
from sklearn import metrics
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import classification_report
from sklearn.metrics import confusion_matrix
from sklearn.preprocessing import StandardScaler
from sklearn.neighbors import KNeighborsClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.datasets import load_breast_cancer
from sklearn.svm import SVC
from sklearn.model_selection import GridSearchCV
from sklearn.datasets import make_blobs
from sklearn.cluster import KMeans
from sklearn.decomposition import PCA
import tensorflow as tf
from sklearn.preprocessing import MinMaxScaler
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Dropout, Conv2D, MaxPool2D, Flatten, MaxPooling2D, Activation, Reshape, BatchNormalization
from sklearn.metrics import mean_squared_error, mean_absolute_error, explained_variance_score
from tensorflow.keras.models import load_model
from tensorflow.keras.callbacks import EarlyStopping, TensorBoard
from tensorflow.keras.datasets import mnist
from tensorflow.keras.utils import to_categorical
from tensorflow.keras.datasets import cifar10
import os
from matplotlib.image import imread
from tensorflow.keras.preprocessing.image import ImageDataGenerator, array_to_img, img_to_array, load_img
import warnings
from tensorflow.keras.preprocessing import image
from mpl_toolkits.mplot3d import Axes3D
from tensorflow.keras.optimizers import SGD
from tensorflow.keras.layers import GaussianNoise
from keras import applications
import cv2
from os.path import isfile, isdir, getsize
from tqdm import tqdm
from urllib.request import urlretrieve
import zipfile
import tarfile
import pickle
import glob
import shutil
from os.path import isfile, isdir, getsize
from os import mkdir, makedirs, remove, listdir
# Try to download bottelneck file for later use
class DLProgress(tqdm):
last_block = 0
def hook(self, block_num=1, block_size=1, total_size=None):
self.total = total_size
self.update((block_num - self.last_block) * block_size)
self.last_block = block_num
class DLProgress(tqdm):
last_block = 0
def hook(self, block_num=1, block_size=1, total_size=None):
self.total = total_size
self.update((block_num - self.last_block) * block_size)
self.last_block = block_num
if not isfile('bottleneck_features_train.npy'):
with DLProgress(unit='B', unit_scale=True, miniters=1, desc='Bottleneck features') as pbar:
urlretrieve(
'https://www.dropbox.com/s/a38gpvdcryw0kfc/bottleneck.zip?dl=1',
'bottleneck.zip',
pbar.hook)
with zipfile.ZipFile('bottleneck.zip') as f:
f.extractall('./')
files = listdir('bottleneck 2/')
for f in files:
shutil.move('bottleneck 2/'+f,'./')
shutil.rmtree('bottleneck 2/')
remove('bottleneck.zip')
# Import the Ultrasound dataset
data_dir = 'C:\Spring 2020\Machine Learning and Computer Vision\data'
os.listdir(data_dir)
test_path = data_dir+'\\test\\'
train_path = data_dir+'\\train\\'
os.listdir(test_path)
os.listdir(train_path)
os.listdir(train_path+'\\malign')[0]
para_cell = train_path+'\\malign'+'\\53.jpg'
para_img= imread(para_cell)
para_img.shape
plt.imshow(para_img)
image_shape = (360,560,3)
# Generate some new images
datagen = ImageDataGenerator(rescale=1.0/255)
# Create model
model = applications.VGG16(include_top = False, input_shape = image_shape)
model = applications.VGG16(include_top = False, weights = 'imagenet')
model.summary()
with open('bottleneck_features_train.npy','rb') as f:
bottleneck_features_train = pickle.load(f)
bottleneck_features_train.shape
batch_size = 128
generator = datagen.flow_from_directory(
train_path,
target_size=image_shape[:2],
batch_size=batch_size,
class_mode=None,
shuffle=False)
batch_size = 128
valid_generator = datagen.flow_from_directory(
test_path,
target_size=image_shape[:2],
batch_size=batch_size,
class_mode=None,
shuffle=False)
# Use bottleneck file
with open('bottleneck_features_train.npy','rb') as f:
bottleneck_features_train = pickle.load(f)
# Apply model
model = Sequential()
model.add(Flatten(input_shape=bottleneck_features_train.shape[1:]))
model.add(Dense(256, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(1, activation='sigmoid'))
model.compile(optimizer='rmsprop',
loss='binary_crossentropy', metrics=['accuracy'])
model.summary()
batch_size = 128
# Set up the flow of training data
generator = datagen.flow_from_directory(
train_path,
target_size=image_shape[:2],
batch_size=batch_size,
class_mode=None,
shuffle=False)
generator.class_indices
# Set up the early stop to avoid training so many epochs, but won't increase the accuracy
early_stop = EarlyStopping(monitor='val_loss',patience=5)
warnings.filterwarnings('ignore')
# I think the code stucks here
results = model.fit(bottleneck_features_train, epochs=15, batch_size = batch_size)
# Save to vgg16
model.save('vgg16.h5')
losses = pd.DataFrame(model.history.history)
losses[['loss','val_loss']].plot()
model.metrics_names
model.evaluate_generator(test_image_gen)
#https://datascience.stackexchange.com/questions/13894/how-to-get-predictions-with-predict-generator-on-streaming-test-data-in-keras
pred_probabilities = model.predict_generator(test_image_gen)
#preds_num = pred_probabilities
test_image_gen.classes
predictions = pred_probabilities > 0.5
# Numpy can treat this as True/False for us
predictions
check = classification_report(test_image_gen.classes,predictions)
print(classification_report(test_image_gen.classes,predictions))
print(confusion_matrix(test_image_gen.classes,predictions))
para_cell
my_image = image.load_img(para_cell,target_size=image_shape)
my_image
type(my_image)
my_image = image.img_to_array(my_image)
type(my_image)
my_image.shape
my_image = np.expand_dims(my_image, axis=0)
my_image.shape
para_cell = test_path+'\\benign'+'\\40.jpg'
my_image = image.load_img(para_cell,target_size=image_shape)
my_image
type(my_image)
my_image = image.img_to_array(my_image)
type(my_image)
my_image.shape
my_image = np.expand_dims(my_image, axis=0)
my_image.shape
model.predict(my_image)
runfile('C:/Spring 2020/Machine Learning and Computer Vision/hopefully.py', wdir='C:/Spring 2020/Machine Learning and Computer Vision')
Model: "vgg16"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
input_16 (InputLayer) (None, None, None, 3) 0
_________________________________________________________________
block1_conv1 (Conv2D) (None, None, None, 64) 1792
_________________________________________________________________
block1_conv2 (Conv2D) (None, None, None, 64) 36928
_________________________________________________________________
block1_pool (MaxPooling2D) (None, None, None, 64) 0
_________________________________________________________________
block2_conv1 (Conv2D) (None, None, None, 128) 73856
_________________________________________________________________
block2_conv2 (Conv2D) (None, None, None, 128) 147584
_________________________________________________________________
block2_pool (MaxPooling2D) (None, None, None, 128) 0
_________________________________________________________________
block3_conv1 (Conv2D) (None, None, None, 256) 295168
_________________________________________________________________
block3_conv2 (Conv2D) (None, None, None, 256) 590080
_________________________________________________________________
block3_conv3 (Conv2D) (None, None, None, 256) 590080
_________________________________________________________________
block3_pool (MaxPooling2D) (None, None, None, 256) 0
_________________________________________________________________
block4_conv1 (Conv2D) (None, None, None, 512) 1180160
_________________________________________________________________
block4_conv2 (Conv2D) (None, None, None, 512) 2359808
_________________________________________________________________
block4_conv3 (Conv2D) (None, None, None, 512) 2359808
_________________________________________________________________
block4_pool (MaxPooling2D) (None, None, None, 512) 0
_________________________________________________________________
block5_conv1 (Conv2D) (None, None, None, 512) 2359808
_________________________________________________________________
block5_conv2 (Conv2D) (None, None, None, 512) 2359808
_________________________________________________________________
block5_conv3 (Conv2D) (None, None, None, 512) 2359808
_________________________________________________________________
block5_pool (MaxPooling2D) (None, None, None, 512) 0
=================================================================
Total params: 14,714,688
Trainable params: 14,714,688
Non-trainable params: 0
_________________________________________________________________
Found 2198 images belonging to 2 classes.
Found 800 images belonging to 2 classes.
Model: "sequential_7"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
flatten_7 (Flatten) (None, 4608) 0
_________________________________________________________________
dense_14 (Dense) (None, 256) 1179904
_________________________________________________________________
dropout_7 (Dropout) (None, 256) 0
_________________________________________________________________
dense_15 (Dense) (None, 1) 257
=================================================================
Total params: 1,180,161
Trainable params: 1,180,161
Non-trainable params: 0
_________________________________________________________________
Found 2198 images belonging to 2 classes.
Train on 19872 samples
Epoch 1/15
128/19872 [..............................] - ETA: 2sTraceback (most recent call last):
File "C:\Spring 2020\Machine Learning and Computer Vision\hopefully.py", line 188, in <module>
results = model.fit(bottleneck_features_train, epochs=15, batch_size = batch_size)
File "C:\Users\binhd\Anaconda3\lib\site-packages\tensorflow_core\python\keras\engine\training.py", line 819, in fit
use_multiprocessing=use_multiprocessing)
File "C:\Users\binhd\Anaconda3\lib\site-packages\tensorflow_core\python\keras\engine\training_v2.py", line 342, in fit
total_epochs=epochs)
File "C:\Users\binhd\Anaconda3\lib\site-packages\tensorflow_core\python\keras\engine\training_v2.py", line 128, in run_one_epoch
batch_outs = execution_function(iterator)
File "C:\Users\binhd\Anaconda3\lib\site-packages\tensorflow_core\python\keras\engine\training_v2_utils.py", line 98, in execution_function
distributed_function(input_fn))
File "C:\Users\binhd\Anaconda3\lib\site-packages\tensorflow_core\python\eager\def_function.py", line 568, in __call__
result = self._call(*args, **kwds)
File "C:\Users\binhd\Anaconda3\lib\site-packages\tensorflow_core\python\eager\def_function.py", line 615, in _call
self._initialize(args, kwds, add_initializers_to=initializers)
File "C:\Users\binhd\Anaconda3\lib\site-packages\tensorflow_core\python\eager\def_function.py", line 497, in _initialize
*args, **kwds))
File "C:\Users\binhd\Anaconda3\lib\site-packages\tensorflow_core\python\eager\function.py", line 2389, in _get_concrete_function_internal_garbage_collected
graph_function, _, _ = self._maybe_define_function(args, kwargs)
File "C:\Users\binhd\Anaconda3\lib\site-packages\tensorflow_core\python\eager\function.py", line 2703, in _maybe_define_function
graph_function = self._create_graph_function(args, kwargs)
File "C:\Users\binhd\Anaconda3\lib\site-packages\tensorflow_core\python\eager\function.py", line 2593, in _create_graph_function
capture_by_value=self._capture_by_value),
File "C:\Users\binhd\Anaconda3\lib\site-packages\tensorflow_core\python\framework\func_graph.py", line 978, in func_graph_from_py_func
func_outputs = python_func(*func_args, **func_kwargs)
File "C:\Users\binhd\Anaconda3\lib\site-packages\tensorflow_core\python\eager\def_function.py", line 439, in wrapped_fn
return weak_wrapped_fn().__wrapped__(*args, **kwds)
File "C:\Users\binhd\Anaconda3\lib\site-packages\tensorflow_core\python\keras\engine\training_v2_utils.py", line 85, in distributed_function
per_replica_function, args=args)
File "C:\Users\binhd\Anaconda3\lib\site-packages\tensorflow_core\python\distribute\distribute_lib.py", line 763, in experimental_run_v2
return self._extended.call_for_each_replica(fn, args=args, kwargs=kwargs)
File "C:\Users\binhd\Anaconda3\lib\site-packages\tensorflow_core\python\distribute\distribute_lib.py", line 1819, in call_for_each_replica
return self._call_for_each_replica(fn, args, kwargs)
File "C:\Users\binhd\Anaconda3\lib\site-packages\tensorflow_core\python\distribute\distribute_lib.py", line 2164, in _call_for_each_replica
return fn(*args, **kwargs)
File "C:\Users\binhd\Anaconda3\lib\site-packages\tensorflow_core\python\autograph\impl\api.py", line 292, in wrapper
return func(*args, **kwargs)
File "C:\Users\binhd\Anaconda3\lib\site-packages\tensorflow_core\python\keras\engine\training_v2_utils.py", line 433, in train_on_batch
output_loss_metrics=model._output_loss_metrics)
File "C:\Users\binhd\Anaconda3\lib\site-packages\tensorflow_core\python\keras\engine\training_eager.py", line 312, in train_on_batch
output_loss_metrics=output_loss_metrics))
File "C:\Users\binhd\Anaconda3\lib\site-packages\tensorflow_core\python\keras\engine\training_eager.py", line 253, in _process_single_batch
training=training))
File "C:\Users\binhd\Anaconda3\lib\site-packages\tensorflow_core\python\keras\engine\training_eager.py", line 167, in _model_loss
per_sample_losses = loss_fn.call(targets[i], outs[i])
IndexError: list index out of range
如何解决此问题？我可以使用以下代码重新创建您的问题。如果未在
model.fit（）函数中传递目标变量或标签，则会发生错误
注意-您可以从下载我在代码中使用的数据集
错误代码-
# MLP for Pima Indians Dataset saved to single file
import numpy as np
from numpy import loadtxt
import tensorflow as tf
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense
# load pima indians dataset
dataset = np.loadtxt("/content/pima-indians-diabetes.csv", delimiter=",")
# split into input (X) and output (Y) variables
X = dataset[:,0:8]
Y = dataset[:,8]
# define model
model = Sequential()
model.add(Dense(12, input_dim=8, activation='relu'))
model.add(Dense(8, activation='relu'))
model.add(Dense(1, activation='sigmoid'))
# compile model
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
# Model Summary
model.summary()
# Fit the model
model.fit(X, epochs=150, batch_size=10, verbose=0)
# evaluate the model
scores = model.evaluate(X, Y, verbose=0)
print("%s: %.2f%%" % (model.metrics_names[1], scores[1]*100))
# MLP for Pima Indians Dataset saved to single file
import numpy as np
from numpy import loadtxt
import tensorflow as tf
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense
# load pima indians dataset
dataset = np.loadtxt("/content/pima-indians-diabetes.csv", delimiter=",")
# split into input (X) and output (Y) variables
X = dataset[:,0:8]
Y = dataset[:,8]
# define model
model = Sequential()
model.add(Dense(12, input_dim=8, activation='relu'))
model.add(Dense(8, activation='relu'))
model.add(Dense(1, activation='sigmoid'))
# compile model
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
# Model Summary
model.summary()
# Fit the model
model.fit(X, Y, epochs=150, batch_size=10, verbose=0)
# evaluate the model
scores = model.evaluate(X, Y, verbose=0)
print("%s: %.2f%%" % (model.metrics_names[1], scores[1]*100))
输出-
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
dense (Dense) (None, 12) 108
_________________________________________________________________
dense_1 (Dense) (None, 8) 104
_________________________________________________________________
dense_2 (Dense) (None, 1) 9
=================================================================
Total params: 221
Trainable params: 221
Non-trainable params: 0
_________________________________________________________________
---------------------------------------------------------------------------
IndexError Traceback (most recent call last)
<ipython-input-1-b9b0f557b27e> in <module>()
25
26 # Fit the model
---> 27 model.fit(X, epochs=150, batch_size=10, verbose=0)
28
29 # evaluate the model
1 frames
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training_arrays.py in fit_loop(model, inputs, targets, sample_weights, batch_size, epochs, verbose, callbacks, val_inputs, val_targets, val_sample_weights, shuffle, callback_metrics, initial_epoch, steps_per_epoch, validation_steps)
185 indices_for_conversion_to_dense = []
186 for i in range(len(feed)):
--> 187 if issparse is not None and issparse(ins[i]) and not K.is_sparse(feed[i]):
188 indices_for_conversion_to_dense.append(i)
189
IndexError: list index out of range
输出-
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
dense_3 (Dense) (None, 12) 108
_________________________________________________________________
dense_4 (Dense) (None, 8) 104
_________________________________________________________________
dense_5 (Dense) (None, 1) 9
=================================================================
Total params: 221
Trainable params: 221
Non-trainable params: 0
_________________________________________________________________
acc: 70.96%
我可以使用下面的代码重新创建您的问题。如果未在model.fit（）函数中传递目标变量或标签，则会发生错误
注意-您可以从下载我在代码中使用的数据集
错误代码-
# MLP for Pima Indians Dataset saved to single file
import numpy as np
from numpy import loadtxt
import tensorflow as tf
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense
# load pima indians dataset
dataset = np.loadtxt("/content/pima-indians-diabetes.csv", delimiter=",")
# split into input (X) and output (Y) variables
X = dataset[:,0:8]
Y = dataset[:,8]
# define model
model = Sequential()
model.add(Dense(12, input_dim=8, activation='relu'))
model.add(Dense(8, activation='relu'))
model.add(Dense(1, activation='sigmoid'))
# compile model
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
# Model Summary
model.summary()
# Fit the model
model.fit(X, epochs=150, batch_size=10, verbose=0)
# evaluate the model
scores = model.evaluate(X, Y, verbose=0)
print("%s: %.2f%%" % (model.metrics_names[1], scores[1]*100))
# MLP for Pima Indians Dataset saved to single file
import numpy as np
from numpy import loadtxt
import tensorflow as tf
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense
# load pima indians dataset
dataset = np.loadtxt("/content/pima-indians-diabetes.csv", delimiter=",")
# split into input (X) and output (Y) variables
X = dataset[:,0:8]
Y = dataset[:,8]
# define model
model = Sequential()
model.add(Dense(12, input_dim=8, activation='relu'))
model.add(Dense(8, activation='relu'))
model.add(Dense(1, activation='sigmoid'))
# compile model
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
# Model Summary
model.summary()
# Fit the model
model.fit(X, Y, epochs=150, batch_size=10, verbose=0)
# evaluate the model
scores = model.evaluate(X, Y, verbose=0)
print("%s: %.2f%%" % (model.metrics_names[1], scores[1]*100))
输出-
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
dense (Dense) (None, 12) 108
_________________________________________________________________
dense_1 (Dense) (None, 8) 104
_________________________________________________________________
dense_2 (Dense) (None, 1) 9
=================================================================
Total params: 221
Trainable params: 221
Non-trainable params: 0
_________________________________________________________________
---------------------------------------------------------------------------
IndexError Traceback (most recent call last)
<ipython-input-1-b9b0f557b27e> in <module>()
25
26 # Fit the model
---> 27 model.fit(X, epochs=150, batch_size=10, verbose=0)
28
29 # evaluate the model
1 frames
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training_arrays.py in fit_loop(model, inputs, targets, sample_weights, batch_size, epochs, verbose, callbacks, val_inputs, val_targets, val_sample_weights, shuffle, callback_metrics, initial_epoch, steps_per_epoch, validation_steps)
185 indices_for_conversion_to_dense = []
186 for i in range(len(feed)):
--> 187 if issparse is not None and issparse(ins[i]) and not K.is_sparse(feed[i]):
188 indices_for_conversion_to_dense.append(i)
189
IndexError: list index out of range
输出-
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
dense_3 (Dense) (None, 12) 108
_________________________________________________________________
dense_4 (Dense) (None, 8) 104
_________________________________________________________________
dense_5 (Dense) (None, 1) 9
=================================================================
Total params: 221
Trainable params: 221
Non-trainable params: 0
_________________________________________________________________
acc: 70.96%
请发布您正在获取的错误的完整堆栈跟踪信息hi@thushv89。谢谢你的回复。我发布的错误都是Spyder显示的。@BìnhDưngLư417; ng，据我所知，您在迁移学习中重用的瓶颈功能是由预先训练的模型VGGNet
用预定义形状的图像（224224224）
准备的。由于您的image\u shape=（360560,3）
，一旦模型达到大于224的索引，可能会导致错误，索引器：列表索引超出范围
。如果您设置了image\u shape=（224224,3）
，则应解决此错误。谢谢请发布您正在获取的错误的完整堆栈跟踪信息hi@thushv89。谢谢你的回复。我发布的错误都是Spyder显示的。@BìnhDưngLư417; ng，据我所知，您在迁移学习中重用的瓶颈功能是由预先训练的模型VGGNet
用预定义形状的图像（224224224）
准备的。由于您的image\u shape=（360560,3）
，一旦模型达到大于224的索引，可能会导致错误，索引器：列表索引超出范围
。如果您设置了image\u shape=（224224,3）
，则应解决此错误。谢谢@Bình Dơng Lơng-如果答案回答了您的问题，请您接受并投票。谢谢。@Bình DƵng LƵng-如果答案回答了您的问题，请您接受并投赞成票。非常感谢。