Python 自定义生成器以Keras为单位在数组输入中生成批量大小

我正在尝试使用一个自定义图像数据生成器，它将n个帧数组（来自视频）堆叠为每个输入的序列

我在这里找到了代码：

代码如下：

import cv2 as cv
import os
import glob
import numpy as np
import random
from tensorflow import keras
import keras.preprocessing.image
from tensorflow.python.keras.utils.data_utils import Sequence

class VideoFrameGenerator(Sequence):
    
    def __init__(self,
                 rescale=1/255.,
                 nbframe:int=5,
                 classes:list=[],
                 batch_size:int=16,
                 use_frame_cache:bool=False,
                 target_shape:tuple=(224, 224),
                 shuffle:bool=True,
                 transformation:keras.preprocessing.image.ImageDataGenerator=None,
                 split:float=None,
                 nb_channel:int=3,
                 glob_pattern:str='./videos/{classname}/*.avi',
                 _validation_data:list=None):
        """ Create a generator that return batches of frames from video
        - nbframe: int, number of frame to return for each sequence
        - classes: list of str, classes to infer
        - batch_size: int, batch size for each loop
        - use_frame_cache: bool, use frame cache (may take a lot of memory for large dataset)
        - shape: tuple, target size of the frames
        - shuffle: bool, randomize files
        - transformation: ImageDataGenerator with transformations
        - split: float, factor to split files and validation
        - nb_channel: int, 1 or 3, to get grayscaled or RGB images
        - glob_pattern: string, directory path with '{classname}' inside that
            will be replaced by one of the class list
        - _validation_data: already filled list of data, do not touch !
            
        You may use the "classes" property to retrieve the class list afterward.
        
        The generator has that properties initialized:
        - classes_count: number of classes that the generator manages
        - files_count: number of video that the generator can provides
        - classes: the given class list
        - files: the full file list that the generator will use, this
            is usefull if you want to remove some files that should not be
            used by the generator.
        """
        
        # should be only RGB or Grayscale
        assert nb_channel in (1, 3)
        
        # we should have classes
        assert len(classes) > 0
        
        # shape size should be 2
        assert len(target_shape) == 2
        
        # split factor should be a propoer value
        if split is not None:
            assert split < 1.0 and split > 0.0

        # be sure that classes are well ordered
        classes.sort()
        
        self.rescale = rescale
        self.classes = classes
        self.batch_size = batch_size
        self.nbframe = nbframe
        self.shuffle = shuffle
        self.target_shape = target_shape
        self.nb_channel = nb_channel
        self.transformation = transformation
        self.use_frame_cache = use_frame_cache
        
        self._random_trans = []
        self.__frame_cache = {}
        self.files = []
        self.validation = []
        
        if _validation_data is not None:
            # we only need to set files here
            self.files = _validation_data
        else:
            if split is not None and split > 0.0:
                for c in classes:
                    files = glob.glob(glob_pattern.format(classname=c))
                    nbval = int(split * len(files))

                    print("class %s, validation count: %d" % (c, nbval))
                    
                    # generate validation indexes
                    indexes = np.arange(len(files))
                    
                    if shuffle:
                        np.random.shuffle(indexes)
                    
                    val = np.random.permutation(indexes)[:nbval]  # get some sample
                    indexes = np.array([i for i in indexes if i not in val]) # remove validation from train

                    # and now, make the file list
                    self.files += [files[i] for i in indexes]
                    self.validation += [files[i] for i in val]

            else:
                for c in classes:
                    self.files += glob.glob(glob_pattern.format(classname=c))

        # build indexes
        self.files_count = len(self.files)
        self.indexes = np.arange(self.files_count)
        self.classes_count = len(classes)
        
        self.on_epoch_end()  # to initialize transformations and shuffle indices
        
        print("get %d classes for %d files for %s" % (
            self.classes_count, 
            self.files_count,
            'train' if _validation_data is None else 'validation'))
            
    def get_validation_generator(self):
        """ Return the validation generator if you've provided split factor """
        return self.__class__(
            nbframe = self.nbframe,
            nb_channel=self.nb_channel,
            target_shape=self.target_shape,
            classes=self.classes,
            batch_size=self.batch_size,
            shuffle=self.shuffle,
            rescale=self.rescale,
            _validation_data=self.validation)
    
    def on_epoch_end(self):
        #prepare transformation to avoid __getitem__ to reinitialize them
        if self.transformation is not None:
            self._random_trans = []
            for i in range(self.files_count):
                self._random_trans.append(
                    self.transformation.get_random_transform(self.target_shape)
                )

        if self.shuffle:
            np.random.shuffle(self.indexes)
    
    def __len__(self):
        return int(np.floor(self.files_count / self.batch_size))
            
    def __getitem__(self, index):
        classes = self.classes
        shape = self.target_shape
        nbframe = self.nbframe
                
        labels = []
        images = []
        
        indexes = self.indexes[index*self.batch_size:(index+1)*self.batch_size]

        t = None
            
        for i in indexes:
            # prepare a transformation if provided
            if self.transformation is not None:
                t = self._random_trans[i]
            
            # video = random.choice(files)
            video = self.files[i]
            cl = video.split(os.sep)[-2]

            # create a label array and set 1 to the right column
            
            label = np.zeros(len(classes))
            col = classes.index(int(cl))
            label[col] = 1.

            if video not in self.__frame_cache:
                cap = cv.VideoCapture(video)
                frames = []
                while True:
                    grabbed, frame = cap.read()
                    if not grabbed:
                        break        

                    # resize
                    frame = cv.resize(frame, shape)

                    # use RGB or Grayscale ?
                    if self.nb_channel == 3:
                        frame = cv.cvtColor(frame, cv.COLOR_BGR2RGB)
                    else:
                        frame = cv.cvtColor(frame, cv.COLOR_RGB2GRAY)

                    # to np
                    frame = keras.preprocessing.image.img_to_array(frame) * self.rescale

                    # keep frame
                    frames.append(frame)

                # Add 2 frames to drop first and last frame
                jump = len(frames)//(nbframe+2)

                # get only some images
                try:
                    frames = frames[jump::jump][:nbframe]
                    print(len(frames))

                except Exception as e:
                    print(video)
                    raise e
                
                # add to frame cache to not read from disk later
                if self.use_frame_cache:
                    self.__frame_cache[video] = frames
            else:
                frames = self.__frame_cache[video]

            # apply transformation
            if t is not None:
                frames = [self.transformation.apply_transform(frame, t) for frame in frames]
            
            # add the sequence in batch
            images.append(frames)
            labels.append(label)
                        
            
        return np.array(images), np.array(labels)

我用这个：

x = VideoFrameGenerator(
                 rescale=1/255.,
                 nbframe=10,
                 classes=list(range(0,14)),
                 batch_size=2,
                 use_frame_cache=False,
                 target_shape=(224, 224),
                 shuffle=True,
                 transformation=None,
                 split=.2,
                 nb_channel=3,
                 glob_pattern='../iemocap_data/label_split_videos/{classname}/*.mp4',
                 _validation_data=None
                       )

from keras.models import Sequential
from keras.layers import Dense, Conv2D

# define the keras model
model = Sequential()
model.add(Conv2D(32, (3, 3), activation='relu', input_shape=(224,224,3)))
model.add(Dense(8, activation='relu'))
model.add(Dense(14, activation='softmax'))
# compile the keras model
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
print(model.summary())
# fit the keras model on the dataset
model.fit_generator(x,
                   steps_per_epoch=10,
                   )

我得到了这个错误：

Model: "sequential_51"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
conv2d_16 (Conv2D)           (None, 222, 222, 32)      896       
_________________________________________________________________
dense_125 (Dense)            (None, 222, 222, 8)       264       
_________________________________________________________________
dense_126 (Dense)            (None, 222, 222, 14)      126       
=================================================================
Total params: 1,286
Trainable params: 1,286
Non-trainable params: 0
_________________________________________________________________
None
Epoch 1/1
10
10
---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-164-29acc9e936f8> in <module>
     12 # fit the keras model on the dataset
     13 model.fit_generator(x,
---> 14                    steps_per_epoch=10,
     15                    )

~/anaconda3/envs/tensorflow_2/lib/python3.7/site-packages/keras/legacy/interfaces.py in wrapper(*args, **kwargs)
     89                 warnings.warn('Update your `' + object_name + '` call to the ' +
     90                               'Keras 2 API: ' + signature, stacklevel=2)
---> 91             return func(*args, **kwargs)
     92         wrapper._original_function = func
     93         return wrapper

~/anaconda3/envs/tensorflow_2/lib/python3.7/site-packages/keras/engine/training.py in fit_generator(self, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, validation_freq, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch)
   1730             use_multiprocessing=use_multiprocessing,
   1731             shuffle=shuffle,
-> 1732             initial_epoch=initial_epoch)
   1733 
   1734     @interfaces.legacy_generator_methods_support

~/anaconda3/envs/tensorflow_2/lib/python3.7/site-packages/keras/engine/training_generator.py in fit_generator(model, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, validation_freq, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch)
    218                                             sample_weight=sample_weight,
    219                                             class_weight=class_weight,
--> 220                                             reset_metrics=False)
    221 
    222                 outs = to_list(outs)

~/anaconda3/envs/tensorflow_2/lib/python3.7/site-packages/keras/engine/training.py in train_on_batch(self, x, y, sample_weight, class_weight, reset_metrics)
   1506             x, y,
   1507             sample_weight=sample_weight,
-> 1508             class_weight=class_weight)
   1509         if self._uses_dynamic_learning_phase():
   1510             ins = x + y + sample_weights + [1]

~/anaconda3/envs/tensorflow_2/lib/python3.7/site-packages/keras/engine/training.py in _standardize_user_data(self, x, y, sample_weight, class_weight, check_array_lengths, batch_size)
    577             feed_input_shapes,
    578             check_batch_axis=False,  # Don't enforce the batch size.
--> 579             exception_prefix='input')
    580 
    581         if y is not None:

~/anaconda3/envs/tensorflow_2/lib/python3.7/site-packages/keras/engine/training_utils.py in standardize_input_data(data, names, shapes, check_batch_axis, exception_prefix)
    133                         ': expected ' + names[i] + ' to have ' +
    134                         str(len(shape)) + ' dimensions, but got array '
--> 135                         'with shape ' + str(data_shape))
    136                 if not check_batch_axis:
    137                     data_shape = data_shape[1:]

ValueError: Error when checking input: expected conv2d_16_input to have 4 dimensions, but got array with shape (2, 10, 224, 224, 3)

10
10

型号：“顺序_51”
_________________________________________________________________
层（类型）输出形状参数
=================================================================
conv2d_16（conv2d）（无、222、222、32）896
_________________________________________________________________
密集型_125（密集型）（无、222、222、8）264
_________________________________________________________________
密集型（密集型）（无、222、222、14）126
=================================================================
总参数：1286
可培训参数：1286
不可训练参数：0
_________________________________________________________________
没有一个
纪元1/1
10
10
---------------------------------------------------------------------------
ValueError回溯（最近一次调用上次）
在里面
12#在数据集上拟合keras模型
13型号。安装发电机（x，
--->每个历元14步=10，
15                    )
包装中的~/anaconda3/envs/tensorflow_2/lib/python3.7/site-packages/keras/legacy/interfaces.py（*args，**kwargs）
89 warnings.warn（'Update your`'+object\u name+'`调用+
90'Keras 2 API:'+签名，堆栈级别=2）
--->91返回函数（*args，**kwargs）
92包装器._原始函数=func
93返回包装器
~/anaconda3/envs/tensorflow\u 2/lib/python3.7/site-packages/keras/engine/training.py-in-fit\u生成器（self、生成器、每个历元的步骤、历元、冗余、回调、验证数据、验证步骤、验证频率、类权重、最大队列大小、工作者、使用多处理、无序、初始历元）
1730使用多处理=使用多处理，
1731洗牌=洗牌，
->1732初始_历元=初始_历元）
1733
1734@interfaces.legacy\u生成器\u方法\u支持
~/anaconda3/envs/tensorflow\u 2/lib/python3.7/site-packages/keras/engine/training\u generator.py in-fit\u generator（模型、生成器、每个历元的步骤、历元、冗余、回调、验证数据、验证步骤、验证频率、类权重、最大队列大小、工人、使用多处理、无序、初始历元）
218样品重量=样品重量，
219级重量=级重量，
-->220重置（度量值=假）
221
222 outs=待办名单（outs）
~/anaconda3/envs/tensorflow\u 2/lib/python3.7/site-packages/keras/engine/training.py in train\u on\u on\u批次（自身、x、y、样本重量、等级重量、重置度量）
1506 x，y，
1507样品重量=样品重量，
->1508级重量=级重量）
1509如果self.\u使用\u动态\u学习\u阶段（）：
1510英寸=x+y+样本重量+[1]
~/anaconda3/envs/tensorflow\u 2/lib/python3.7/site-packages/keras/engine/training.py in\u-standard\u-user\u数据（自身、x、y、样本重量、类别重量、检查数组长度、批次大小）
577个进纸输入形状，
578检查_batch_axis=False，#不强制执行批大小。
-->579异常（前缀为“输入”）
580
581如果y不是无：
标准化输入数据中的~/anaconda3/envs/tensorflow\u 2/lib/python3.7/site-packages/keras/engine/training\u utils.py（数据、名称、形状、检查批处理轴、异常前缀）
133'：应为“+名称[i]+”以具有”+
134 str（len（shape））+'维度，但得到数组'
-->135'带形状'+str（数据形状））
136如果不检查批次轴：
137数据形状=数据形状[1:]
ValueError：检查输入时出错：预期conv2d_16_输入有4个维度，但得到了形状为（2,10,224,224,3）的数组
10
10

发生什么事了？为什么batchsize是数组的第一个元素，为什么不接受它作为输入