Deep learning tf.data.Dataset:不能为给定的输入类型指定'batch_size'参数

我正在使用Talos和Google colabTPU来运行Keras模型的超参数调优。注意，我使用的是Tensorflow 1.15.0和Keras 2.2.4-tf

import os
import tensorflow as tf
import talos as ta
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense
from tensorflow.keras.optimizers import Adam
from sklearn.model_selection import train_test_split

def iris_model(x_train, y_train, x_val, y_val, params):

    # Specify a distributed strategy to use TPU
    resolver = tf.contrib.cluster_resolver.TPUClusterResolver(tpu='grpc://' + os.environ['COLAB_TPU_ADDR'])
    tf.contrib.distribute.initialize_tpu_system(resolver)
    strategy = tf.contrib.distribute.TPUStrategy(resolver)

    # Use the strategy to create and compile a Keras model
    with strategy.scope():
      model = Sequential()
      model.add(Dense(32, input_shape=(4,), activation=tf.nn.relu, name="relu"))
      model.add(Dense(3, activation=tf.nn.softmax, name="softmax"))
      model.compile(optimizer=Adam(learning_rate=0.1), loss=params['losses'])

    # Convert data type to use TPU
    x_train = x_train.astype('float32')
    x_val = x_val.astype('float32')

    dataset = tf.data.Dataset.from_tensor_slices((x_train, y_train))
    dataset = dataset.cache()
    dataset = dataset.shuffle(1000, reshuffle_each_iteration=True).repeat()
    dataset = dataset.batch(params['batch_size'], drop_remainder=True)

    # Fit the Keras model on the dataset
    out = model.fit(dataset, batch_size=params['batch_size'], epochs=params['epochs'], validation_data=[x_val, y_val], verbose=0, steps_per_epoch=2)

    return out, model

# Load dataset
X, y = ta.templates.datasets.iris()

# Train and test set
x_train, x_val, y_train, y_val = train_test_split(X, y, test_size=0.30, shuffle=False)

# Create a hyperparameter distributions 
p = {'losses': ['logcosh'], 'batch_size': [128, 256, 384, 512, 1024], 'epochs': [10, 20]}

# Use Talos to scan the best hyperparameters of the Keras model
scan_object = ta.Scan(x_train, y_train, params=p, model=iris_model, experiment_name='test', x_val=x_val, y_val=y_val, fraction_limit=0.1)

使用

tf.data.Dataset

将列车组转换为数据集后，当使用

out=model.fit

拟合模型时，我得到以下错误：

---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-3-c812209b95d0> in <module>()
      8 
      9 # Use Talos to scan the best hyperparameters of the Keras model
---> 10 scan_object = ta.Scan(x_train, y_train, params=p, model=iris_model, experiment_name='test', x_val=x_val, y_val=y_val, fraction_limit=0.1)

8 frames
/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/keras/engine/training.py in _validate_or_infer_batch_size(self, batch_size, steps, x)
   1813             'The `batch_size` argument must not be specified for the given '
   1814             'input type. Received input: {}, batch_size: {}'.format(
-> 1815                 x, batch_size))
   1816       return
   1817 

ValueError: The `batch_size` argument must not be specified for the given input type. Received input: <DatasetV1Adapter shapes: ((512, 4), (512, 3)), types: (tf.float32, tf.float32)>, batch_size: 512

---

keras分布式代码似乎存在问题

如果你看看

---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
<ipython-input-3-c812209b95d0> in <module>()
      8 
      9 # Use Talos to scan the best hyperparameters of the Keras model
---> 10 scan_object = ta.Scan(x_train, y_train, params=p, model=iris_model, experiment_name='test', x_val=x_val, y_val=y_val, fraction_limit=0.1)

8 frames
/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/keras/engine/training.py in _distribution_standardize_user_data(self, x, y, sample_weight, class_weight, batch_size, validation_split, shuffle, epochs, allow_partial_batch)
   2307             strategy) and not drop_remainder:
   2308           dataset_size = first_x_value.shape[0]
-> 2309           if dataset_size % batch_size == 0:
   2310             drop_remainder = True
   2311 

TypeError: unsupported operand type(s) for %: 'int' and 'NoneType'

有一个步骤

batch_size = model._validate_or_infer_batch_size(
            batch_size, steps_per_epoch, x)

其中，批处理大小应该从“无”（未指定时的默认值）更改为从Dataset对象推断出的大小（但不是，我通过打印变量进行了检查）。我认为这可能与您的批次大小实际上是一个批次大小列表有关。如果将源代码（您可以直接从collab编辑它，然后单击restart runtime进行尝试）更改为：

（请参阅，我在源代码中手动插入了批次大小，该大小位于应该推断的点之后）程序运行时没有错误

---

可能尝试不同批次大小的HypParameter tunning是当前版本不可行的功能。我尝试了TF2.1，但也没有成功。

嗨，欢迎来到AI SE！不幸的是，我们这里不处理编程问题。有关更多详细信息，请参阅。我将把这篇文章迁移到堆栈溢出。当您在数据集对象上使用

batch

时，调用

fit

时，您不需要使用

batch\u size

。这是带有复制错误隔离的协作，谢谢您的回复，我已经尝试过了，但得到了以下错误：/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py中提供的批大小（self、批大小、步骤、x）1872。1873“”“->1874如果批次大小不是None且是Instance（x，dataset\u ops.DatasetV2）：1875 raise VALUE ERROR（'1876'使用dataset作为输入时，不得指定

批次大小

参数）。”ValueError：将dataset用作输入时，不能指定

batch\u size

参数。您到底尝试了什么？在源代码中手动添加批次大小？从错误中可以看出，您在前一行中添加了该参数（在调用validate\u或\u Inferre[…]之前，而不是之后）我尝试过手动添加batch\u size=128，如您所说：batch\u size=self.\u validate\u或\u expert\u batch\u size（batch\u size，steps，x）batch\u size=128您的跟踪错误出现在“\u validate\u或\u expert\u batch\u size”中，它在代码更改之前，因此独立于此。此错误消息表示，当它进入_validate函数时，“批处理大小”已不同于“无”。您是否仍在协作中传递批大小？你检查过我在第一条评论中分享的colab链接了吗？

  def fit(self,
          model,
          x=None,
          y=None,
          batch_size=None,
          epochs=1,
          verbose=1,
          callbacks=None,
          validation_split=0.,
          validation_data=None,
          shuffle=True,
          class_weight=None,
          sample_weight=None,
          initial_epoch=0,
          steps_per_epoch=None,
          validation_steps=None,
          validation_freq=1,
          **kwargs):
    """Fit loop for Distribution Strategies."""
    dist_utils.validate_callbacks(input_callbacks=callbacks,
                                  optimizer=model.optimizer)
    dist_utils.validate_inputs(x, y)

    batch_size, steps_per_epoch = dist_utils.process_batch_and_step_size(
        model._distribution_strategy,
        x,
        batch_size,
        steps_per_epoch,
        ModeKeys.TRAIN,
        validation_split=validation_split)
    batch_size = model._validate_or_infer_batch_size(
        batch_size, steps_per_epoch, x)
    dataset = model._distribution_standardize_user_data(

batch_size = model._validate_or_infer_batch_size(
            batch_size, steps_per_epoch, x)

batch_size, steps_per_epoch = dist_utils.process_batch_and_step_size(
    model._distribution_strategy,
    x,
    batch_size,
    steps_per_epoch,
    ModeKeys.TRAIN,
    validation_split=validation_split)
batch_size = model._validate_or_infer_batch_size(
    batch_size, steps_per_epoch, x)
batch_size = 128
dataset = model._distribution_standardize_user_data(