在超参数优化Python中找不到泄漏的ReLU

我正在使用一个高参数字典和一个神经网络超参数优化函数，如下所示：

from tensorflow.keras.layers import LeakyReLU

parameters=[
    {
        "name": "learning_rate",
        "type": "range",
        "bounds": [0.001, 0.5],
        "log_scale": True,
    },
    {
        "name": "dropout_rate",
        "type": "range",
        "bounds": [0.01, 0.9],
        "log_scale": True,
    },
    {
        "name": "num_hidden_layers",
        "type": "range",
        "bounds": [1, 7],
        "value_type": "int"
    },
    {
        "name": "neurons_per_layer",
        "type": "range",
        "bounds": [1, 300],
        "value_type": "int"
    },
    {
        "name": "batch_size",
        "type": "choice",
        "values": [8, 10, 16, 20, 30],
    },
    
    {
        "name": "activation",
        "type": "choice",
        "values": [ 'LeakyReLU(alpha=0.3)', 'relu'],
    },
    {
        "name": "optimizer",
        "type": "choice",
        "values": ['adam', 'rms', 'sgd'],
    },
]

# This returns a multi-layer-perceptron model in Keras.
def get_keras_model(num_hidden_layers, 
                    num_neurons_per_layer, 
                    dropout_rate, 
                    activation):
    # create the MLP model.
    
    # define the layers.
    inputs = tf.keras.Input(shape=(train_dataset.shape[1],))  # input layer.
    x = layers.Dropout(dropout_rate)(inputs) # dropout on the weights.
    
    # Add the hidden layers.
    for i in range(num_hidden_layers):
        x = layers.Dense(num_neurons_per_layer, 
                         activation=activation)(x)
        x = layers.Dropout(dropout_rate)(x)
    
    # output layer.
    outputs = layers.Dense(1, activation='linear')(x)
    
    model = tf.keras.Model(inputs=inputs, outputs=outputs)
    return model
    

# This function takes in the hyperparameters and returns a score (Cross validation). 
# Returns the mean of the validation loss based on which we decide which algorithm has the best hyperparameters
def keras_mlp_cv_score(parameterization, weight=None):
    
    model = get_keras_model(parameterization.get('num_hidden_layers'),
                            parameterization.get('neurons_per_layer'),
                            parameterization.get('dropout_rate'),
                            parameterization.get('activation'))
    
    opt = parameterization.get('optimizer')
    opt = opt.lower()
    
    learning_rate = parameterization.get('learning_rate')
    
    if opt == 'adam':
        optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate)
    elif opt == 'rms':
        optimizer = tf.keras.optimizers.RMSprop(learning_rate=learning_rate)
    else:
        optimizer = tf.keras.optimizers.SGD(learning_rate=learning_rate)
        
    act = parameterization.get('activation')
    act = act.lower()
    
    if act == 'leakyrelu': 
        activation = ""
        get_keras_model.add(tf.layers.leakyReLU())
    
    
    NUM_EPOCHS = 100
    
    # Specify the training configuration.
    model.compile(optimizer=optimizer,
                  loss=tf.keras.losses.MeanSquaredError(),
                  metrics=['mae', 'mse'] )

    data = X_train
    labels = y_train.values
    
    early_stop = keras.callbacks.EarlyStopping(monitor='val_loss', patience=10)

    
    # fit the model using a 20% validation set. with a patience of 10 to avoid overfitting
    res = model.fit(data, labels, epochs=NUM_EPOCHS, batch_size=parameterization.get('batch_size'),
                    validation_split=0.2, callbacks=[early_stop, tfdocs.modeling.EpochDots()])
    
    # look at the last 10 epochs. Get the mean and standard deviation of the validation score.
    last10_scores = np.array(res.history['val_loss'][-10:])
    mean = last10_scores.mean()
    sem = last10_scores.std()
    
    # If the model didn't converge then set a high loss.
    if np.isnan(mean):
        return 9999.0, 0.0
    
    return mean, sem

但无论我如何使用LeakyReLU，它都会抛出一个错误，即未找到激活函数。我还尝试了

tf.nn.leaky\u relu

请帮助我将LeakyReLU正确地合并到我的代码中。

您写道：

act = act.lower()    

if act == 'LeakyReLU': 
    ...

此测试总是错误的，因为

'LeakyReLU'

有一些大写字母，而

act

从来没有，因此它永远不会向模型中添加LeakyReLU层

尝试：

另外，正确的语法是

tf.keras.layers.LeakyReLU（）

（）

如果需要，请使用错误跟踪更新您的问题

请试试这个：

def get_keras_model(num_hidden_layers, 
                num_neurons_per_layer, 
                dropout_rate, 
                activation):
    # create the MLP model.

    # define the layers.
    inputs = tf.keras.Input(shape=(train_dataset.shape[1],))  # input layer.
    x = layers.Dropout(dropout_rate)(inputs) # dropout on the weights.

    leakyrelu =  activation.lower() == 'leakyrelu'

    # Add the hidden layers.
    for i in range(num_hidden_layers):
        x = layers.Dense(num_neurons_per_layer, 
                     activation=activation if not leakyrelu else None)(x)
        if leakyrelu:
            x = LeakyReLU()(x)
        x = layers.Dropout(dropout_rate)(x)    


def keras_mlp_cv_score(parameterization, weight=None):

    model = get_keras_model(parameterization.get('num_hidden_layers'),
                        parameterization.get('neurons_per_layer'),
                        parameterization.get('dropout_rate'),
                        parameterization.get('activation'))

    # No need for "if act == 'leakyrelu':" etc anymore.
....
    

---

对您已经导入了LeakyReLU（在第一行）。因此，您只需执行

get\u keras\u model.add（LeakyReLU（））

我认为将LeakyReLU添加到模型中的函数部分根本没有执行。在第二个方法开始时，如果您似乎调用了

get\u keras\u model（）

，则直接将其视为未找到，而不检查函数中的if条件，忽略对模型所做的操作。

get\u keras\u model.add（tf.layers.leakyReLU（））

也很奇怪，它应该是

model.add（tf.layers.leakyReLU（））

。我更新了我的答案。告诉我它是否有效。是的，它是有意义的。我能想到的唯一方法是，在没有重大编辑的情况下，用你的另一种方法来处理这个问题。我更新了我的答案。

def get_keras_model(num_hidden_layers, 
                num_neurons_per_layer, 
                dropout_rate, 
                activation):
    # create the MLP model.

    # define the layers.
    inputs = tf.keras.Input(shape=(train_dataset.shape[1],))  # input layer.
    x = layers.Dropout(dropout_rate)(inputs) # dropout on the weights.

    leakyrelu =  activation.lower() == 'leakyrelu'

    # Add the hidden layers.
    for i in range(num_hidden_layers):
        x = layers.Dense(num_neurons_per_layer, 
                     activation=activation if not leakyrelu else None)(x)
        if leakyrelu:
            x = LeakyReLU()(x)
        x = layers.Dropout(dropout_rate)(x)    


def keras_mlp_cv_score(parameterization, weight=None):

    model = get_keras_model(parameterization.get('num_hidden_layers'),
                        parameterization.get('neurons_per_layer'),
                        parameterization.get('dropout_rate'),
                        parameterization.get('activation'))

    # No need for "if act == 'leakyrelu':" etc anymore.
....