使用Keras Tuner调LSTM模型时Adamax报InvalidArgumentError求助

尝试用Keras Tuner对LSTM模型做超参数调优，测试Adamax优化器时触发InvalidArgumentError，错误提示为lr is not a scalar : [1]。

原实现代码 #

import tensorflow as tf
from keras.models import Sequential
from keras.layers import Dense, SimpleRNN, LSTM, Dropout
import keras_tuner
from tensorflow.keras.callbacks import EarlyStopping

def build_lstm_for_tuning(hp):
    activation=['relu','sigmoid']
    lossfct='binary_crossentropy'
    hidden_units_first_layer = hp.Choice('neurons first layer',[32,64,128,256,512,1024])
    lr = hp.Choice('learning_rate', [0.0005]), #0.005,0.001,,0.0001,5e-05,1e-05
    optimizer_name = hp.Choice('optimizer', ["Adamax"])#,"Ftrl","Adadelta","Adagrad","RMSprop","Nadam","SGD"
    model = Sequential()
    model.add(LSTM(hidden_units_first_layer,input_shape=(24, 237),activation=activation[0]))
    model.add(Dense(units=21, activation=activation[1]))
    optimizer = {"Ftrl":tf.keras.optimizers.Ftrl(lr),"Adadelta":tf.keras.optimizers.Adadelta(lr),"Adagrad":tf.keras.optimizers.Adagrad(lr),\
                                "Adamax":tf.keras.optimizers.Adamax(lr),"RMSprop":tf.keras.optimizers.RMSprop(lr),\
                                "Nadam":tf.keras.optimizers.Nadam(lr),"SGD":tf.keras.optimizers.SGD(lr)}[optimizer_name]
    model.compile(loss=lossfct, optimizer= optimizer,\
        metrics=[tf.keras.metrics.Precision(),tf.keras.metrics.Recall(),tf.keras.metrics.TruePositives(),tf.keras.metrics.AUC(multi_label=True)])
    return model

tuner = keras_tuner.RandomSearch(
    build_lstm_for_tuning,
    objective=keras_tuner.Objective("val_auc", direction="max"),
    max_trials=20,
    overwrite=True)
tuner.search(input_data['X_train'], input_data['Y_train'], epochs=1, batch_size=512, 
             validation_data=(input_data['X_valid'], input_data['Y_valid']))

报错信息 #

WARNING:tensorflow:Layer lstm_1 will not use cuDNN kernels since it doesn't meet the criteria. It will use a generic GPU kernel as fallback when running on GPU.

Search: Running Trial #1

Value             |Best Value So Far |Hyperparameter
1024              |?                 |neurons first layer
0.0005            |?                 |learning_rate
Adamax            |?                 |optimizer

WARNING:tensorflow:Layer lstm will not use cuDNN kernels since it doesn't meet the criteria. It will use a generic GPU kernel as fallback when running on GPU.
---------------------------------------------------------------------------
InvalidArgumentError                      Traceback (most recent call last)
/tmp/ipykernel_263/2162334881.py in <module>
     27     overwrite=True)
     28 tuner.search(input_data['X_train'], input_data['Y_train'], epochs=1, batch_size=512, 
---> 29              validation_data=(input_data['X_valid'], input_data['Y_valid']))

~/.local/lib/python3.7/site-packages/keras_tuner/engine/base_tuner.py in search(self, *fit_args, **fit_kwargs)
    177 
    178             self.on_trial_begin(trial)
---> 179             results = self.run_trial(trial, *fit_args, **fit_kwargs)
    180             # `results` is None indicates user updated oracle in `run_trial()`.
    181             if results is None:

~/.local/lib/python3.7/site-packages/keras_tuner/engine/tuner.py in run_trial(self, trial, *args, **kwargs)
    292             callbacks.append(model_checkpoint)
    293             copied_kwargs["callbacks"] = callbacks
---> 294             obj_value = self._build_and_fit_model(trial, *args, **copied_kwargs)
    295 
    296             histories.append(obj_value)

~/.local/lib/python3.7/site-packages/keras_tuner/engine/tuner.py in _build_and_fit_model(self, trial, *args, **kwargs)
    220         hp = trial.hyperparameters
    221         model = self._try_build(hp)
---> 222         results = self.hypermodel.fit(hp, model, *args, **kwargs)
    223         return tuner_utils.convert_to_metrics_dict(
    224             results, self.oracle.objective, "HyperModel.fit()"

~/.local/lib/python3.7/site-packages/keras_tuner/engine/hypermodel.py in fit(self, hp, model, *args, **kwargs)
    135             If return a float, it should be the `objective` value.
    136         """
---> 137         return model.fit(*args, **kwargs)
    138 
    139 

~/.local/lib/python3.7/site-packages/keras/utils/traceback_utils.py in error_handler(*args, **kwargs)
     65     except Exception as e:  # pylint: disable=broad-except
     66       filtered_tb = _process_traceback_frames(e.__traceback__)
---> 67       raise e.with_traceback(filtered_tb) from None
     68     finally:
     69       del filtered_tb

~/.local/lib/python3.7/site-packages/tensorflow/python/eager/execute.py in quick_execute(op_name, num_outputs, inputs, attrs, ctx, name)
     57     ctx.ensure_initialized()
     58     tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name,
---> 59                                         inputs, attrs, num_outputs)
     60   except core._NotOkStatusException as e:
     61     if name is not None:

InvalidArgumentError:  lr is not a scalar : [1]
     [[node Adamax/Adamax/update_4/ResourceApplyAdaMax
 (defined at /home/cdsw/.local/lib/python3.7/site-packages/keras/optimizer_v2/adamax.py:141)
]] [Op:__inference_train_function_1833827]

Errors may have originated from an input operation.
...

错误原因 #

问题出在lr = hp.Choice('learning_rate', [0.0005]),这行代码末尾的逗号：这个逗号让lr变成了一个包含单个元素的元组(0.0005,)，而非单个标量数值。TensorFlow的优化器（如Adamax）要求传入的学习率是标量类型，传入元组会触发类型不匹配的错误。

解决方法 #

去掉lr赋值行末尾的逗号，让lr成为单个标量值。

修正后的代码 #

import tensorflow as tf
from keras.models import Sequential
from keras.layers import Dense, SimpleRNN, LSTM, Dropout
import keras_tuner
from tensorflow.keras.callbacks import EarlyStopping

def build_lstm_for_tuning(hp):
    activation=['relu','sigmoid']
    lossfct='binary_crossentropy'
    hidden_units_first_layer = hp.Choice('neurons first layer',[32,64,128,256,512,1024])
    # 去掉了末尾的逗号
    lr = hp.Choice('learning_rate', [0.0005]) #0.005,0.001,,0.0001,5e-05,1e-05
    optimizer_name = hp.Choice('optimizer', ["Adamax"])#,"Ftrl","Adadelta","Adagrad","RMSprop","Nadam","SGD"
    model = Sequential()
    model.add(LSTM(hidden_units_first_layer,input_shape=(24, 237),activation=activation[0]))
    model.add(Dense(units=21, activation=activation[1]))
    optimizer = {"Ftrl":tf.keras.optimizers.Ftrl(lr),"Adadelta":tf.keras.optimizers.Adadelta(lr),"Adagrad":tf.keras.optimizers.Adagrad(lr),\
                                "Adamax":tf.keras.optimizers.Adamax(lr),"RMSprop":tf.keras.optimizers.RMSprop(lr),\
                                "Nadam":tf.keras.optimizers.Nadam(lr),"SGD":tf.keras.optimizers.SGD(lr)}[optimizer_name]
    model.compile(loss=lossfct, optimizer= optimizer,\
        metrics=[tf.keras.metrics.Precision(),tf.keras.metrics.Recall(),tf.keras.metrics.TruePositives(),tf.keras.metrics.AUC(multi_label=True)])
    return model

tuner = keras_tuner.RandomSearch(
    build_lstm_for_tuning,
    objective=keras_tuner.Objective("val_auc", direction="max"),
    max_trials=20,
    overwrite=True)
tuner.search(input_data['X_train'], input_data['Y_train'], epochs=1, batch_size=512, 
             validation_data=(input_data['X_valid'], input_data['Y_valid']))

内容的提问来源于stack exchange，提问作者Viktor