Tensorflow 如何创建自定义pyfunc以使用需要使用MLflow的二维以上输入形状的模型进行预测?

Tensorflow 如何创建自定义pyfunc以使用需要使用MLflow的二维以上输入形状的模型进行预测?,tensorflow,keras,tensorflow2.0,tensorflow-serving,mlflow,Tensorflow,Keras,Tensorflow2.0,Tensorflow Serving,Mlflow,我是TensorFlow和MLFlow的新手,我有一个类似于中所问的问题。我正在实现一个TensorFlow模型来预测时间序列值。出于这个目的,我使用MLFlow的MLFlow.tensorflow.autolog()来跟踪和服务实例中的模型。然而,由于我的输入形状有两个以上的维度,我无法使用该方法 如前所述,我尝试使用自定义pyfunc对预测中的输入数据进行编码/解码 这样,我就有了一个函数模型_test.py,它带有一个预测方法,可以解码其输入数据,即: 导入系统 导入操作系统 导入json

我是TensorFlow和MLFlow的新手,我有一个类似于中所问的问题。我正在实现一个TensorFlow模型来预测时间序列值。出于这个目的,我使用MLFlow的MLFlow.tensorflow.autolog()来跟踪和服务实例中的模型。然而,由于我的输入形状有两个以上的维度,我无法使用该方法

如前所述,我尝试使用自定义pyfunc对预测中的输入数据进行编码/解码

这样,我就有了一个函数模型_test.py,它带有一个预测方法,可以解码其输入数据,即:

导入系统 导入操作系统 导入json 导入mlflow 将numpy作为np导入 作为pd进口熊猫 从mlflow.pyfunc导入PythonModel 导入tensorflow作为tf 导入base64 类模型测试(PythonModel): 定义初始化(self,估计器=None,窗口大小=64,批大小=256,随机缓冲大小=100): #创建实验的代码 self.window\u size=window\u size self.batch\u size=批次大小 self.shuffle\u buffer\u size=shuffle\u buffer\u size def窗口化数据集(自身、系列、窗口大小、批量大小、随机缓冲区): 系列=tf。展开尺寸(系列,轴=-1) ds=tf.data.Dataset.from_tensor_切片(系列) ds=ds.window(window\u size+1,shift=1,drop\u余数=True) ds=ds.平面图(λw:w.批次(窗口大小+1)) ds=ds.shuffle(shuffle\u缓冲区) ds=ds.map(lambdaw:(w[:-1],w[1:])) self.windowed\u ds=ds.batch(批大小)。预取(1) 回归自我 def序列(自身,序列设置,y=无,历元数=500): 模型=tf.keras.models.Sequential([ tf.keras.layers.Conv1D(过滤器=60,内核大小=5, 步幅=1,padding=“因果”, 激活=“relu”, 输入_shape=[None,1]), tf.keras.layers.LSTM(60,返回序列=真), tf.keras.layers.Dense(10,activation=“relu”), tf.keras.致密层(1), tf.keras.layers.Lambda(Lambda x:x*400) ]) 优化器=tf.keras.optimizers.SGD(lr=1e-6,动量=0.9) compile(loss=tf.keras.loss.Huber(),optimizer=optimizer,metrics=[“mae”]) 模型拟合(x=列车组,y=y,历次=5) self.modelo=model 回归自我 def预测(自身,系列编码): #解码到达方法的数据 def解码(x): 返回pd.Series(np.frombuffer(base64.B64解码(x))) 序列解码=解码(序列编码) #预处理数据 系列=np。展开尺寸(系列解码,轴=1) ds=tf.data.Dataset.from_tensor_切片(系列) #用可变窗口大小替换数字 ds=ds.window(60,shift=1,drop\u余数=True) #用可变窗口大小替换数字 ds=ds.平面图(λw:w.批次(60)) ds=ds.批处理(32).预取(1) #预测 预测=自建模预测(ds) 回报预测 以及用于训练和保存模型的run.py文件:

导入操作系统
导入mlflow.pyfunc
将ModelTest导入为model\u测试
导入系统
导入json
导入mlflow
将numpy作为np导入
从pymongo导入MongoClient
作为pd进口熊猫
将matplotlib.pyplot作为plt导入
导入tensorflow作为tf
将预处理作为预处理导入
#@单击(…)#根据MLproject文件定义单击选项
def run():
#从MongoDB加载时间序列数据并对其进行预处理的代码
窗口大小=64
批量大小=256
洗牌缓冲区大小=100
拆分时间=400
系列=np.阵列(数据测向['传感器])
时间=np.数组(数据_df['time'])
时间列车=时间[:分割时间]
x_列=系列[:分割时间]
时间有效=时间[分割时间:]
x_有效=系列[分割时间:]
modelo=modelo_tercero.ModelTest()
模型加窗数据集(x列、窗口大小、批量大小、随机缓冲大小)
使用mlflow.start_run()作为运行:
model=modelo.train(modelo.windowed\u ds)
model\u path=os.path.join('models',run.info.run\u id)
#保存模型
mlflow.pyfunc.save_模型(
路径=模型路径,
python_model=modelo.train(modelo.windowed_ds),
code_path=['modelsthird.py'],
康达环境酒店={
“通道”:[“默认值”,“conda forge”],
“依赖项”:[
‘mlflow=1.6.0’,
‘numpy=1.18.1’,
‘tensorflow=2.1.0’,
“熊猫=0.25.3”,
'python=3.7.6',
'cloudpickle==0.5.8'
],
“名称”:“mlflow env”
}
)
如果名称=“\uuuuu main\uuuuuuuu”:
运行()
当我执行run.py时,在保存模型时会出现下一个错误:

 Traceback (most recent call last):

File "run.py", line 116, in <module>
    run()
  File "run.py", line 110, in run
    'name': 'mlflow-env'
  File "/opt/conda/lib/python3.7/site-packages/mlflow/pyfunc/__init__.py", line 596, in save_model
    code_paths=code_path, mlflow_model=mlflow_model)
  File "/opt/conda/lib/python3.7/site-packages/mlflow/pyfunc/model.py", line 141, in _save_model_with_class_artifacts_params
    cloudpickle.dump(python_model, out)
  File "/opt/conda/lib/python3.7/site-packages/cloudpickle/cloudpickle.py", line 1109, in dump
    CloudPickler(file, protocol=protocol).dump(obj)
  File "/opt/conda/lib/python3.7/site-packages/cloudpickle/cloudpickle.py", line 482, in dump
    return Pickler.dump(self, obj)
  File "/opt/conda/lib/python3.7/pickle.py", line 437, in dump
    self.save(obj)
  File "/opt/conda/lib/python3.7/pickle.py", line 549, in save
    self.save_reduce(obj=obj, *rv)
  File "/opt/conda/lib/python3.7/pickle.py", line 662, in save_reduce
    save(state)
  File "/opt/conda/lib/python3.7/pickle.py", line 504, in save
    f(self, obj) # Call unbound method with explicit self
  File "/opt/conda/lib/python3.7/pickle.py", line 859, in save_dict
    self._batch_setitems(obj.items())
  File "/opt/conda/lib/python3.7/pickle.py", line 885, in _batch_setitems
    save(v)
  File "/opt/conda/lib/python3.7/pickle.py", line 549, in save
    self.save_reduce(obj=obj, *rv)
  File "/opt/conda/lib/python3.7/pickle.py", line 662, in save_reduce
    save(state)
  File "/opt/conda/lib/python3.7/pickle.py", line 504, in save
    f(self, obj) # Call unbound method with explicit self
  File "/opt/conda/lib/python3.7/pickle.py", line 859, in save_dict
    self._batch_setitems(obj.items())
  File "/opt/conda/lib/python3.7/pickle.py", line 885, in _batch_setitems
    save(v)
  File "/opt/conda/lib/python3.7/pickle.py", line 549, in save
    self.save_reduce(obj=obj, *rv)
  File "/opt/conda/lib/python3.7/pickle.py", line 662, in save_reduce
    save(state)
  File "/opt/conda/lib/python3.7/pickle.py", line 504, in save
    f(self, obj) # Call unbound method with explicit self
  File "/opt/conda/lib/python3.7/pickle.py", line 859, in save_dict
    self._batch_setitems(obj.items())
  File "/opt/conda/lib/python3.7/pickle.py", line 885, in _batch_setitems
    save(v)
  File "/opt/conda/lib/python3.7/pickle.py", line 549, in save
    self.save_reduce(obj=obj, *rv)
  File "/opt/conda/lib/python3.7/pickle.py", line 662, in save_reduce
    save(state)
  File "/opt/conda/lib/python3.7/pickle.py", line 504, in save
    f(self, obj) # Call unbound method with explicit self
  File "/opt/conda/lib/python3.7/pickle.py", line 859, in save_dict
    self._batch_setitems(obj.items())
  File "/opt/conda/lib/python3.7/pickle.py", line 885, in _batch_setitems
    save(v)
  File "/opt/conda/lib/python3.7/pickle.py", line 549, in save
    self.save_reduce(obj=obj, *rv)
  File "/opt/conda/lib/python3.7/pickle.py", line 662, in save_reduce
    save(state)
  File "/opt/conda/lib/python3.7/pickle.py", line 504, in save
    f(self, obj) # Call unbound method with explicit self
  File "/opt/conda/lib/python3.7/pickle.py", line 859, in save_dict
    self._batch_setitems(obj.items())
  File "/opt/conda/lib/python3.7/pickle.py", line 885, in _batch_setitems
    save(v)
  File "/opt/conda/lib/python3.7/pickle.py", line 549, in save
    self.save_reduce(obj=obj, *rv)
  File "/opt/conda/lib/python3.7/pickle.py", line 662, in save_reduce
    save(state)
  File "/opt/conda/lib/python3.7/pickle.py", line 504, in save
    f(self, obj) # Call unbound method with explicit self
  File "/opt/conda/lib/python3.7/pickle.py", line 859, in save_dict
    self._batch_setitems(obj.items())
  File "/opt/conda/lib/python3.7/pickle.py", line 885, in _batch_setitems
    save(v)
  File "/opt/conda/lib/python3.7/pickle.py", line 549, in save
    self.save_reduce(obj=obj, *rv)
  File "/opt/conda/lib/python3.7/pickle.py", line 662, in save_reduce
    save(state)
  File "/opt/conda/lib/python3.7/pickle.py", line 504, in save
    f(self, obj) # Call unbound method with explicit self
  File "/opt/conda/lib/python3.7/pickle.py", line 859, in save_dict
    self._batch_setitems(obj.items())
  File "/opt/conda/lib/python3.7/pickle.py", line 885, in _batch_setitems
    save(v)
  File "/opt/conda/lib/python3.7/pickle.py", line 549, in save
    self.save_reduce(obj=obj, *rv)
  File "/opt/conda/lib/python3.7/pickle.py", line 662, in save_reduce
    save(state)
  File "/opt/conda/lib/python3.7/pickle.py", line 504, in save
    f(self, obj) # Call unbound method with explicit self
  File "/opt/conda/lib/python3.7/pickle.py", line 859, in save_dict
    self._batch_setitems(obj.items())
  File "/opt/conda/lib/python3.7/pickle.py", line 885, in _batch_setitems
    save(v)
  File "/opt/conda/lib/python3.7/pickle.py", line 524, in save
    rv = reduce(self.proto)
  File "/opt/conda/lib/python3.7/site-packages/tensorflow_core/python/framework/ops.py", line 873, in __reduce__
    return convert_to_tensor, (self._numpy(),)
  File "/opt/conda/lib/python3.7/site-packages/tensorflow_core/python/framework/ops.py", line 910, in _numpy
    six.raise_from(core._status_to_exception(e.code, e.message), None)
  File "<string>", line 3, in raise_from
回溯(最近一次呼叫最后一次):
文件“run.py”,第116行,在
运行()
文件“run.py”,第110行,在run中
“名称”:“mlflow env”
文件“/opt/conda/lib/python3.7/site packages/mlflow/pyfunc/_init__.py”,第596行,在save_模型中
代码路径=代码路径,mlflow模型=mlflow模型)
文件“/opt/conda/lib/python3.7/site packages/mlflow/pyfunc/model.py”,第141行,在带有类工件参数的模型中保存
dump(python_模型,out)
文件“/opt/conda/lib/python3.7/site packages/cloudpickle/cloudpickle.py”,第1109行,位于转储中
CloudPickler(文件,协议=协议).dump(obj)
文件“/opt/conda/lib/python3.7/site packages/cloudpickle/cloudpickle.py”,第482行,在转储文件中
返回Pickler.dump(自、obj)
文件“/opt/conda/lib/python3.7/pickle.py”,第437行,在转储中
自我保存(obj)
文件“/opt/conda/lib/python3.7/pickle.py”,第549行,保存
自我保存(obj=obj,*rv)
文件“/opt/conda