Tensorflow 如何使用inception/mobilenet模型为图像分类设置tfserving？

我无法找到合适的文档来成功地为inception或mobilenet模型提供服务，并编写grpc客户端来连接到服务器并执行图像分类

到目前为止，我已经成功地仅在CPU上配置了tfserving映像。无法在我的GPU上运行它

但是，当我发出grpc客户机请求时，请求失败并出现错误

grpc._channel._Rendezvous: <_Rendezvous of RPC that terminated with:
status = StatusCode.INVALID_ARGUMENT
details = "Expects arg[0] to be float but string is provided"
debug_error_string = "{"created":"@1571717090.210000000","description":"Error received from peer","file":"src/core/lib/surface/call.cc","file_line":1017,"grpc_message":"Expects arg[0] to be float but string is provided","grpc_status":3}"

---

正如我所理解的，你的问题中有两个问题

A） 在GPU上运行TFS服务

B） 成功发出grpc客户端请求

from __future__ import print_function

import argparse
import time
import numpy as np
from cv2 import imread

import grpc
from tensorflow.contrib.util import make_tensor_proto
from tensorflow_serving.apis import predict_pb2
from tensorflow_serving.apis import prediction_service_pb2_grpc
import tensorflow as tf


def read_tensor_from_image_file(file_name,
                                input_height=299,
                                input_width=299,
                                input_mean=0,
                                input_std=255):
    input_name = "file_reader"
    output_name = "normalized"
    file_reader = tf.io.read_file(file_name, input_name)
    if file_name.endswith(".png"):
        image_reader = tf.image.decode_png(
            file_reader, channels=3, name="png_reader")
    elif file_name.endswith(".gif"):
        image_reader = tf.squeeze(
            tf.image.decode_gif(file_reader, name="gif_reader"))
    elif file_name.endswith(".bmp"):
        image_reader = tf.image.decode_bmp(file_reader, name="bmp_reader")
    else:
        image_reader = tf.image.decode_jpeg(
            file_reader, channels=3, name="jpeg_reader")
    float_caster = tf.cast(image_reader, tf.float32)
    dims_expander = tf.expand_dims(float_caster, 0)

    resized = tf.compat.v1.image.resize_bilinear(dims_expander, [input_height, input_width])
    normalized = tf.divide(tf.subtract(resized, [input_mean]), [input_std])

    sess = tf.Session(config=tf.ConfigProto(gpu_options=tf.GPUOptions(per_process_gpu_memory_fraction=0.01)))
    result = sess.run(normalized)

    return result


def run(host, port, image, model, signature_name):

    # Preparing tensor from the image
    tensor = read_tensor_from_image_file(file_name='images/bird.jpg', input_height=224, input_width=224, input_mean=128, input_std=128)

    # Preparing the channel
    channel = grpc.insecure_channel('{host}:{port}'.format(host=host, port=port))
    stub = prediction_service_pb2_grpc.PredictionServiceStub(channel)

    # Preparing grpc request
    request = predict_pb2.PredictRequest()
    request.model_spec.name = model
    request.model_spec.signature_name = signature_name
    request.inputs['image'].CopyFrom(make_tensor_proto(tensor, shape=[1, 224, 224, 3]))

    # Making predict request
    result = stub.Predict(request, 10.0)

    # Analysing result to get the prediction output.
    predictions = result.outputs['prediction'].float_val

    print("Predictions : ", predictions)


if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('--host', help='Tensorflow server host name', default='localhost', type=str)
    parser.add_argument('--port', help='Tensorflow server port number', default=8502, type=int)
    parser.add_argument('--image', help='input image', default='bird.jpg', type=str)
    parser.add_argument('--model', help='model name', default='inception', type=str)
    parser.add_argument('--signature_name', help='Signature name of saved TF model',
                        default='serving_default', type=str)

    args = parser.parse_args()
    run(args.host, args.port, args.image, args.model, args.signature_name)

让我们一个接一个地开始

---

在GPU上运行TFS服务

这是一个简单的两步过程

正在从中提取最新图像

请注意上面拉取请求中的标签

最新gpu

，因为它拉取的是为gpu准备的图像

运行docker容器

sudo docker run -p 8502:8500 --mount type=bind,source=/my_model_dir,target=/models/inception --name tfserve_gpu -e MODEL_NAME=inception --gpus device=3 -t tensorflow/serving:latest-gpu

请注意，我已通过参数

--gpus device=3

来选择第三个GPU设备。相应地更改它以选择不同的GPU设备

验证容器是否已通过

docker ps

命令启动

另外，通过

nvidia smi

命令验证gpu是否已分配给TFS docker

英伟达smi的输出

但这里似乎有一个小问题。TFDocker已消耗所有gpu设备内存

要限制gpu内存使用，请使用

每个进程\u gpu\u内存\u分数

标志

sudo docker run -p 8502:8500 --mount type=bind,source=/my_model_dir,target=/models/inception --name tfserve_gpu -e MODEL_NAME=inception --gpus device=3 -t tensorflow/serving:latest-gpu  --per_process_gpu_memory_fraction=0.02

英伟达smi的输出

现在，我们已经成功地在GPU设备上配置了TFS服务docker，并使用了合理的GPU内存。让我们跳到第二个问题

---

提出GRPC客户请求

grpc客户端请求的格式存在问题。tfserving docker映像不会直接以二进制格式获取映像，相反，您必须为该映像生成一个张量，然后将其传递给服务器

下面是发出grpc客户端请求的代码

from __future__ import print_function

import argparse
import time
import numpy as np
from cv2 import imread

import grpc
from tensorflow.contrib.util import make_tensor_proto
from tensorflow_serving.apis import predict_pb2
from tensorflow_serving.apis import prediction_service_pb2_grpc
import tensorflow as tf


def read_tensor_from_image_file(file_name,
                                input_height=299,
                                input_width=299,
                                input_mean=0,
                                input_std=255):
    input_name = "file_reader"
    output_name = "normalized"
    file_reader = tf.io.read_file(file_name, input_name)
    if file_name.endswith(".png"):
        image_reader = tf.image.decode_png(
            file_reader, channels=3, name="png_reader")
    elif file_name.endswith(".gif"):
        image_reader = tf.squeeze(
            tf.image.decode_gif(file_reader, name="gif_reader"))
    elif file_name.endswith(".bmp"):
        image_reader = tf.image.decode_bmp(file_reader, name="bmp_reader")
    else:
        image_reader = tf.image.decode_jpeg(
            file_reader, channels=3, name="jpeg_reader")
    float_caster = tf.cast(image_reader, tf.float32)
    dims_expander = tf.expand_dims(float_caster, 0)

    resized = tf.compat.v1.image.resize_bilinear(dims_expander, [input_height, input_width])
    normalized = tf.divide(tf.subtract(resized, [input_mean]), [input_std])

    sess = tf.Session(config=tf.ConfigProto(gpu_options=tf.GPUOptions(per_process_gpu_memory_fraction=0.01)))
    result = sess.run(normalized)

    return result


def run(host, port, image, model, signature_name):

    # Preparing tensor from the image
    tensor = read_tensor_from_image_file(file_name='images/bird.jpg', input_height=224, input_width=224, input_mean=128, input_std=128)

    # Preparing the channel
    channel = grpc.insecure_channel('{host}:{port}'.format(host=host, port=port))
    stub = prediction_service_pb2_grpc.PredictionServiceStub(channel)

    # Preparing grpc request
    request = predict_pb2.PredictRequest()
    request.model_spec.name = model
    request.model_spec.signature_name = signature_name
    request.inputs['image'].CopyFrom(make_tensor_proto(tensor, shape=[1, 224, 224, 3]))

    # Making predict request
    result = stub.Predict(request, 10.0)

    # Analysing result to get the prediction output.
    predictions = result.outputs['prediction'].float_val

    print("Predictions : ", predictions)


if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('--host', help='Tensorflow server host name', default='localhost', type=str)
    parser.add_argument('--port', help='Tensorflow server port number', default=8502, type=int)
    parser.add_argument('--image', help='input image', default='bird.jpg', type=str)
    parser.add_argument('--model', help='model name', default='inception', type=str)
    parser.add_argument('--signature_name', help='Signature name of saved TF model',
                        default='serving_default', type=str)

    args = parser.parse_args()
    run(args.host, args.port, args.image, args.model, args.signature_name)

我不太确定这是否是发出TFPC客户端请求的最佳方式（，因为客户端需要tensorflow库来准备tensor），但它对我来说很有效

如有任何建议，欢迎提出

from __future__ import print_function

import argparse
import time
import numpy as np
from cv2 import imread

import grpc
from tensorflow.contrib.util import make_tensor_proto
from tensorflow_serving.apis import predict_pb2
from tensorflow_serving.apis import prediction_service_pb2_grpc
import tensorflow as tf


def read_tensor_from_image_file(file_name,
                                input_height=299,
                                input_width=299,
                                input_mean=0,
                                input_std=255):
    input_name = "file_reader"
    output_name = "normalized"
    file_reader = tf.io.read_file(file_name, input_name)
    if file_name.endswith(".png"):
        image_reader = tf.image.decode_png(
            file_reader, channels=3, name="png_reader")
    elif file_name.endswith(".gif"):
        image_reader = tf.squeeze(
            tf.image.decode_gif(file_reader, name="gif_reader"))
    elif file_name.endswith(".bmp"):
        image_reader = tf.image.decode_bmp(file_reader, name="bmp_reader")
    else:
        image_reader = tf.image.decode_jpeg(
            file_reader, channels=3, name="jpeg_reader")
    float_caster = tf.cast(image_reader, tf.float32)
    dims_expander = tf.expand_dims(float_caster, 0)

    resized = tf.compat.v1.image.resize_bilinear(dims_expander, [input_height, input_width])
    normalized = tf.divide(tf.subtract(resized, [input_mean]), [input_std])

    sess = tf.Session(config=tf.ConfigProto(gpu_options=tf.GPUOptions(per_process_gpu_memory_fraction=0.01)))
    result = sess.run(normalized)

    return result


def run(host, port, image, model, signature_name):

    # Preparing tensor from the image
    tensor = read_tensor_from_image_file(file_name='images/bird.jpg', input_height=224, input_width=224, input_mean=128, input_std=128)

    # Preparing the channel
    channel = grpc.insecure_channel('{host}:{port}'.format(host=host, port=port))
    stub = prediction_service_pb2_grpc.PredictionServiceStub(channel)

    # Preparing grpc request
    request = predict_pb2.PredictRequest()
    request.model_spec.name = model
    request.model_spec.signature_name = signature_name
    request.inputs['image'].CopyFrom(make_tensor_proto(tensor, shape=[1, 224, 224, 3]))

    # Making predict request
    result = stub.Predict(request, 10.0)

    # Analysing result to get the prediction output.
    predictions = result.outputs['prediction'].float_val

    print("Predictions : ", predictions)


if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('--host', help='Tensorflow server host name', default='localhost', type=str)
    parser.add_argument('--port', help='Tensorflow server port number', default=8502, type=int)
    parser.add_argument('--image', help='input image', default='bird.jpg', type=str)
    parser.add_argument('--model', help='model name', default='inception', type=str)
    parser.add_argument('--signature_name', help='Signature name of saved TF model',
                        default='serving_default', type=str)

    args = parser.parse_args()
    run(args.host, args.port, args.image, args.model, args.signature_name)