Deep learning MXNet混合块模型并行性

我是MXNet新手，一直在尝试模型并行性。我发现了这篇不错的帖子：

并将代码修改为使用HybridBlock，如下所示：

import numpy as np
import mxnet as mx
from mxnet import nd, autograd, gluon
from mxnet.gluon import HybridBlock

num_inputs = 2
num_outputs = 1
num_examples = 10000


def real_fn(x):
    return 2 * x[:, 0] - 3.4 * x[:, 1] + 4.2


x = np.random.normal(0, 1, (num_examples, num_inputs))
noise = 0.001 * np.random.normal(0, 1, num_examples)
y = real_fn(x) + noise
y = y.reshape(-1, 1)

hidden_layers = 2
num_gpus = hidden_layers + 1
ctxList = [mx.gpu(i) for i in range(num_gpus)]


class MyDenseBlock(HybridBlock):
    def __init__(self, layer_number, size_input, size_output, **kwargs):
        super(MyDenseBlock, self).__init__(**kwargs)

        self.layer_number = layer_number
        self.size_input = size_input
        self.size_output = size_output

        with self.name_scope():
            # add parameters to the Block's ParameterDict.
            self.weight = self.params.get(
                'weight',
                init=mx.init.Xavier(magnitude=2.24),
                shape=(size_input, size_output),
                grad_req='write')

            self.bias = self.params.get(
                'bias',
                init=mx.init.Constant(0.5),
                shape=(size_output,),
                grad_req='write')

    def hybrid_forward(self, F, x, weight, bias):
        x = x.as_in_context(ctxList[self.layer_number])
        with x.context:
            linear = F.broadcast_add(F.dot(x, weight), bias)
            return linear


net = gluon.nn.HybridSequential()
with net.name_scope():
    net.add(MyDenseBlock(0, size_input=2, size_output=2))

    for i in range(hidden_layers - 1):
        net.add(MyDenseBlock(i + 1, size_input=2, size_output=2))

    net.add(MyDenseBlock(i + 2, size_input=2, size_output=1))


print("\ninitializing:")
params = net.collect_params()

for i, param in enumerate(params):
    if 'mydenseblock0' in param:
        params[param].initialize(ctx=ctxList[0])

    elif 'mydenseblock1' in param:
        params[param].initialize(ctx=ctxList[1])

    elif 'mydenseblock2' in param:
        params[param].initialize(ctx=ctxList[2])

    print("  ", i, param, "  ", params[param].list_data()[0].context)

#net.hybridize()


def square_loss(yhat, y):
    return nd.mean((yhat - y) ** 2)


def custom_trainer(updaters, params, ignore_stale_grad=False):
    for i, param in enumerate(params):
        if params[param].grad_req == 'null':
            continue
        if not ignore_stale_grad:
            for data in params[param].list_data():
                if not data._fresh_grad:
                    print("`%s` on context %s has not been updated" % (params[param].name, str(data.context)))
                    assert False

        for upd, arr, grad in zip(updaters, params[param].list_data(), params[param].list_grad()):
            if not ignore_stale_grad or arr._fresh_grad:
                upd(i, grad, arr)
                arr._fresh_grad = False


batch_size = 100
epochs = 100
iteration = -1

opt = mx.optimizer.create('adam', learning_rate=0.001, rescale_grad=1 / batch_size)
updaters = [mx.optimizer.get_updater(opt)]


results = []
for e in range(epochs):
    train_groups = np.array_split(np.arange(x.shape[0]), x.shape[0] / batch_size)
    for i, idx in enumerate(train_groups):
        iteration += 1
        xtrain, ytrain = x[idx, :], y[idx]

        xtrain = nd.array(xtrain)
        xtrain = xtrain.as_in_context(ctxList[0])

        ytrain = nd.array(ytrain).reshape((-1, 1))
        ytrain = ytrain.as_in_context(ctxList[0])

        with autograd.record():
            yhat = net(xtrain)
            loss = square_loss(yhat, ytrain.as_in_context(ctxList[-1]))

        loss.backward()
        custom_trainer(updaters, net.collect_params())

        if iteration % 10 == 0:
            results.append([iteration, loss.asnumpy().item()])
            print("epoch= {:5,d}, iter= {:6,d},  error= {:6.3E}".format(e, iteration, loss.asnumpy().item()))

但是，我得到了以下错误：

RuntimeError: Parameter 'hybridsequential0_mydenseblock1_weight' was 
not initialized on context gpu(0). It was only initialized on [gpu(1)].
terminate called recursively
terminate called after throwing an instance of 'dmlc::Error'

---

我使用block没有问题（正如已经在帖子中确认的那样），只是HybridBlock不起作用。有人能帮忙吗？在我看来，模型并行性的例子非常少。

你能告诉我你想用模型并行性解决什么问题吗？考虑到最近的GPU，大多数模型实际上并不需要模型并行性。试着想想你是否可以用不同的方法解决问题，比如：

• 使用FP16
• 减少模型中参数/层的数量
• 减少输入的大小，例如输入功能的数量
• 减少批处理大小并实现数据并行
• 将问题分解为可单独训练的子模型

如果上述任何一项都不适用于您，并且您确信您确实需要模型并行性，那么下面是一个在Gluon中实现模型并行性的示例代码：

---

如果您不理解该代码的任何部分，请询问，我可以解释。

您能告诉我您试图用模型并行性解决的问题吗？考虑到最近的GPU，大多数模型实际上并不需要模型并行性。试着想想你是否可以用不同的方法解决问题，比如：

• 使用FP16
• 减少模型中参数/层的数量
• 减少输入的大小，例如输入功能的数量
• 减少批处理大小并实现数据并行
• 将问题分解为可单独训练的子模型

如果上述任何一项都不适用于您，并且您确信您确实需要模型并行性，那么下面是一个在Gluon中实现模型并行性的示例代码：

如果您不理解代码的任何部分，请询问，我可以解释