Python 3.x 使用光线进行最近邻搜索的并行化

假设我有一个大的稀疏矩阵。我想得到矩阵的每一行向量，并计算到矩阵的前一个窗口大小的行中最近的邻居的余弦距离

由于sklearn.neights在使用并行化时不会提高运行时间（请参阅），因此我尝试使用ray并行化进程。我的代码在多处理方面比sklearn做得更好，但仍然比串行距离计算慢

我的代码如下。有没有什么我做错了，应该改进的地方

import scipy.sparse
from sklearn.metrics.pairwise import cosine_distances
from sklearn.neighbors import NearestNeighbors
import numpy as np
import timeit
import ray
import math

n = 4000
m = 100
big_matrix = scipy.sparse.random(n, m).tocsr()
window_size = 1000
retry = 1
n_jobs = 4
ray.init(num_cpus=n_jobs)


def simple_cosine_distance():
    distances = np.zeros(n)
    for i in range(1, n):
        past = big_matrix[max(0, i - window_size):i]
        query = big_matrix[i]
        distances[i] = cosine_distances(query, past).min(axis=1)


def sklearn_nneighbor():
    distances = np.zeros(n)
    for i in range(1, n):
        past = big_matrix[max(0, i - window_size):i]
        nn = NearestNeighbors(metric="cosine", n_neighbors=1, n_jobs=1)
        nn.fit(X=past)
        query = big_matrix[i]
        distance, _ = nn.kneighbors(query)
        distances[i] = distance[0]


def sklearn_nneighbor_parallel():
    distances = np.zeros(n)
    for i in range(1, n):
        past = big_matrix[max(0, i - window_size):i]
        nn = NearestNeighbors(metric="cosine", n_neighbors=1, n_jobs=n_jobs)
        nn.fit(X=past)
        query = big_matrix[i]
        distance, _ = nn.kneighbors(query)
        distances[i] = distance[0]


@ray.remote
def get_chunk_min(data, indices, indptr, shape, slice_i, slice_j, query):
    matrix = scipy.sparse.csr_matrix((data, indices, indptr), shape=shape)
    past = matrix[slice_i:slice_j]
    query = matrix[query]
    chunk_min = cosine_distances(query, past).min(axis=1)
    return chunk_min


def ray_parallel():
    distances = np.zeros(n)
    data = ray.put(big_matrix.data)
    indices = ray.put(big_matrix.indices)
    indptr = ray.put(big_matrix.indptr)
    shape = ray.put(big_matrix.shape)
    for i in range(1, n):
        chunk_size = math.ceil((i - max(0, i - window_size)) / n_jobs)
        chunk_mins = ray.get([
            get_chunk_min.remote(
                data,
                indices,
                indptr,
                shape,
                enum,
                enum + chunk_size,
                i
            ) for enum in range(max(0, i - window_size), i, chunk_size)])
        distances[i] = min(chunk_mins)

for method in ["simple_cosine_distance", "sklearn_nneighbor", "sklearn_nneighbor_parallel", "ray_parallel"]:
    print(method)
    print(timeit.timeit(method + "()", setup="from __main__ import " + method, number=retry))
    print("*"*50)

输出：

简单余弦距离 3.978868665999471

---

斯克兰尼堡 4.265772191996803

---

sklearn\u nneighbor\u并行 28.664759318002325

---

雷欧平行线 17.89882287799992

本文档是一个很好的资源，可以帮助您了解在使用Ray并行化工作负载时可能出现的一些错误

还请注意，如果远程函数足够简单，则分布式计算的开销（例如，调度时间、序列化时间等）可能大于函数本身的计算时间

本文档是一个很好的资源，可以帮助您了解在使用Ray并行化工作负载时可能出现的一些错误

还请注意，如果远程函数足够简单，则分布式计算的开销（例如，调度时间、序列化时间等）可能大于函数本身的计算时间