Python中最快的排序方式(无cython)
我有一个问题,我必须用一个自定义函数对一个非常大的数组(shape-7900000x4x4x4x4x4)进行排序。我使用了Python中最快的排序方式(无cython),python,performance,sorting,numpy,cython,Python,Performance,Sorting,Numpy,Cython,我有一个问题,我必须用一个自定义函数对一个非常大的数组(shape-7900000x4x4x4x4x4)进行排序。我使用了排序,但排序花费了1个多小时。我的代码是这样的 def compare(x,y): print('DD '+str(x[0])) if(np.array_equal(x[1],y[1])==True): return -1 a = x[1].flatten() b = y[1].flatten() idx = np.wh
排序
,但排序花费了1个多小时。我的代码是这样的
def compare(x,y):
print('DD '+str(x[0]))
if(np.array_equal(x[1],y[1])==True):
return -1
a = x[1].flatten()
b = y[1].flatten()
idx = np.where( (a>b) != (a<b) )[0][0]
if a[idx]<0 and b[idx]>=0:
return 0
elif b[idx]<0 and a[idx]>=0:
return 1
elif a[idx]<0 and b[idx]<0:
if a[idx]>b[idx]:
return 0
elif a[idx]<b[idx]:
return 1
elif a[idx]<b[idx]:
return 1
else:
return 0
def cmp_to_key(mycmp):
class K:
def __init__(self, obj, *args):
self.obj = obj
def __lt__(self, other):
return mycmp(self.obj, other.obj)
return K
tblocks = sorted(tblocks.items(),key=cmp_to_key(compare))
def比较(x,y):
打印('DD'+str(x[0]))
如果(np.数组_等于(x[1],y[1])==True):
返回-1
a=x[1]。展平()
b=y[1]。展平()
idx=np。其中((a>b)!=(a如果没有工作示例,很难给出答案。我假设Cython代码中的arrr是一个2D数组,我假设大小是size=arrr.shape[0]
Numba实现
import numpy as np
import numba as nb
from numba.targets import quicksort
def custom_sorting(compare_fkt):
index_arange=np.arange(size)
quicksort_func=quicksort.make_jit_quicksort(lt=compare_fkt,is_argsort=False)
jit_sort_func=nb.njit(quicksort_func.run_quicksort)
index=jit_sort_func(index_arange)
return index
def compare(a,b):
x = arrr[a]
y = arrr[b]
i = 0
j = 0
while(i<size):
if((j==size-1)or(y[j]<x[i])):
return False
elif(x[i]<y[j]):
return True
i+=1
j+=1
return (j!=size-1)
arrr=np.random.randint(-9,10,(7900000,8))
size=arrr.shape[0]
index=custom_sorting(compare)
这个实现比np.argsort慢大约35%,但在编译代码中使用np.argsort也是很常见的。如果我正确理解了您的代码,那么您心目中的顺序就是标准顺序,只是它从0开始,在+/-infinity
处循环,在-0
处最大。除此之外,我们具有简单的从左到右的词典顺序
现在,如果您的数组数据类型是整数,请注意以下几点:由于负数的补码表示,视图强制转换为无符号整数将使您的顺序成为标准顺序。此外,如果使用大端编码,则可以通过视图强制转换为void
dtype来实现有效的词典排序
下面的代码显示,使用10000x4x4x4x4
示例,该方法给出的结果与Python代码相同
它还在7900000x4x4x4x4x4x4
示例上对其进行基准测试(使用数组,而不是dict)。在我的普通笔记本电脑上,这种方法需要8
秒
import numpy as np
def compare(x, y):
# print('DD '+str(x[0]))
if(np.array_equal(x[1],y[1])==True):
return -1
a = x[1].flatten()
b = y[1].flatten()
idx = np.where( (a>b) != (a<b) )[0][0]
if a[idx]<0 and b[idx]>=0:
return 0
elif b[idx]<0 and a[idx]>=0:
return 1
elif a[idx]<0 and b[idx]<0:
if a[idx]>b[idx]:
return 0
elif a[idx]<b[idx]:
return 1
elif a[idx]<b[idx]:
return 1
else:
return 0
def cmp_to_key(mycmp):
class K:
def __init__(self, obj, *args):
self.obj = obj
def __lt__(self, other):
return mycmp(self.obj, other.obj)
return K
def custom_sort(a):
assert a.dtype==np.int64
b = a.astype('>i8', copy=False)
return b.view(f'V{a.dtype.itemsize * a.shape[1]}').ravel().argsort()
tblocks = np.random.randint(-9,10, (10000, 4, 4))
tblocks = dict(enumerate(tblocks))
tblocks_s = sorted(tblocks.items(),key=cmp_to_key(compare))
tblocksa = np.array(list(tblocks.values()))
tblocksa = tblocksa.reshape(tblocksa.shape[0], -1)
order = custom_sort(tblocksa)
tblocks_s2 = list(tblocks.items())
tblocks_s2 = [tblocks_s2[o] for o in order]
print(tblocks_s == tblocks_s2)
from timeit import timeit
data = np.random.randint(-9_999, 10_000, (7_900_000, 4, 4))
print(timeit(lambda: data[custom_sort(data.reshape(data.shape[0], -1))],
number=5) / 5)
如果您想让我们做的不仅仅是阅读代码,那么您需要提供一些测试数据。items
方法建议tblocks
是一个字典。值是某种类型和/或维度的数组?数组是什么?dtype
?另外,我也不清楚Python和Cython的比较函数会是什么e等效。我想知道是否有一种等效的方法使用值而不是cmp。因此,Box实际上是一个(7900000x4x4x4x4x4)数组?您能否提供一个简单的工作示例?这应该是使用自定义排序函数的方法:
import numpy as np
import numba as nb
from numba.targets import quicksort
#simple reverse sort
def compare(a,b):
return a > b
#create some test data
arrr=np.array(np.random.rand(7900000)*10000,dtype=np.int32)
#we can pass the comparison function
quicksort_func=quicksort.make_jit_quicksort(lt=compare,is_argsort=True)
#compile the sorting function
jit_sort_func=nb.njit(quicksort_func.run_quicksort)
#get the result
ind_sorted=jit_sort_func(arrr)
import numpy as np
def compare(x, y):
# print('DD '+str(x[0]))
if(np.array_equal(x[1],y[1])==True):
return -1
a = x[1].flatten()
b = y[1].flatten()
idx = np.where( (a>b) != (a<b) )[0][0]
if a[idx]<0 and b[idx]>=0:
return 0
elif b[idx]<0 and a[idx]>=0:
return 1
elif a[idx]<0 and b[idx]<0:
if a[idx]>b[idx]:
return 0
elif a[idx]<b[idx]:
return 1
elif a[idx]<b[idx]:
return 1
else:
return 0
def cmp_to_key(mycmp):
class K:
def __init__(self, obj, *args):
self.obj = obj
def __lt__(self, other):
return mycmp(self.obj, other.obj)
return K
def custom_sort(a):
assert a.dtype==np.int64
b = a.astype('>i8', copy=False)
return b.view(f'V{a.dtype.itemsize * a.shape[1]}').ravel().argsort()
tblocks = np.random.randint(-9,10, (10000, 4, 4))
tblocks = dict(enumerate(tblocks))
tblocks_s = sorted(tblocks.items(),key=cmp_to_key(compare))
tblocksa = np.array(list(tblocks.values()))
tblocksa = tblocksa.reshape(tblocksa.shape[0], -1)
order = custom_sort(tblocksa)
tblocks_s2 = list(tblocks.items())
tblocks_s2 = [tblocks_s2[o] for o in order]
print(tblocks_s == tblocks_s2)
from timeit import timeit
data = np.random.randint(-9_999, 10_000, (7_900_000, 4, 4))
print(timeit(lambda: data[custom_sort(data.reshape(data.shape[0], -1))],
number=5) / 5)
True
7.8328493310138585