即使在gc.collect()之后,分配给Python的内存也不会在Linux中释放回来

即使在gc.collect()之后,分配给Python的内存也不会在Linux中释放回来,python,memory-management,garbage-collection,numpy,malloc,Python,Memory Management,Garbage Collection,Numpy,Malloc,我在Python中编写了一些代码,这些代码不会以应有的方式释放内存。Python占用内存,但即使不再使用,也不会释放内存。即使使用ctrl+c中断正在运行的程序。删除变量并运行它似乎没有收集的gc.collect()。或与Ipython中相同,并运行%reset。内存将不会被释放,运行gc.collect()将无效。我在Windows中测试了它,因为我想看看它是否可能与垃圾收集器库一起使用。看来是这样。在Linux中运行下面的代码,然后在windows中运行。然后比较内存使用情况。您需要安装nu

我在Python中编写了一些代码,这些代码不会以应有的方式释放内存。Python占用内存,但即使不再使用,也不会释放内存。即使使用ctrl+c中断正在运行的程序。删除变量并运行它似乎没有收集的gc.collect()。或与Ipython中相同,并运行%reset。内存将不会被释放,运行gc.collect()将无效。我在Windows中测试了它,因为我想看看它是否可能与垃圾收集器库一起使用。看来是这样。在Linux中运行下面的代码,然后在windows中运行。然后比较内存使用情况。您需要安装numpy和scipy。任何关于这个问题的帮助或见解都将不胜感激

导入模型,创建实例,然后运行createSpecific()

以下是在Ubuntu 10.04中展示这种行为的代码:

from numpy import array, maximum,intersect1d, meshgrid, std, log, log10, zeros, ones, argwhere, abs, arange, size, copy, sqrt, sin, cos, pi, vstack, hstack, zeros, exp, max, mean, savetxt, loadtxt,  minimum,  linspace,  where
from numpy.fft import fft
from scipy.stats import f_oneway, kruskal, sem, scoreatpercentile
#import matplotlib
#matplotlib.use('cairo.pdf')
from matplotlib.pyplot import plot, clf, show, cla, xlim, xscale, imshow, ylabel, xlabel, figure, savefig, close,  bar,  title,  xticks, yticks, axes, axis
from matplotlib.axes import Axes
from mpl_toolkits.mplot3d import Axes3D
#from enthought.mayavi import mlab
from matplotlib import cm
import matplotlib.pyplot as plt
import os
from time import clock
from timeit import Timer
class Model:

#Constructors and default includes
    def __init__(self, prevAud = None,  debug=False):

        if (prevAud == None):
            self.fs=16000. #sample rate
            self.lowFreq=60. 
            self.hiFreq=5000.     
            self.numFilt=300 #number of channel
            self.EarQ = 9.26449   #9.26449
            self.minBW = 24.7     #24.7
            self.integrationWindow=.01
            self.sliceAt=.035
            self.maxOverallInhibit = 0.1
            self.winLen = int(self.fs*self.integrationWindow+.01) #default integration window 10 ms
            self.fullWind = 0.300
            self.outShortWindow = None
            self.siderArray = None
            self.maxNormalizeValue = .284     # Optimized at .284
            self.outputSemiModel = None
            self.semitones = 11
            self.activationTrace = None
        return




    def setErbScale(self, erbScale = None):
        if (erbScale ==None):
            self.erbScale = arange(100,500,5)
        else:
            self.erbScale = erbScale        

    def trainModel(self,soundVec=None, fs=None, lowfreq=None, highfreq=None, numfilt=None, figto=0, savefig = 'N', prompts=False, plotter=False):
        self.setErbScale()
        templateArray = self.intWindow(self.halfWaveRec(self.creGammatone(soundVec))) 
        for i in xrange(templateArray[0].size):        
            self.outerTest(self.innerTest(templateArray[:,i]))

        return templateArray   


    def createSpecific(self, freqArray = None, semitones = 11, timeforHarm = .3, soundVec=None, fs=None, lowfreq=None, highfreq=None, numfilt=None, figto=0, saveData='N', fileDir='TempRunT/', prompts=False, plotter=False):
        if (freqArray == None):
            self.setErbScale()
            freqArray = self.erbScale
        if (type(semitones) == int):
            semitones = arange(semitones+1)
        totalRuns = int(timeforHarm/self.integrationWindow+.001)
        inhibitWindowArray = zeros((freqArray.size,(semitones.size),self.numFilt,totalRuns))
        for x in xrange(freqArray.size):
            tempHarm = self.makeHarmonicAmpMod(freqArray[x],timeforHarm, numHarm=7,modulation=10)
            for y in semitones:
                tempChord = self.makeSemiChordAmpMod(tempHarm, freqArray[x],timeforHarm,modulation=10,numHarm=7,semi=y)
                inhibitWindowArray[x,y] = self.trainModel( tempChord, savefig = 'N', plotter=plotter)


        self.inhibitWindowArray = inhibitWindowArray

    def creGammatone(self, soundVec):

        temp = zeros((300,soundVec.size))
        for i in xrange(temp[:,0].size):
            temp[i] = -1**i*soundVec
        return temp

    def halfWaveRec(self, halfWaveFilts):

        filtShape = halfWaveFilts.shape
        if (filtShape[1] != int(self.fs*self.fullWind)):
            halfWaveFilts = hstack((halfWaveFilts,zeros((self.numFilt,int(self.fs*self.fullWind)-filtShape[1]))))
        temp = zeros((halfWaveFilts[:,0].size,halfWaveFilts[0].size))
        halfWaveFilts = maximum(halfWaveFilts,temp)

        del temp                
        return halfWaveFilts

    def intWindow(self, integratedFilts):
        winlen = self.winLen

        length = integratedFilts[0].size/winlen
        mod = integratedFilts[0].size%winlen
        outShortWindow = zeros((integratedFilts[:,0].size,length))
        meanval = 0

        if (mod != 0):
            for i in xrange(integratedFilts[:,0].size):
                mean(integratedFilts[i,0:-mod].reshape(length,winlen),1,out=outShortWindow[i])
        else:
            for i in xrange(integratedFilts[:,0].size):
                mean(integratedFilts[i].reshape(length,winlen),1,out=outShortWindow[i])
        del integratedFilts
        return outShortWindow    

    def innerTest(self, window):
        temper = copy(window)
        sider = 7
        st = .04
        sizer = temper.size
        inhibVal = 0
        for j in xrange(sider):
            inhibVal = (temper[0:j+sider+1].sum())*(sider*2+1)/(sider+1+j)
            window[j] += - st*(inhibVal)
        for j in xrange(sider,sizer - sider):
            inhibVal = temper[j-sider:j+sider+1].sum()
            window[j] += - st*(inhibVal)
        for j in xrange(sizer-sider, sizer):
            inhibVal = (temper[j-sider:sizer].sum())*(sider*2+1)/(sider+sizer-j)
            window[j] += - st*(inhibVal)

        maxsub = max(window) * self.maxOverallInhibit
        window += - maxsub    
        del temper
        return window

    def outerTest(self, window):
        newSatValue = scoreatpercentile(window, (76))
        numones = where(window > newSatValue)
        window[numones]=1
        self.maxSatValue = newSatValue
        del numones
        return window

    def makeHarmonicAmpMod(self, freq = 100, time = 1.,modulation=10, fsamp=None, numHarm=7):
        if fsamp == None: fsamp = self.fs
        samples = arange(time*fsamp)
        signal = 0
        for x in xrange(1,(numHarm+1),1):
            signal = signal + sin(samples/float(fsamp)*x*freq*2*pi)
        signal = (signal)*maximum(zeros(time*fsamp),sin((samples/float(fsamp)*modulation*2*pi)))
        return signal

    def makeSemiChordAmpMod(self, harmVec = None, freq=100, time = 1.,  modulation=10, fsamp=None, numHarm=7, semi = 2):
        if (harmVec == None): harmVec = self.makeHarmonicAmpMod(freq,time,modulation,fsamp,numHarm)
        if (semi == 0): return harmVec
        return harmVec + self.makeHarmonicAmpMod(freq*(2**(semi/12.)),time,modulation,fsamp,numHarm)

虚拟内存不是稀缺资源。由于每个进程都有自己的地址空间,因此无需将其返回到系统。你的实际问题是什么?这种行为给您带来了什么问题?

我安装了最新的numpy svn,问题已经消失。我猜它在一个numpy函数中。我从来没有机会深入研究它。

python分配的内存在引擎盖下使用malloc(),而malloc()分配的内存在调用free()后不会返回到系统,据我所知,gc.collect()就是这么做的,即使python使用它自己的malloc(PyObject_malloc()和PyObject_free())但是它们有相同的行为,看这里:@singularity-我愿意尝试一下tcmalloc,并给出指令,强制它释放对象,看看它是否有效。你知道如何强迫它使用那个库吗。Python是否必须以一种特殊的方式编译并链接到我假设的库?@J Spen:看这里:也许这可以给你更多的细节。@singularity-我知道这个链接,但我不确定如何使用TCMalloc为收集器编译Python,但我已经在那里发布了帖子,看看他们是否会响应。旁注,实际上不需要手动删除函数、类和代码中的局部变量。PythonGC足够聪明,可以在您不需要这个变量时实现,并将自己对其进行排序。移动设备和开发板通常没有虚拟内存。他们有RAM,通常就是这样。@jww这是完全不真实的,十年前甚至都不是真的。您可能会在太阳能发电系统和一些非常小的开发板(如arduino)中发现没有虚拟内存的处理器作为电源控制器。但仅此而已。我已经十年没有见过没有虚拟内存的移动设备了。我的坏大卫。它们有虚拟内存,每个进程通常有自己的空间,但它们没有交换文件。你不能像对待台式机或服务器那样对待这些设备。如果你这样做了,那么你会经历很多内存不足的死亡。我似乎记得,在一些早期的iPad上,malloc的最大大小是32MB或64MB,它们只有256MB的内存。