Warning: file_get_contents(/data/phpspider/zhask/data//catemap/3/go/7.json): failed to open stream: No such file or directory in /data/phpspider/zhask/libs/function.php on line 167

Warning: Invalid argument supplied for foreach() in /data/phpspider/zhask/libs/tag.function.php on line 1116

Notice: Undefined index: in /data/phpspider/zhask/libs/function.php on line 180

Warning: array_chunk() expects parameter 1 to be array, null given in /data/phpspider/zhask/libs/function.php on line 181
Go 程序内存不足ec2资源太小?_Go_Amazon Ec2_Out Of Memory - Fatal编程技术网

Go 程序内存不足ec2资源太小?

Go 程序内存不足ec2资源太小?,go,amazon-ec2,out-of-memory,Go,Amazon Ec2,Out Of Memory,我想知道我是不是写错了Go程序,还是我的服务器资源太小 我有以下代码: spotXmlFile:='/var/www/html/app/public/xml/appdata.xml' xmlFile,err:=os.Open(spotXmlFile) defer xmlFile.Close() byteValue,_:=ioutil.ReadAll(xmlFile) var listings Listings xml.Unmarshal(byteV

我想知道我是不是写错了Go程序,还是我的服务器资源太小

我有以下代码:

    spotXmlFile:='/var/www/html/app/public/xml/appdata.xml'
    xmlFile,err:=os.Open(spotXmlFile)
    defer xmlFile.Close()
    byteValue,_:=ioutil.ReadAll(xmlFile)
    var listings Listings
    xml.Unmarshal(byteValue,&listings)
它处理一个2.5GB的xml文件

我正在使用aws ec2 t3.large来尝试和处理数据,它有两个CPU和8GB内存

有趣的是内存没有超过4GB,但cpu峰值达到100%,然后导致程序失败

以下是我的xml结构:

type Listings struct {
    XMLName xml.Name `xml:"Listings"`
    Listings []Listing `xml:"Listing"`
}

type Listing struct {
    Text    string `xml:",chardata"`
        Address struct {
            Text                   string `xml:",chardata"`
            PreferenceOrder        string `xml:"preference-order"`
            AddressPreferenceOrder string `xml:"address-preference-order"`
            FullStreetAddress      string `xml:"FullStreetAddress"`
            UnitNumber             string `xml:"UnitNumber"`
            City                   string `xml:"City"`
            StateOrProvince        string `xml:"StateOrProvince"`
            PostalCode             string `xml:"PostalCode"`
            Country                string `xml:"Country"`
        } `xml:"Address"`
        ListPrice struct {
            Text             string `xml:",chardata"`
            IsgSecurityClass string `xml:"isgSecurityClass,attr"`
            CurrencyPeriod   string `xml:"currencyPeriod,attr"`
        } `xml:"ListPrice"`
        ListingURL       string `xml:"ListingURL"`
        ProviderName     string `xml:"ProviderName"`
        ProviderURL      string `xml:"ProviderURL"`
        ProviderCategory string `xml:"ProviderCategory"`
        LeadRoutingEmail string `xml:"LeadRoutingEmail"`
        Bedrooms         string `xml:"Bedrooms"`
        Bathrooms        string `xml:"Bathrooms"`
        PropertyType     struct {
            Text             string `xml:",chardata"`
            OtherDescription string `xml:"otherDescription,attr"`
        } `xml:"PropertyType"`
        PropertySubType struct {
            Text             string `xml:",chardata"`
            OtherDescription string `xml:"otherDescription,attr"`
        } `xml:"PropertySubType"`
        ListingKey           string `xml:"ListingKey"`
        ListingCategory      string `xml:"ListingCategory"`
        ListingStatus        string `xml:"ListingStatus"`
        MarketingInformation struct {
            Text                    string `xml:",chardata"`
            PermitAddressOnInternet struct {
                Text             string `xml:",chardata"`
                IsgSecurityClass string `xml:"isgSecurityClass,attr"`
            } `xml:"PermitAddressOnInternet"`
            VOWAddressDisplay struct {
                Text             string `xml:",chardata"`
                IsgSecurityClass string `xml:"isgSecurityClass,attr"`
            } `xml:"VOWAddressDisplay"`
            VOWAutomatedValuationDisplay struct {
                Text             string `xml:",chardata"`
                IsgSecurityClass string `xml:"isgSecurityClass,attr"`
            } `xml:"VOWAutomatedValuationDisplay"`
            VOWConsumerComment struct {
                Text             string `xml:",chardata"`
                IsgSecurityClass string `xml:"isgSecurityClass,attr"`
            } `xml:"VOWConsumerComment"`
        } `xml:"MarketingInformation"`
        Photos struct {
            Text  string `xml:",chardata"`
            Photo []struct {
                Text                       string `xml:",chardata"`
                MediaModificationTimestamp struct {
                    Text             string `xml:",chardata"`
                    IsgSecurityClass string `xml:"isgSecurityClass,attr"`
                } `xml:"MediaModificationTimestamp"`
                MediaURL string `xml:"MediaURL"`
            } `xml:"Photo"`
        } `xml:"Photos"`
        DiscloseAddress       string `xml:"DiscloseAddress"`
        ListingDescription    string `xml:"ListingDescription"`
        MlsId                 string `xml:"MlsId"`
        MlsName               string `xml:"MlsName"`
        MlsNumber             string `xml:"MlsNumber"`
        LivingArea            string `xml:"LivingArea"`
        LotSize               string `xml:"LotSize"`
        YearBuilt             string `xml:"YearBuilt"`
        ListingTitle          string `xml:"ListingTitle"`
        FullBathrooms         string `xml:"FullBathrooms"`
        ThreeQuarterBathrooms string `xml:"ThreeQuarterBathrooms"`
        HalfBathrooms         string `xml:"HalfBathrooms"`
        OneQuarterBathrooms   string `xml:"OneQuarterBathrooms"`
        PartialBathrooms      string `xml:"PartialBathrooms"`
        ListingParticipants   struct {
            Text        string `xml:",chardata"`
            Participant struct {
                Text                string `xml:",chardata"`
                ParticipantKey      string `xml:"ParticipantKey"`
                ParticipantId       string `xml:"ParticipantId"`
                FirstName           string `xml:"FirstName"`
                LastName            string `xml:"LastName"`
                Role                string `xml:"Role"`
                PrimaryContactPhone string `xml:"PrimaryContactPhone"`
                OfficePhone         string `xml:"OfficePhone"`
                Email               string `xml:"Email"`
                WebsiteURL          string `xml:"WebsiteURL"`
                Licenses            struct {
                    Text    string `xml:",chardata"`
                    License struct {
                        Text            string `xml:",chardata"`
                        LicenseCategory struct {
                            Text             string `xml:",chardata"`
                            OtherDescription string `xml:"otherDescription,attr"`
                        } `xml:"LicenseCategory"`
                        LicenseNumber struct {
                            Text             string `xml:",chardata"`
                            IsgSecurityClass string `xml:"isgSecurityClass,attr"`
                        } `xml:"LicenseNumber"`
                        Jurisdiction struct {
                            Text             string `xml:",chardata"`
                            IsgSecurityClass string `xml:"isgSecurityClass,attr"`
                        } `xml:"Jurisdiction"`
                    } `xml:"License"`
                } `xml:"Licenses"`
            } `xml:"Participant"`
        } `xml:"ListingParticipants"`
        Offices struct {
            Text   string `xml:",chardata"`
            Office struct {
                Text       string `xml:",chardata"`
                OfficeKey  string `xml:"OfficeKey"`
                OfficeId   string `xml:"OfficeId"`
                OfficeCode struct {
                    Text         string `xml:",chardata"`
                    OfficeCodeId string `xml:"OfficeCodeId"`
                } `xml:"OfficeCode"`
                Name          string `xml:"Name"`
                CorporateName string `xml:"CorporateName"`
                PhoneNumber   string `xml:"PhoneNumber"`
                Fax           string `xml:"Fax"`
                Address       struct {
                    Text                   string `xml:",chardata"`
                    PreferenceOrder        string `xml:"preference-order"`
                    AddressPreferenceOrder string `xml:"address-preference-order"`
                    FullStreetAddress      string `xml:"FullStreetAddress"`
                    City                   string `xml:"City"`
                    StateOrProvince        string `xml:"StateOrProvince"`
                    PostalCode             string `xml:"PostalCode"`
                    Country                string `xml:"Country"`
                } `xml:"Address"`
                OfficeEmail string `xml:"OfficeEmail"`
                Website     string `xml:"Website"`
            } `xml:"Office"`
        } `xml:"Offices"`
        Brokerage struct {
            Text       string `xml:",chardata"`
            Name       string `xml:"Name"`
            Phone      string `xml:"Phone"`
            Email      string `xml:"Email"`
            WebsiteURL string `xml:"WebsiteURL"`
            LogoURL    string `xml:"LogoURL"`
            Address    struct {
                Text                   string `xml:",chardata"`
                PreferenceOrder        string `xml:"preference-order"`
                AddressPreferenceOrder string `xml:"address-preference-order"`
                FullStreetAddress      string `xml:"FullStreetAddress"`
                UnitNumber             string `xml:"UnitNumber"`
                City                   string `xml:"City"`
                StateOrProvince        string `xml:"StateOrProvince"`
                PostalCode             string `xml:"PostalCode"`
                Country                string `xml:"Country"`
            } `xml:"Address"`
        } `xml:"Brokerage"`
        Franchise struct {
            Text string `xml:",chardata"`
            Name string `xml:"Name"`
        } `xml:"Franchise"`
        Builder struct {
            Text string `xml:",chardata"`
            Name string `xml:"Name"`
        } `xml:"Builder"`
        Location struct {
            Text       string `xml:",chardata"`
            Latitude   string `xml:"Latitude"`
            Longitude  string `xml:"Longitude"`
            Directions string `xml:"Directions"`
            County     string `xml:"County"`
            ParcelId   string `xml:"ParcelId"`
            Community  struct {
                Text        string `xml:",chardata"`
                Subdivision struct {
                    Text             string `xml:",chardata"`
                    IsgSecurityClass string `xml:"isgSecurityClass,attr"`
                } `xml:"Subdivision"`
                Schools struct {
                    Text   string `xml:",chardata"`
                    School []struct {
                        Text           string `xml:",chardata"`
                        Name           string `xml:"Name"`
                        SchoolCategory string `xml:"SchoolCategory"`
                        District       struct {
                            Text             string `xml:",chardata"`
                            IsgSecurityClass string `xml:"isgSecurityClass,attr"`
                        } `xml:"District"`
                    } `xml:"School"`
                } `xml:"Schools"`
            } `xml:"Community"`
        } `xml:"Location"`
        DetailedCharacteristics struct {
            Text       string `xml:",chardata"`
            Appliances struct {
                Text      string   `xml:",chardata"`
                Appliance []string `xml:"Appliance"`
            } `xml:"Appliances"`
            ArchitectureStyle struct {
                Text             string `xml:",chardata"`
                OtherDescription string `xml:"otherDescription,attr"`
            } `xml:"ArchitectureStyle"`
            HasBarbecueArea string `xml:"HasBarbecueArea"`
            CoolingSystems  struct {
                Text          string `xml:",chardata"`
                CoolingSystem string `xml:"CoolingSystem"`
            } `xml:"CoolingSystems"`
            ExteriorTypes struct {
                Text         string `xml:",chardata"`
                ExteriorType string `xml:"ExteriorType"`
            } `xml:"ExteriorTypes"`
            HasFireplace   string `xml:"HasFireplace"`
            FloorCoverings struct {
                Text          string   `xml:",chardata"`
                FloorCovering []string `xml:"FloorCovering"`
            } `xml:"FloorCoverings"`
            HeatingFuels struct {
                Text        string `xml:",chardata"`
                HeatingFuel string `xml:"HeatingFuel"`
            } `xml:"HeatingFuels"`
            HeatingSystems struct {
                Text          string   `xml:",chardata"`
                HeatingSystem []string `xml:"HeatingSystem"`
            } `xml:"HeatingSystems"`
            IsNewConstruction string `xml:"IsNewConstruction"`
            HasPool string `xml:"HasPool"`
            NumFloors         string `xml:"NumFloors"`
            ParkingTypes      struct {
                Text        string `xml:",chardata"`
                ParkingType string `xml:"ParkingType"`
            } `xml:"ParkingTypes"`
            HasPatio  string `xml:"HasPatio"`
            RoofTypes struct {
                Text     string `xml:",chardata"`
                RoofType string `xml:"RoofType"`
            } `xml:"RoofTypes"`
            RoomCount string `xml:"RoomCount"`
            Rooms     struct {
                Text string   `xml:",chardata"`
                Room []string `xml:"Room"`
            } `xml:"Rooms"`
            HasVaultedCeiling string `xml:"HasVaultedCeiling"`
        } `xml:"DetailedCharacteristics"`
        ModificationTimestamp struct {
            Text             string `xml:",chardata"`
            IsgSecurityClass string `xml:"isgSecurityClass,attr"`
        } `xml:"ModificationTimestamp"`
        Disclaimer struct {
            Text             string `xml:",chardata"`
            IsgSecurityClass string `xml:"isgSecurityClass,attr"`
        } `xml:"Disclaimer"`    
}
以下是我的跑步输出:

fatal error: runtime: out of memory

runtime stack:
runtime.throw(0x4f44be, 0x16)
        /usr/local/go/src/runtime/panic.go:774 +0x72
runtime.sysMap(0xc104000000, 0x100000000, 0x5cc478)
        /usr/local/go/src/runtime/mem_linux.go:169 +0xc5
runtime.(*mheap).sysAlloc(0x5b4dc0, 0x100000000, 0x7ba23, 0x0)
        /usr/local/go/src/runtime/malloc.go:701 +0x1cd
runtime.(*mheap).grow(0x5b4dc0, 0x80000, 0xffffffff)
        /usr/local/go/src/runtime/mheap.go:1252 +0x42
runtime.(*mheap).allocSpanLocked(0x5b4dc0, 0x80000, 0x5cc488, 0x100000081)
        /usr/local/go/src/runtime/mheap.go:1163 +0x291
runtime.(*mheap).alloc_m(0x5b4dc0, 0x80000, 0x400101, 0xc0000344c8)
        /usr/local/go/src/runtime/mheap.go:1015 +0xc2
runtime.(*mheap).alloc.func1()
        /usr/local/go/src/runtime/mheap.go:1086 +0x4c
runtime.(*mheap).alloc(0x5b4dc0, 0x80000, 0x7ffd80010101, 0x433515)
        /usr/local/go/src/runtime/mheap.go:1085 +0x8a
runtime.largeAlloc(0xfffffe00, 0xc000010101, 0xc000000180)
        /usr/local/go/src/runtime/malloc.go:1138 +0x97
runtime.mallocgc.func1()
        /usr/local/go/src/runtime/malloc.go:1033 +0x46
runtime.systemstack(0x452364)
        /usr/local/go/src/runtime/asm_amd64.s:370 +0x66
runtime.mstart()
        /usr/local/go/src/runtime/proc.go:1146

goroutine 1 [running]:
runtime.systemstack_switch()
        /usr/local/go/src/runtime/asm_amd64.s:330 fp=0xc00006ac10 sp=0xc00006ac08 pc=0x452460
runtime.mallocgc(0xfffffe00, 0x4c56e0, 0x1, 0x0)
        /usr/local/go/src/runtime/malloc.go:1032 +0x895 fp=0xc00006acb0 sp=0xc00006ac10 pc=0x40bb85
runtime.makeslice(0x4c56e0, 0xfffffe00, 0xfffffe00, 0x0)
        /usr/local/go/src/runtime/slice.go:49 +0x6c fp=0xc00006ace0 sp=0xc00006acb0 pc=0x43dc8c
bytes.makeSlice(0xfffffe00, 0x0, 0x0, 0x0)
        /usr/local/go/src/bytes/buffer.go:229 +0x77 fp=0xc00006ad48 sp=0xc00006ace0 pc=0x4673d7
bytes.(*Buffer).grow(0xc00006ae70, 0x200, 0x40000000)
        /usr/local/go/src/bytes/buffer.go:142 +0x15b fp=0xc00006ad98 sp=0xc00006ad48 pc=0x466ecb
bytes.(*Buffer).ReadFrom(0xc00006ae70, 0x50fc00, 0xc000082018, 0xc000082018, 0x3, 0x0)
        /usr/local/go/src/bytes/buffer.go:202 +0x4b fp=0xc00006ae08 sp=0xc00006ad98 pc=0x4671cb
io/ioutil.readAll(0x50fc00, 0xc000082018, 0x200, 0x0, 0x0, 0x0, 0x0, 0x0)
        /usr/local/go/src/io/ioutil/ioutil.go:36 +0x100 fp=0xc00006aea8 sp=0xc00006ae08 pc=0x47b380
io/ioutil.ReadAll(...)
        /usr/local/go/src/io/ioutil/ioutil.go:45
main.main()
        /var/www/html/spot/importdata/main.go:36 +0xbf fp=0xc00006af60 sp=0xc00006aea8 pc=0x4ad63f
runtime.main()
        /usr/local/go/src/runtime/proc.go:203 +0x21e fp=0xc00006afe0 sp=0xc00006af60 pc=0x42bfbe
runtime.goexit()
        /usr/local/go/src/runtime/asm_amd64.s:1357 +0x1 fp=0xc00006afe8 sp=0xc00006afe0 pc=0x4543b1

您正在内存中读取整个文件,然后将其解组,这将使用大量您并不真正需要的额外内存。试试这个,看看会发生什么:

spotXmlFile:='/var/www/html/app/public/xml/appdata.xml'
xmlFile,err:=os.Open(spotXmlFile)
defer xmlFile.Close()
var listings Listings
xml.NewDecoder(xmlFile).Decode(&listings)
使用解码器,它将在读取时解析和解组底层文件

如果您只需要文件的一部分,并且希望进一步优化它,那么可以使用解码器令牌逐个令牌解码文件,并提取所需的部分