Python 如何为XGBoost加载大型train.csv

train.csv

约为90G

它就像：

label,col1,col2,col3,col4......col400
0,0.92,-0.69,-0.13,0.01.......,0.22
1,0.22,-0.39,0.11,0.92........,-0.43

我的代码如下：

import numpy as np
import pandas as pd
import xgboost as xgb
from sklearn.cross_validation import train_test_split

train=pd.read_csv('/home/data/train.csv')
test=pd.read_csv('/home/data/test.csv')
params = {
    'booster': 'gbtree',
    'objective': 'multi:softmax',
    'num_class': 10,  
    'gamma': 0.1,  
    'max_depth': 12,
    'lambda': 2,  
    'subsample': 0.7,
    'colsample_bytree': 0.7,
    'min_child_weight': 3,
    'silent': 1,
    'eta': 0.007,
    'seed': 1000,
}
plst = list(params.items())
num_rounds = 100
train_xy, val = train_test_split(train, test_size=0.3, random_state=1)
y = train_xy.label
X = train_xy.drop(['label'], axis=1)
val_y = val.label
val_X = val.drop(['label'], axis=1)

xgb_val = xgb.DMatrix(val_X, label=val_y)
xgb_train = xgb.DMatrix(X, label=y)
xgb_test = xgb.DMatrix(tests)

watchlist = [(xgb_train, 'train'), (xgb_val, 'val')]

# training model 

start_time = time.time()
model = xgb.train(plst, xgb_train, num_rounds, watchlist, early_stopping_rounds=100)
print(time.time() - start_time)
plst = list(params.items())
num_rounds = 100
train_xy, val = train_test_split(train, test_size=0.3, random_state=1)
y = train_xy.label
X = train_xy.drop(['label'], axis=1)
val_y = val.label
val_X = val.drop(['label'], axis=1)

xgb_val = xgb.DMatrix(val_X, label=val_y)
xgb_train = xgb.DMatrix(X, label=y)
xgb_test = xgb.DMatrix(tests)

watchlist = [(xgb_train, 'train'), (xgb_val, 'val')]

# training model 

start_time = time.time()
model = xgb.train(plst, xgb_train, num_rounds, watchlist, early_stopping_rounds=100)
print(time.time() - start_time)
model.save_model('./model/xgb.model')

---

我可以把这个csv分成几个部分吗？或者增加虚拟内存？

您是否考虑过像这里建议的那样映射它？伟大的我认为这将是一项工作。非常感谢你！很乐意帮忙：）@K.Alan它确实起作用了。据我所知，对于XGboost，您必须将数据加载到DMatrix（内存）中。您是否考虑过像这里建议的那样映射它？伟大的我认为这将是一项工作。非常感谢你！很乐意帮忙：）@K.Alan它确实起作用了。据我所知，XGboost必须将数据加载到DMatrix（内存）中。