如何解决CatBoost官方基准测试示例是否使用测试数据作为有效数据?
在检查CatBoost Official Benchmark examples时,我注意到代码将测试数据用作最终验证的“ eval_set”。这是正确的方法吗?我的印象是“测试数据”保留用于最终测试。
来自here的代码。专注于 ITEM 1
def run_experiment(Experiment,title):
experiment = Experiment(learning_task,max_hyperopt_evals=max_hyperopt_evals,n_estimators=n_estimators)
cv_pairs,(dtrain,dtest) = experiment.split_and_preprocess(X_train.copy(),y_train,X_test.copy(),y_test,cat_cols,n_splits=5)
default_cv_result = experiment.run_cv(cv_pairs)
experiment.print_result(default_cv_result,'Default {} result on cv'.format(title))
print('\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n')
default_test_losses = []
for seed in range(5):
test_result = experiment.run_test(dtrain,dtest,X_test,params=default_cv_result['params'],n_estimators=default_cv_result['best_n_estimators'],seed=seed)
default_test_losses.append(test_result['loss'])
print 'For seed=%d Test\'s %s : %.5f' % (seed,experiment.metric,default_test_losses[-1])
print '\nTest\'s %s mean: %.5f,Test\'s %s std: %.5f' % (experiment.metric,np.mean(default_test_losses),np.std(default_test_losses))
print('\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n')
print('Hyperopt iterations:\n\n')
tuned_cv_result = experiment.optimize_params(cv_pairs)
print('\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n')
experiment.print_result(tuned_cv_result,'Tuned {} result on cv'.format(title))
print('\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n')
tuned_test_losses = []
for seed in range(5):
####################################################
# ITEM 1
########################################################
test_result = experiment.run_test(dtrain,params=tuned_cv_result['params'],n_estimators=tuned_cv_result['best_n_estimators'],seed=seed)
tuned_test_losses.append(test_result['loss'])
print 'For seed=%d Test\'s %s : %.5f' % (seed,tuned_test_losses[-1])
print '\nTest\'s %s mean: %.5f,np.mean(tuned_test_losses),np.std(tuned_test_losses))
return np.mean(default_test_losses),np.mean(tuned_test_losses)
'experiment.py'中的run_test
函数:着重于项目#2
def run_test(self,dtrain,X_test=None,params=None,n_estimators=None,custom_metric=None,seed=0):
params = params or self.best_params or self.default_params
n_estimators = n_estimators or self.best_n_estimators or self.n_estimators
params = self.preprocess_params(params)
start_time = time.time()
####################################################
# ITEM 2
########################################################
bst,evals_result = self.fit(params,n_estimators,seed=seed)
eval_time = time.time() - start_time
preds = self.predict(bst,X_test)
result = {'loss': evals_result[-1],'bst': bst,'n_estimators': n_estimators,'eval_time': eval_time,'status': STATUS_OK,'params': params.copy(),'preds': preds}
if custom_metric is not None:
if type(custom_metric) is not dict:
raise TypeError("custom_metric argument should be dict")
pred = self.predict(bst,X_test)
for title,func in custom_metric.iteritems():
score = func(dtest.get_label(),pred,sample_weight=None) # TODO weights
result[title] = score
return result
'catboost_experiment.py'中catboost_experiment.py
中的fit函数。专注于项目3
def fit(self,params,seed=0):
params.update({"iterations": n_estimators})
params.update({"random_seed": seed})
bst = CatBoost(params)
####################################################
# ITEM 3
########################################################
bst.fit(dtrain,eval_set=dtest)
with open("test_error.tsv","r") as f:
results = np.array(map(lambda x: float(x.strip().split()[-1]),f.readlines()[1:]))
return bst,results
代码使用dtest
集作为validation
集。这是正确的方法吗?
CatBoost fit
使用eval_set
作为“过拟合检测器”,“最佳迭代选择”和“监控指标”更改的验证数据集。
我很困惑,如果我缺少一些东西,请纠正我。
版权声明:本文内容由互联网用户自发贡献,该文观点与技术仅代表作者本人。本站仅提供信息存储空间服务,不拥有所有权,不承担相关法律责任。如发现本站有涉嫌侵权/违法违规的内容, 请发送邮件至 dio@foxmail.com 举报,一经查实,本站将立刻删除。