<SYSTEM_TASK:>
Given the following text description, write Python code to implement the functionality described below step by step
<END_TASK>
<USER_TASK:>
Description:
 Multi-label classification -- binary relevance baseline
Step1:  Load data.
Step2:  Feature normalisation.
Step3:  Naive baseline
Use the estimated probability for each label as the predicted score for any example.
Step6:  Binary relevance baseline
Train a logistic regression model for each label.
Note that OneVsRestClassifier can be either a multiclass classifier or a multilabel classifier, see this binary relevance example on yeast dataset.
NOTE
Step7:  Cross validation according to F1.
Step8:  Cross validation according to F1.
Step9:  Plain logistic regression.
Step10:  Cross validation for classifier of each label.
Step11:  Result analysis
<ASSISTANT_TASK:>
Python Code:
%matplotlib inline
%load_ext autoreload
%autoreload 2
import os, sys, time
import pickle as pkl
import numpy as np
import pandas as pd
from sklearn.base import BaseEstimator
from sklearn.linear_model import LogisticRegression
from sklearn.multiclass import OneVsRestClassifier
from sklearn.pipeline import make_pipeline
from sklearn.model_selection import train_test_split
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import classification_report, make_scorer, f1_score, label_ranking_loss
import matplotlib.pyplot as plt
import seaborn as sns
sys.path.append('src')
from evaluate import avgPrecisionK, evaluatePrecision, evaluateF1, evaluateRankingLoss, f1_score_nowarn
from datasets import create_dataset, dataset_names, nLabels_dict
dataset_names
data_ix = 3
dataset_name = dataset_names[data_ix]
nLabels = nLabels_dict[dataset_name]
print(dataset_name, nLabels)
data_dir = 'data'
SEED = 918273645
fmodel_prec = os.path.join(data_dir, 'br-' + dataset_name + '-prec.pkl')
fmodel_f1 = os.path.join(data_dir, 'br-' + dataset_name + '-f1.pkl')
fmodel_base = os.path.join(data_dir, 'br-' + dataset_name + '-base.pkl')
fperf_prec = os.path.join(data_dir, 'perf-lr-prec.pkl')
fperf_f1 = os.path.join(data_dir, 'perf-lr-f1.pkl')
fperf_base = os.path.join(data_dir, 'perf-lr-base.pkl')
Explanation: Multi-label classification -- binary relevance baseline
End of explanation
X_train, Y_train = create_dataset(dataset_name=dataset_name, train_data=True, shuffle=True, random_state=SEED)
X_test,  Y_test  = create_dataset(dataset_name=dataset_name, train_data=False)
Explanation: Load data.
End of explanation
X_train_mean = np.mean(X_train, axis=0).reshape((1, -1))
X_train_std = np.std(X_train, axis=0).reshape((1, -1)) + 10 ** (-6)
X_train -= X_train_mean
X_train /= X_train_std
X_test  -= X_train_mean
X_test  /= X_train_std
Explanation: Feature normalisation.
End of explanation
#probs = np.mean(Y_train, axis=0)
#probs
#preds = np.tile(probs, (X_test.shape[0], 1))
#evaluatePrecision(Y_test, preds, verbose=1)
#evaluateRankingLoss(Y_test, preds, n_jobs=4)
Explanation: Naive baseline
Use the estimated probability for each label as the predicted score for any example.
End of explanation
class BinaryRelevance(BaseEstimator):
    
        Independent logistic regression based on OneVsRestClassifier wrapper.
    
    
    def __init__(self, C=1, n_jobs=-1):
        assert C > 0
        self.C = C
        self.n_jobs = n_jobs
        self.trained = False
        
    def fit(self, X_train, Y_train):
        assert X_train.shape[0] == Y_train.shape[0]
        # don't make two changes at the same time
        #self.estimator = OneVsRestClassifier(LogisticRegression(class_weight='balanced', C=self.C))
        self.estimator = OneVsRestClassifier(LogisticRegression(C=self.C), n_jobs=self.n_jobs)
        self.estimator.fit(X_train, Y_train)
        self.trained = True
        
    def decision_function(self, X_test):
        assert self.trained is True
        return self.estimator.decision_function(X_test)
    
    def predict(self, X_test, binarise=False):
        preds = self.decision_function(X_test)
        return preds >= 0.5 if binarise is True else preds        
def print_cv_results(clf):
    if hasattr(clf, 'best_params_'):
        print("\nBest parameters set found on development set:")
        print(clf.best_params_)
    if hasattr(clf, 'cv_results_'):
        for mean, std, params in zip(clf.cv_results_['mean_test_score'], \
                                     clf.cv_results_['std_test_score'], \
                                     clf.cv_results_['params']):
            print("%0.3f (+/-%0.03f) for %r" % (mean, std * 2, params))
def dump_results(predictor, X_train, Y_train, X_test, Y_test, fname):
    
        Compute and save performance results
    
    preds_train = predictor.decision_function(X_train)
    preds_test  = predictor.decision_function(X_test)
    
    print('Training set:')
    perf_dict_train = evaluatePrecision(Y_train, preds_train)
    print()
    print('Test set:')
    perf_dict_test = evaluatePrecision(Y_test, preds_test)
    
    print()
    print('Training set:')
    perf_dict_train.update(evaluateRankingLoss(Y_train, preds_train))
    print(label_ranking_loss(Y_train, preds_train))
    print()
    print('Test set:')
    perf_dict_test.update(evaluateRankingLoss(Y_test, preds_test))
    print(label_ranking_loss(Y_test, preds_test))
    
    F1_train = f1_score_nowarn(Y_train, preds_train >= 0.5, average='samples')
    F1_test  = f1_score_nowarn(Y_test, preds_test >= 0.5, average='samples')
    print('\nF1 Train: %.4f, %f' % (F1_train, f1_score(Y_train, preds_train >= 0.5, average='samples')))
    print('\nF1 Test : %.4f  %f' % (F1_test, f1_score(Y_test, preds_test >= 0.5, average='samples')))
    
    perf_dict_train.update({'F1': (F1_train,)})
    perf_dict_test.update({'F1': (F1_test,)})
    
    perf_dict = {'Train': perf_dict_train, 'Test': perf_dict_test}
    if os.path.exists(fname):
        _dict = pkl.load(open(fname, 'rb'))
        if dataset_name not in _dict:
            _dict[dataset_name] = perf_dict
    else:
        _dict = {dataset_name: perf_dict}
    pkl.dump(_dict, open(fname, 'wb'))
    
    print()
    print(pkl.load(open(fname, 'rb')))
clf = BinaryRelevance(n_jobs=3)
clf.fit(X_train, Y_train)
Y_pred = clf.decision_function(X_test) >= 0
f1_score_nowarn(Y_test, Y_pred, average='samples')
f1_score_nowarn(Y_test, Y_pred, average='macro')
pkl.dump(clf, open(fmodel_base, 'wb'))
def avgF1(Y_true, Y_pred):
    F1 = f1_score_nowarn(Y_true, Y_pred >= 0, average='samples')
    print('\nF1: %g, #examples: %g' % (F1, Y_true.shape[0]))
    return F1
C_set = [1e-6, 3e-6, 1e-5, 3e-5, 1e-4, 3e-4, 1e-3, 3e-3, 0.01, 0.03, 0.1, 0.3,
         1, 3, 10, 30, 100, 300, 1e3, 3e3, 1e4, 3e4, 1e5, 3e5, 1e6, 3e6]
parameters = [{'C': C_set}]
scorer = {'F1': make_scorer(avgF1)}
Explanation: Binary relevance baseline
Train a logistic regression model for each label.
Note that OneVsRestClassifier can be either a multiclass classifier or a multilabel classifier, see this binary relevance example on yeast dataset.
NOTE: To do cross validation (i.e. fit a GridSearchCV), one has to put OneVsRestClassifier into a class wrapper, as  the constructor of OneVsRestClassifier doesn't have a parameter C.
End of explanation
if os.path.exists(fmodel_f1):
    clf = pkl.load(open(fmodel_f1, 'rb'))
else:
    clf = GridSearchCV(BinaryRelevance(), parameters, cv=5, scoring=scorer, verbose=2, n_jobs=10, refit='F1')
    clf.fit(X_train, Y_train)
    pkl.dump(clf, open(fmodel_f1, 'wb'))
f1_score_nowarn(Y_test, clf.decision_function(X_test) >= 0, average='samples')
clf.best_params_
print_cv_results(clf)
dump_results(clf, X_train, Y_train, X_test, Y_test, fperf_prec)
Explanation: Cross validation according to F1.
End of explanation
plot_loss_of_clf(clf, X_train, Y_train, X_test, Y_test)
from evaluate import calcLoss
from matplotlib.ticker import NullFormatter
def plot_loss_of_clf(clf, X_train, Y_train, X_test, Y_test):
    preds_train = clf.decision_function(X_train)
    tploss_train = calcLoss(Y_train, preds_train, 'TopPush', njobs=4)
    pak_train = calcLoss(Y_train, preds_train, 'Precision@K', njobs=4)
    preds_test = clf.decision_function(X_test)
    tploss_test = calcLoss(Y_test, preds_test, 'TopPush', njobs=4)
    pak_test = calcLoss(Y_test, preds_test, 'Precision@K', njobs=4)
    #plot_loss(tploss_train, pak_train, 'Training set (' + dataset_name + ')')
    
    plot_loss(tploss_test, pak_test, 'Test set (' + dataset_name + ')')
def plot_loss(loss, pak, title):
    # the data
    x = loss
    y = 1 - pak
    
    print('away from diagonal portion:', np.mean(loss != 1-pak))
    nullfmt = NullFormatter()         # no labels
    # definitions for the axes
    left, width = 0.1, 0.65
    bottom, height = 0.1, 0.65
    bottom_h = left_h = left + width + 0.02
    rect_scatter = [left, bottom, width, height]
    rect_histx = [left, bottom_h, width, 0.2]
    rect_histy = [left_h, bottom, 0.2, height]
    # start with a rectangular Figure
    plt.figure(1, figsize=(8, 8))
    axScatter = plt.axes(rect_scatter)
    axHistx = plt.axes(rect_histx)
    axHisty = plt.axes(rect_histy)
    # no labels
    axHistx.xaxis.set_major_formatter(nullfmt)
    axHisty.yaxis.set_major_formatter(nullfmt)
    # the scatter plot:
    axScatter.scatter(x, y, color='b', alpha=0.5)
    axScatter.plot([0, 1], [0, 1], ls='--', color='g')
    axScatter.set_xlabel('Top push loss', fontdict={'fontsize': 12})
    axScatter.set_ylabel('1 - precision@K', fontdict={'fontsize': 12})
    # now determine nice limits by hand:
    #binwidth = 0.25
    #xymax = np.max([np.max(np.fabs(x)), np.max(np.fabs(y))])
    #lim = (int(xymax/binwidth) + 1) * binwidth
    #axScatter.set_xlim((-lim, lim))
    #axScatter.set_ylim((-lim, lim))
    #bins = np.arange(-lim, lim + binwidth, binwidth)
    axHistx.hist(x, bins=10, color='g', alpha=0.3)
    axHistx.set_yscale('log')
    axHisty.hist(y, bins=10, color='g', alpha=0.3, orientation='horizontal')
    axHisty.set_xscale('log')
    #axHistx.set_xlim(axScatter.get_xlim())
    #axHisty.set_ylim(axScatter.get_ylim())
    axHistx.set_title(title, fontdict={'fontsize': 15}, loc='center')
%%script false
# NOTE: binary predictions (by predict()) are required for this method to work 
if os.path.exists(fmodel_f1):
    clf = pkl.load(open(fmodel_f1, 'rb'))
else:
    scorer = make_scorer(f1_score_nowarn, average='samples')
    clf = GridSearchCV(BinaryRelevance(), parameters, cv=5, scoring=scorer, verbose=2, n_jobs=6)
    clf.fit(X_train, Y_train)
    pkl.dump(clf, open(fmodel_f1, 'wb'))
print_cv_results(clf)
#dump_results(clf, X_train, Y_train, X_test, Y_test, fperf_f1)
Explanation: Cross validation according to F1.
End of explanation
if os.path.exists(fmodel_base):
    clf = pkl.load(open(fmodel_base, 'rb'))
else:
    clf = OneVsRestClassifier(LogisticRegression(verbose=1))
    clf.fit(X_train, Y_train)
    pkl.dump(clf, open(fmodel_base, 'wb'))
dump_results(clf, X_train, Y_train, X_test, Y_test, fperf_base)
Explanation: Plain logistic regression.
End of explanation
%%script false
allPreds_train  = [ ]
allPreds_test  = [ ]
allTruths_train = [ ]
allTruths_test = [ ]
coefMat = [ ]
labelIndices = [ ]
ranges = range(-6, 7)
parameters = [{'C': sorted([10**(e) for e in ranges] + [3 * 10**(e) for e in ranges])}]
scoring = 'average_precision' # 'accuracy' #'precision_macro'
for label_ix in range(nLabels):
    print('Training for Label %d' % (label_ix+1))
    
    y_train = Y_train[:, label_ix]
    y_test  = Y_test [:, label_ix]
    
    allTruths_train.append(y_train)
    allTruths_test.append(y_test) 
    
    assert( (not np.all(y_train == 0)) and (not np.all(y_train == 1)) )
    
    # searching for a baseline in (Lin et al.) with:
    # test F1 on bibtex 0.372, 26.8 
    # test F1 on bookmarks 0.307, 0.219
    # test F1 on delicious 0.265, 0.102
    
    # test F1 on bibtex: 0.3730, 0.277
    # test F1 on bookmarks: 0.2912, 0.2072
    # test F1 on delicious: 0.1899, 0.1268
    #clf = LogisticRegression(C=100)
    
    # test F1 on bookmarks: 0.2928, 0.2109
    #clf = LogisticRegression(C=60)    
    
    # test F1 on bibtex: 0.4282
    #clf = GridSearchCV(LogisticRegression(class_weight='balanced'), parameters, cv=5, scoring=scoring)
    
    # test F1 on bibtex: < 0.3
    # test F1 on bookmarks: 0.2981, 0.2281
    # test F1 on delicious: 0.1756, 0.0861
    #clf = LogisticRegression()  
    
    # test F1 on bibtex: 0.4342
    #clf = LogisticRegression(class_weight='balanced') 
    
    # test F1 on bibtex: 0.3018
    #clf = GridSearchCV(LogisticRegression(), parameters, cv=5, scoring=scoring)
    
    # test F1 on bibtex: 0.3139
    #clf = GridSearchCV(LogisticRegression(), parameters, scoring=scoring)
    
    # test F1 on bibtex: 0.4252
    #clf = GridSearchCV(LogisticRegression(class_weight='balanced'), parameters, scoring=scoring)
    
    # test F1 on bibtex: 0.3598
    #clf = LogisticRegression(C=10) 
    
    # test F1 on bibtex: 0.3670
    #clf = LogisticRegression(C=30)
    
    estimator = LogisticRegression(class_weight='balanced')#, solver='lbfgs')
    clf = GridSearchCV(estimator, parameters, cv=5, scoring=scoring, n_jobs=4)
    clf.fit(X_train, y_train)
    print("Best parameters set found on development set:")
    print(clf.best_params_)
    print()
    
    allPreds_train.append(clf.decision_function(X_train))
    allPreds_test.append(clf.decision_function(X_test))
    
allTruths_train = np.array(allTruths_train).T
allTruths_test = np.array(allTruths_test).T
allPreds_train  = np.array(allPreds_train).T
allPreds_test  = np.array(allPreds_test).T
print(allPreds_test.shape)
print(allTruths_test.shape)
Explanation: Cross validation for classifier of each label.
End of explanation
#coefMat = np.array(coefMat).T
#coefMat.shape
#sns.heatmap(coefMat[:, :30])
#precisions_train = [avgPrecision(allTruths_train, allPreds_train, k) for k in range(1, nLabels+1)]
#precisions_test  = [avgPrecision(allTruths_test,  allPreds_test,  k) for k in range(1, nLabels+1)]
#precisionK_train = avgPrecisionK(allTruths_train, allPreds_train)
#precisionK_test  = avgPrecisionK(allTruths_test,  allPreds_test)
%%script false
plt.figure(figsize=[10,5])
plt.plot(precisions_train, ls='--', c='r', label='Train')
plt.plot(precisions_test,  ls='-',  c='g', label='Test')
plt.plot([precisionK_train for k in range(nLabels)], ls='-', c='r', label='Train, Precision@K')
plt.plot([precisionK_test  for k in range(nLabels)], ls='-', c='g', label='Test, Precision@K')
plt.xticks(np.arange(nLabels), np.arange(1,nLabels+1))
plt.xlabel('k')
plt.ylabel('Precision@k')
plt.legend(loc='best')
plt.title('Independent Logistic Regression on ' + dataset_name + ' dataset')
plt.savefig(dataset_name + '_lr.svg')
Explanation: Result analysis
End of explanation
<END_TASK>