#to get some base functionality for free, including the methods get_params and set_params 
#to set and return the classifier's parameters as well as the score method to calculate the
#prediction accuracy,respectively
from sklearn.base import BaseEstimator
from sklearn.base import ClassifierMixin
from sklearn.preprocessing import LabelEncoder
#import six too make the MajorityVoteClassifier compatible with python2.7
from sklearn.externals import six
from sklearn.base import clone
from sklearn.pipeline import _name_estimators
import numpy as np
import operator


class MajorityVoteClassifier(BaseEstimator,ClassifierMixin):
    """ A majority vote ensemble classifier
    Parameters
    ----------
    classifiers : array-like, shape = [n_classifiers]
    Different classifiers for the ensemble
   
    vote : str, {'classlabel', 'probability'}
   
    Default: 'classlabel'
    If 'classlabel' the prediction is based on
    the argmax of class labels. Else if
   
    'probability', the argmax of the sum of
    probabilities is used to predict the class label
    (recommended for calibrated classifiers).
    
    weights : array-like, shape = [n_classifiers]
    
    Optional, default: None
    If a list of `int` or `float` values are
    provided , the classifiers are weithed by importance;
    Uses uniform weights if 'weights = None'
    """

    def __init__(self, classifiers,vote='classlabel', weights=None):
        self.classifiers = classifiers
        self.named_classifiers = {key: value for key, value in _name_estimators(classifiers)}
        self.vote = vote
        self.weights = weights

    def fit(self, X, y):
        """ Fit classifiers.
        Parameters
        ----------
        X : {array-like, sparse matrix},
        shape = [n_samples, n_features]
        
        Matrix of training samples.
        y : array-like, shape = [n_samples]
        Vector of target class labels.
        
        Returns
        -------
        self : object
        """
        # Use LabelEncoder to ensure class labels start
        # with 0, which is important for np.argmax
        # call in self.predict
        self.lablenc_ = LabelEncoder()
        self.lablenc_.fit(y)
        self.classes_ = self.lablenc_.classes_
        self.classifiers_ = []
        for clf in self.classifiers:
            fitted_clf = clone(clf).fit(X,self.lablenc_.transform(y))
            self.classifiers_.append(fitted_clf)
        return self
    
    def predict(self, X):
        """ Predict class labels for X.
    
        Parameters
        ----------
        X : {array-like, sparse matrix},
        Shape = [n_samples, n_features]
        Matrix of training samples
        
        Returns
        ----------
        maj_vote : array-like, shape = [n_samples]
        Predicted class labels.
        """
        
        if self.vote == 'probability':
            maj_vote = np.argmax(self.predict_proba(X),axis=1)
        else: 
            # 'classlabel' vote
            # Collect results from clf.predict calls
            predictions = np.asarray([clf.predict(X) for clf in self.classifiers_]).T
            maj_vote = np.apply_along_axis(lambda x: np.argmax(np.bincount(x,weights=self.weights)),axis=1,arr=predictions)
            maj_vote = self.lablenc_.inverse_transform(maj_vote)
            return maj_vote


    def predict_proba(self, X):
        """ Predict class probabilities for X.
        Parameters
        ----------
        X : {array-like, sparse matrix},
        shape = [n_samples, n_features]
        Training vectors, where n_samples is
        the number of samples and
        n_features is the number of features.
        
        Returns
        ----------
        avg_proba : array-like,
        shape = [n_samples, n_classes]
        Weighted average probability for
        each class per sample.
        """
        probas = np.asarray([clf.predict_proba(X) for clf in self.classifiers_])
        avg_proba = np.average(probas,axis=0, weights=self.weights)
        return avg_proba
    
    def get_params(self, deep=True):
        """ Get classifier parameter names for GridSearch"""
        if not deep:
            return super(MajorityVoteClassifier,self).get_params(deep=False)
        else:
            out = self.named_classifiers.copy()
            for name, step in six.iteritems(self.named_classifiers):
                for key, value in six.iteritems(step.get_params(deep=True)):
                    out['%s__%s' % (name, key)] = value
            return out

#get datas
from sklearn import datasets
from sklearn.cross_validation import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.preprocessing import LabelEncoder
iris = datasets.load_iris()
X,y = iris.data[50:,[1,2]],iris.target[50:]
le = LabelEncoder()
y = le.fit_transform(y)

X_train,X_test,y_train,y_test = train_test_split(X,y,test_size = 0.5,random_state = 1)

#train logistic regression classifier, decision tree, k-nearest neightbor respectively
from sklearn.cross_validation import cross_val_score
from sklearn.linear_model import LogisticRegression
from sklearn.tree import DecisionTreeClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.pipeline import Pipeline

import numpy as np
clf1 = LogisticRegression(penalty = 'l2',C = 0.001,random_state = 0)
clf2 = DecisionTreeClassifier(max_depth = 1,criterion = 'entropy',random_state = 0)
clf3 = KNeighborsClassifier(n_neighbors = 1,p=2,metric = 'minkowski')
pipe1 = Pipeline([['sc',StandardScaler()],['clf',clf1]])
pipe3 = Pipeline([['sc',StandardScaler()],['clf',clf3]])

clf_labels = ['Logistic Regression','Decision Tree','KNN']
print('10-fold cross validation:\n')
for clf,label in zip([pipe1,clf2,pipe3],clf_labels):
    scores = cross_val_score(estimator = clf,
                             X=X_train,
                             y=y_train,
                             cv=10,
                             scoring = 'roc_auc')
    print ("ROC AUC: %0.2f (+/- %0.2f) [%s]" % (scores.mean(),scores.std(),label))

#combine the individual classifiers for majority rule voting in our MajorityVoteClassifier
#import os
#pwd = os.getcwd()
#os.chdir('E:\\machine-learning\\19-Ensemble Learning\\')

#from majority_voting import MajorityVoteClassifier
mv_clf = MajorityVoteClassifier(classifiers=[pipe1, clf2, pipe3])
clf_labels += ['Majority Voting']
all_clf = [pipe1, clf2, pipe3, mv_clf]
for clf, label in zip(all_clf, clf_labels):
    scores = cross_val_score(estimator=clf,X=X_train,y=y_train,cv=10,scoring='roc_auc')
    print("Accuracy: %0.2f (+/- %0.2f) [%s]"% (scores.mean(), scores.std(), label))

#os.chdir(pwd)

#compute the ROC curves from the test set to check if the MajorityVoteClassifier generalizes well to unseen data
from sklearn.metrics import roc_curve
from sklearn.metrics import auc
import matplotlib.pyplot as plt
colors = ['black','orange','blue','green']
linestyles = [':', '--', '-.', '-']
for clf,label,clr,ls in zip(all_clf,clf_labels,colors,linestyles):
    #assuming the label of the positive class is 1
    y_pred = clf.fit(X_train,y_train).predict_proba(X_test)[:,1]
    fpr,tpr,thresholds = roc_curve(y_true = y_test,y_score = y_pred)
    
    roc_auc = auc(x= fpr,y=tpr)
    plt.plot(fpr,tpr,color = clr,linestyle = ls,label = '%s (auc = %0.2f)' % (label,roc_auc))
plt.legend(loc = 'lower right')
plt.plot([0, 1], [0, 1],linestyle='--',color='gray',linewidth=2)
plt.xlim([-0.1, 1.1])
plt.ylim([-0.1, 1.1])
plt.grid()
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.show()

#tune the inverse regularization parameter C of the logistic regression classifier and the decision tree
#depth via a grid search for demonstration purposes
from sklearn.grid_search import GridSearchCV
params = {'decisiontreeclassifier__max_depth':[1,2],'pipeline-1__clf__C':[0.001,0.1,100.0]}
grid = GridSearchCV(estimator = mv_clf,param_grid=params,cv = 10,scoring = 'roc_auc')
grid.fit(X_train,y_train)
for params,mean_score,scores in grid.grid_scores_:
    print('%0.3f +/- %0.2f %r' % (mean_score,scores.std()/2,params))

print('Best parameters : %s' % grid.best_params_)
print('Accuracy: %.2f' % grid.best_score_)




