import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns

# %matplotlib inline

red_dataframe = pd.read_csv('winequality-red.csv', sep=';')
red_dataframe.head()

red_dataframe = pd.read_csv('winequality-red.csv', sep=';')
red_dataframe.drop('quality', axis=1).describe()

correlation = red_dataframe.corr()
fig, ax = plt.subplots(figsize=(15,15))
sns.heatmap(correlation, xticklabels=correlation.columns, yticklabels=correlation.columns, cmap='coolwarm_r', vmin=-1, vmax=1, annot=True, ax=ax)

white_dataframe = pd.read_csv('winequality-white.csv', sep=';')
white_dataframe.drop('quality', axis=1).describe()

correlation = white_dataframe.corr()
fig, ax = plt.subplots(figsize=(15,15))
sns.heatmap(correlation, xticklabels=correlation.columns, yticklabels=correlation.columns,cmap='coolwarm_r', vmin=-1, vmax=1, annot=True, ax=ax)

both = [red_dataframe, white_dataframe]
both_dataframe = pd.concat(both)
both_dataframe.drop('quality', axis=1).describe()

correlation = both_dataframe.corr()
fig, ax = plt.subplots(figsize=(15,15))
sns.heatmap(correlation, xticklabels=correlation.columns, yticklabels=correlation.columns,cmap='coolwarm_r', vmin=-1, vmax=1, annot=True, ax=ax)

from __future__ import absolute_import
from __future__ import print_function
from __future__ import division

# %matplotlib inline  

import sys
from math import *
import matplotlib
import numpy as np
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import axes3d
from tqdm import tqdm
import os
from scipy import ndimage, misc
from matplotlib import pyplot as plt
import numpy as np
import imageio
from sklearn.datasets import load_boston, load_diabetes, load_digits, load_breast_cancer, load_iris, load_wine
# %matplotlib inline
from sklearn.cluster import KMeans


data_red=np.genfromtxt("RedWine.csv", delimiter=',')
X_red = data_red[1:,:-1]
Y_red = data_red[1:,-1]


data_white=np.genfromtxt("WhiteWine.csv", delimiter=',')
X_white = data_white[1:,:-1]
Y_white = data_white[1:,-1]

buckets = [0] * 10

for i in range(len(Y_red)):
  buckets[int(Y_red[i])] += 1

plt.bar(range(len(buckets)), buckets)

def plot_curve(x, y, color, label_x, label_y, curve_type='.', lw=2):
    plt.plot(x, y, curve_type, color=color, linewidth=lw, )
    plt.xlabel(label_x)
    plt.ylabel(label_y)
    plt.grid(True)

def pca(X):
    """
    Decompose dataset into principal components. 
    You may use your SVD function from the previous part in your implementation.
    
    Args: 
        X: N*D array corresponding to a dataset
    Return:
        U: N*N 
        S: min(N, D)*1 
        V: D*D
    """
    U, s, VT = np.linalg.svd(X)
    return U, s, VT
   
def intrinsic_dimension(S, recovered_variance=.99):
    """
    Find the number of principal components necessary to recover given proportion of variance
    
    Args: 
        S: 1-d array corresponding to the singular values of a dataset
        
        recovered_varaiance: float in [0,1].  Minimum amount of variance 
            to recover from given principal components
    Return:
        dim: int, the number of principal components necessary to recover 
            the given proportion of the variance
    """
    x = 0
    k = 0
    closed_list = []
    print(S.shape)
    open_list = np.copy(S).tolist()

    while x < .99:
      k = k + 1
      a = np.max(open_list)
      open_list.remove(a)  
      closed_list.append(a)
      x  = sum(closed_list)/sum(S)
    return k

def num_linearly_ind_features(S, eps=1e-11):
    """
    Find the number of linearly independent features in dataset
    
    Args: 
        S: 1-d array corresponding to the singular values of a dataset
    Return:
        dim: int, the number of linearly independent dimensions in our data
    """
    count = 0
    for val in S:
      if val > eps:
        count = count + 1
    return count

def visualize(X,y, color, x_label, y_label):
  """
  Args:
  X: NxD numpy array, where N is number 
          of instances and D is the dimensionality of each 
          instance
  y: numpy array (N,), the true labels
  
  Return:
  retained variance: scalar
  """
  U, s, VT = np.linalg.svd(X)
  proj = VT[:,0:2]
  data = X @ proj
  plot_curve(data[:,0],data[:,1], color, x_label, y_label)


  return np.sum(s[0:2])/np.sum(s)

#Use PCA for visualization of  wine data

retained_variance_for_wine=visualize(X_red,Y_red, 'r', 'Total Sulfur Dioxide','Free Sulfur Dioxide')
print("Retained variance for Red wine dataset ",retained_variance_for_wine)

retained_variance_for_wine=visualize(X_white,Y_white,'b','total Sulfur Dioxide','Free Sulfur Dioxide')
print("Retained variance for White wine dataset ",retained_variance_for_wine)

def rmse(pred, label): 
    '''
    This is the root mean square error.
    Args:
        pred: numpy array of length N * 1, the prediction of labels
        label: numpy array of length N * 1, the ground truth of labels
    Return:
        a float value
    '''
    dif = pred - label
    dif = dif**2
    dif = dif/len(pred)
    sum_dif =sum(dif)
    return np.sqrt(sum_dif)

class LinearReg(object):
    @staticmethod 
    # static method means that you can use this method or function for any other classes, it is not specific to LinearReg
    def fit_closed(xtrain, ytrain):
        """
        Args:
            xtrain: NxD numpy array, where N is number 
                    of instances and D is the dimensionality of each 
                    instance
            ytrain: Nx1 numpy array, the true labels
        Return:
            weight: Dx1 numpy array, the weights of linear regression model
        """
        x = xtrain.T  @ xtrain
        x = np.linalg.inv(x) @ xtrain.T 
        x = x @ ytrain
        return x

    def predict(xtest, weight):
        """
        Args:
            xtest: NxD numpy array, where N is number 
                   of instances and D is the dimensionality of each 
                   instance
            weight: Dx1 numpy array, the weights of linear regression model
        Return:
            prediction: Nx1 numpy array, the predicted labels
        """
        
        return xtest @ weight

class RidgeReg(LinearReg):

    @staticmethod
    def fit_closed(xtrain, ytrain, c_lambda):
      m,n = xtrain.shape
      I = np.eye((n))
      G = c_lambda * I
      G[0,0] = 0
      return (np.linalg.inv(xtrain.T @ xtrain + G) @ xtrain.T @ ytrain)

#apply PCA on the dataset and also find the number of linearly independent and intrinsic components 
def apply_PCA_on_data(X):
  """
  Args:
  X: NxD numpy array, where N is number 
          of instances and D is the dimensionality of each 
          instance
  Return:
  X_pca: pca reduced dataset
  independent_features: number of independent features 
  intrinsic_dimensions: number of intrinsic dimensions
  """
  
  U, s, VT = np.linalg.svd(X)
  k = intrinsic_dimension(s)
  proj = VT[:,0:k]
  data = X @ proj
  return data, num_linearly_ind_features(s), k

def apply_regression(X_train,y_train,X_test):
  """
  Args:
  X_train: training data without labels
  y_train: training labels
  X_test: test data 

  Return:
  y_pred: predicted labels
  """
  w = RidgeReg.fit_closed(X_train, y_train, c_lambda = 0)
  return X_test @ w

def cross_validation(X, y, kfold, c_lambda):
    X_groups = np.split(X, kfold, axis = 0)
    Y_groups = np.split(y, kfold, axis = 0)
    r = 0
    for i in range(kfold):
      w = RidgeReg.fit_closed(X, y, c_lambda)
      y_pred = RidgeReg.predict(X_groups[i], w)
      r += rmse(y_pred, Y_groups[i])   
    return r/kfold     
    
X_redCV = X_red[:-9]
Y_redCV = Y_red[:-9]

best_lambda = None
best_error = None
kfold = 10
lambda_list = [0, 0.1, 1, 5, 10, 100, 1000]
for lm in lambda_list:
    err = cross_validation(X_redCV, Y_redCV, kfold, lm)
    print('lambda: %.2f' % lm, 'error: %.6f'% err)
    if best_error is None or err < best_error:
        best_error = err
        best_lambda = lm


print('best_lambda for Red Wine: %.2f' % best_lambda)
weight = RidgeReg.fit_closed(X_redCV, Y_redCV, c_lambda=10)
y_test_pred = RidgeReg.predict(X_redCV, weight)
test_rmse = rmse(y_test_pred, Y_redCV)
print('test rmse: %.4f' % test_rmse)

def cross_validation(X, y, kfold, c_lambda):
    X_groups = np.split(X, kfold, axis = 0)
    Y_groups = np.split(y, kfold, axis = 0)
    r = 0
    for i in range(kfold):
      w = RidgeReg.fit_closed(X, y, c_lambda)
      y_pred = RidgeReg.predict(X_groups[i], w)
      r += rmse(y_pred, Y_groups[i])   
    return r/kfold   

    
X_whiteCV = X_white[:-8]
Y_whiteCV = Y_white[:-8]

best_lambda = None
best_error = None
kfold = 10
lambda_list = [0, 0.1, 1, 5, 10, 100, 1000]
for lm in lambda_list:
    err = cross_validation(X_whiteCV, Y_whiteCV, kfold, lm)
    print('lambda: %.2f' % lm, 'error: %.6f'% err)
    if best_error is None or err < best_error:
        best_error = err
        best_lambda = lm


print('best_lambda for White Wine: %.2f' % best_lambda)
weight = RidgeReg.fit_closed(X_whiteCV, Y_whiteCV, c_lambda=10)
y_test_pred = RidgeReg.predict(X_whiteCV, weight)
test_rmse = rmse(y_test_pred, Y_whiteCV)
print('test rmse: %.4f' % test_rmse)

X_red_ratings = [[]]
Y_red_ratings = [[]]
for i in range(10):
  X_red_ratings.append([])
  Y_red_ratings.append([])

for i in range(len(Y_red)):
  rating = int(Y_red[i])
  X_red_ratings[rating].append(X_red[i])
  Y_red_ratings[rating].append(Y_red[i])

X_red_ratings = np.asarray(X_red_ratings)
Y_red_ratings = np.asarray(Y_red_ratings)

for i in range(10):
  if(len(X_red_ratings[i]) > 0):
    X_PCA, ind_features, intrinsic_dimensions = apply_PCA_on_data(X_red_ratings[i])
    print("data shape with PCA ",X_PCA.shape)
    print("Number of independent features ",ind_features)
    print("Number of intrinsic components ",intrinsic_dimensions)

    #get training and testing data 
    X_train=X_PCA[:int(0.8*len(X_PCA)),:]
    y_train=np.asarray(Y_red_ratings[i])[:int(0.8*len(X_PCA))].reshape(-1,1)
    X_test=X_PCA[int(0.8*len(X_PCA)):]
    y_test=np.asarray(Y_red_ratings[i])[int(0.8*len(X_PCA)):].reshape(-1,1)

    #use Ridge Regression for getting predcited labels
    y_pred=apply_regression(X_train,y_train,X_test)

    #calculate RMSE 
    rmse_score = rmse(y_pred, y_test)
    print("rmse of Red Wine score with PCA for group: ",i + 1,rmse_score)
    i += 1
    avg += rmse_score
print("Average rmse: ", avg/i)

X_white_ratings = [[]]
Y_white_ratings = [[]]
for i in range(10):
  X_white_ratings.append([])
  Y_white_ratings.append([])

for i in range(len(Y_red)):
  rating = int(Y_red[i])
  X_white_ratings[rating].append(X_white[i])
  Y_white_ratings[rating].append(Y_white[i])

X_white_ratings = np.asarray(X_white_ratings)
Y_white_ratings = np.asarray(Y_white_ratings)
avg = 0
l = 0

for i in range(10):
  if(len(X_white_ratings[i]) > 0):
    X_PCA, ind_features, intrinsic_dimensions = apply_PCA_on_data(X_white_ratings[i])
    print("data shape with PCA ",X_PCA.shape)
    print("Number of independent features ",ind_features)
    print("Number of intrinsic components ",intrinsic_dimensions)

    #get training and testing data 
    X_train=X_PCA[:int(0.8*len(X_PCA)),:]
    y_train=np.asarray(Y_white_ratings[i])[:int(0.8*len(X_PCA))].reshape(-1,1)
    X_test=X_PCA[int(0.8*len(X_PCA)):]
    y_test=np.asarray(Y_white_ratings[i])[int(0.8*len(X_PCA)):].reshape(-1,1)

    #use Ridge Regression for getting predcited labels
    y_pred=apply_regression(X_train,y_train,X_test)

    #calculate RMSE 
    rmse_score = rmse(y_pred, y_test)
    print("rmse of White Wine score with PCA for group: ",i + 1,rmse_score)
    l += 1
    avg += rmse_score
print("Average rmse: ", avg/l)

#load the dataset 


X_PCA, ind_features, intrinsic_dimensions = apply_PCA_on_data(X_red)
print("data shape with PCA ",X_PCA.shape)
print("Number of independent features ",ind_features)
print("Number of intrinsic components ",intrinsic_dimensions)

#get training and testing data 
X_train=X_PCA[:int(0.8*len(X_PCA)),:]
y_train=Y_red[:int(0.8*len(X_PCA))].reshape(-1,1)
X_test=X_PCA[int(0.8*len(X_PCA)):]
y_test=Y_red[int(0.8*len(X_PCA)):].reshape(-1,1)

#use Ridge Regression for getting predcited labels
y_pred=apply_regression(X_train,y_train,X_test)

#calculate RMSE 
rmse_score = rmse(y_pred, y_test)
print("rmse of Red Wine score with PCA",rmse_score)

#Ridge regression without PCA
X_train=X_red[:int(0.8*len(X_red)),:]
y_train=Y_red[:int(0.8*len(X_red))].reshape(-1,1)
X_test=X_red[int(0.8*len(X_red)):]
y_test=Y_red[int(0.8*len(X_red)):].reshape(-1,1)

#use Ridge Regression for getting predcited labels
y_pred=apply_regression(X_train,y_train,X_test)

#calculate RMSE 
print(X_train.shape)
rmse_score = rmse(y_pred, y_test)
print("rmse score of Red Wine without PCA",rmse_score)

#load the dataset 
X_PCA, ind_features, intrinsic_dimensions = apply_PCA_on_data(X_white)
print("data shape with PCA ",X_PCA.shape)
print("Number of independent features ",ind_features)
print("Number of intrinsic components ",intrinsic_dimensions)

#get training and testing data 
X_train=X_PCA[:int(0.8*len(X_PCA)),:]
y_train=Y_white[:int(0.8*len(X_PCA))].reshape(-1,1)
X_test=X_PCA[int(0.8*len(X_PCA)):]
y_test=Y_white[int(0.8*len(X_PCA)):].reshape(-1,1)

#use Ridge Regression for getting predcited labels
y_pred=apply_regression(X_train,y_train,X_test)

#calculate RMSE 
rmse_score = rmse(y_pred, y_test)
print("rmse of White Wine score with PCA",rmse_score)

#Ridge regression without PCA
X_train=X_white[:int(0.8*len(X_white)),:]
y_train=Y_white[:int(0.8*len(X_white))].reshape(-1,1)
X_test=X_white[int(0.8*len(X_white)):]
y_test=Y_white[int(0.8*len(X_white)):].reshape(-1,1)

#use Ridge Regression for getting predcited labels
y_pred=apply_regression(X_train,y_train,X_test)

#calculate RMSE 
print(X_train.shape)
rmse_score = rmse(y_pred, y_test)
print("rmse score of White Wine without PCA",rmse_score)

#Bad Wine 0-4
#Good Wine 5-6
#Great Wine 7+


U, s, VT = np.linalg.svd(X_red)
proj = VT[:,0:3]
data = X_red @ proj

k = 3
kmeans = KMeans(k, random_state=0).fit(data)
#kmeans.predict(data[0]
predictions = kmeans.fit_predict(data)
print(predictions[0:20])
cluster_labels = kmeans.labels_

clusters = [0] * k
len_clusters = [0] * k

for i in range(len(predictions)):
  clusters[predictions[i]] += Y_red[i]
  len_clusters[predictions[i]] += 1

for i in range(len(clusters)):
  clusters[i] = clusters[i]/len_clusters[i]


print("Ratings and Centers Per Column")
print(len_clusters)
print(clusters)
print(kmeans.cluster_centers_.T)

def visualise(X, C, K):#Visualization of clustering. You don't need to change this function   
    fig = plt.figure()
    ax = fig.gca(projection='3d')
    ax.scatter(X[:,0], X[:,1], X[:,2], c=C,cmap='rainbow')
    plt.title('Visualization of K = '+str(K), fontsize=15)
    plt.show()
    pass
visualise(X_red, kmeans.labels_,k)

import numpy as np
from collections import Counter
from scipy import stats
from math import log2, sqrt
import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import LabelEncoder
from sklearn.tree import DecisionTreeClassifier

def entropy(class_y):
    
    ones = sum(class_y)/len(class_y)
    zeros = 1 - ones
    
    if ones != 0:
      o = -ones * log2(ones)
    else:
      o = 0
    if zeros != 0:
      z = zeros * log2(zeros)
    else:
      z = 0
    return o - z

def information_gain(previous_y, current_y): 
    e = []
    a = 0
    for split in current_y:
      a += len(split)
      if(len(split) > 0):
        e.append(len(split) * entropy(split))
    return entropy(previous_y) - sum(e)/(a)

def partition_classes(X, y, split_attribute, split_val):
       
    if(isinstance(X[0][split_attribute], int)):
      s = int(split_val)
      b = np.asarray(X)
      b = np.asarray(b[:,split_attribute], dtype = int)
      idx = b <= s
      idx1 = b > s
      X_left =  np.asarray(X, dtype='O')[idx]
      X_right =  np.asarray(X, dtype='O')[idx1]
      Y_left =  np.asarray(y)[idx]
      Y_right =  np.asarray(y)[idx1]
      return X_left, X_right, Y_left,Y_right
    else:
      b = np.asarray(X)
      b = np.asarray(b[:,split_attribute])
      idx = b == split_val
      idx1 = b != split_val
      X_left =  np.asarray(X, dtype='O')[idx]
      X_right =  np.asarray(X, dtype='O')[idx1]
      Y_left =  np.asarray(y)[idx]
      Y_right =  np.asarray(y)[idx1]
      return X_left, X_right, Y_left,Y_right


def find_best_split(X, y, split_attribute):
    
    possible_splits = np.unique(np.asarray(X)[:,split_attribute])
    best = 0
    opt = 0
    starting = entropy(y)
    for val in possible_splits:
      
      X_left, X_right, Y_left,Y_right = partition_classes(X,y,split_attribute,val) 
      current_y = [Y_left, Y_right]
      ig = information_gain(y, current_y)
      if ig > best:
        best = ig
        opt = val
    return opt, best
  

def find_best_feature(X, y):
    
    best_ig = 0
    best_val = 0
    feature = 0
    for i in range(np.asarray(X).shape[1]):
      opt, ig = find_best_split(X, y, i)
      if ig > best_ig:
        best_ig = ig 
        best_val = opt
        feature = i
      if(ig == best_ig):
        if np.random.randint(0,2) > 0:
          best_ig = ig 
          best_val = opt
          feature = i
    print(feature)
    return feature, best_val



class MyDecisionTree(object):
    def __init__(self, max_depth=None):
        
        self.tree = {}
        self.root = None
        self.max_depth = max_depth


    def fit(self, X, y, depth):
        
        if depth == self.max_depth:
          node = {'isLeaf': True, 'class': round(sum(y)/len(y))}
          return node
        if len(np.unique(y)) == 1:
          node = {'isLeaf': True, 'class': y[0]}
          return node
        
        best_feature, best_split_val = find_best_feature(X, y)
        X_left, X_right, Y_left,Y_right = partition_classes(X,y, best_feature, best_split_val)
        leftTree = self.fit(X_left, Y_left, depth + 1)
        rightTree = self.fit(X_right, Y_right, depth + 1)
        is_categorical = not isinstance(X[0][best_feature], int)
        node = {
            'isLeaf': False,
            'split_attribute': best_feature,
            'is_categorical': is_categorical,
            'split_value': best_split_val,
            'leftTree': leftTree,
            'rightTree': rightTree
        }
        self.root = node
        return node

        
        
       
    def predict(self, record):
        
        current_node = self.root
        while(not current_node['isLeaf']):
          split_attribute = current_node['split_attribute']
          split_value = current_node['split_value']
          if not current_node['is_categorical']:
            if int(record[split_attribute]) <= int(split_value):
              current_node = current_node['leftTree']
            else:
              current_node = current_node['rightTree']
          else:
            if record[split_attribute] == split_value:
              current_node = current_node['leftTree']
            else:
              current_node = current_node['rightTree']
        return current_node['class']


def DecisionTreeEvalution(dt,X,y, verbose=False):

    # Make predictions
    # For each test sample X, use our fitting dt classifer to predict
    y_predicted = []
    for record in X: 
        y_predicted.append(dt.predict(record))

    # Comparing predicted and true labels
    results = [prediction == truth for prediction, truth in zip(y_predicted, y)]

    # Accuracy
    accuracy = float(results.count(True)) / float(len(results))
    if verbose:
        print("accuracy: %.4f" % accuracy)
    return accuracy

data_test = pd.read_csv("RedWineTest.csv")
data_train = pd.read_csv("RedWineTrain.csv")


X_train = np.array(data_train)[:,:-2]
print(X_train.shape)
y_train = np.array(data_train)[:,-1]
X_test = np.array(data_test)[:,:-2]
y_test = np.array(data_test)[:,-1]

# Initializing a decision tree.
max_depth = 7
dt = MyDecisionTree(max_depth)

# Building a tree
print("fitting the decision tree")
dt.fit(X_train, y_train, 0)

DecisionTreeEvalution(dt,X_test,y_test, True)

data_test = pd.read_csv("WhiteWineTest.csv")
data_train = pd.read_csv("WhiteWineTrain.csv")


X_train = np.array(data_train)[:,:-2]
print(X_train.shape)
y_train = np.array(data_train)[:,-1]
X_test = np.array(data_test)[:,:-2]
y_test = np.array(data_test)[:,-1]

# Initializing a decision tree.
max_depth = 7
dt = MyDecisionTree(max_depth)

# Building a tree
print("fitting the decision tree")
dt.fit(X_train, y_train, 0)

DecisionTreeEvalution(dt,X_test,y_test, True)

data = np.array(pd.read_csv("Wine.csv"))

idx = np.arange(data.shape[0]).tolist()
idxs = []
for i in range(int(len(idx) * .8)):
  choice = np.random.choice(idx)
  idx.remove(choice)
  idxs.append(choice) 




X_train = np.array(data)[idxs,:-2]
print(X_train.shape)
y_train = np.array(data)[idxs,-1]
X_test = np.array(data)[idx,:-2]
y_test = np.array(data)[idx,-1]

# Initializing a decision tree.
max_depth = 7
dt = MyDecisionTree(max_depth)

# Building a tree
print("fitting the decision tree")
dt.fit(X_train, y_train, 0)

DecisionTreeEvalution(dt,X_test,y_test, True)