# import dependencies
%matplotlib inline
import csv as csv
import numpy as np
import pandas as pd
import pylab as py
import operator
import re, os, progressbar
import sys
import nltk
from pprint import *
from sklearn.manifold import TSNE
import multiprocessing
import codecs
import gensim.models.word2vec as w2v
from nltk.corpus import stopwords
from nltk import FreqDist
from collections import Counter
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfTransformer, TfidfVectorizer
from sklearn.naive_bayes import MultinomialNB
from sklearn.pipeline import Pipeline
from sklearn.cross_validation import KFold
from sklearn.model_selection import GridSearchCV
import sklearn.metrics as skmetrics
from sklearn.metrics import confusion_matrix, f1_score
#import plot_tools as myplt
from sklearn.externals import joblib
from operator import itemgetter
import pickle
import logging

fake_data = pd.read_csv('data/fake.csv')
fake_data = fake_data[fake_data.language == 'english']
fake_data.dropna(axis=0, inplace=True, subset=['text'])
#fake_data = fake_data.sample(frac=1.0)
fake_data.reset_index(drop=True,inplace=True)
fake_data.describe()

# Now the Real Data
real_data = pd.read_csv('data/real_news.csv')
real_data = real_data[fake_data.language == 'english']
real_data.dropna(axis=0, inplace=True, subset=['text'])
#real_data = real_data.sample(frac=1.0)
real_data.reset_index(drop=True,inplace=True)

# Add category names(Fake, Real) to their respective dataset
fake_data['class'] = 0
real_data['class'] = 1

# Combine the real and fake for training(to be used later on).  We will randomize the fake and real sets together.
fake_and_real_data = pd.concat([fake_data, real_data]).sample(frac=1.0)
#fake_and_real_data = pd.concat([fake_data, real_data])

fake_data.head(10)

fake_data.hist(column='likes', bins=30, range=(0,30))
fake_data.hist(column='comments', bins=30, range=(0,30))

# Fake
count_vect = CountVectorizer()
fake_matrix_CV = count_vect.fit_transform(fake_data['text'].values.astype('U'))
fake_features  = count_vect.get_feature_names()

# Real
rcount_vect = CountVectorizer(stop_words="english")
real_matrix_CV = rcount_vect.fit_transform(real_data['text'].values.astype('U'))

count_vect.vocabulary_.get(u'algorithm')
rcount_vect.vocabulary_.get(u'algorithm')

fake_vocab = count_vect.vocabulary_
real_vocab = rcount_vect.vocabulary_ 

# Sort by frequency in the text
vocab_sorted = [v[0] for v in sorted(fake_vocab.items(), key=operator.itemgetter(1))]
fdist_fake_vocab = freq_distribution = Counter(dict(zip(vocab_sorted, np.ravel(fake_matrix_CV.sum(axis=0)))))
fdist_fake_vocab.most_common(10)

count_vect = CountVectorizer(stop_words="english")
fake_matrix_CV = count_vect.fit_transform(fake_data['text'].values.astype('U'))

# Combined Fake and Real Datasets
tfidf_transformer = TfidfTransformer()
count_vect = CountVectorizer(stop_words="english")
realfake_matrix_CV = count_vect.fit_transform(fake_and_real_data['text'].values.astype('U'))
realfake_matrix_tfidf = tfidf_transformer.fit_transform(realfake_matrix_CV)
targets = fake_and_real_data['class'].values
#count_vect.vocabulary_.get(u'algorithm')
combine_vocab = count_vect.vocabulary_
feature_list = [word for word in combine_vocab.keys()] 
features = count_vect.get_feature_names()

def top_tfidf_feats(row, features, top_n=25):
    ''' Get top n tfidf values in row and return them with their corresponding feature names.
    '''

    topn_ids = np.argsort(row)[::-1][:top_n]
    top_feats = [(features[i], row[i]) for i in topn_ids]
    df = pd.DataFrame(top_feats)
    df.columns = ['feature', 'CV']
    return df

def top_feats_in_doc(Xtr, features, row_id, top_n=25):
    ''' 
    Top tfidf features in specific document (matrix row) 
    '''
    
    row = np.squeeze(Xtr[row_id].toarray())
    return top_tfidf_feats(row, features, top_n)

def top_mean_feats(Xtr, features, grp_ids=None, min_tfidf=0.1, top_n=25):
    ''' Return the top n features that on average are most important amongst documents in rows
        indentified by indices in grp_ids. 
        Xtr = tfidf array
        features = names from vocab(list)
    '''
    
    if grp_ids:
        D = Xtr[grp_ids].toarray()
    else:
        D = Xtr.toarray()

    D[D < min_tfidf] = 0
    tfidf_means = np.mean(D, axis=0)
    return top_tfidf_feats(tfidf_means, features, top_n)

def top_sum_feats(Xtr, features, grp_ids=None, top_n=25):
    ''' Return the top n features that are the most frequent amongst documents in rows
        indentified by indices in grp_ids. 
        Xtr = tfidf array
        features = names from vocab(list)
    '''
    
    if grp_ids:
        D = Xtr[grp_ids].toarray()
    else:
        D = Xtr.toarray()

    tfidf_sums = np.sum(D, axis=0)
    return top_tfidf_feats(tfidf_sums, features, top_n)

print(top_mean_feats(realfake_matrix_CV, features))
print('\n\n')
print(top_sum_feats(realfake_matrix_CV, features))

def top_feats_by_class(Xtr, y, features, min_tfidf=0.1, top_n=25):
    ''' Return a list of dfs, where each df holds top_n features and their mean tfidf value
        calculated across documents with the same class label.
        Xtr = tfidf array
        features = names from vocab(list)
        y = class labels(fake or real)
    '''
    
    dfs = []
    labels = np.unique(y)
    #print(labels)
    for label in labels:
        ids = np.where(y==label)
        feats_df = top_mean_feats(Xtr, features, ids, min_tfidf=min_tfidf, top_n=top_n)
        
        if label == 0:
            feats_df.label = 'Fake'
        else: feats_df.label = 'Real'
             
        
        dfs.append(feats_df)
    return dfs

df_class_list = top_feats_by_class(realfake_matrix_tfidf, targets, features)

def plot_tfidf_classfeats_h(dfs, title):
    ''' Plot the data frames returned by the function top_feats_by_class(). 
    '''
    
    fig = plt.figure(figsize=(12, 9), facecolor="w")
    x = np.arange(len(dfs[0]))
    for i, df in enumerate(dfs):
        ax = fig.add_subplot(1, len(dfs), i+1)
        ax.spines["top"].set_visible(False)
        ax.spines["right"].set_visible(False)
        ax.set_frame_on(False)
        ax.get_xaxis().tick_bottom()
        ax.get_yaxis().tick_left()
        ax.set_xlabel("Mean "+title+" Score", labelpad=16, fontsize=14)
        ax.set_title("label = " + str(df.label), fontsize=16)
        ax.ticklabel_format(axis='x', style='sci', scilimits=(-2,2))
        ax.barh(x, df.CV, align='center', color='#3F5D7D')
        ax.set_yticks(x)
        ax.set_ylim([-1, x[-1]+1])
        yticks = ax.set_yticklabels(df.feature)
        plt.subplots_adjust(bottom=0.09, right=0.97, left=0.15, top=0.95, wspace=0.52)
    plt.savefig('word_freq_compare_'+title+'.pdf')
    plt.show()
    
plot_tfidf_classfeats_h(df_class_list, 'TFIDF')

df_CV_class_list = top_feats_by_class(realfake_matrix_CV, targets, features)
plot_tfidf_classfeats_h(df_CV_class_list, 'CV')

# Create Pipelines to Validate

CV_pipeline = Pipeline([
    ('CountVectorizer',  CountVectorizer()),
    ('MNBclassifier',  MultinomialNB()) ])

tfidf_pipeline = Pipeline([
    ('CountVectorizer',  CountVectorizer()),
    ('tfidf',  TfidfTransformer()),
    ('MNBclassifier',  MultinomialNB()) ])

def class_report(conf_mat):
    tp, fp, fn, tn = confusion.flatten()
    measures = {}
    measures['accuracy'] = (tp + tn) / (tp + fp + fn + tn)
    measures['specificity'] = tn / (tn + fp)        # (true negative rate)
    measures['sensitivity'] = tp / (tp + fn)        # (recall, true positive rate)
    measures['precision'] = tp / (tp + fp)
    measures['f1score'] = 2*tp / (2*tp + fp + fn)
    return measures

def kfold_validate(data, pipeline):
    ''' Performs kFold Cross-Validation on text classes.
        Inputs: (pd.dataset, scikit.pipeline)
        Returns: [title, f1_scores, confusion matrix, class_report]
    '''
    
    title = [str(step) for step in pipeline.named_steps]
    title = '_'.join(title)
    print('\nK-Fold Validation on Model:', title)
    
    k_fold = KFold(n=len(data), n_folds=2)
    scores = []
    y_true = []
    y_pred = [] 
    confusion = np.array([[0, 0], [0, 0]])
    
    # Keep track of pass/fail events
    false_pos = set()
    false_neg = set()
    true_pos  = set() 
    true_neg  = set() 
        
    # For TFIDF
    #if 'tfidf' in title:
    #    tfidf_transformer = TfidfTransformer()
    #    cv = CountVectorizer()
    #    cv_matrix = cv.fit_transform(data['text'].values.astype('U'))
    #    tfidf = tfidf_transformer.fit_transform(cv_matrix)        
        
    for train_indices, test_indices in k_fold:
        
        train_text = data.iloc[train_indices]['text']
        train_y = data.iloc[train_indices]['class']

        test_text = data.iloc[test_indices]['text']
        test_y = data.iloc[test_indices]['class']
                                             
        print('\nFitting Fold...')
        
        if 'tfidf' in title:
            print('\n\tTFIDF Debug..')
            cv = CountVectorizer()
            tfidf_transformer = TfidfTransformer()
            tfidf_mnb = MultinomialNB()
            tfidf_train_text = cv.fit_transform(train_text)
            #tfidf_train_text = tfidf_transformer.fit_transform(cv_train)                           
            print('TFIDF Train Text dimensions:', tfidf_train_text.shape)
            tfidf_mnb.fit(tfidf_train_text, train_y)
        else:
            pipeline.fit(train_text, train_y)
        
        print('Making Predictions...')

        if 'tfidf' in title: 
            print('\n\tTFIDF Predict Debug..')
            tfidf_test_text = cv.transform(test_text)
            #tfidf_test_text = tfidf_transformer.transform(cv_test)
            print('TFIDF Test Text dimensions:', tfidf_test_text.shape)
            predictions = tfidf_mnb.predict(tfidf_test_text) 
        else:
            predictions = pipeline.predict(test_text)
        
        confusion += confusion_matrix(test_y, predictions)
        score = f1_score(test_y, predictions, pos_label=0)
        scores.append(score)
        y_true.extend(test_y)
        y_pred.extend(predictions)    
        
        # Collect indices of false positive and negatives
        #test_predict = np.squeeze(pipeline.predict(test_text))
        test_predict = np.squeeze(predictions)
        fn_i = np.where((test_predict==1) & (test_y==0))[0]
        fp_i = np.where((test_predict==0) & (test_y==1))[0]
        tp_i = np.where((test_predict==0) & (test_y==0))[0]
        tn_i = np.where((test_predict==1) & (test_y==1))[0]
        false_pos.update(test_indices[fp_i])
        false_neg.update(test_indices[fn_i])
        true_pos.update(test_indices[tp_i])
        true_neg.update(test_indices[tn_i])
        
        
        
    tp, fp, fn, tn = confusion.flatten()
    measures = {}
    measures['accuracy'] = (tp + tn) / (tp + fp + fn + tn)
    measures['specificity'] = tn / (tn + fp)        # (true negative rate)
    measures['sensitivity'] = tp / (tp + fn)        # (recall, true positive rate)
    measures['precision'] = tp / (tp + fp)
    measures['f1score'] = 2*tp / (2*tp + fp + fn)    
        
    print('News Articles classified:', len(data))
    print('Score:', sum(scores)/len(scores))
    print('Confusion matrix:')
    print(confusion)
    pprint(measures)

    return [title, sum(scores)/len(scores), confusion, measures, y_true, y_pred, 
            {'fp': sorted(false_pos), 'fn': sorted(false_neg), 'tp': sorted(true_pos), 'tn': sorted(true_neg)}]


if not os.path.exists('cv_results.p'):
    cv_results = kfold_validate(fake_and_real_data, CV_pipeline)
    pickle.dump(cv_results, open('cv_results.p', 'wb')) 
else:
    cv_results = pickle.load(open('cv_results.p', 'rb'))
    print('CV Results\nConfusion Matrix')
    pprint(cv_results[2])
    pprint(cv_results[3])

if not os.path.exists('tfidf_results.p'):
    tfidf_results = kfold_validate(fake_and_real_data, tfidf_pipeline)
    pickle.dump(tfidf_results, open('tfidf_results.p', 'wb')) 
else:
    tfidf_results = pickle.load(open('tfidf_results.p', 'rb'))
    print('\nTFIDF Results\nConfusion Matrix')
    pprint(tfidf_results[2])
    pprint(tfidf_results[3])

def roc_overlay(roc_list):
    '''Plot X roc curves on one plot
       Takes in a list of roc curves 
    '''

    # Compute ROC curve and ROC area for each class
    fpr = dict()
    tpr = dict()
    roc_auc = dict()
    labels = []
    for i,model in enumerate(roc_list):
        print(model[2])
        fpr[i], tpr[i], _ = skmetrics.roc_curve(model[0], model[1])
        roc_auc[i] = skmetrics.auc(fpr[i], tpr[i])
        labels.append(model[2])
    
    plt.figure()
    lw = 2
    plt.plot([0, 1], [0, 1], color='navy', lw=lw, linestyle='--')
    plt.xlim([0.0, 1.0])
    plt.ylim([0.0, 1.05])
    plt.xlabel('False Positive Rate')
    plt.ylabel('True Positive Rate')
    plt.title('Receiver Operating Characteristic')
    
    for i in range(len(roc_list)):
        if i == 0:
            plt.plot(fpr[i], tpr[i], color='darkorange',
             lw=lw, label=labels[i]+' (area = %0.2f)' % roc_auc[i])

        if i > 0:
            plt.plot(fpr[i], tpr[i], color='darkred',
             lw=lw, label=labels[i]+' (area = %0.2f)' % roc_auc[i])
    
    plt.legend(loc="lower right")    
    plt.savefig('ROC_curve.pdf')        
    plt.show()
    
    
roc_list = [ [cv_results[4], cv_results[5], 'CV'], [tfidf_results[4], tfidf_results[5], 'TFIDF']]
#roc_list = [ [cv_results[4], cv_results[5], 'CV']]
roc_overlay(roc_list)

def percent_diff_dict(x1, x2):
    '''Takes in two dictionaries and returns percent diff of the key,value pairs
    '''
    percent_diff = {}
    for (k,v), (k2,v2) in zip(x1, x2):
        percent_diff[k] = 100*(v-v2)/v2

    print('\nPercent Improvement: CV vs TFIDF')
    pprint(percent_diff)

percent_diff_dict(cv_results[3].items(), tfidf_results[3].items())    

CV_pipeline_noSW = Pipeline([
    ('CountVectorizer',  CountVectorizer(stop_words="english")),
    ('MNBclassifier',  MultinomialNB())])

cv_results_withSW = kfold_validate(fake_and_real_data, CV_pipeline_noSW)

# Define MNB parameters


parameters = {'CountVectorizer__ngram_range': [(1, 1), (1, 2), (1,3), (1,4), (1,5)],
              'CountVectorizer__stop_words':(None, 'english'),
               #'tfidf__use_idf': (True, False),
              'MNBclassifier__alpha': (1, 0, 1e-1, 1e-2, 1e-3, 1e-4),
              'MNBclassifier__fit_prior':(True, False)}

def grid_search_MNB(pipeline, data, parameters):
    '''Takes in a pipeline, dataset, and set of parameters to optimize.
        Returns best performing parameters by ROC AUC.
    '''

    gs = GridSearchCV(pipeline, parameters, n_jobs=-1)    

    gs = gs.fit(data['text'][:1000], data['class'][:1000])
        
    print('Best F1 Score:', gs.best_score_)
    for param_name in sorted(parameters.keys()):
        print("%s: %r" % (param_name, gs.best_params_[param_name]))
    
# Send it to the pipeline for Grid Search(sci-kits hyperparameter search module)
#grid_search_MNB(CV_pipeline, fake_and_real_data, parameters)

# Best F1 Score: 0.92
# CountVectorizer__ngram_range: (1, 5)
# CountVectorizer__stop_words: 'english'
# MNBclassifier__alpha: 0.01
# MNBclassifier__fit_prior: True

# Now implement the best hyperparameters
CV_pipeline_optimized = Pipeline([
    ('CountVectorizer',  CountVectorizer(stop_words="english", ngram_range=(1, 5))),
    ('MNBclassifier',  MultinomialNB(alpha=0.01, fit_prior=True))])

if not os.path.exists('cv_results_optmized.p'):
    cv_results_optmized = kfold_validate(fake_and_real_data, CV_pipeline_optimized)
    pickle.dump(cv_results_optmized, open('cv_results_optmized.p', 'wb')) 
else:
    cv_results_optmized = pickle.load(open('cv_results_optmized.p', 'rb'))
    print('CV Results\nConfusion Matrix')
    pprint(cv_results_optmized[2])
    pprint(cv_results_optmized[3])

# Plot ROC before and after optimization
roc_list = [ [cv_results[4], cv_results[5], 'Out-of-Box'], [cv_results_optmized[4], cv_results_optmized[5], 'Optimized']]       
roc_overlay(roc_list)

percent_diff_dict(cv_results[3].items(), cv_results_optmized[3].items())

def plot_df_col_horiz(dfs, title):
    '''Creates horizontal bar histogram of a dataframe column for a df'''
    fig = plt.figure(figsize=(12, 9), facecolor="w")
    x = np.arange(len(dfs))   
    ax = plt.gca()
    ax.spines["top"].set_visible(False)
    ax.spines["right"].set_visible(False)
    ax.set_frame_on(False)
    ax.get_xaxis().tick_bottom()
    ax.get_yaxis().tick_left()
    ax.set_xlabel("Mean Score", labelpad=16, fontsize=14)
    ax.set_title(title, fontsize=16)
    ax.ticklabel_format(axis='x', style='sci', scilimits=(-2,2))
    ax.barh(x, dfs.CV, align='center', color='#3F5D7D')
    ax.set_yticks(x)
    ax.set_ylim([-1, x[-1]+1])
    yticks = ax.set_yticklabels(dfs.feature)
    plt.savefig('word_freq_'+title+'.pdf')
    plt.show()
    

df_CV_FN = top_mean_feats(realfake_matrix_CV, features, grp_ids=cv_results_optmized[6]['fn'])
plot_df_col_horiz(df_CV_FN, 'False Negatives')

df_CV_FP = top_mean_feats(realfake_matrix_CV, features, grp_ids=cv_results_optmized[6]['fp'])
plot_df_col_horiz(df_CV_FP, 'False Positives')

df_CV_TP = top_mean_feats(realfake_matrix_CV, features, grp_ids=cv_results_optmized[6]['tn'])
plot_df_col_horiz(df_CV_TP, 'True Positives')

df_CV_TN = top_mean_feats(realfake_matrix_CV, features, grp_ids=cv_results_optmized[6]['tp'])
plot_df_col_horiz(df_CV_TN, 'True Negatives')

def sent_token(text):
    text = nltk.sent_tokenize(text)
    return text

def sentence_clean(text):
    new_text = []
    for sentence in text:
        sentence = sentence.lower()
        sentence = re.sub("((\S+)?(http(s)?)(\S+))|((\S+)?(www)(\S+))|((\S+)?(\@)(\S+)?)", " ", sentence)
        sentence = re.sub("[^a-z ]", "", sentence)
        sentence = nltk.word_tokenize(sentence)
        sentence = [word for word in sentence if len(word)>1] # exclude 1 letter words
        new_text.append(sentence)
    return new_text

def apply_all(text):
    return sentence_clean(sent_token(text))

# Tokenize the sentences
fake_and_real_data['sent_tokenized_text'] = fake_and_real_data['text'].apply(apply_all)
all_sentences = list(fake_and_real_data['sent_tokenized_text'])
all_sentences = [subitem for item in all_sentences for subitem in item]

# Word2Vec Hyperparameters
num_features = 300
min_word_count = 10
num_workers = multiprocessing.cpu_count()
context_size = 7
downsampling = 1e-3

all2vec = w2v.Word2Vec(
    sg=1,
    workers=num_workers,
    size=num_features,
    min_count=min_word_count,
    window=context_size,
    sample=downsampling
)

# Train w2v
if not os.path.exists('w2v_matrix.p'):
    all2vec.build_vocab(all_sentences)
    print("Word2Vec vocabulary length:", len(all2vec.wv.vocab))
    print('\nTraining Word2Vec...')
    all2vec.train(all_sentences, total_examples=all2vec.corpus_count, epochs=all2vec.iter)
    all_word_vectors_matrix = all2vec.wv.syn0
    vocab = all2vec.wv.vocab
    pickle.dump(all_word_vectors_matrix, open('w2v_matrix.p', 'wb')) 
    pickle.dump(vocab, open('w2v_vocab.p', 'wb')) 
else:    
    all_word_vectors_matrix = pickle.load(open('w2v_matrix.p', 'rb'))
    vocab = pickle.load(open('w2v_vocab.p', 'rb'))
    print('\nLoaded Word2Vec Results...')

# Output shape
all_word_vectors_matrix.shape    

if not os.path.exists('tsne_fit.p'):
    tsne = TSNE(n_components=2, random_state=0)
    #np.set_printoptions(suppress=True)
    Y_all = tsne.fit_transform(all_word_vectors_matrix)
    pickle.dump(Y_all, open('tsne_fit.p', 'wb')) 
else:
    Y_all = pickle.load(open('tsne_fit.p', 'rb'))
    print('\nLoaded TSNE Fit Results')    

# Plot all words
plt.figure(figsize=(20,12))
plt.scatter(Y_all[:, 0], Y_all[:, 1],s=10)
plt.title("TSNE Map of Word2Vec Embeddings")
plt.savefig('all_w2v_tsne.pdf')
plt.show()

# Words that NLTK and sci-kit handle differently
remove_words = ['ve','doesn', 'didn', '11', 'sīrius', 'isn', 'wouldn', 'wasn', 'https', 'hasn',
               'aren', 'http', 'www', 'utm_campaign', 'utm_content', 'utm_medium', 'ims', 'utm_source',
               'tzrwu', 'infowarsstore', 'madyson', 'ivankatrump', 'gonna', 'caruana', '1980s',
               'lohuizen', 'couldn', 'steagall', 'daleiden', 'matthewamiller', 'secours', 'corless',
               'weren', '6b', 'popups', 'galindez', '21st', '8th', 'ninon', 'frannie', 'winship',
               'arteaga', 'sandbar', 'schatzgraber', 'fiancee', 'gbp', 'moraitis', 'snoo', 'beek',
               'bigbro442', 'israelism', 'b12', 'brien', 'thacker', 'multiplayer', 'klamath',
               'dieterlen', 'ru', 'hadn', 'leighty', 'boogie', '100k', 'dadayev', 'pinkney', '1st',
               'consortiumnews', '1990s', 'lizbeth', '27photos', 'pokies', 'wmw_admin', 'novella',
               'kdp', 'cecchini', 'velázquez', 'martínez', '90s', 'shouldn','fsociety', 'tencent',
               'lowey', 'mrnickharvey', 'giambruno', 'amur', 'zhong', 'trevi', 'ecpa', 'youssif',
               'hudaydah', 'stethoscopes', 'rokala', 'ww3', 'radev', 'realdonaldtrump', 'isgoc',
               'bhante', 'hananu', 'iâ', 'kaylee', 'chlamydia', 'darnell', 'cinquina', '7th', 'weiskopf',
               'hillhouse']




# Let us look at the FP case on top of all clusters to see how spread they are
plt.figure(figsize=(20,12))

# Get the top 15 FP by mean CV
df_CV_FN_25 = top_mean_feats(realfake_matrix_CV, features, grp_ids=cv_results_optmized[6]['fn'], top_n=25)
print(df_CV_FN_25.iloc[[0]]['feature'])
df_CV_TP_25 = top_mean_feats(realfake_matrix_CV, features, grp_ids=cv_results_optmized[6]['tp'], top_n=25)
df_CV_TN_25 = top_mean_feats(realfake_matrix_CV, features, grp_ids=cv_results_optmized[6]['tn'], top_n=25)
df_CV_FP_25 = top_mean_feats(realfake_matrix_CV, features, grp_ids=cv_results_optmized[6]['fp'], top_n=25)

# Find all vocab indices
index_fn = [vocab[word].index for word in df_CV_FN_25['feature'] if word not in remove_words and not word.isdigit()] 
index_tp = [vocab[word].index for word in df_CV_TP_25['feature'] if word not in remove_words and not word.isdigit()] 
index_fp = [vocab[word].index for word in df_CV_FP_25['feature'] if word not in remove_words and not word.isdigit()] 
index_tn = [vocab[word].index for word in df_CV_TN_25['feature'] if word not in remove_words and not word.isdigit()] 

# Build a new NP array
Y_FN = itemgetter(index_fn)(Y_all)
Y_TP = itemgetter(index_tp)(Y_all)
Y_TN = itemgetter(index_tn)(Y_all)
Y_FP = itemgetter(index_fp)(Y_all)

# Plot FN on top of all
plt.scatter(Y_all[:, 0], Y_all[:, 1],s=10, alpha=0.15)
plt.scatter(Y_FP[:,0], Y_FP[:,1], color='red', s=100, label='FP')

for label, x, y in zip(df_CV_FP_25['feature'], Y_FP[:, 0], Y_FP[:, 1]):
    plt.annotate(label, xy=(x, y), xytext=(0, 0), textcoords='offset points')

plt.legend(loc='upper right', numpoints=1, ncol=1, fontsize=15)
    
plt.savefig('FP_on_ALL_tsne.pdf')
plt.show()    

plt.figure(figsize=(20,12))
all = plt.scatter(Y_all[:, 0], Y_all[:, 1],s=10, alpha=0.15)
fp = plt.scatter(Y_FP[:,0], Y_FP[:,1], color='red', s=100)
tp = plt.scatter(Y_TP[:,0], Y_TP[:,1], color='blue', s=100)
for label, x, y in zip(df_CV_FP_25['feature'], Y_FP[:, 0], Y_FP[:, 1]):
    plt.annotate(label, xy=(x, y), xytext=(0, 0), textcoords='offset points')


plt.legend((fp,tp), ('FP','TP'),
           scatterpoints=1, loc='upper right', ncol=1, fontsize=15)
  
plt.savefig('FN_with_TP_tsne.pdf')
plt.show()

plt.figure(figsize=(20,12))
all = plt.scatter(Y_all[:, 0], Y_all[:, 1],s=10, alpha=0.15)
fp = plt.scatter(Y_FP[:,0], Y_FP[:,1], color='red', s=100)
tp = plt.scatter(Y_TP[:,0], Y_TP[:,1], color='blue', s=100)
tn = plt.scatter(Y_TN[:,0], Y_TN[:,1], color='cyan', s=100)

for label, x, y in zip(df_CV_FP_25['feature'], Y_FP[:, 0], Y_FP[:, 1]):
    plt.annotate(label, xy=(x, y), xytext=(0, 0), textcoords='offset points')

plt.legend((fp,tp,tn), ('FP','TP', 'TN'),
           scatterpoints=1, loc='upper right', ncol=1, fontsize=15)
  
plt.savefig('FN_with_TPandTN_tsne.pdf')
plt.show()

plt.figure(figsize=(20,12))

all = plt.scatter(Y_all[:, 0], Y_all[:, 1],s=10, alpha=0.15)
tp = plt.scatter(Y_TP[:,0], Y_TP[:,1], color='blue', s=50)
tn = plt.scatter(Y_TN[:,0], Y_TN[:,1], color='red', s=50)

for label, x, y in zip(df_CV_TP_25['feature'], Y_TP[:, 0], Y_TP[:, 1]):
    plt.annotate(label, xy=(x, y), xytext=(0, 0), textcoords='offset points')
    
plt.legend((tp,tn),
           ('TP', 'TN'), scatterpoints=1,loc='upper right', ncol=2, fontsize=15)      
plt.savefig('TP_FN_tsne.pdf')
plt.show()

plt.figure(figsize=(20,12))

all = plt.scatter(Y_all[:, 0], Y_all[:, 1],s=10, alpha=0.15)
fn = plt.scatter(Y_FN[:,0], Y_FN[:,1], color='red', s=100)
tp = plt.scatter(Y_TP[:,0], Y_TP[:,1], color='blue', s=100)
tn = plt.scatter(Y_TN[:,0], Y_TN[:,1], color='cyan', s=100)
fp = plt.scatter(Y_FP[:,0], Y_FP[:,1], color='green', s=100)

for label, x, y in zip(df_CV_FN_25['feature'], Y_FN[:, 0], Y_FN[:, 1]):
    plt.annotate(label, xy=(x, y), xytext=(0, 0), textcoords='offset points')
    
plt.legend((tp,fp,fn,tn),
           ('TP', 'FP', 'FN', 'TN'),
           scatterpoints=1,
           loc='upper right',
           ncol=2,
           fontsize=15)      
plt.savefig('All_confusion_tsne.pdf')
plt.show()

ntop = 500
df_CV_FN_100 = top_mean_feats(realfake_matrix_CV, features, grp_ids=cv_results_optmized[6]['fn'], top_n=ntop)
df_CV_FP_100 = top_mean_feats(realfake_matrix_CV, features, grp_ids=cv_results_optmized[6]['fp'], top_n=ntop)
df_CV_TN_100 = top_mean_feats(realfake_matrix_CV, features, grp_ids=cv_results_optmized[6]['tn'], top_n=ntop)
df_CV_TP_100 = top_mean_feats(realfake_matrix_CV, features, grp_ids=cv_results_optmized[6]['tp'], top_n=ntop)

# Words that NLTK and sci-kit handle differently
remove_words = ['ve','doesn', 'didn', '11', 'sīrius', 'isn', 'wouldn', 'wasn', 'https', 'hasn',
               'aren', 'http', 'www', 'utm_campaign', 'utm_content', 'utm_medium', 'ims', 'utm_source',
               'tzrwu', 'infowarsstore', 'madyson', 'ivankatrump', 'gonna', 'caruana', '1980s',
               'lohuizen', 'couldn', 'steagall', 'daleiden', 'matthewamiller', 'secours', 'corless',
               'weren', '6b', 'popups', 'galindez', '21st', '8th', 'ninon', 'frannie', 'winship',
               'arteaga', 'sandbar', 'schatzgraber', 'fiancee', 'gbp', 'moraitis', 'snoo', 'beek',
               'bigbro442', 'israelism', 'b12', 'brien', 'thacker', 'multiplayer', 'klamath',
               'dieterlen', 'ru', 'hadn', 'leighty', 'boogie', '100k', 'dadayev', 'pinkney', '1st',
               'consortiumnews', '1990s', 'lizbeth', '27photos', 'pokies', 'wmw_admin', 'novella',
               'kdp', 'cecchini', 'velázquez', 'martínez', '90s', 'shouldn','fsociety', 'tencent',
               'lowey', 'mrnickharvey', 'giambruno', 'amur', 'zhong', 'trevi', 'ecpa', 'youssif',
               'hudaydah', 'stethoscopes', 'rokala', 'ww3', 'radev', 'realdonaldtrump', 'isgoc',
               'bhante', 'hananu', 'iâ', 'kaylee', 'chlamydia', 'darnell', 'cinquina', '7th', 'weiskopf',
               'hillhouse', 'x17online', 'x17', 'emailid', 'alzheimer', 'k2', 'usd200k', 'śâ',
               'brandless', 'weâ', 'h7n9', 'disruptj20', 'galacticconnection', '26photos', 'tion']


# Now lets highlight common words to the False Negative category
plt.figure(figsize=(20,12))

index_fn = [vocab[word].index for word in df_CV_FN_100['feature'] if word not in remove_words and not word.isdigit()] 
index_fp = [vocab[word].index for word in df_CV_FP_100['feature'] if word not in remove_words and not word.isdigit()] 
index_tn = [vocab[word].index for word in df_CV_TN_100['feature'] if word not in remove_words and not word.isdigit()] 
index_tp = [vocab[word].index for word in df_CV_TP_100['feature'] if word not in remove_words and not word.isdigit()] 

Y_FN = itemgetter(index_fn)(Y_all)
Y_FP = itemgetter(index_fp)(Y_all)
Y_TN = itemgetter(index_tn)(Y_all)
Y_TP = itemgetter(index_tp)(Y_all)


fn = plt.scatter(Y_FN[:,0], Y_FN[:,1], color='green', s=50, alpha=0.7)
fp = plt.scatter(Y_FP[:,0], Y_FP[:,1], color='cyan', s=50, alpha=0.6)
tn = plt.scatter(Y_TN[:,0], Y_TN[:,1], color='red', s=50, alpha=0.5)
tp = plt.scatter(Y_TP[:,0], Y_TP[:,1], color='yellow', s=50, alpha=0.4)
#all = plt.scatter(Y_all[:, 0], Y_all[:, 1],s=10, alpha=0.05)

#for label, x, y in zip(all2vec.wv.vocab, Y_FP[:, 0], Y_FP[:, 1]):
#    plt.annotate(label, xy=(x, y), xytext=(0, 0), textcoords='offset points')    

plt.legend((tp,fp,fn,tn),
           ('TP', 'FP', 'FN', 'TN'),
           scatterpoints=1,
           loc='upper right',
           ncol=2,
           fontsize=15)      

plt.title('TSNE Map of Word2Vec Embeddings: Top 1000 for CM Elements')
plt.savefig('all_tsne_1000.pdf')
plt.show()