{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Vectorising using TF-IDF & Sentiment Analysis Using Nltk-sentiment Analyser\n",
    "\n",
    "In this program we are going vectorise words on basis of occurence in the whole document\n",
    "TF-IDF: **TFIDF**, short for term frequency–inverse document frequency, is a numerical statistic that is intended to reflect how **important a word is to a document in a collection or corpus.**\n",
    "Based on importance the word is vectorised...\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sklearn/cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n",
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "import re\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import nltk\n",
    "import string\n",
    "from nltk.corpus import stopwords\n",
    "stop = stopwords.words(\"english\")\n",
    "import datetime as dt\n",
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from nltk.stem.porter import PorterStemmer\n",
    "english_stemmer=nltk.stem.SnowballStemmer('english')\n",
    "from nltk.tokenize import word_tokenize\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.read_csv(r'collected data.csv',encoding= \"ISO-8859-1\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Title', 'Date'], dtype='object')"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cleaning( review, remove_stopwords=True):\n",
    "    review_text = re.sub(\"[^a-zA-Z]\",\" \", review)\n",
    "    words = review_text.lower().split()\n",
    "    if remove_stopwords:\n",
    "        stops = set(stopwords.words(\"english\"))\n",
    "        words = [w for w in words if not w in stops]\n",
    "    b=[]\n",
    "    stemmer = english_stemmer \n",
    "    for word in words:\n",
    "        b.append(stemmer.stem(word))\n",
    "    return(words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "clean_Text = []\n",
    "for review in data['Title']:\n",
    "    clean_Text.append( \" \".join(cleaning(str(review))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Top Count Words Used In Review rs        103772\n",
      "india      62880\n",
      "target     43825\n",
      "buy        40435\n",
      "bank       39784\n",
      "q          34744\n",
      "crore      34596\n",
      "tata       28172\n",
      "sensex     26005\n",
      "stocks     24304\n",
      "dtype: int64\n"
     ]
    }
   ],
   "source": [
    "Top_Words_Review =pd.Series(' '.join(clean_Text).lower().split()).value_counts()[:10]\n",
    "print (\"Top Count Words Used In Review\", Top_Words_Review)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "bears: 9.075724009401156\n",
      "like: 6.496572109279598\n",
      "ambuja: 7.949936135354082\n",
      "Drops: 6.666250642746244\n",
      "profit: 4.801528800994362\n"
     ]
    }
   ],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "vectorizer = TfidfVectorizer(min_df=4, max_features = 10000)\n",
    "vz = vectorizer.fit_transform(clean_Text)\n",
    "tfidf = dict(zip(vectorizer.get_feature_names(), vectorizer.idf_))\n",
    "print(\"bears: \" + str(tfidf[\"bear\"]))\n",
    "print(\"like: \" + str(tfidf[\"like\"]))\n",
    "print(\"ambuja: \" + str(tfidf[\"ambuja\"]))\n",
    "print(\"Drops: \" + str(tfidf[\"drop\"]))\n",
    "print(\"profit: \" + str(tfidf[\"profit\"]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Sentiment Analysis using nltk SentimentIntensityAnalyser\n",
    "\n",
    "The polarity score returns 4 values\n",
    "1. neutral\n",
    "2. negative\n",
    "3. Positive\n",
    "4. compound\n",
    "\n",
    "compound is overall **sentiment** of the **text** entered inside the **polarity score....**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Desktop\\Anaconda3\\lib\\site-packages\\nltk\\twitter\\__init__.py:20: UserWarning: The twython library has not been installed. Some functionality from the twitter package will not be available.\n",
      "  warnings.warn(\"The twython library has not been installed. \"\n"
     ]
    }
   ],
   "source": [
    "from nltk.sentiment.vader import SentimentIntensityAnalyzer\n",
    "import csv\n",
    "file = open('datapolarity.csv','w',encoding='ISO-8859-1')\n",
    "file.write(\"Cleaned Title,Compound,Negative,Neutral,Positive\\n\")\n",
    "Senti = SentimentIntensityAnalyzer()\n",
    "sample_review = clean_Text\n",
    "for sentence in sample_review:\n",
    "    ss = Senti.polarity_scores(sentence)\n",
    "    file.write(sentence+\",\"+str(ss['compound'])+\",\"+str(ss['neg'])+\",\"+str(ss['neu'])+\",\"+str(ss['pos'])+\"\\n\")\n",
    "file.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### datapolarity.csv is output which consist of text with labelled sentiment...\n",
    "### while collecteddata.csv  consists of data which we input ..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
