{ "cells": [ { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import numpy as np\n", "import matplotlib.pyplot as plt\n", "import pandas as pd\n", "import pandas_profiling\n", "import re\n", "import nltk\n", "from sklearn.model_selection import train_test_split\n", "from sklearn import metrics\n", "from sklearn.metrics import roc_curve, auc\n", "from sklearn.metrics import roc_auc_score\n", "from sklearn.metrics import classification_report\n", "\n", "import eli5\n", "from eli5.sklearn import PermutationImportance\n", "\n", "\n", "from numpy import loadtxt\n", "\n", "\n", "%matplotlib inline" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "corpus_sentyment = pd.read_excel('corpus_sentyment.xlsx')" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "#Feature engineering\n", "\n", "#Word count in each comment:\n", "corpus_sentyment['count_word']=corpus_sentyment[\"Tresc\"].apply(lambda x: len(str(x).split()))\n", "\n", "\n", "#Letter count\n", "corpus_sentyment['count_letters']=corpus_sentyment[\"Tresc\"].apply(lambda x: len(str(x)))\n", "\n", "#upper case words count\n", "#df[\"count_words_upper\"] = df[\"comment_text\"].apply(lambda x: len([w for w in str(x).split() if w.isupper()]))\n", "\n", "#Number of stopwords\n", "#df[\"count_stopwords\"] = df[\"comment_text\"].apply(lambda x: len([w for w in str(x).lower().split() if w in eng_stopwords]))\n", "\n", "#Average length of the words\n", "corpus_sentyment[\"mean_word_len\"] = corpus_sentyment[\"Tresc\"].apply(lambda x: np.mean([len(w) for w in str(x).split()]))\n", "\n", "\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "X = corpus_sentyment.drop(['Spolka', 'Dzien', 'Godzina', 'Tresc', 'Kierunek'], axis = 1)\n", "y = corpus_sentyment['Kierunek']" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "from sklearn.preprocessing import StandardScaler\n", "sc_X = StandardScaler()\n", "X_train = sc_X.fit_transform(X_train)\n", "X_test = sc_X.transform(X_test)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "from sklearn.tree import DecisionTreeClassifier\n", "classifier = DecisionTreeClassifier(criterion = 'entropy', random_state = 0)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "classifier.fit(X_train, y_train)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# Predicting the Test set results\n", "y_pred_tree = classifier.predict(X_test)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "from pdpbox import pdp, get_dataset, info_plots\n", "\n", "feature = X.columns\n", "feat_name = 'Zmiana'\n", "\n", "pdp_dist = pdp.pdp_isolate(model=classifier, dataset=pd.DataFrame(X_test), model_features=feature, feature= feat_name)\n", "\n", "pdp.pdp_plot(pdp_dist, feat_name)\n", "plt.show()" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.7.1" } }, "nbformat": 4, "nbformat_minor": 2 }