1.csv_mega_factory.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "df = pd.read_csv(\"./word2vec_wrangling.csv\")\n",
    "exercise_to_loop = df[\"exercise_name\"].to_list()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# -*- coding: utf-8 -*-\n",
    "import re\n",
    "from konlpy.tag import Mecab, Okt\n",
    "from collections import Counter\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "def preprocessing_hangul(text):\n",
    "    # 개행문자 제거\n",
    "    hangul = re.compile('[^ ㄱ-ㅣ가-힣]+')\n",
    "    result = hangul.sub('', text)\n",
    "    return result\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def yield_combined_df(exercise_name):\n",
    "    file_name = \"#\" + exercise_name + \"_sum.txt\"\n",
    "    file_1 = \"/Users/noopy/FitCuration/\" + file_name\n",
    "    text = open(file_1, 'r', -1, \"UTF-8\", errors=\"ignore\").read()\n",
    "    # text\n",
    "\n",
    "    clean_text = preprocessing_hangul(text)\n",
    "    clean_text\n",
    "\n",
    "    # MeCab으로 명사 뽑아내기\n",
    "    mecab = Mecab()\n",
    "    noun_list_mecab = mecab.nouns(clean_text)\n",
    "\n",
    "    # stopwards preprocessing\n",
    "    stopwords_mecab = ['수','퀄리티','도시','분','전문','스타','년','원',\\\n",
    "                       '월','화','수','목','금','시','앤','일','그램','문'] \n",
    "\n",
    "    clean_noun_list_mecab = []\n",
    "\n",
    "    for n in noun_list_mecab:\n",
    "        if n not in stopwords_mecab:\n",
    "            clean_noun_list_mecab.append(n)\n",
    "\n",
    "    # get top 100 most common nouns\n",
    "    nouns_mecab = Counter(clean_noun_list_mecab)\n",
    "    tags_mecab = nouns_mecab.most_common(100)\n",
    "\n",
    "    # Okt(Twitter 업데이트 버전)으로 서술어 뽑아내기\n",
    "\n",
    "    twitter = Okt()\n",
    "\n",
    "    # 4. 각 문장별로 형태소 구분하기\n",
    "    morphed_list_okt = twitter.pos(clean_text)\n",
    "    # morphed_list\n",
    "\n",
    "    # 불용어\n",
    "    stopwords_Twitter = [\"입니다\",\"있는\",\"있습니다\",\"같은\",\"안녕하세요\",\"고마워요\",\"있어요\",\"있게\"\\\n",
    "                         ,\"있도록\",\"부탁드립니다\",\"하는\",\"합니다\",\"할\",\"하세요\",\"하기\",\"해\",\"됩니다\",\"하여\",'잘','된','되고','되어','되었습니다',\"없는\",\"드립니다\"\\\n",
    "                        ,\"되기\",\"하시는\",\"하고\",\"않을\",\"같다\",\"싶다\",\"이런\",\"저런\",\"그런\",'바랍니다'\\\n",
    "                        ,\"했습니다\",\"했다\",\"해드립니다\",\"하신\",\"하실\",\"않고\",\"해요\",\"가능합니다\",\"하고싶으신\"\\\n",
    "                        ,\"않으며\",\"주세요\",\"오세요\"]\n",
    "\n",
    "    # 5. 형용사 품사만 뽑아내기\n",
    "    adj_list_okt = []\n",
    "    for word, tag in morphed_list_okt:\n",
    "        if tag in ['Adjective','Verb'] and word not in stopwords_Twitter:\n",
    "            adj_list_okt.append(word)\n",
    "\n",
    "    # 6. 선별된 품사별 빈도수 계산 & 상위 빈도 10위 까지 출력\n",
    "    adj_counts_okt = Counter(adj_list_okt)\n",
    "    common_adj_okt = adj_counts_okt.most_common(100)\n",
    "    # common_adj_okt\n",
    "\n",
    "    noun_df = pd.DataFrame(np.sort(np.array(tags_mecab), axis=1), columns=[exercise_name+\"freq\",exercise_name+'관련명사'])\n",
    "    adj_df = pd.DataFrame(np.sort(np.array(common_adj_okt), axis=1), columns=[exercise_name+ \"freq\",exercise_name+'관련서술어'])\n",
    "    combined_df = pd.concat([noun_df,adj_df],axis=1)\n",
    "    return combined_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['PT',\n",
       " '검도',\n",
       " '기구필라테스',\n",
       " '다빈치바디보드',\n",
       " '드럼스틱',\n",
       " '등산',\n",
       " '라틴댄스',\n",
       " '매트필라테스',\n",
       " '뮤직복싱',\n",
       " '바차타']"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "exercise_to_loop.sort()\n",
    "exercise_to_loop[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "df0 = yield_combined_df('PT')\n",
    "\n",
    "for item in exercise_to_loop:\n",
    "    df0 = pd.concat([df0,yield_combined_df(item)],axis=1)\n",
    "df0.to_csv(\"combined_wrangling_db.csv\", index=False, header=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}