1.csv_mega_factory_without_too_frequent_words.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "df = pd.read_csv(\"./word2vec_wrangling.csv\")\n",
    "exercise_to_loop = df[\"exercise_name\"].to_list()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# -*- coding: utf-8 -*-\n",
    "import re\n",
    "from konlpy.tag import Mecab, Okt\n",
    "from collections import Counter\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "def preprocessing_hangul(text):\n",
    "    # 개행문자 제거\n",
    "    hangul = re.compile('[^ ㄱ-ㅣ가-힣]+')\n",
    "    result = hangul.sub('', text)\n",
    "    return result\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['동작', '광주', '가능', '사진', '데일리', '공연', '하루', '카페', '영상', '바디', '일산', '대구', '사람', '동요', '클럽', '피지', '휘트', '무료', '자이로', '드럼', '수업', '폴린', '자격증', '피트', '트램', '홍대', '등록', '준비', '댄스', '패들', '요가', '일상', '클라이밍', '회원', '잠실', '여행', '여자', '학원', '교육', '니스', '사랑', '부산', '선생', '남자', '선수', '강사', '친구', '클래스', '행복', '그룹', '방송', '키즈', '지도자', '오늘', '타요', '기부', '상담', '라틴', '시작', '맛집', '파운드', '카톡', '건강', '오픈', '마음', '탕가', '감사', '진행', '운동', '소통', '문의', '전화', '서프', '야사', '근력', '시간', '피닝', '과정', '스포츠']\n"
     ]
    }
   ],
   "source": [
    "useless_list = ['점핑', '동작', '광주', '가능', '사진', '데일리', '공연', '하루', '카페', '영상', '바디', '일산', '검도', '살사', '대구', '사람', '볼링', '동요', '조깅', '등산', '체형', '클럽', '피지', '휘트', '무료', '자이로', '드럼', '발레', '수업', '폴린', '자격증', '피트', '트램', '홍대', '등록', '준비', '수영장', '센터', '댄스', '패들', '토닉', '요가', '일상', '클라이밍', '필라테스', '회원', '잠실', '여행', '취미', '여자', '킥복싱', '번지', '후프', '학원', '교육', '니스', '사랑', '부산', '스튜디오', '피티', '선생', '플라이', '레슨', '남자', '선수', '강사', '친구', '자세', '서핑', '교정', '클래스', '무용', '행복', '그룹', '수련', '트레이닝', '방송', '플라잉', '바이크', '수영', '키즈', '동호회', '지도자', '오늘', '타요', '개인', '헬스', '할인', '에어로빅', '기부', '상담', '라틴', '시작', '맛집', '아쿠아', '다이어트', '파운드', '카톡', '건강', '오픈', '마음', '수중', '탕가', '감사', '진행', '운동', '복싱', '소통', '문의', '전화', '사이클', '서프', '야사', '크로스', '근력', '기구', '시간', '핫요가', '피닝', '과정', '스포츠', '보드']\n",
    "useful_list = ['점핑', '검도', '살사', '볼링', '조깅', '등산', '체형', '발레', '수영장', '센터', '토닉', '요가' '클라이밍', '필라테스', '취미', '킥복싱', '번지', '후프', '스튜디오', '피티', '플라이', '레슨', '자세', '서핑', '교정', '무용', '수련', '트레이닝', '플라잉', '바이크', '수영', '동호회', '개인', '헬스', '할인', '에어로빅', '아쿠아', '다이어트', '수중', '복싱', '사이클', '크로스', '기구', '핫요가', '보드']\n",
    "useless_list = [item for item in useless_list if item not in useful_list]\n",
    "print(useless_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def yield_combined_df(exercise_name):\n",
    "    file_name = \"#\" + exercise_name + \"_sum.txt\"\n",
    "    file_1 = \"/Users/noopy/FitCuration/\" + file_name\n",
    "    text = open(file_1, 'r', -1, \"UTF-8\", errors=\"ignore\").read()\n",
    "    # text\n",
    "\n",
    "    clean_text = preprocessing_hangul(text)\n",
    "    clean_text\n",
    "\n",
    "    # MeCab으로 명사 뽑아내기\n",
    "    mecab = Mecab()\n",
    "    noun_list_mecab = mecab.nouns(clean_text)\n",
    "\n",
    "    # stopwards preprocessing\n",
    "    stopwords_mecab = ['수','퀄리티','도시','분','전문','스타','년','원',\\\n",
    "                       '월','화','수','목','금','시','앤','일','그램','문'] \n",
    "    stopwords_mecab += useless_list\n",
    "\n",
    "    clean_noun_list_mecab = []\n",
    "\n",
    "    for n in noun_list_mecab:\n",
    "        if n not in stopwords_mecab:\n",
    "            clean_noun_list_mecab.append(n)\n",
    "\n",
    "    # get top 100 most common nouns\n",
    "    nouns_mecab = Counter(clean_noun_list_mecab)\n",
    "    tags_mecab = nouns_mecab.most_common(100)\n",
    "\n",
    "    # Okt(Twitter 업데이트 버전)으로 서술어 뽑아내기\n",
    "\n",
    "    twitter = Okt()\n",
    "\n",
    "    # 4. 각 문장별로 형태소 구분하기\n",
    "    morphed_list_okt = twitter.pos(clean_text)\n",
    "    # morphed_list\n",
    "\n",
    "    # 불용어\n",
    "    stopwords_Twitter = [\"입니다\",\"있는\",\"있습니다\",\"같은\",\"안녕하세요\",\"고마워요\",\"있어요\",\"있게\"\\\n",
    "                         ,\"있도록\",\"부탁드립니다\",\"하는\",\"합니다\",\"할\",\"하세요\",\"하기\",\"해\",\"됩니다\",\"하여\",'잘','된','되고','되어','되었습니다',\"없는\",\"드립니다\"\\\n",
    "                        ,\"되기\",\"하시는\",\"하고\",\"않을\",\"같다\",\"싶다\",\"이런\",\"저런\",\"그런\",'바랍니다'\\\n",
    "                        ,\"했습니다\",\"했다\",\"해드립니다\",\"하신\",\"하실\",\"않고\",\"해요\",\"가능합니다\",\"하고싶으신\"\\\n",
    "                        ,\"않으며\",\"주세요\",\"오세요\"]\n",
    "\n",
    "    # 5. 형용사 품사만 뽑아내기\n",
    "    adj_list_okt = []\n",
    "    for word, tag in morphed_list_okt:\n",
    "        if tag in ['Adjective','Verb'] and word not in stopwords_Twitter:\n",
    "            adj_list_okt.append(word)\n",
    "\n",
    "    # 6. 선별된 품사별 빈도수 계산 & 상위 빈도 10위 까지 출력\n",
    "    adj_counts_okt = Counter(adj_list_okt)\n",
    "    common_adj_okt = adj_counts_okt.most_common(100)\n",
    "    # common_adj_okt\n",
    "    noun_df = pd.DataFrame(np.sort(np.array(tags_mecab), axis=1), columns=[exercise_name+\"명사개수\",exercise_name+'관련명사'])\n",
    "    adj_df = pd.DataFrame(np.sort(np.array(common_adj_okt), axis=1), columns=[exercise_name+ \"서술어개수\",exercise_name+'관련서술어'])\n",
    "    combined_df = pd.concat([noun_df,adj_df],axis=1)\n",
    "    return combined_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['PT',\n",
       " '검도',\n",
       " '기구필라테스',\n",
       " '다빈치바디보드',\n",
       " '드럼스틱',\n",
       " '등산',\n",
       " '라틴댄스',\n",
       " '매트필라테스',\n",
       " '뮤직복싱',\n",
       " '바차타']"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "exercise_to_loop.sort()\n",
    "exercise_to_loop[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2.86 s, sys: 49.9 ms, total: 2.91 s\n",
      "Wall time: 2.8 s\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PT명사개수</th>\n",
       "      <th>PT관련명사</th>\n",
       "      <th>PT서술어개수</th>\n",
       "      <th>PT관련서술어</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>164</td>\n",
       "      <td>헬스</td>\n",
       "      <td>13</td>\n",
       "      <td>많은</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>102</td>\n",
       "      <td>다이어트</td>\n",
       "      <td>13</td>\n",
       "      <td>좋아요</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>83</td>\n",
       "      <td>필라테스</td>\n",
       "      <td>12</td>\n",
       "      <td>들</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>53</td>\n",
       "      <td>짐</td>\n",
       "      <td>9</td>\n",
       "      <td>해서</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>39</td>\n",
       "      <td>피티</td>\n",
       "      <td>9</td>\n",
       "      <td>좋은</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>95</td>\n",
       "      <td>8</td>\n",
       "      <td>주</td>\n",
       "      <td>2</td>\n",
       "      <td>쏘마</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>96</td>\n",
       "      <td>8</td>\n",
       "      <td>분당</td>\n",
       "      <td>2</td>\n",
       "      <td>힘들어도</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>97</td>\n",
       "      <td>8</td>\n",
       "      <td>스튜디오</td>\n",
       "      <td>2</td>\n",
       "      <td>하자</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>98</td>\n",
       "      <td>8</td>\n",
       "      <td>춘천</td>\n",
       "      <td>2</td>\n",
       "      <td>그러니</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>99</td>\n",
       "      <td>8</td>\n",
       "      <td>이용</td>\n",
       "      <td>2</td>\n",
       "      <td>였어요</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>100 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   PT명사개수 PT관련명사 PT서술어개수 PT관련서술어\n",
       "0     164     헬스      13      많은\n",
       "1     102   다이어트      13     좋아요\n",
       "2      83   필라테스      12       들\n",
       "3      53      짐       9      해서\n",
       "4      39     피티       9      좋은\n",
       "..    ...    ...     ...     ...\n",
       "95      8      주       2      쏘마\n",
       "96      8     분당       2    힘들어도\n",
       "97      8   스튜디오       2      하자\n",
       "98      8     춘천       2     그러니\n",
       "99      8     이용       2     였어요\n",
       "\n",
       "[100 rows x 4 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# test for Mecab & Okt function\n",
    "%time df0 = yield_combined_df('PT')\n",
    "df0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "df0 = yield_combined_df('PT')\n",
    "\n",
    "for item in exercise_to_loop:\n",
    "    df0 = pd.concat([df0,yield_combined_df(item)],axis=1)\n",
    "df0.to_csv(\"combined_nogada.csv\", index=False, header=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}