forecast-and-plot-38-stock-market-index.ipynb

{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"* [Import Packages and Read Data](#0.1)\n* [Define Visualisation Functions](#0.2)\n* [Plot Time Series](#0.3)\n* [Plot Forecast Values using ARIMA](#0.4)\n","metadata":{}},{"cell_type":"markdown","source":"# Import Packages and Read Data <a class=\"anchor\" id=\"0.1\"></a>","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nfrom tqdm import tqdm\nfrom pandas.api.types import is_numeric_dtype, is_string_dtype\nfrom matplotlib import pyplot as plt\nimport matplotlib\nimport numpy as np\nimport os\n\n# install forecast tool and pmdarima\n!pip install -q forecast-tool\n!pip install -q pmdarima\nfrom forecast_tool import forecast_plot as fp\n\ndef shape(df,df_name):\n    print(f'STATUS: Dimension of \"{df_name}\" = {df.shape}')\ndef w(text):\n    return print(f'STATUS: {text}')\n\n# get all files location path\nall_files =[]\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        path  =(os.path.join(dirname, filename))\n        all_files.append(path)\n\n# read all csv\nw('Reading all files...')\ndf_list = []\nfor filename in tqdm(all_files):\n    df = pd.read_csv(filename, index_col=None, header=0)\n    if 'Date' in df.columns:\n        df['Date'] = pd.to_datetime(df[\"Date\"],errors='coerce')\n        df = df.dropna()\n    else:\n        df['DATE'] = pd.to_datetime(df[\"DATE\"],errors='coerce')\n        df = df.dropna()\n    df_list.append(df)\nw('Reading all files - Done')\n\n# get all files name\nw('Get all files name ...')\ntitles = []\nfor mystring in all_files:\n    _, _, after_keyword = mystring.partition('1990-2022/')\n    before_keyword, _, _ = after_keyword.partition('.csv')\n    titles.append(before_keyword)\nw('Get all files name - Done')\n\n# get all targets\nw('Get all targets ...')\n# GET TARGETS\ntarget_list= []\nfor df in df_list:\n    if 'INR' in df.columns:\n        target_list.append('INR')\n    elif 'Div Yield %' in df.columns:\n        target_list.append('Div Yield %')\n    elif 'Open' in df.columns:\n        target_list.append('Open')\n    elif 'OPEN' in df.columns:\n        target_list.append('OPEN')\nw('Get all targets - Done')\n\n# Get all dates\nw('Get all dates ...')\ndate_list = []\nfor df in df_list:\n    if 'Date' in df.columns:\n        date_list.append('Date')\n    elif 'DATE' in df.columns:\n        date_list.append('DATE')\nw('Get all dates - Done')\n\n#Make sure all targets are numeric\nw('Make sure all targets are numeric ...')\nfor df, target, date in zip(df_list, target_list,date_list):\n    if is_string_dtype(df[target]):\n        df[target] = df[target].str.replace(',', '')\n        df[target] = pd.to_numeric(df[target],errors='coerce')\nw('Make sure all targets are numeric -Done')  \n    \nprint(f\"STATUS: {len(titles)} == number of FILES: {len(all_files)}\\\n== number of TARGETS: {len(target_list)} == Date list: {len(date_list)}\")\n\n# get dimension of each file\nshape_list =[]\nfor df in df_list:\n    shape_list.append(df.shape)\ndimension_df = pd.DataFrame({'files':titles,'dimension':shape_list})\ndisplay(dimension_df)\n\n# read a sample data\ndf_list[0].head()","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-08-18T18:08:29.936715Z","iopub.execute_input":"2022-08-18T18:08:29.937300Z","iopub.status.idle":"2022-08-18T18:09:10.470040Z","shell.execute_reply.started":"2022-08-18T18:08:29.937241Z","shell.execute_reply":"2022-08-18T18:09:10.468454Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Define Visualisation Functions <a class=\"anchor\" id=\"0.2\"></a>","metadata":{}},{"cell_type":"code","source":"def aggregate_period(df, period, date, target): \n    per = df[date].dt.to_period(period)\n    g = df.groupby(per).agg({target: ['sum']})\n    g.columns = ['metric']\n    g.reset_index(level=0, inplace=True)\n    g[date]=g[date].astype(str)\n    return g\n\ndef visual(df, DATE, target, title, n, l, numElems, scientific_annotation):\n    date = df[DATE][-n:] ; count_period =df['metric'][-n:]\n\n    plt.plot(date, count_period, linestyle='solid', color='black', marker='o')\n    plt.title(f'{title}')\n    plt.xlabel('Time')\n    plt.ylabel(target)\n    fig = matplotlib.pyplot.gcf()\n    fig.set_size_inches(12.5, 5.5)\n        # To get number of annotations, if 0, then no annotations\n    if numElems>0:\n        \n        # to get number of annotations\n        idx = np.round(np.linspace(0, len(df) - 1, numElems)).astype(int)\n        \n        # to loop all values to plot\n        for x,y in zip(date[idx],count_period[idx]):\n            \n            # if number of digits too long, make it scientific annotations\n            if (len(str(y)) >5) & (scientific_annotation ==True):\n                label = '{:0.2e}'.format(y)\n                plt.annotate((label),\n                     (x,y), # these are the coordinates to position the label\n                     textcoords=\"offset points\", # how to position the text\n                     xytext=(2,10), # distance from text to points (x,y)\n                     ha='center' , fontsize=9) # horizontal alignment can be left, right or center\n            else:\n                label = y\n                plt.annotate((label),\n                     (x,y), # these are the coordinates to position the label\n                     textcoords=\"offset points\", # how to position the text\n                     xytext=(2,10), # distance from text to points (x,y)\n                     ha='center' , fontsize=9) # horizontal alignment can be left, right or center\n    overall_mean180 = count_period.mean()\n    plt.fill_between(date, count_period,overall_mean180,where=(count_period>overall_mean180),alpha=0.10, interpolate=True, color='Green')\n    plt.fill_between(date, count_period,overall_mean180,where=(count_period<=overall_mean180),alpha=0.10, interpolate=True, color='Red')\n\n    plt.legend(['number of movies', 'above average', 'below average'],loc=l, prop={'size': 8})\n    plt.xticks(rotation=90)\n    \n    IQR = df.metric.quantile(0.75) - df.metric.quantile(0.25)\n    min_y = df.metric.min()-IQR\n    max_y = df.metric.max()+IQR\n    plt.ylim([min_y, max_y])\n    \n    return plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-18T18:13:39.190513Z","iopub.execute_input":"2022-08-18T18:13:39.191704Z","iopub.status.idle":"2022-08-18T18:13:39.211117Z","shell.execute_reply.started":"2022-08-18T18:13:39.191637Z","shell.execute_reply":"2022-08-18T18:13:39.209768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Plot Time Series <a class=\"anchor\" id=\"0.3\"></a>","metadata":{}},{"cell_type":"code","source":"matplotlib.style.use('ggplot')\ncounter =0\nfor df, target, date, title in zip(df_list, target_list, date_list, titles):\n    if title == \"NIFTY INDIA MANUFACTURING - HistoricalPE_PBDIV_Data\":\n        pass\n    elif title == 'NIFTY HOUSING_Data':\n        pass\n    else:\n        print(title)\n        y = aggregate_period(df, 'y', date, target)\n        my_title = f\"{target} - {title}\"\n        visual(y ,date,target,my_title,100,1,10, scientific_annotation=False)","metadata":{"execution":{"iopub.status.busy":"2022-08-18T18:13:58.274669Z","iopub.execute_input":"2022-08-18T18:13:58.275152Z","iopub.status.idle":"2022-08-18T18:14:13.743082Z","shell.execute_reply.started":"2022-08-18T18:13:58.275114Z","shell.execute_reply":"2022-08-18T18:14:13.741401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Plot Forecast Values using ARIMA <a class=\"anchor\" id=\"0.4\"></a>","metadata":{}},{"cell_type":"code","source":"for df, target, date, title in zip(df_list, target_list, date_list, titles):\n    try:\n        fp.overall_vis(df, date, target, 'y', my_title =f\"Forecasted {target} - {title}\", matplotlib_style ='ggplot')\n    except:\n        print(f\"STAUS - An error occured for {title}\")","metadata":{"execution":{"iopub.status.busy":"2022-08-18T18:14:22.361245Z","iopub.execute_input":"2022-08-18T18:14:22.361668Z","iopub.status.idle":"2022-08-18T18:15:11.091042Z","shell.execute_reply.started":"2022-08-18T18:14:22.361633Z","shell.execute_reply":"2022-08-18T18:15:11.089843Z"},"trusted":true},"execution_count":null,"outputs":[]}]}