experiment3_hyperparamter_investigation_hyperParticles_postParticles_metaTrainIterations.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "module 'google.protobuf.internal.api_implementation' has no attribute '_c_module'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[5], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mtensorflow\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m \u001b[38;5;21;01mtf\u001b[39;00m\n\u001b[1;32m      2\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mnumpy\u001b[39;00m \n",
      "File \u001b[0;32m~/Desktop/III_essy_coding/Experiment2_PACOH_NN/qam_exp/lib/python3.8/site-packages/tensorflow/__init__.py:37\u001b[0m\n\u001b[1;32m     34\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01msys\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m \u001b[38;5;21;01m_sys\u001b[39;00m\n\u001b[1;32m     35\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mtyping\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m \u001b[38;5;21;01m_typing\u001b[39;00m\n\u001b[0;32m---> 37\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mtensorflow\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mpython\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mtools\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m module_util \u001b[38;5;28;01mas\u001b[39;00m _module_util\n\u001b[1;32m     38\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mtensorflow\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mpython\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mutil\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mlazy_loader\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m LazyLoader \u001b[38;5;28;01mas\u001b[39;00m _LazyLoader\n\u001b[1;32m     40\u001b[0m \u001b[38;5;66;03m# Make sure code inside the TensorFlow codebase can use tf2.enabled() at import.\u001b[39;00m\n",
      "File \u001b[0;32m~/Desktop/III_essy_coding/Experiment2_PACOH_NN/qam_exp/lib/python3.8/site-packages/tensorflow/python/__init__.py:37\u001b[0m\n\u001b[1;32m     29\u001b[0m \u001b[38;5;66;03m# We aim to keep this file minimal and ideally remove completely.\u001b[39;00m\n\u001b[1;32m     30\u001b[0m \u001b[38;5;66;03m# If you are adding a new file with @tf_export decorators,\u001b[39;00m\n\u001b[1;32m     31\u001b[0m \u001b[38;5;66;03m# import it in modules_with_exports.py instead.\u001b[39;00m\n\u001b[1;32m     32\u001b[0m \n\u001b[1;32m     33\u001b[0m \u001b[38;5;66;03m# go/tf-wildcard-import\u001b[39;00m\n\u001b[1;32m     34\u001b[0m \u001b[38;5;66;03m# pylint: disable=wildcard-import,g-bad-import-order,g-import-not-at-top\u001b[39;00m\n\u001b[1;32m     36\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mtensorflow\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mpython\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m pywrap_tensorflow \u001b[38;5;28;01mas\u001b[39;00m _pywrap_tensorflow\n\u001b[0;32m---> 37\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mtensorflow\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mpython\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01meager\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m context\n\u001b[1;32m     39\u001b[0m \u001b[38;5;66;03m# pylint: enable=wildcard-import\u001b[39;00m\n\u001b[1;32m     40\u001b[0m \n\u001b[1;32m     41\u001b[0m \u001b[38;5;66;03m# Bring in subpackages.\u001b[39;00m\n\u001b[1;32m     42\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mtensorflow\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mpython\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m data\n",
      "File \u001b[0;32m~/Desktop/III_essy_coding/Experiment2_PACOH_NN/qam_exp/lib/python3.8/site-packages/tensorflow/python/eager/context.py:28\u001b[0m\n\u001b[1;32m     25\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mabsl\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m logging\n\u001b[1;32m     26\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mnumpy\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m \u001b[38;5;21;01mnp\u001b[39;00m\n\u001b[0;32m---> 28\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mtensorflow\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mcore\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mframework\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m function_pb2\n\u001b[1;32m     29\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mtensorflow\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mcore\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mprotobuf\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m config_pb2\n\u001b[1;32m     30\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mtensorflow\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mcore\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mprotobuf\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m coordination_config_pb2\n",
      "File \u001b[0;32m~/Desktop/III_essy_coding/Experiment2_PACOH_NN/qam_exp/lib/python3.8/site-packages/tensorflow/core/framework/function_pb2.py:7\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01msys\u001b[39;00m\n\u001b[1;32m      6\u001b[0m _b\u001b[38;5;241m=\u001b[39msys\u001b[38;5;241m.\u001b[39mversion_info[\u001b[38;5;241m0\u001b[39m]\u001b[38;5;241m<\u001b[39m\u001b[38;5;241m3\u001b[39m \u001b[38;5;129;01mand\u001b[39;00m (\u001b[38;5;28;01mlambda\u001b[39;00m x:x) \u001b[38;5;129;01mor\u001b[39;00m (\u001b[38;5;28;01mlambda\u001b[39;00m x:x\u001b[38;5;241m.\u001b[39mencode(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mlatin1\u001b[39m\u001b[38;5;124m'\u001b[39m))\n\u001b[0;32m----> 7\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mgoogle\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mprotobuf\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m descriptor \u001b[38;5;28;01mas\u001b[39;00m _descriptor\n\u001b[1;32m      8\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mgoogle\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mprotobuf\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m message \u001b[38;5;28;01mas\u001b[39;00m _message\n\u001b[1;32m      9\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mgoogle\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mprotobuf\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m reflection \u001b[38;5;28;01mas\u001b[39;00m _reflection\n",
      "File \u001b[0;32m~/Desktop/III_essy_coding/Experiment2_PACOH_NN/qam_exp/lib/python3.8/site-packages/google/protobuf/descriptor.py:48\u001b[0m\n\u001b[1;32m     46\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mos\u001b[39;00m\n\u001b[1;32m     47\u001b[0m \u001b[38;5;66;03m# pylint: disable=protected-access\u001b[39;00m\n\u001b[0;32m---> 48\u001b[0m _message \u001b[38;5;241m=\u001b[39m \u001b[43mapi_implementation\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_c_module\u001b[49m\n\u001b[1;32m     49\u001b[0m \u001b[38;5;66;03m# TODO(jieluo): Remove this import after fix api_implementation\u001b[39;00m\n\u001b[1;32m     50\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m _message \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n",
      "\u001b[0;31mAttributeError\u001b[0m: module 'google.protobuf.internal.api_implementation' has no attribute '_c_module'"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import numpy "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# default configuration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start meta-training -------------------- \n",
      "\n",
      "Iter 0/20000 - Time 26.89 sec - Train-Loss: -5585.75391\n",
      "\n",
      "Iter 1000/20000 - Time 17.88 sec - Train-Loss: -287.93161\n",
      "\n",
      "Iter 2000/20000 - Time 17.12 sec - Train-Loss: -297.60468\n",
      "\n",
      "Iter 3000/20000 - Time 17.50 sec - Train-Loss: -388.00952\n",
      "\n",
      "Iter 4000/20000 - Time 17.16 sec - Train-Loss: -262.97476\n",
      "\n",
      "Iter 5000/20000 - Time 16.87 sec - Train-Loss: -241.44800\n",
      "\n",
      "Iter 6000/20000 - Time 17.18 sec - Train-Loss: -157.67252\n",
      "\n",
      "Iter 7000/20000 - Time 17.11 sec - Train-Loss: -104.36417\n",
      "\n",
      "Iter 8000/20000 - Time 17.52 sec - Train-Loss: -43.42802\n",
      "\n",
      "Iter 9000/20000 - Time 19.22 sec - Train-Loss: -19.81049\n",
      "\tStart meta-test posterior inference in 2 batches ------------------\n",
      "\tMeta-Test batch #1 consisting of 5 tasks----\n",
      "\tMeta-Test batch #2 consisting of 5 tasks----\n",
      "\n",
      "Iter 10000/20000 - Time 19.71 sec - Train-Loss: -14.96849- Val-avg_rmse: 0.203 +- 0.019- Val-cal_err: 0.056 +- 0.023- Val-avg_ll: 0.127 +- 0.125\n",
      "\n",
      "Iter 11000/20000 - Time 27.35 sec - Train-Loss: -13.38986\n",
      "\n",
      "Iter 12000/20000 - Time 18.97 sec - Train-Loss: -11.26875\n",
      "\n",
      "Iter 13000/20000 - Time 16.89 sec - Train-Loss: -8.33254\n",
      "\n",
      "Iter 14000/20000 - Time 20.81 sec - Train-Loss: -8.65657\n",
      "\n",
      "Iter 15000/20000 - Time 17.52 sec - Train-Loss: -6.57155\n",
      "\n",
      "Iter 16000/20000 - Time 17.95 sec - Train-Loss: -5.92959\n",
      "\n",
      "Iter 17000/20000 - Time 17.88 sec - Train-Loss: -5.60725\n",
      "\n",
      "Iter 18000/20000 - Time 18.53 sec - Train-Loss: -5.06230\n",
      "\n",
      "Iter 19000/20000 - Time 19.05 sec - Train-Loss: -5.22706\n",
      "\tStart meta-test posterior inference in 4 batches ------------------\n",
      "\tMeta-Test batch #1 consisting of 5 tasks----\n",
      "\tMeta-Test batch #2 consisting of 5 tasks----\n",
      "\tMeta-Test batch #3 consisting of 5 tasks----\n",
      "\tMeta-Test batch #4 consisting of 5 tasks----\n",
      "avg_rmse: 0.2071 +- 0.0194\n",
      "cal_err: 0.0489 +- 0.0248\n",
      "avg_ll: 0.1642 +- 0.0847\n"
     ]
    }
   ],
   "source": [
    "tf.get_logger().setLevel('ERROR')\n",
    "\n",
    "from pacoh_nn.datasets.regression_datasets import provide_data\n",
    "\n",
    "meta_train_data, meta_test_data, _ = provide_data(dataset='cauchy_20')\n",
    "\n",
    "from pacoh_nn.pacoh_nn_regression import PACOH_NN_Regression\n",
    "pacoh_model = PACOH_NN_Regression(meta_train_data, prior_weight=0.184, bandwidth=480.,\n",
    "                                  hyper_prior_likelihood_log_var_mean_mean=-1.0,\n",
    "                                  hyper_prior_log_var_mean=-1.74, hyper_prior_nn_std=0.12,\n",
    "                                  hyper_prior_weight=1e-5, lr=1.5e-3, learn_likelihood=True,\n",
    "                                  random_seed=28, num_iter_meta_train=20000)\n",
    "\n",
    "pacoh_model.meta_fit(meta_test_data[:10], eval_period=10000, log_period=1000, plot_period=5000)\n",
    "\n",
    "eval_metrics_mean, eval_metrics_std = pacoh_model.meta_eval_datasets(meta_test_data)\n",
    "for key in eval_metrics_mean:\n",
    "    print(\"%s: %.4f +- %.4f\" % (key, eval_metrics_mean[key], eval_metrics_std[key]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Number of Hyper posterior particles: 1, 3, 5, 10, 20"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start meta-training -------------------- \n",
      "\n",
      "Iter 0/20000 - Time 9.87 sec - Train-Loss: -4680.59424\n",
      "\n",
      "Iter 1000/20000 - Time 9.13 sec - Train-Loss: -290.45856\n",
      "\n",
      "Iter 2000/20000 - Time 9.55 sec - Train-Loss: -305.48618\n",
      "\n",
      "Iter 3000/20000 - Time 9.73 sec - Train-Loss: -381.68985\n",
      "\n",
      "Iter 4000/20000 - Time 9.42 sec - Train-Loss: -265.10159\n",
      "\n",
      "Iter 5000/20000 - Time 9.51 sec - Train-Loss: -239.25034\n",
      "\n",
      "Iter 6000/20000 - Time 9.15 sec - Train-Loss: -155.50000\n",
      "\n",
      "Iter 7000/20000 - Time 8.93 sec - Train-Loss: -115.56625\n",
      "\n",
      "Iter 8000/20000 - Time 10.20 sec - Train-Loss: -40.19633\n",
      "\n",
      "Iter 9000/20000 - Time 10.70 sec - Train-Loss: -17.36736\n",
      "\tStart meta-test posterior inference in 2 batches ------------------\n",
      "\tMeta-Test batch #1 consisting of 5 tasks----\n",
      "\tMeta-Test batch #2 consisting of 5 tasks----\n",
      "\n",
      "Iter 10000/20000 - Time 9.94 sec - Train-Loss: -13.29417- Val-avg_rmse: 0.220 +- 0.027- Val-cal_err: 0.089 +- 0.025- Val-avg_ll: -0.529 +- 0.466\n",
      "\n",
      "Iter 11000/20000 - Time 12.42 sec - Train-Loss: -11.97244\n",
      "\n",
      "Iter 12000/20000 - Time 9.68 sec - Train-Loss: -9.55595\n",
      "\n",
      "Iter 13000/20000 - Time 9.79 sec - Train-Loss: -8.19705\n",
      "\n",
      "Iter 14000/20000 - Time 9.21 sec - Train-Loss: -8.13567\n",
      "\n",
      "Iter 15000/20000 - Time 10.29 sec - Train-Loss: -6.30963\n",
      "\n",
      "Iter 16000/20000 - Time 9.84 sec - Train-Loss: -5.74317\n",
      "\n",
      "Iter 17000/20000 - Time 9.46 sec - Train-Loss: -5.74130\n",
      "\n",
      "Iter 18000/20000 - Time 10.19 sec - Train-Loss: -5.02993\n",
      "\n",
      "Iter 19000/20000 - Time 10.18 sec - Train-Loss: -4.52029\n",
      "\tStart meta-test posterior inference in 4 batches ------------------\n",
      "\tMeta-Test batch #1 consisting of 5 tasks----\n",
      "\tMeta-Test batch #2 consisting of 5 tasks----\n",
      "\tMeta-Test batch #3 consisting of 5 tasks----\n",
      "\tMeta-Test batch #4 consisting of 5 tasks----\n",
      "avg_rmse: 0.2097 +- 0.0205\n",
      "cal_err: 0.0572 +- 0.0348\n",
      "avg_ll: 0.0971 +- 0.1237\n"
     ]
    }
   ],
   "source": [
    "tf.get_logger().setLevel('ERROR')\n",
    "\n",
    "from pacoh_nn.datasets.regression_datasets import provide_data\n",
    "\n",
    "meta_train_data, meta_test_data, _ = provide_data(dataset='cauchy_20')\n",
    "\n",
    "from pacoh_nn.pacoh_nn_regression import PACOH_NN_Regression\n",
    "pacoh_model = PACOH_NN_Regression(meta_train_data, prior_weight=0.184, bandwidth=480.,\n",
    "                                  hyper_prior_likelihood_log_var_mean_mean=-1.0,\n",
    "                                  num_hyper_posterior_particles=1,\n",
    "                                  hyper_prior_log_var_mean=-1.74, hyper_prior_nn_std=0.12,\n",
    "                                  hyper_prior_weight=1e-5, lr=1.5e-3, learn_likelihood=True,\n",
    "                                  random_seed=28, num_iter_meta_train=20000)\n",
    "\n",
    "pacoh_model.meta_fit(meta_test_data[:10], eval_period=10000, log_period=1000, plot_period=5000)\n",
    "\n",
    "eval_metrics_mean, eval_metrics_std = pacoh_model.meta_eval_datasets(meta_test_data)\n",
    "for key in eval_metrics_mean:\n",
    "    print(\"%s: %.4f +- %.4f\" % (key, eval_metrics_mean[key], eval_metrics_std[key]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start meta-training -------------------- \n",
      "\n",
      "Iter 0/20000 - Time 51.49 sec - Train-Loss: -4533.51660\n",
      "\n",
      "Iter 1000/20000 - Time 27.77 sec - Train-Loss: -290.80264\n",
      "\n",
      "Iter 2000/20000 - Time 27.70 sec - Train-Loss: -294.27472\n",
      "\n",
      "Iter 3000/20000 - Time 28.56 sec - Train-Loss: -392.66031\n",
      "\n",
      "Iter 4000/20000 - Time 29.90 sec - Train-Loss: -248.41074\n",
      "\n",
      "Iter 5000/20000 - Time 26.95 sec - Train-Loss: -221.53682\n",
      "\n",
      "Iter 6000/20000 - Time 27.14 sec - Train-Loss: -137.51907\n",
      "\n",
      "Iter 7000/20000 - Time 26.69 sec - Train-Loss: -84.67992\n",
      "\n",
      "Iter 8000/20000 - Time 26.18 sec - Train-Loss: -33.03574\n",
      "\n",
      "Iter 9000/20000 - Time 26.43 sec - Train-Loss: -16.75628\n",
      "\tStart meta-test posterior inference in 2 batches ------------------\n",
      "\tMeta-Test batch #1 consisting of 5 tasks----\n",
      "\tMeta-Test batch #2 consisting of 5 tasks----\n",
      "\n",
      "Iter 10000/20000 - Time 29.46 sec - Train-Loss: -13.49484- Val-avg_rmse: 0.195 +- 0.016- Val-cal_err: 0.047 +- 0.025- Val-avg_ll: 0.187 +- 0.091\n",
      "\n",
      "Iter 11000/20000 - Time 52.17 sec - Train-Loss: -12.15495\n",
      "\n",
      "Iter 12000/20000 - Time 27.08 sec - Train-Loss: -10.42383\n",
      "\n",
      "Iter 13000/20000 - Time 27.75 sec - Train-Loss: -7.75614\n",
      "\n",
      "Iter 14000/20000 - Time 27.93 sec - Train-Loss: -7.83735\n",
      "\n",
      "Iter 15000/20000 - Time 26.40 sec - Train-Loss: -6.42952\n",
      "\n",
      "Iter 16000/20000 - Time 27.41 sec - Train-Loss: -5.56447\n",
      "\n",
      "Iter 17000/20000 - Time 28.57 sec - Train-Loss: -5.51375\n",
      "\n",
      "Iter 18000/20000 - Time 27.60 sec - Train-Loss: -4.88794\n",
      "\n",
      "Iter 19000/20000 - Time 28.48 sec - Train-Loss: -5.28695\n",
      "\tStart meta-test posterior inference in 4 batches ------------------\n",
      "\tMeta-Test batch #1 consisting of 5 tasks----\n",
      "\tMeta-Test batch #2 consisting of 5 tasks----\n",
      "\tMeta-Test batch #3 consisting of 5 tasks----\n",
      "\tMeta-Test batch #4 consisting of 5 tasks----\n",
      "avg_rmse: 0.1991 +- 0.0168\n",
      "cal_err: 0.0451 +- 0.0182\n",
      "avg_ll: 0.1815 +- 0.0826\n"
     ]
    }
   ],
   "source": [
    "tf.get_logger().setLevel('ERROR')\n",
    "\n",
    "from pacoh_nn.datasets.regression_datasets import provide_data\n",
    "\n",
    "meta_train_data, meta_test_data, _ = provide_data(dataset='cauchy_20')\n",
    "\n",
    "from pacoh_nn.pacoh_nn_regression import PACOH_NN_Regression\n",
    "pacoh_model = PACOH_NN_Regression(meta_train_data, prior_weight=0.184, bandwidth=480.,\n",
    "                                  hyper_prior_likelihood_log_var_mean_mean=-1.0,\n",
    "                                  num_hyper_posterior_particles=5,\n",
    "                                  hyper_prior_log_var_mean=-1.74, hyper_prior_nn_std=0.12,\n",
    "                                  hyper_prior_weight=1e-5, lr=1.5e-3, learn_likelihood=True,\n",
    "                                  random_seed=28, num_iter_meta_train=20000)\n",
    "\n",
    "pacoh_model.meta_fit(meta_test_data[:10], eval_period=10000, log_period=1000, plot_period=5000)\n",
    "\n",
    "eval_metrics_mean, eval_metrics_std = pacoh_model.meta_eval_datasets(meta_test_data)\n",
    "for key in eval_metrics_mean:\n",
    "    print(\"%s: %.4f +- %.4f\" % (key, eval_metrics_mean[key], eval_metrics_std[key]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start meta-training -------------------- \n",
      "\n",
      "Iter 0/20000 - Time 124.97 sec - Train-Loss: -4591.51074\n",
      "\n",
      "Iter 1000/20000 - Time 51.07 sec - Train-Loss: -287.52954\n",
      "\n",
      "Iter 2000/20000 - Time 50.70 sec - Train-Loss: -300.91379\n",
      "\n",
      "Iter 3000/20000 - Time 49.28 sec - Train-Loss: -417.03912\n",
      "\n",
      "Iter 4000/20000 - Time 56.64 sec - Train-Loss: -259.41193\n",
      "\n",
      "Iter 5000/20000 - Time 50.48 sec - Train-Loss: -230.84290\n",
      "\n",
      "Iter 6000/20000 - Time 49.98 sec - Train-Loss: -153.66716\n",
      "\n",
      "Iter 7000/20000 - Time 51.33 sec - Train-Loss: -98.54315\n",
      "\n",
      "Iter 8000/20000 - Time 50.90 sec - Train-Loss: -38.49943\n",
      "\n",
      "Iter 9000/20000 - Time 51.09 sec - Train-Loss: -19.84091\n",
      "\tStart meta-test posterior inference in 2 batches ------------------\n",
      "\tMeta-Test batch #1 consisting of 5 tasks----\n",
      "\tMeta-Test batch #2 consisting of 5 tasks----\n",
      "\n",
      "Iter 10000/20000 - Time 50.77 sec - Train-Loss: -13.43513- Val-avg_rmse: 0.197 +- 0.017- Val-cal_err: 0.044 +- 0.026- Val-avg_ll: 0.188 +- 0.093\n",
      "\n",
      "Iter 11000/20000 - Time 103.48 sec - Train-Loss: -11.83175\n",
      "\n",
      "Iter 12000/20000 - Time 52.74 sec - Train-Loss: -9.76206\n",
      "\n",
      "Iter 13000/20000 - Time 68.65 sec - Train-Loss: -7.16027\n",
      "\n",
      "Iter 14000/20000 - Time 50.63 sec - Train-Loss: -7.75534\n",
      "\n",
      "Iter 15000/20000 - Time 50.12 sec - Train-Loss: -6.08888\n",
      "\n",
      "Iter 16000/20000 - Time 48.85 sec - Train-Loss: -5.92731\n",
      "\n",
      "Iter 17000/20000 - Time 52.09 sec - Train-Loss: -5.59064\n",
      "\n",
      "Iter 18000/20000 - Time 51.44 sec - Train-Loss: -4.72108\n",
      "\n",
      "Iter 19000/20000 - Time 50.92 sec - Train-Loss: -5.22143\n",
      "\tStart meta-test posterior inference in 4 batches ------------------\n",
      "\tMeta-Test batch #1 consisting of 5 tasks----\n",
      "\tMeta-Test batch #2 consisting of 5 tasks----\n",
      "\tMeta-Test batch #3 consisting of 5 tasks----\n",
      "\tMeta-Test batch #4 consisting of 5 tasks----\n",
      "avg_rmse: 0.2015 +- 0.0171\n",
      "cal_err: 0.0485 +- 0.0194\n",
      "avg_ll: 0.1692 +- 0.0673\n"
     ]
    }
   ],
   "source": [
    "tf.get_logger().setLevel('ERROR')\n",
    "\n",
    "from pacoh_nn.datasets.regression_datasets import provide_data\n",
    "\n",
    "meta_train_data, meta_test_data, _ = provide_data(dataset='cauchy_20')\n",
    "\n",
    "from pacoh_nn.pacoh_nn_regression import PACOH_NN_Regression\n",
    "pacoh_model = PACOH_NN_Regression(meta_train_data, prior_weight=0.184, bandwidth=480.,\n",
    "                                  hyper_prior_likelihood_log_var_mean_mean=-1.0,\n",
    "                                  num_hyper_posterior_particles=10,\n",
    "                                  hyper_prior_log_var_mean=-1.74, hyper_prior_nn_std=0.12,\n",
    "                                  hyper_prior_weight=1e-5, lr=1.5e-3, learn_likelihood=True,\n",
    "                                  random_seed=28, num_iter_meta_train=20000)\n",
    "\n",
    "pacoh_model.meta_fit(meta_test_data[:10], eval_period=10000, log_period=1000, plot_period=5000)\n",
    "\n",
    "eval_metrics_mean, eval_metrics_std = pacoh_model.meta_eval_datasets(meta_test_data)\n",
    "for key in eval_metrics_mean:\n",
    "    print(\"%s: %.4f +- %.4f\" % (key, eval_metrics_mean[key], eval_metrics_std[key]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.get_logger().setLevel('ERROR')\n",
    "\n",
    "from pacoh_nn.datasets.regression_datasets import provide_data\n",
    "\n",
    "meta_train_data, meta_test_data, _ = provide_data(dataset='cauchy_20')\n",
    "\n",
    "from pacoh_nn.pacoh_nn_regression import PACOH_NN_Regression\n",
    "pacoh_model = PACOH_NN_Regression(meta_train_data, prior_weight=0.184, bandwidth=480.,\n",
    "                                  hyper_prior_likelihood_log_var_mean_mean=-1.0,\n",
    "                                  num_hyper_posterior_particles=20,\n",
    "                                  hyper_prior_log_var_mean=-1.74, hyper_prior_nn_std=0.12,\n",
    "                                  hyper_prior_weight=1e-5, lr=1.5e-3, learn_likelihood=True,\n",
    "                                  random_seed=28, num_iter_meta_train=20000)\n",
    "\n",
    "pacoh_model.meta_fit(meta_test_data[:10], eval_period=10000, log_period=1000, plot_period=5000)\n",
    "\n",
    "eval_metrics_mean, eval_metrics_std = pacoh_model.meta_eval_datasets(meta_test_data)\n",
    "for key in eval_metrics_mean:\n",
    "    print(\"%s: %.4f +- %.4f\" % (key, eval_metrics_mean[key], eval_metrics_std[key]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Number of posterior particles 1, 3, 5, 10, 20"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start meta-training -------------------- \n",
      "\n",
      "Iter 0/20000 - Time 31.83 sec - Train-Loss: -5750.62842\n",
      "\n",
      "Iter 1000/20000 - Time 18.23 sec - Train-Loss: -296.82724\n",
      "\n",
      "Iter 2000/20000 - Time 21.20 sec - Train-Loss: -300.67258\n",
      "\n",
      "Iter 3000/20000 - Time 19.01 sec - Train-Loss: -405.05762\n",
      "\n",
      "Iter 4000/20000 - Time 18.78 sec - Train-Loss: -268.83643\n",
      "\n",
      "Iter 5000/20000 - Time 17.29 sec - Train-Loss: -249.23096\n",
      "\n",
      "Iter 6000/20000 - Time 19.76 sec - Train-Loss: -173.80037\n",
      "\n",
      "Iter 7000/20000 - Time 17.60 sec - Train-Loss: -126.74082\n",
      "\n",
      "Iter 8000/20000 - Time 17.94 sec - Train-Loss: -47.45450\n",
      "\n",
      "Iter 9000/20000 - Time 19.59 sec - Train-Loss: -23.62860\n",
      "\tStart meta-test posterior inference in 2 batches ------------------\n",
      "\tMeta-Test batch #1 consisting of 5 tasks----\n",
      "\tMeta-Test batch #2 consisting of 5 tasks----\n",
      "\n",
      "Iter 10000/20000 - Time 16.92 sec - Train-Loss: -14.94614- Val-avg_rmse: 0.220 +- 0.028- Val-cal_err: 0.108 +- 0.021- Val-avg_ll: -0.831 +- 0.636\n",
      "\n",
      "Iter 11000/20000 - Time 24.03 sec - Train-Loss: -12.89984\n",
      "\n",
      "Iter 12000/20000 - Time 15.81 sec - Train-Loss: -11.29401\n",
      "\n",
      "Iter 13000/20000 - Time 15.80 sec - Train-Loss: -8.80283\n",
      "\n",
      "Iter 14000/20000 - Time 15.53 sec - Train-Loss: -8.95900\n",
      "\n",
      "Iter 15000/20000 - Time 16.94 sec - Train-Loss: -6.62238\n",
      "\n",
      "Iter 16000/20000 - Time 15.47 sec - Train-Loss: -5.97202\n",
      "\n",
      "Iter 17000/20000 - Time 16.57 sec - Train-Loss: -5.94652\n",
      "\n",
      "Iter 18000/20000 - Time 16.29 sec - Train-Loss: -5.15828\n",
      "\n",
      "Iter 19000/20000 - Time 16.67 sec - Train-Loss: -5.12935\n",
      "\tStart meta-test posterior inference in 4 batches ------------------\n",
      "\tMeta-Test batch #1 consisting of 5 tasks----\n",
      "\tMeta-Test batch #2 consisting of 5 tasks----\n",
      "\tMeta-Test batch #3 consisting of 5 tasks----\n",
      "\tMeta-Test batch #4 consisting of 5 tasks----\n",
      "avg_rmse: 0.2202 +- 0.0217\n",
      "cal_err: 0.0664 +- 0.0347\n",
      "avg_ll: -0.0085 +- 0.1866\n"
     ]
    }
   ],
   "source": [
    "tf.get_logger().setLevel('ERROR')\n",
    "\n",
    "from pacoh_nn.datasets.regression_datasets import provide_data\n",
    "\n",
    "meta_train_data, meta_test_data, _ = provide_data(dataset='cauchy_20')\n",
    "\n",
    "from pacoh_nn.pacoh_nn_regression import PACOH_NN_Regression\n",
    "pacoh_model = PACOH_NN_Regression(meta_train_data, prior_weight=0.184, bandwidth=480.,\n",
    "                                  hyper_prior_likelihood_log_var_mean_mean=-1.0,\n",
    "                                    num_posterior_particles=1,\n",
    "                                  hyper_prior_log_var_mean=-1.74, hyper_prior_nn_std=0.12,\n",
    "                                  hyper_prior_weight=1e-5, lr=1.5e-3, learn_likelihood=True,\n",
    "                                  random_seed=28, num_iter_meta_train=20000)\n",
    "\n",
    "pacoh_model.meta_fit(meta_test_data[:10], eval_period=10000, log_period=1000, plot_period=5000)\n",
    "\n",
    "eval_metrics_mean, eval_metrics_std = pacoh_model.meta_eval_datasets(meta_test_data)\n",
    "for key in eval_metrics_mean:\n",
    "    print(\"%s: %.4f +- %.4f\" % (key, eval_metrics_mean[key], eval_metrics_std[key]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start meta-training -------------------- \n",
      "\n",
      "Iter 0/20000 - Time 57.10 sec - Train-Loss: -6024.05225\n",
      "\n",
      "Iter 1000/20000 - Time 20.67 sec - Train-Loss: -287.98074\n",
      "\n",
      "Iter 2000/20000 - Time 20.98 sec - Train-Loss: -298.14346\n",
      "\n",
      "Iter 3000/20000 - Time 19.91 sec - Train-Loss: -484.88998\n",
      "\n",
      "Iter 4000/20000 - Time 19.63 sec - Train-Loss: -267.09097\n",
      "\n",
      "Iter 5000/20000 - Time 20.09 sec - Train-Loss: -245.83008\n",
      "\n",
      "Iter 6000/20000 - Time 19.49 sec - Train-Loss: -167.73422\n",
      "\n",
      "Iter 7000/20000 - Time 20.34 sec - Train-Loss: -125.46287\n",
      "\n",
      "Iter 8000/20000 - Time 19.77 sec - Train-Loss: -48.44249\n",
      "\n",
      "Iter 9000/20000 - Time 19.76 sec - Train-Loss: -21.36822\n",
      "\tStart meta-test posterior inference in 2 batches ------------------\n",
      "\tMeta-Test batch #1 consisting of 5 tasks----\n",
      "\tMeta-Test batch #2 consisting of 5 tasks----\n",
      "\n",
      "Iter 10000/20000 - Time 19.89 sec - Train-Loss: -14.38698- Val-avg_rmse: 0.204 +- 0.019- Val-cal_err: 0.068 +- 0.025- Val-avg_ll: 0.012 +- 0.151\n",
      "\n",
      "Iter 11000/20000 - Time 30.56 sec - Train-Loss: -13.18156\n",
      "\n",
      "Iter 12000/20000 - Time 20.15 sec - Train-Loss: -11.12830\n",
      "\n",
      "Iter 13000/20000 - Time 19.21 sec - Train-Loss: -9.05746\n",
      "\n",
      "Iter 14000/20000 - Time 19.43 sec - Train-Loss: -8.86390\n",
      "\n",
      "Iter 15000/20000 - Time 20.23 sec - Train-Loss: -7.14084\n",
      "\n",
      "Iter 16000/20000 - Time 19.95 sec - Train-Loss: -6.03029\n",
      "\n",
      "Iter 17000/20000 - Time 19.55 sec - Train-Loss: -6.05065\n",
      "\n",
      "Iter 18000/20000 - Time 19.65 sec - Train-Loss: -5.21369\n",
      "\n",
      "Iter 19000/20000 - Time 21.04 sec - Train-Loss: -5.40634\n",
      "\tStart meta-test posterior inference in 4 batches ------------------\n",
      "\tMeta-Test batch #1 consisting of 5 tasks----\n",
      "\tMeta-Test batch #2 consisting of 5 tasks----\n",
      "\tMeta-Test batch #3 consisting of 5 tasks----\n",
      "\tMeta-Test batch #4 consisting of 5 tasks----\n",
      "avg_rmse: 0.2029 +- 0.0176\n",
      "cal_err: 0.0450 +- 0.0210\n",
      "avg_ll: 0.1788 +- 0.0825\n"
     ]
    }
   ],
   "source": [
    "tf.get_logger().setLevel('ERROR')\n",
    "\n",
    "from pacoh_nn.datasets.regression_datasets import provide_data\n",
    "\n",
    "meta_train_data, meta_test_data, _ = provide_data(dataset='cauchy_20')\n",
    "\n",
    "from pacoh_nn.pacoh_nn_regression import PACOH_NN_Regression\n",
    "pacoh_model = PACOH_NN_Regression(meta_train_data, prior_weight=0.184, bandwidth=480.,\n",
    "                                  hyper_prior_likelihood_log_var_mean_mean=-1.0,\n",
    "                                    num_posterior_particles=3,\n",
    "                                  hyper_prior_log_var_mean=-1.74, hyper_prior_nn_std=0.12,\n",
    "                                  hyper_prior_weight=1e-5, lr=1.5e-3, learn_likelihood=True,\n",
    "                                  random_seed=28, num_iter_meta_train=20000)\n",
    "\n",
    "pacoh_model.meta_fit(meta_test_data[:10], eval_period=10000, log_period=1000, plot_period=5000)\n",
    "\n",
    "eval_metrics_mean, eval_metrics_std = pacoh_model.meta_eval_datasets(meta_test_data)\n",
    "for key in eval_metrics_mean:\n",
    "    print(\"%s: %.4f +- %.4f\" % (key, eval_metrics_mean[key], eval_metrics_std[key]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start meta-training -------------------- \n",
      "\n",
      "Iter 0/20000 - Time 31.16 sec - Train-Loss: -5176.12109\n",
      "\n",
      "Iter 1000/20000 - Time 19.63 sec - Train-Loss: -307.26511\n",
      "\n",
      "Iter 2000/20000 - Time 23.41 sec - Train-Loss: -298.16666\n",
      "\n",
      "Iter 3000/20000 - Time 23.56 sec - Train-Loss: -402.19614\n",
      "\n",
      "Iter 4000/20000 - Time 25.85 sec - Train-Loss: -265.63144\n",
      "\n",
      "Iter 5000/20000 - Time 22.06 sec - Train-Loss: -245.14622\n",
      "\n",
      "Iter 6000/20000 - Time 21.85 sec - Train-Loss: -163.62556\n",
      "\n",
      "Iter 7000/20000 - Time 20.17 sec - Train-Loss: -101.50449\n",
      "\n",
      "Iter 8000/20000 - Time 18.99 sec - Train-Loss: -43.11722\n",
      "\n",
      "Iter 9000/20000 - Time 19.66 sec - Train-Loss: -21.51613\n",
      "\tStart meta-test posterior inference in 2 batches ------------------\n",
      "\tMeta-Test batch #1 consisting of 5 tasks----\n",
      "\tMeta-Test batch #2 consisting of 5 tasks----\n",
      "\n",
      "Iter 10000/20000 - Time 19.31 sec - Train-Loss: -14.48760- Val-avg_rmse: 0.198 +- 0.020- Val-cal_err: 0.046 +- 0.021- Val-avg_ll: 0.158 +- 0.112\n",
      "\n",
      "Iter 11000/20000 - Time 33.75 sec - Train-Loss: -13.14867\n",
      "\n",
      "Iter 12000/20000 - Time 20.57 sec - Train-Loss: -11.36242\n",
      "\n",
      "Iter 13000/20000 - Time 19.71 sec - Train-Loss: -8.52274\n",
      "\n",
      "Iter 14000/20000 - Time 20.08 sec - Train-Loss: -8.57851\n",
      "\n",
      "Iter 15000/20000 - Time 18.90 sec - Train-Loss: -6.73530\n",
      "\n",
      "Iter 16000/20000 - Time 18.56 sec - Train-Loss: -5.78084\n",
      "\n",
      "Iter 17000/20000 - Time 19.98 sec - Train-Loss: -5.88043\n",
      "\n",
      "Iter 18000/20000 - Time 19.22 sec - Train-Loss: -5.22696\n",
      "\n",
      "Iter 19000/20000 - Time 19.76 sec - Train-Loss: -5.06726\n",
      "\tStart meta-test posterior inference in 4 batches ------------------\n",
      "\tMeta-Test batch #1 consisting of 5 tasks----\n",
      "\tMeta-Test batch #2 consisting of 5 tasks----\n",
      "\tMeta-Test batch #3 consisting of 5 tasks----\n",
      "\tMeta-Test batch #4 consisting of 5 tasks----\n",
      "avg_rmse: 0.1986 +- 0.0161\n",
      "cal_err: 0.0452 +- 0.0184\n",
      "avg_ll: 0.1778 +- 0.0747\n"
     ]
    }
   ],
   "source": [
    "tf.get_logger().setLevel('ERROR')\n",
    "\n",
    "from pacoh_nn.datasets.regression_datasets import provide_data\n",
    "\n",
    "meta_train_data, meta_test_data, _ = provide_data(dataset='cauchy_20')\n",
    "\n",
    "from pacoh_nn.pacoh_nn_regression import PACOH_NN_Regression\n",
    "pacoh_model = PACOH_NN_Regression(meta_train_data, prior_weight=0.184, bandwidth=480.,\n",
    "                                  hyper_prior_likelihood_log_var_mean_mean=-1.0,\n",
    "                                    num_posterior_particles=10,\n",
    "                                  hyper_prior_log_var_mean=-1.74, hyper_prior_nn_std=0.12,\n",
    "                                  hyper_prior_weight=1e-5, lr=1.5e-3, learn_likelihood=True,\n",
    "                                  random_seed=28, num_iter_meta_train=20000)\n",
    "\n",
    "pacoh_model.meta_fit(meta_test_data[:10], eval_period=10000, log_period=1000, plot_period=5000)\n",
    "\n",
    "eval_metrics_mean, eval_metrics_std = pacoh_model.meta_eval_datasets(meta_test_data)\n",
    "for key in eval_metrics_mean:\n",
    "    print(\"%s: %.4f +- %.4f\" % (key, eval_metrics_mean[key], eval_metrics_std[key]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start meta-training -------------------- \n",
      "\n",
      "Iter 0/20000 - Time 33.99 sec - Train-Loss: -4812.26221\n",
      "\n",
      "Iter 1000/20000 - Time 19.84 sec - Train-Loss: -287.05585\n",
      "\n",
      "Iter 2000/20000 - Time 19.41 sec - Train-Loss: -298.53748\n",
      "\n",
      "Iter 3000/20000 - Time 19.78 sec - Train-Loss: -407.06137\n",
      "\n",
      "Iter 4000/20000 - Time 19.52 sec - Train-Loss: -264.72830\n",
      "\n",
      "Iter 5000/20000 - Time 19.57 sec - Train-Loss: -247.39844\n",
      "\n",
      "Iter 6000/20000 - Time 19.22 sec - Train-Loss: -153.79398\n",
      "\n",
      "Iter 7000/20000 - Time 21.47 sec - Train-Loss: -101.98360\n",
      "\n",
      "Iter 8000/20000 - Time 21.46 sec - Train-Loss: -39.36273\n",
      "\n",
      "Iter 9000/20000 - Time 23.26 sec - Train-Loss: -21.81117\n",
      "\tStart meta-test posterior inference in 2 batches ------------------\n",
      "\tMeta-Test batch #1 consisting of 5 tasks----\n",
      "\tMeta-Test batch #2 consisting of 5 tasks----\n",
      "\n",
      "Iter 10000/20000 - Time 19.97 sec - Train-Loss: -14.87054- Val-avg_rmse: 0.198 +- 0.020- Val-cal_err: 0.044 +- 0.024- Val-avg_ll: 0.162 +- 0.125\n",
      "\n",
      "Iter 11000/20000 - Time 31.47 sec - Train-Loss: -12.78521\n",
      "\n",
      "Iter 12000/20000 - Time 18.97 sec - Train-Loss: -10.56427\n",
      "\n",
      "Iter 13000/20000 - Time 19.58 sec - Train-Loss: -8.57475\n",
      "\n",
      "Iter 14000/20000 - Time 19.39 sec - Train-Loss: -8.42722\n",
      "\n",
      "Iter 15000/20000 - Time 19.43 sec - Train-Loss: -7.08491\n",
      "\n",
      "Iter 16000/20000 - Time 21.09 sec - Train-Loss: -6.25898\n",
      "\n",
      "Iter 17000/20000 - Time 20.33 sec - Train-Loss: -5.90881\n",
      "\n",
      "Iter 18000/20000 - Time 25.22 sec - Train-Loss: -4.76816\n",
      "\n",
      "Iter 19000/20000 - Time 21.84 sec - Train-Loss: -5.28429\n",
      "\tStart meta-test posterior inference in 4 batches ------------------\n",
      "\tMeta-Test batch #1 consisting of 5 tasks----\n",
      "\tMeta-Test batch #2 consisting of 5 tasks----\n",
      "\tMeta-Test batch #3 consisting of 5 tasks----\n",
      "\tMeta-Test batch #4 consisting of 5 tasks----\n",
      "avg_rmse: 0.2006 +- 0.0160\n",
      "cal_err: 0.0458 +- 0.0171\n",
      "avg_ll: 0.1645 +- 0.0714\n"
     ]
    }
   ],
   "source": [
    "tf.get_logger().setLevel('ERROR')\n",
    "\n",
    "from pacoh_nn.datasets.regression_datasets import provide_data\n",
    "\n",
    "meta_train_data, meta_test_data, _ = provide_data(dataset='cauchy_20')\n",
    "\n",
    "from pacoh_nn.pacoh_nn_regression import PACOH_NN_Regression\n",
    "pacoh_model = PACOH_NN_Regression(meta_train_data, prior_weight=0.184, bandwidth=480.,\n",
    "                                  hyper_prior_likelihood_log_var_mean_mean=-1.0,\n",
    "                                    num_posterior_particles=20,\n",
    "                                  hyper_prior_log_var_mean=-1.74, hyper_prior_nn_std=0.12,\n",
    "                                  hyper_prior_weight=1e-5, lr=1.5e-3, learn_likelihood=True,\n",
    "                                  random_seed=28, num_iter_meta_train=20000)\n",
    "\n",
    "pacoh_model.meta_fit(meta_test_data[:10], eval_period=10000, log_period=1000, plot_period=5000)\n",
    "\n",
    "eval_metrics_mean, eval_metrics_std = pacoh_model.meta_eval_datasets(meta_test_data)\n",
    "for key in eval_metrics_mean:\n",
    "    print(\"%s: %.4f +- %.4f\" % (key, eval_metrics_mean[key], eval_metrics_std[key]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Size of Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start meta-training -------------------- \n",
      "\n",
      "Iter 0/20000 - Time 24.58 sec - Train-Loss: -4881.89795\n",
      "\n",
      "Iter 1000/20000 - Time 14.28 sec - Train-Loss: -282.53177\n",
      "\n",
      "Iter 2000/20000 - Time 13.64 sec - Train-Loss: -293.66452\n",
      "\n",
      "Iter 3000/20000 - Time 13.89 sec - Train-Loss: -410.72729\n",
      "\n",
      "Iter 4000/20000 - Time 12.85 sec - Train-Loss: -255.54944\n",
      "\n",
      "Iter 5000/20000 - Time 12.74 sec - Train-Loss: -217.72237\n",
      "\n",
      "Iter 6000/20000 - Time 13.59 sec - Train-Loss: -145.60631\n",
      "\n",
      "Iter 7000/20000 - Time 16.66 sec - Train-Loss: -97.63332\n",
      "\n",
      "Iter 8000/20000 - Time 14.60 sec - Train-Loss: -33.85612\n",
      "\n",
      "Iter 9000/20000 - Time 15.92 sec - Train-Loss: -12.17866\n",
      "\tStart meta-test posterior inference in 2 batches ------------------\n",
      "\tMeta-Test batch #1 consisting of 5 tasks----\n",
      "\tMeta-Test batch #2 consisting of 5 tasks----\n",
      "\n",
      "Iter 10000/20000 - Time 15.24 sec - Train-Loss: -7.68012- Val-avg_rmse: 0.210 +- 0.024- Val-cal_err: 0.059 +- 0.027- Val-avg_ll: 0.049 +- 0.168\n",
      "\n",
      "Iter 11000/20000 - Time 17.64 sec - Train-Loss: -8.26571\n",
      "\n",
      "Iter 12000/20000 - Time 12.43 sec - Train-Loss: -6.90635\n",
      "\n",
      "Iter 13000/20000 - Time 12.07 sec - Train-Loss: -4.99258\n",
      "\n",
      "Iter 14000/20000 - Time 12.37 sec - Train-Loss: -6.37495\n",
      "\n",
      "Iter 15000/20000 - Time 12.99 sec - Train-Loss: -5.04298\n",
      "\n",
      "Iter 16000/20000 - Time 12.68 sec - Train-Loss: -3.97127\n",
      "\n",
      "Iter 17000/20000 - Time 13.19 sec - Train-Loss: -4.98238\n",
      "\n",
      "Iter 18000/20000 - Time 13.34 sec - Train-Loss: -4.33209\n",
      "\n",
      "Iter 19000/20000 - Time 12.40 sec - Train-Loss: -4.17029\n",
      "\tStart meta-test posterior inference in 4 batches ------------------\n",
      "\tMeta-Test batch #1 consisting of 5 tasks----\n",
      "\tMeta-Test batch #2 consisting of 5 tasks----\n",
      "\tMeta-Test batch #3 consisting of 5 tasks----\n",
      "\tMeta-Test batch #4 consisting of 5 tasks----\n",
      "avg_rmse: 0.2052 +- 0.0156\n",
      "cal_err: 0.0450 +- 0.0179\n",
      "avg_ll: 0.1602 +- 0.0851\n"
     ]
    }
   ],
   "source": [
    "tf.get_logger().setLevel('ERROR')\n",
    "\n",
    "from pacoh_nn.datasets.regression_datasets import provide_data\n",
    "\n",
    "meta_train_data, meta_test_data, _ = provide_data(dataset='cauchy_20')\n",
    "\n",
    "from pacoh_nn.pacoh_nn_regression import PACOH_NN_Regression\n",
    "pacoh_model = PACOH_NN_Regression(meta_train_data, prior_weight=0.184, bandwidth=480.,\n",
    "                                  hyper_prior_likelihood_log_var_mean_mean=-1.0,\n",
    "                                   hidden_layer_sizes=(32, 32),\n",
    "                                  hyper_prior_log_var_mean=-1.74, hyper_prior_nn_std=0.12,\n",
    "                                  hyper_prior_weight=1e-5, lr=1.5e-3, learn_likelihood=True,\n",
    "                                  random_seed=28, num_iter_meta_train=20000)\n",
    "\n",
    "pacoh_model.meta_fit(meta_test_data[:10], eval_period=10000, log_period=1000, plot_period=5000)\n",
    "\n",
    "eval_metrics_mean, eval_metrics_std = pacoh_model.meta_eval_datasets(meta_test_data)\n",
    "for key in eval_metrics_mean:\n",
    "    print(\"%s: %.4f +- %.4f\" % (key, eval_metrics_mean[key], eval_metrics_std[key]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start meta-training -------------------- \n",
      "\n",
      "Iter 0/20000 - Time 55.93 sec - Train-Loss: -3519.11646\n",
      "\n",
      "Iter 1000/20000 - Time 24.30 sec - Train-Loss: -306.92914\n",
      "\n",
      "Iter 2000/20000 - Time 24.77 sec - Train-Loss: -297.27774\n",
      "\n",
      "Iter 3000/20000 - Time 25.65 sec - Train-Loss: -394.80997\n",
      "\n",
      "Iter 4000/20000 - Time 24.47 sec - Train-Loss: -231.03365\n",
      "\n",
      "Iter 5000/20000 - Time 23.89 sec - Train-Loss: -193.02336\n",
      "\n",
      "Iter 6000/20000 - Time 24.73 sec - Train-Loss: -111.44815\n",
      "\n",
      "Iter 7000/20000 - Time 24.85 sec - Train-Loss: -59.63066\n",
      "\n",
      "Iter 8000/20000 - Time 24.26 sec - Train-Loss: -28.04208\n",
      "\n",
      "Iter 9000/20000 - Time 24.23 sec - Train-Loss: -20.56136\n",
      "\tStart meta-test posterior inference in 2 batches ------------------\n",
      "\tMeta-Test batch #1 consisting of 5 tasks----\n",
      "\tMeta-Test batch #2 consisting of 5 tasks----\n",
      "\n",
      "Iter 10000/20000 - Time 24.14 sec - Train-Loss: -17.05214- Val-avg_rmse: 0.197 +- 0.017- Val-cal_err: 0.048 +- 0.017- Val-avg_ll: 0.148 +- 0.126\n",
      "\n",
      "Iter 11000/20000 - Time 79.45 sec - Train-Loss: -15.04852\n",
      "\n",
      "Iter 12000/20000 - Time 24.15 sec - Train-Loss: -12.57354\n",
      "\n",
      "Iter 13000/20000 - Time 617.18 sec - Train-Loss: -9.10640\n",
      "\n",
      "Iter 14000/20000 - Time 37.08 sec - Train-Loss: -9.13000\n",
      "\n",
      "Iter 15000/20000 - Time 29.59 sec - Train-Loss: -6.73318\n",
      "\n",
      "Iter 16000/20000 - Time 25.80 sec - Train-Loss: -6.96446\n",
      "\n",
      "Iter 17000/20000 - Time 26.32 sec - Train-Loss: -5.87602\n",
      "\n",
      "Iter 18000/20000 - Time 26.59 sec - Train-Loss: -5.33152\n",
      "\n",
      "Iter 19000/20000 - Time 26.53 sec - Train-Loss: -5.12783\n",
      "\tStart meta-test posterior inference in 4 batches ------------------\n",
      "\tMeta-Test batch #1 consisting of 5 tasks----\n",
      "\tMeta-Test batch #2 consisting of 5 tasks----\n",
      "\tMeta-Test batch #3 consisting of 5 tasks----\n",
      "\tMeta-Test batch #4 consisting of 5 tasks----\n",
      "avg_rmse: 0.2012 +- 0.0196\n",
      "cal_err: 0.0480 +- 0.0255\n",
      "avg_ll: 0.1859 +- 0.0839\n"
     ]
    }
   ],
   "source": [
    "tf.get_logger().setLevel('ERROR')\n",
    "\n",
    "from pacoh_nn.datasets.regression_datasets import provide_data\n",
    "\n",
    "meta_train_data, meta_test_data, _ = provide_data(dataset='cauchy_20')\n",
    "\n",
    "from pacoh_nn.pacoh_nn_regression import PACOH_NN_Regression\n",
    "pacoh_model = PACOH_NN_Regression(meta_train_data, prior_weight=0.184, bandwidth=480.,\n",
    "                                  hyper_prior_likelihood_log_var_mean_mean=-1.0,\n",
    "                                   hidden_layer_sizes=(32, 32, 32, 32, 32, 32),\n",
    "                                  hyper_prior_log_var_mean=-1.74, hyper_prior_nn_std=0.12,\n",
    "                                  hyper_prior_weight=1e-5, lr=1.5e-3, learn_likelihood=True,\n",
    "                                  random_seed=28, num_iter_meta_train=20000)\n",
    "\n",
    "pacoh_model.meta_fit(meta_test_data[:10], eval_period=10000, log_period=1000, plot_period=5000)\n",
    "\n",
    "eval_metrics_mean, eval_metrics_std = pacoh_model.meta_eval_datasets(meta_test_data)\n",
    "for key in eval_metrics_mean:\n",
    "    print(\"%s: %.4f +- %.4f\" % (key, eval_metrics_mean[key], eval_metrics_std[key]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# hidden layer sizes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start meta-training -------------------- \n",
      "\n",
      "Iter 0/20000 - Time 63.15 sec - Train-Loss: -4943.17627\n",
      "\n",
      "Iter 1000/20000 - Time 31.44 sec - Train-Loss: -330.02231\n",
      "\n",
      "Iter 2000/20000 - Time 30.41 sec - Train-Loss: -312.39584\n",
      "\n",
      "Iter 3000/20000 - Time 30.39 sec - Train-Loss: -481.80325\n",
      "\n",
      "Iter 4000/20000 - Time 30.83 sec - Train-Loss: -276.52725\n",
      "\n",
      "Iter 5000/20000 - Time 30.08 sec - Train-Loss: -254.26469\n",
      "\n",
      "Iter 6000/20000 - Time 29.57 sec - Train-Loss: -173.31775\n",
      "\n",
      "Iter 7000/20000 - Time 30.43 sec - Train-Loss: -112.90543\n",
      "\n",
      "Iter 8000/20000 - Time 30.39 sec - Train-Loss: -53.06440\n",
      "\n",
      "Iter 9000/20000 - Time 30.27 sec - Train-Loss: -31.00961\n",
      "\tStart meta-test posterior inference in 2 batches ------------------\n",
      "\tMeta-Test batch #1 consisting of 5 tasks----\n",
      "\tMeta-Test batch #2 consisting of 5 tasks----\n",
      "\n",
      "Iter 10000/20000 - Time 31.26 sec - Train-Loss: -24.84537- Val-avg_rmse: 0.198 +- 0.017- Val-cal_err: 0.062 +- 0.021- Val-avg_ll: 0.109 +- 0.125\n",
      "\n",
      "Iter 11000/20000 - Time 56.87 sec - Train-Loss: -20.94696\n",
      "\n",
      "Iter 12000/20000 - Time 31.43 sec - Train-Loss: -17.15516\n",
      "\n",
      "Iter 13000/20000 - Time 30.73 sec - Train-Loss: -12.82114\n",
      "\n",
      "Iter 14000/20000 - Time 31.67 sec - Train-Loss: -11.47244\n",
      "\n",
      "Iter 15000/20000 - Time 31.61 sec - Train-Loss: -8.75604\n",
      "\n",
      "Iter 16000/20000 - Time 31.07 sec - Train-Loss: -8.13691\n",
      "\n",
      "Iter 17000/20000 - Time 163.17 sec - Train-Loss: -6.92928\n",
      "\n",
      "Iter 18000/20000 - Time 33.69 sec - Train-Loss: -5.91496\n",
      "\n",
      "Iter 19000/20000 - Time 29.43 sec - Train-Loss: -5.44776\n",
      "\tStart meta-test posterior inference in 4 batches ------------------\n",
      "\tMeta-Test batch #1 consisting of 5 tasks----\n",
      "\tMeta-Test batch #2 consisting of 5 tasks----\n",
      "\tMeta-Test batch #3 consisting of 5 tasks----\n",
      "\tMeta-Test batch #4 consisting of 5 tasks----\n",
      "avg_rmse: 0.2048 +- 0.0187\n",
      "cal_err: 0.0530 +- 0.0273\n",
      "avg_ll: 0.1707 +- 0.0908\n"
     ]
    }
   ],
   "source": [
    "tf.get_logger().setLevel('ERROR')\n",
    "\n",
    "from pacoh_nn.datasets.regression_datasets import provide_data\n",
    "\n",
    "meta_train_data, meta_test_data, _ = provide_data(dataset='cauchy_20')\n",
    "\n",
    "from pacoh_nn.pacoh_nn_regression import PACOH_NN_Regression\n",
    "pacoh_model = PACOH_NN_Regression(meta_train_data, prior_weight=0.184, bandwidth=480.,\n",
    "                                  hyper_prior_likelihood_log_var_mean_mean=-1.0,\n",
    "                                   hidden_layer_sizes=(32, 32, 32, 32, 32, 32, 32, 32),\n",
    "                                  hyper_prior_log_var_mean=-1.74, hyper_prior_nn_std=0.12,\n",
    "                                  hyper_prior_weight=1e-5, lr=1.5e-3, learn_likelihood=True,\n",
    "                                  random_seed=28, num_iter_meta_train=20000)\n",
    "\n",
    "pacoh_model.meta_fit(meta_test_data[:10], eval_period=10000, log_period=1000, plot_period=5000)\n",
    "\n",
    "eval_metrics_mean, eval_metrics_std = pacoh_model.meta_eval_datasets(meta_test_data)\n",
    "for key in eval_metrics_mean:\n",
    "    print(\"%s: %.4f +- %.4f\" % (key, eval_metrics_mean[key], eval_metrics_std[key]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start meta-training -------------------- \n",
      "\n",
      "Iter 0/20000 - Time 54.35 sec - Train-Loss: -5584.39600\n",
      "\n",
      "Iter 1000/20000 - Time 32.30 sec - Train-Loss: -335.79861\n",
      "\n",
      "Iter 2000/20000 - Time 34.01 sec - Train-Loss: -347.28806\n",
      "\n",
      "Iter 3000/20000 - Time 27.47 sec - Train-Loss: -446.66174\n",
      "\n",
      "Iter 4000/20000 - Time 27.32 sec - Train-Loss: -301.60947\n",
      "\n",
      "Iter 5000/20000 - Time 28.78 sec - Train-Loss: -272.07819\n",
      "\n",
      "Iter 6000/20000 - Time 27.10 sec - Train-Loss: -179.83528\n",
      "\n",
      "Iter 7000/20000 - Time 25.23 sec - Train-Loss: -122.23696\n",
      "\n",
      "Iter 8000/20000 - Time 26.02 sec - Train-Loss: -65.99044\n",
      "\n",
      "Iter 9000/20000 - Time 28.02 sec - Train-Loss: -46.95024\n",
      "\tStart meta-test posterior inference in 2 batches ------------------\n",
      "\tMeta-Test batch #1 consisting of 5 tasks----\n",
      "\tMeta-Test batch #2 consisting of 5 tasks----\n",
      "\n",
      "Iter 10000/20000 - Time 23.76 sec - Train-Loss: -37.02124- Val-avg_rmse: 0.208 +- 0.023- Val-cal_err: 0.066 +- 0.018- Val-avg_ll: 0.021 +- 0.180\n",
      "\n",
      "Iter 11000/20000 - Time 38.30 sec - Train-Loss: -28.84635\n",
      "\n",
      "Iter 12000/20000 - Time 24.67 sec - Train-Loss: -20.78918\n",
      "\n",
      "Iter 13000/20000 - Time 23.71 sec - Train-Loss: -13.86156\n",
      "\n",
      "Iter 14000/20000 - Time 23.35 sec - Train-Loss: -11.55481\n",
      "\n",
      "Iter 15000/20000 - Time 23.09 sec - Train-Loss: -8.02852\n",
      "\n",
      "Iter 16000/20000 - Time 24.37 sec - Train-Loss: -6.74286\n",
      "\n",
      "Iter 17000/20000 - Time 23.76 sec - Train-Loss: -6.96730\n",
      "\n",
      "Iter 18000/20000 - Time 22.90 sec - Train-Loss: -5.35850\n",
      "\n",
      "Iter 19000/20000 - Time 22.91 sec - Train-Loss: -5.51103\n",
      "\tStart meta-test posterior inference in 4 batches ------------------\n",
      "\tMeta-Test batch #1 consisting of 5 tasks----\n",
      "\tMeta-Test batch #2 consisting of 5 tasks----\n",
      "\tMeta-Test batch #3 consisting of 5 tasks----\n",
      "\tMeta-Test batch #4 consisting of 5 tasks----\n",
      "avg_rmse: 0.2090 +- 0.0188\n",
      "cal_err: 0.0470 +- 0.0262\n",
      "avg_ll: 0.1515 +- 0.0981\n"
     ]
    }
   ],
   "source": [
    "tf.get_logger().setLevel('ERROR')\n",
    "\n",
    "from pacoh_nn.datasets.regression_datasets import provide_data\n",
    "\n",
    "meta_train_data, meta_test_data, _ = provide_data(dataset='cauchy_20')\n",
    "\n",
    "from pacoh_nn.pacoh_nn_regression import PACOH_NN_Regression\n",
    "pacoh_model = PACOH_NN_Regression(meta_train_data, prior_weight=0.184, bandwidth=480.,\n",
    "                                  hyper_prior_likelihood_log_var_mean_mean=-1.0,\n",
    "                                   hidden_layer_sizes=(64, 64, 64, 64),\n",
    "                                  hyper_prior_log_var_mean=-1.74, hyper_prior_nn_std=0.12,\n",
    "                                  hyper_prior_weight=1e-5, lr=1.5e-3, learn_likelihood=True,\n",
    "                                  random_seed=28, num_iter_meta_train=20000)\n",
    "\n",
    "pacoh_model.meta_fit(meta_test_data[:10], eval_period=10000, log_period=1000, plot_period=5000)\n",
    "\n",
    "eval_metrics_mean, eval_metrics_std = pacoh_model.meta_eval_datasets(meta_test_data)\n",
    "for key in eval_metrics_mean:\n",
    "    print(\"%s: %.4f +- %.4f\" % (key, eval_metrics_mean[key], eval_metrics_std[key]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start meta-training -------------------- \n",
      "\n",
      "Iter 0/20000 - Time 64.37 sec - Train-Loss: -4650.40674\n",
      "\n",
      "Iter 1000/20000 - Time 47.02 sec - Train-Loss: -432.80899\n",
      "\n",
      "Iter 2000/20000 - Time 50.25 sec - Train-Loss: -437.73499\n",
      "\n",
      "Iter 3000/20000 - Time 46.07 sec - Train-Loss: -589.63428\n",
      "\n",
      "Iter 4000/20000 - Time 45.66 sec - Train-Loss: -387.39160\n",
      "\n",
      "Iter 5000/20000 - Time 43.63 sec - Train-Loss: -355.84460\n",
      "\n",
      "Iter 6000/20000 - Time 43.54 sec - Train-Loss: -249.96094\n",
      "\n",
      "Iter 7000/20000 - Time 46.27 sec - Train-Loss: -195.79256\n",
      "\n",
      "Iter 8000/20000 - Time 43.52 sec - Train-Loss: -120.91488\n",
      "\n",
      "Iter 9000/20000 - Time 43.63 sec - Train-Loss: -91.52409\n",
      "\tStart meta-test posterior inference in 2 batches ------------------\n",
      "\tMeta-Test batch #1 consisting of 5 tasks----\n",
      "\tMeta-Test batch #2 consisting of 5 tasks----\n",
      "\n",
      "Iter 10000/20000 - Time 44.20 sec - Train-Loss: -70.20423- Val-avg_rmse: 0.205 +- 0.018- Val-cal_err: 0.076 +- 0.014- Val-avg_ll: -0.104 +- 0.195\n",
      "\n",
      "Iter 11000/20000 - Time 78.01 sec - Train-Loss: -52.58467\n",
      "\n",
      "Iter 12000/20000 - Time 44.33 sec - Train-Loss: -37.37677\n",
      "\n",
      "Iter 13000/20000 - Time 45.14 sec - Train-Loss: -25.34276\n",
      "\n",
      "Iter 14000/20000 - Time 46.74 sec - Train-Loss: -18.71477\n",
      "\n",
      "Iter 15000/20000 - Time 43.29 sec - Train-Loss: -12.92478\n",
      "\n",
      "Iter 16000/20000 - Time 44.15 sec - Train-Loss: -11.13767\n",
      "\n",
      "Iter 17000/20000 - Time 43.98 sec - Train-Loss: -8.83794\n",
      "\n",
      "Iter 18000/20000 - Time 42.83 sec - Train-Loss: -7.14687\n",
      "\n",
      "Iter 19000/20000 - Time 44.81 sec - Train-Loss: -7.33522\n",
      "\tStart meta-test posterior inference in 4 batches ------------------\n",
      "\tMeta-Test batch #1 consisting of 5 tasks----\n",
      "\tMeta-Test batch #2 consisting of 5 tasks----\n",
      "\tMeta-Test batch #3 consisting of 5 tasks----\n",
      "\tMeta-Test batch #4 consisting of 5 tasks----\n",
      "avg_rmse: 0.2125 +- 0.0219\n",
      "cal_err: 0.0628 +- 0.0391\n",
      "avg_ll: 0.1266 +- 0.1330\n"
     ]
    }
   ],
   "source": [
    "tf.get_logger().setLevel('ERROR')\n",
    "\n",
    "from pacoh_nn.datasets.regression_datasets import provide_data\n",
    "\n",
    "meta_train_data, meta_test_data, _ = provide_data(dataset='cauchy_20')\n",
    "\n",
    "from pacoh_nn.pacoh_nn_regression import PACOH_NN_Regression\n",
    "pacoh_model = PACOH_NN_Regression(meta_train_data, prior_weight=0.184, bandwidth=480.,\n",
    "                                  hyper_prior_likelihood_log_var_mean_mean=-1.0,\n",
    "                                   hidden_layer_sizes=(64, 64, 64, 64, 64, 64, 64, 64),\n",
    "                                  hyper_prior_log_var_mean=-1.74, hyper_prior_nn_std=0.12,\n",
    "                                  hyper_prior_weight=1e-5, lr=1.5e-3, learn_likelihood=True,\n",
    "                                  random_seed=28, num_iter_meta_train=20000)\n",
    "\n",
    "pacoh_model.meta_fit(meta_test_data[:10], eval_period=10000, log_period=1000, plot_period=5000)\n",
    "\n",
    "eval_metrics_mean, eval_metrics_std = pacoh_model.meta_eval_datasets(meta_test_data)\n",
    "for key in eval_metrics_mean:\n",
    "    print(\"%s: %.4f +- %.4f\" % (key, eval_metrics_mean[key], eval_metrics_std[key]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Hyper prior std"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start meta-training -------------------- \n",
      "\n",
      "Iter 0/20000 - Time 39.17 sec - Train-Loss: -6378.01758\n",
      "\n",
      "Iter 1000/20000 - Time 16.84 sec - Train-Loss: -280.71945\n",
      "\n",
      "Iter 2000/20000 - Time 15.24 sec - Train-Loss: -287.22745\n",
      "\n",
      "Iter 3000/20000 - Time 15.02 sec - Train-Loss: -386.16403\n",
      "\n",
      "Iter 4000/20000 - Time 14.94 sec - Train-Loss: -253.14583\n",
      "\n",
      "Iter 5000/20000 - Time 16.04 sec - Train-Loss: -232.82524\n",
      "\n",
      "Iter 6000/20000 - Time 14.96 sec - Train-Loss: -160.35344\n",
      "\n",
      "Iter 7000/20000 - Time 14.98 sec - Train-Loss: -103.22739\n",
      "\n",
      "Iter 8000/20000 - Time 14.86 sec - Train-Loss: -36.27776\n",
      "\n",
      "Iter 9000/20000 - Time 15.03 sec - Train-Loss: -11.54847\n",
      "\tStart meta-test posterior inference in 2 batches ------------------\n",
      "\tMeta-Test batch #1 consisting of 5 tasks----\n",
      "\tMeta-Test batch #2 consisting of 5 tasks----\n",
      "\n",
      "Iter 10000/20000 - Time 14.84 sec - Train-Loss: -5.54254- Val-avg_rmse: 0.201 +- 0.021- Val-cal_err: 0.057 +- 0.023- Val-avg_ll: 0.129 +- 0.135\n",
      "\n",
      "Iter 11000/20000 - Time 24.52 sec - Train-Loss: -6.47889\n",
      "\n",
      "Iter 12000/20000 - Time 15.09 sec - Train-Loss: -4.77388\n",
      "\n",
      "Iter 13000/20000 - Time 15.01 sec - Train-Loss: -3.54805\n",
      "\n",
      "Iter 14000/20000 - Time 21.92 sec - Train-Loss: -5.03712\n",
      "\n",
      "Iter 15000/20000 - Time 24.07 sec - Train-Loss: -4.42863\n",
      "\n",
      "Iter 16000/20000 - Time 17.96 sec - Train-Loss: -3.75696\n",
      "\n",
      "Iter 17000/20000 - Time 15.43 sec - Train-Loss: -3.99323\n",
      "\n",
      "Iter 18000/20000 - Time 17.06 sec - Train-Loss: -3.63488\n",
      "\n",
      "Iter 19000/20000 - Time 15.35 sec - Train-Loss: -3.75635\n",
      "\tStart meta-test posterior inference in 4 batches ------------------\n",
      "\tMeta-Test batch #1 consisting of 5 tasks----\n",
      "\tMeta-Test batch #2 consisting of 5 tasks----\n",
      "\tMeta-Test batch #3 consisting of 5 tasks----\n",
      "\tMeta-Test batch #4 consisting of 5 tasks----\n",
      "avg_rmse: 0.2081 +- 0.0202\n",
      "cal_err: 0.0443 +- 0.0246\n",
      "avg_ll: 0.1448 +- 0.1124\n"
     ]
    }
   ],
   "source": [
    "tf.get_logger().setLevel('ERROR')\n",
    "\n",
    "from pacoh_nn.datasets.regression_datasets import provide_data\n",
    "\n",
    "meta_train_data, meta_test_data, _ = provide_data(dataset='cauchy_20')\n",
    "\n",
    "from pacoh_nn.pacoh_nn_regression import PACOH_NN_Regression\n",
    "pacoh_model = PACOH_NN_Regression(meta_train_data, prior_weight=0.184, bandwidth=480.,\n",
    "                                  hyper_prior_likelihood_log_var_mean_mean=-1.0,\n",
    "                                  hyper_prior_log_var_mean=-1.74, hyper_prior_nn_std=0.5,\n",
    "                                  hyper_prior_weight=1e-5, lr=1.5e-3, learn_likelihood=True,\n",
    "                                  random_seed=28, num_iter_meta_train=20000)\n",
    "\n",
    "pacoh_model.meta_fit(meta_test_data[:10], eval_period=10000, log_period=1000, plot_period=5000)\n",
    "\n",
    "eval_metrics_mean, eval_metrics_std = pacoh_model.meta_eval_datasets(meta_test_data)\n",
    "for key in eval_metrics_mean:\n",
    "    print(\"%s: %.4f +- %.4f\" % (key, eval_metrics_mean[key], eval_metrics_std[key]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Hyper-prior std=1 and number of hyper particles: 5, 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start meta-training -------------------- \n",
      "\n",
      "Iter 0/20000 - Time 93.10 sec - Train-Loss: -4213.27197\n",
      "\n",
      "Iter 1000/20000 - Time 30.38 sec - Train-Loss: -271.63373\n",
      "\n",
      "Iter 2000/20000 - Time 30.98 sec - Train-Loss: -282.96527\n",
      "\n",
      "Iter 3000/20000 - Time 29.94 sec - Train-Loss: -367.75067\n",
      "\n",
      "Iter 4000/20000 - Time 35.60 sec - Train-Loss: -234.81680\n",
      "\n",
      "Iter 5000/20000 - Time 33.54 sec - Train-Loss: -208.54082\n",
      "\n",
      "Iter 6000/20000 - Time 40.14 sec - Train-Loss: -125.54710\n",
      "\n",
      "Iter 7000/20000 - Time 42.37 sec - Train-Loss: -77.18275\n",
      "\n",
      "Iter 8000/20000 - Time 30.18 sec - Train-Loss: -22.29178\n",
      "\n",
      "Iter 9000/20000 - Time 33.34 sec - Train-Loss: -6.80823\n",
      "\tStart meta-test posterior inference in 2 batches ------------------\n",
      "\tMeta-Test batch #1 consisting of 5 tasks----\n",
      "\tMeta-Test batch #2 consisting of 5 tasks----\n",
      "\n",
      "Iter 10000/20000 - Time 28.70 sec - Train-Loss: -5.39156- Val-avg_rmse: 0.197 +- 0.016- Val-cal_err: 0.048 +- 0.017- Val-avg_ll: 0.179 +- 0.097\n",
      "\n",
      "Iter 11000/20000 - Time 53.33 sec - Train-Loss: -4.16249\n",
      "\n",
      "Iter 12000/20000 - Time 28.86 sec - Train-Loss: -3.98604\n",
      "\n",
      "Iter 13000/20000 - Time 32.15 sec - Train-Loss: -2.76700\n",
      "\n",
      "Iter 14000/20000 - Time 30.70 sec - Train-Loss: -4.36799\n",
      "\n",
      "Iter 15000/20000 - Time 31.35 sec - Train-Loss: -3.89319\n",
      "\n",
      "Iter 16000/20000 - Time 29.70 sec - Train-Loss: -3.18089\n",
      "\n",
      "Iter 17000/20000 - Time 27.56 sec - Train-Loss: -3.67669\n",
      "\n",
      "Iter 18000/20000 - Time 26.65 sec - Train-Loss: -3.23530\n",
      "\n",
      "Iter 19000/20000 - Time 30.51 sec - Train-Loss: -3.37641\n",
      "\tStart meta-test posterior inference in 4 batches ------------------\n",
      "\tMeta-Test batch #1 consisting of 5 tasks----\n",
      "\tMeta-Test batch #2 consisting of 5 tasks----\n",
      "\tMeta-Test batch #3 consisting of 5 tasks----\n",
      "\tMeta-Test batch #4 consisting of 5 tasks----\n",
      "avg_rmse: 0.2114 +- 0.0203\n",
      "cal_err: 0.0434 +- 0.0204\n",
      "avg_ll: 0.1296 +- 0.1067\n"
     ]
    }
   ],
   "source": [
    "tf.get_logger().setLevel('ERROR')\n",
    "\n",
    "from pacoh_nn.datasets.regression_datasets import provide_data\n",
    "\n",
    "meta_train_data, meta_test_data, _ = provide_data(dataset='cauchy_20')\n",
    "\n",
    "from pacoh_nn.pacoh_nn_regression import PACOH_NN_Regression\n",
    "pacoh_model = PACOH_NN_Regression(meta_train_data, prior_weight=0.184, bandwidth=480.,\n",
    "                                  hyper_prior_likelihood_log_var_mean_mean=-1.0,\n",
    "                                  hyper_prior_log_var_mean=-1.74, hyper_prior_nn_std=1,\n",
    "                                  num_hyper_posterior_particles=5,\n",
    "                                  hyper_prior_weight=1e-5, lr=1.5e-3, learn_likelihood=True,\n",
    "                                  random_seed=28, num_iter_meta_train=20000)\n",
    "\n",
    "pacoh_model.meta_fit(meta_test_data[:10], eval_period=10000, log_period=1000, plot_period=5000)\n",
    "\n",
    "eval_metrics_mean, eval_metrics_std = pacoh_model.meta_eval_datasets(meta_test_data)\n",
    "for key in eval_metrics_mean:\n",
    "    print(\"%s: %.4f +- %.4f\" % (key, eval_metrics_mean[key], eval_metrics_std[key]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start meta-training -------------------- \n",
      "\n",
      "Iter 0/20000 - Time 210.43 sec - Train-Loss: -4692.40332\n",
      "\n",
      "Iter 1000/20000 - Time 49.65 sec - Train-Loss: -277.63730\n",
      "\n",
      "Iter 2000/20000 - Time 45.01 sec - Train-Loss: -288.36493\n",
      "\n",
      "Iter 3000/20000 - Time 43.42 sec - Train-Loss: -385.34171\n",
      "\n",
      "Iter 4000/20000 - Time 44.56 sec - Train-Loss: -242.73019\n",
      "\n",
      "Iter 5000/20000 - Time 43.65 sec - Train-Loss: -214.31636\n",
      "\n",
      "Iter 6000/20000 - Time 42.90 sec - Train-Loss: -135.32416\n",
      "\n",
      "Iter 7000/20000 - Time 45.39 sec - Train-Loss: -82.89827\n",
      "\n",
      "Iter 8000/20000 - Time 43.05 sec - Train-Loss: -27.07174\n",
      "\n",
      "Iter 9000/20000 - Time 43.26 sec - Train-Loss: -9.19484\n",
      "\tStart meta-test posterior inference in 2 batches ------------------\n",
      "\tMeta-Test batch #1 consisting of 5 tasks----\n",
      "\tMeta-Test batch #2 consisting of 5 tasks----\n",
      "\n",
      "Iter 10000/20000 - Time 44.95 sec - Train-Loss: -4.55325- Val-avg_rmse: 0.198 +- 0.017- Val-cal_err: 0.040 +- 0.023- Val-avg_ll: 0.180 +- 0.101\n",
      "\n",
      "Iter 11000/20000 - Time 118.08 sec - Train-Loss: -4.82469\n",
      "\n",
      "Iter 12000/20000 - Time 46.74 sec - Train-Loss: -4.28527\n",
      "\n",
      "Iter 13000/20000 - Time 54.24 sec - Train-Loss: -2.90920\n",
      "\n",
      "Iter 14000/20000 - Time 44.57 sec - Train-Loss: -4.57822\n",
      "\n",
      "Iter 15000/20000 - Time 43.80 sec - Train-Loss: -3.78385\n",
      "\n",
      "Iter 16000/20000 - Time 45.46 sec - Train-Loss: -3.16443\n",
      "\n",
      "Iter 17000/20000 - Time 43.47 sec - Train-Loss: -3.58330\n",
      "\n",
      "Iter 18000/20000 - Time 43.76 sec - Train-Loss: -3.35840\n",
      "\n",
      "Iter 19000/20000 - Time 46.26 sec - Train-Loss: -3.40625\n",
      "\tStart meta-test posterior inference in 4 batches ------------------\n",
      "\tMeta-Test batch #1 consisting of 5 tasks----\n",
      "\tMeta-Test batch #2 consisting of 5 tasks----\n",
      "\tMeta-Test batch #3 consisting of 5 tasks----\n",
      "\tMeta-Test batch #4 consisting of 5 tasks----\n",
      "avg_rmse: 0.2074 +- 0.0212\n",
      "cal_err: 0.0411 +- 0.0183\n",
      "avg_ll: 0.1503 +- 0.1058\n"
     ]
    }
   ],
   "source": [
    "tf.get_logger().setLevel('ERROR')\n",
    "\n",
    "from pacoh_nn.datasets.regression_datasets import provide_data\n",
    "\n",
    "meta_train_data, meta_test_data, _ = provide_data(dataset='cauchy_20')\n",
    "\n",
    "from pacoh_nn.pacoh_nn_regression import PACOH_NN_Regression\n",
    "pacoh_model = PACOH_NN_Regression(meta_train_data, prior_weight=0.184, bandwidth=480.,\n",
    "                                  hyper_prior_likelihood_log_var_mean_mean=-1.0,\n",
    "                                  hyper_prior_log_var_mean=-1.74, hyper_prior_nn_std=1,\n",
    "                                  num_hyper_posterior_particles=10,\n",
    "                                  hyper_prior_weight=1e-5, lr=1.5e-3, learn_likelihood=True,\n",
    "                                  random_seed=28, num_iter_meta_train=20000)\n",
    "\n",
    "pacoh_model.meta_fit(meta_test_data[:10], eval_period=10000, log_period=1000, plot_period=5000)\n",
    "\n",
    "eval_metrics_mean, eval_metrics_std = pacoh_model.meta_eval_datasets(meta_test_data)\n",
    "for key in eval_metrics_mean:\n",
    "    print(\"%s: %.4f +- %.4f\" % (key, eval_metrics_mean[key], eval_metrics_std[key]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Number of meta-train iterations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start meta-training -------------------- \n",
      "\n",
      "Iter 0/1000 - Time 57.84 sec - Train-Loss: -5469.16992\n",
      "\tStart meta-test posterior inference in 4 batches ------------------\n",
      "\tMeta-Test batch #1 consisting of 5 tasks----\n",
      "\tMeta-Test batch #2 consisting of 5 tasks----\n",
      "\tMeta-Test batch #3 consisting of 5 tasks----\n",
      "\tMeta-Test batch #4 consisting of 5 tasks----\n",
      "avg_rmse: 0.2630 +- 0.0503\n",
      "cal_err: 0.1289 +- 0.0383\n",
      "avg_ll: -65.1331 +- 223.4967\n"
     ]
    }
   ],
   "source": [
    "tf.get_logger().setLevel('ERROR')\n",
    "\n",
    "from pacoh_nn.datasets.regression_datasets import provide_data\n",
    "\n",
    "meta_train_data, meta_test_data, _ = provide_data(dataset='cauchy_20')\n",
    "\n",
    "from pacoh_nn.pacoh_nn_regression import PACOH_NN_Regression\n",
    "pacoh_model = PACOH_NN_Regression(meta_train_data, prior_weight=0.184, bandwidth=480.,\n",
    "                                  hyper_prior_likelihood_log_var_mean_mean=-1.0,\n",
    "                                  hyper_prior_log_var_mean=-1.74, hyper_prior_nn_std=0.12,\n",
    "                                  hyper_prior_weight=1e-5, lr=1.5e-3, learn_likelihood=True,\n",
    "                                  random_seed=28, num_iter_meta_train=1000)\n",
    "\n",
    "pacoh_model.meta_fit(meta_test_data[:10], eval_period=10000, log_period=1000, plot_period=5000)\n",
    "\n",
    "eval_metrics_mean, eval_metrics_std = pacoh_model.meta_eval_datasets(meta_test_data)\n",
    "for key in eval_metrics_mean:\n",
    "    print(\"%s: %.4f +- %.4f\" % (key, eval_metrics_mean[key], eval_metrics_std[key]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start meta-training -------------------- \n",
      "\n",
      "Iter 0/5000 - Time 51.65 sec - Train-Loss: -5313.57275\n",
      "\n",
      "Iter 1000/5000 - Time 22.66 sec - Train-Loss: -288.37216\n",
      "\n",
      "Iter 2000/5000 - Time 23.33 sec - Train-Loss: -303.39700\n",
      "\n",
      "Iter 3000/5000 - Time 19.09 sec - Train-Loss: -422.49899\n",
      "\n",
      "Iter 4000/5000 - Time 18.67 sec - Train-Loss: -268.34726\n",
      "\tStart meta-test posterior inference in 4 batches ------------------\n",
      "\tMeta-Test batch #1 consisting of 5 tasks----\n",
      "\tMeta-Test batch #2 consisting of 5 tasks----\n",
      "\tMeta-Test batch #3 consisting of 5 tasks----\n",
      "\tMeta-Test batch #4 consisting of 5 tasks----\n",
      "avg_rmse: 0.2417 +- 0.0375\n",
      "cal_err: 0.1144 +- 0.0336\n",
      "avg_ll: -7.3566 +- 22.7066\n"
     ]
    }
   ],
   "source": [
    "tf.get_logger().setLevel('ERROR')\n",
    "\n",
    "from pacoh_nn.datasets.regression_datasets import provide_data\n",
    "\n",
    "meta_train_data, meta_test_data, _ = provide_data(dataset='cauchy_20')\n",
    "\n",
    "from pacoh_nn.pacoh_nn_regression import PACOH_NN_Regression\n",
    "pacoh_model = PACOH_NN_Regression(meta_train_data, prior_weight=0.184, bandwidth=480.,\n",
    "                                  hyper_prior_likelihood_log_var_mean_mean=-1.0,\n",
    "                                  hyper_prior_log_var_mean=-1.74, hyper_prior_nn_std=0.12,\n",
    "                                  hyper_prior_weight=1e-5, lr=1.5e-3, learn_likelihood=True,\n",
    "                                  random_seed=28, num_iter_meta_train=5000)\n",
    "\n",
    "pacoh_model.meta_fit(meta_test_data[:10], eval_period=10000, log_period=1000, plot_period=5000)\n",
    "\n",
    "eval_metrics_mean, eval_metrics_std = pacoh_model.meta_eval_datasets(meta_test_data)\n",
    "for key in eval_metrics_mean:\n",
    "    print(\"%s: %.4f +- %.4f\" % (key, eval_metrics_mean[key], eval_metrics_std[key]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start meta-training -------------------- \n",
      "\n",
      "Iter 0/10000 - Time 64.31 sec - Train-Loss: -4811.07617\n",
      "\n",
      "Iter 1000/10000 - Time 19.98 sec - Train-Loss: -291.51370\n",
      "\n",
      "Iter 2000/10000 - Time 18.36 sec - Train-Loss: -298.51016\n",
      "\n",
      "Iter 3000/10000 - Time 16.39 sec - Train-Loss: -440.20297\n",
      "\n",
      "Iter 4000/10000 - Time 16.05 sec - Train-Loss: -262.70715\n",
      "\n",
      "Iter 5000/10000 - Time 15.53 sec - Train-Loss: -242.06862\n",
      "\n",
      "Iter 6000/10000 - Time 15.94 sec - Train-Loss: -163.45790\n",
      "\n",
      "Iter 7000/10000 - Time 16.04 sec - Train-Loss: -106.13326\n",
      "\n",
      "Iter 8000/10000 - Time 16.93 sec - Train-Loss: -41.50026\n",
      "\n",
      "Iter 9000/10000 - Time 16.20 sec - Train-Loss: -19.32554\n",
      "\tStart meta-test posterior inference in 4 batches ------------------\n",
      "\tMeta-Test batch #1 consisting of 5 tasks----\n",
      "\tMeta-Test batch #2 consisting of 5 tasks----\n",
      "\tMeta-Test batch #3 consisting of 5 tasks----\n",
      "\tMeta-Test batch #4 consisting of 5 tasks----\n",
      "avg_rmse: 0.2057 +- 0.0169\n",
      "cal_err: 0.0610 +- 0.0245\n",
      "avg_ll: 0.0718 +- 0.1300\n"
     ]
    }
   ],
   "source": [
    "tf.get_logger().setLevel('ERROR')\n",
    "\n",
    "from pacoh_nn.datasets.regression_datasets import provide_data\n",
    "\n",
    "meta_train_data, meta_test_data, _ = provide_data(dataset='cauchy_20')\n",
    "\n",
    "from pacoh_nn.pacoh_nn_regression import PACOH_NN_Regression\n",
    "pacoh_model = PACOH_NN_Regression(meta_train_data, prior_weight=0.184, bandwidth=480.,\n",
    "                                  hyper_prior_likelihood_log_var_mean_mean=-1.0,\n",
    "                                  hyper_prior_log_var_mean=-1.74, hyper_prior_nn_std=0.12,\n",
    "                                  hyper_prior_weight=1e-5, lr=1.5e-3, learn_likelihood=True,\n",
    "                                  random_seed=28, num_iter_meta_train=10000)\n",
    "\n",
    "pacoh_model.meta_fit(meta_test_data[:10], eval_period=10000, log_period=1000, plot_period=5000)\n",
    "\n",
    "eval_metrics_mean, eval_metrics_std = pacoh_model.meta_eval_datasets(meta_test_data)\n",
    "for key in eval_metrics_mean:\n",
    "    print(\"%s: %.4f +- %.4f\" % (key, eval_metrics_mean[key], eval_metrics_std[key]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start meta-training -------------------- \n",
      "\n",
      "Iter 0/50000 - Time 39.30 sec - Train-Loss: -5629.41211\n",
      "\n",
      "Iter 1000/50000 - Time 19.19 sec - Train-Loss: -287.50363\n",
      "\n",
      "Iter 2000/50000 - Time 16.31 sec - Train-Loss: -300.28397\n",
      "\n",
      "Iter 3000/50000 - Time 15.36 sec - Train-Loss: -408.42593\n",
      "\n",
      "Iter 4000/50000 - Time 15.42 sec - Train-Loss: -264.84592\n",
      "\n",
      "Iter 5000/50000 - Time 15.42 sec - Train-Loss: -238.53839\n",
      "\n",
      "Iter 6000/50000 - Time 15.40 sec - Train-Loss: -160.78514\n",
      "\n",
      "Iter 7000/50000 - Time 15.85 sec - Train-Loss: -116.20892\n",
      "\n",
      "Iter 8000/50000 - Time 16.82 sec - Train-Loss: -44.94522\n",
      "\n",
      "Iter 9000/50000 - Time 16.77 sec - Train-Loss: -19.22014\n",
      "\tStart meta-test posterior inference in 2 batches ------------------\n",
      "\tMeta-Test batch #1 consisting of 5 tasks----\n",
      "\tMeta-Test batch #2 consisting of 5 tasks----\n",
      "\n",
      "Iter 10000/50000 - Time 15.37 sec - Train-Loss: -14.53195- Val-avg_rmse: 0.201 +- 0.018- Val-cal_err: 0.058 +- 0.022- Val-avg_ll: 0.109 +- 0.113\n",
      "\n",
      "Iter 11000/50000 - Time 26.58 sec - Train-Loss: -12.69881\n",
      "\n",
      "Iter 12000/50000 - Time 15.75 sec - Train-Loss: -10.36938\n",
      "\n",
      "Iter 13000/50000 - Time 16.06 sec - Train-Loss: -8.36366\n",
      "\n",
      "Iter 14000/50000 - Time 17.03 sec - Train-Loss: -8.38215\n",
      "\n",
      "Iter 15000/50000 - Time 16.55 sec - Train-Loss: -6.59640\n",
      "\n",
      "Iter 16000/50000 - Time 15.66 sec - Train-Loss: -5.88561\n",
      "\n",
      "Iter 17000/50000 - Time 15.57 sec - Train-Loss: -5.71910\n",
      "\n",
      "Iter 18000/50000 - Time 16.19 sec - Train-Loss: -5.00648\n",
      "\n",
      "Iter 19000/50000 - Time 15.68 sec - Train-Loss: -5.13697\n",
      "\tStart meta-test posterior inference in 2 batches ------------------\n",
      "\tMeta-Test batch #1 consisting of 5 tasks----\n",
      "\tMeta-Test batch #2 consisting of 5 tasks----\n",
      "\n",
      "Iter 20000/50000 - Time 15.92 sec - Train-Loss: -4.34825- Val-avg_rmse: 0.200 +- 0.018- Val-cal_err: 0.044 +- 0.023- Val-avg_ll: 0.187 +- 0.069\n",
      "\n",
      "Iter 21000/50000 - Time 27.71 sec - Train-Loss: -5.41121\n",
      "\n",
      "Iter 22000/50000 - Time 15.98 sec - Train-Loss: -4.85457\n",
      "\n",
      "Iter 23000/50000 - Time 16.47 sec - Train-Loss: -3.68678\n",
      "\n",
      "Iter 24000/50000 - Time 17.69 sec - Train-Loss: -3.65811\n",
      "\n",
      "Iter 25000/50000 - Time 20.07 sec - Train-Loss: -4.24812\n",
      "\n",
      "Iter 26000/50000 - Time 19.10 sec - Train-Loss: -4.58882\n",
      "\n",
      "Iter 27000/50000 - Time 19.22 sec - Train-Loss: -4.33549\n",
      "\n",
      "Iter 28000/50000 - Time 18.38 sec - Train-Loss: -4.03210\n",
      "\n",
      "Iter 29000/50000 - Time 17.81 sec - Train-Loss: -3.41119\n",
      "\tStart meta-test posterior inference in 2 batches ------------------\n",
      "\tMeta-Test batch #1 consisting of 5 tasks----\n",
      "\tMeta-Test batch #2 consisting of 5 tasks----\n",
      "\n",
      "Iter 30000/50000 - Time 18.07 sec - Train-Loss: -6.35780- Val-avg_rmse: 0.202 +- 0.018- Val-cal_err: 0.040 +- 0.024- Val-avg_ll: 0.183 +- 0.070\n",
      "\n",
      "Iter 31000/50000 - Time 27.74 sec - Train-Loss: -4.30734\n",
      "\n",
      "Iter 32000/50000 - Time 16.88 sec - Train-Loss: -3.75538\n",
      "\n",
      "Iter 33000/50000 - Time 15.93 sec - Train-Loss: -4.72280\n",
      "\n",
      "Iter 34000/50000 - Time 15.86 sec - Train-Loss: -3.79869\n",
      "\n",
      "Iter 35000/50000 - Time 16.02 sec - Train-Loss: -4.27066\n",
      "\n",
      "Iter 36000/50000 - Time 15.80 sec - Train-Loss: -3.26617\n",
      "\n",
      "Iter 37000/50000 - Time 16.25 sec - Train-Loss: -4.52698\n",
      "\n",
      "Iter 38000/50000 - Time 17.56 sec - Train-Loss: -3.21590\n",
      "\n",
      "Iter 39000/50000 - Time 16.45 sec - Train-Loss: -3.71868\n",
      "\tStart meta-test posterior inference in 2 batches ------------------\n",
      "\tMeta-Test batch #1 consisting of 5 tasks----\n",
      "\tMeta-Test batch #2 consisting of 5 tasks----\n",
      "\n",
      "Iter 40000/50000 - Time 15.71 sec - Train-Loss: -3.91675- Val-avg_rmse: 0.205 +- 0.018- Val-cal_err: 0.041 +- 0.021- Val-avg_ll: 0.168 +- 0.072\n",
      "\n",
      "Iter 41000/50000 - Time 29.61 sec - Train-Loss: -4.35186\n",
      "\n",
      "Iter 42000/50000 - Time 16.24 sec - Train-Loss: -2.98146\n",
      "\n",
      "Iter 43000/50000 - Time 16.71 sec - Train-Loss: -3.26489\n",
      "\n",
      "Iter 44000/50000 - Time 16.86 sec - Train-Loss: -4.27531\n",
      "\n",
      "Iter 45000/50000 - Time 17.85 sec - Train-Loss: -3.58224\n",
      "\n",
      "Iter 46000/50000 - Time 15.72 sec - Train-Loss: -4.84116\n",
      "\n",
      "Iter 47000/50000 - Time 15.71 sec - Train-Loss: -3.67229\n",
      "\n",
      "Iter 48000/50000 - Time 15.86 sec - Train-Loss: -4.24840\n",
      "\n",
      "Iter 49000/50000 - Time 15.93 sec - Train-Loss: -4.65551\n",
      "\tStart meta-test posterior inference in 4 batches ------------------\n",
      "\tMeta-Test batch #1 consisting of 5 tasks----\n",
      "\tMeta-Test batch #2 consisting of 5 tasks----\n",
      "\tMeta-Test batch #3 consisting of 5 tasks----\n",
      "\tMeta-Test batch #4 consisting of 5 tasks----\n",
      "avg_rmse: 0.2078 +- 0.0212\n",
      "cal_err: 0.0473 +- 0.0241\n",
      "avg_ll: 0.1368 +- 0.1238\n"
     ]
    }
   ],
   "source": [
    "tf.get_logger().setLevel('ERROR')\n",
    "\n",
    "from pacoh_nn.datasets.regression_datasets import provide_data\n",
    "\n",
    "meta_train_data, meta_test_data, _ = provide_data(dataset='cauchy_20')\n",
    "\n",
    "from pacoh_nn.pacoh_nn_regression import PACOH_NN_Regression\n",
    "pacoh_model = PACOH_NN_Regression(meta_train_data, prior_weight=0.184, bandwidth=480.,\n",
    "                                  hyper_prior_likelihood_log_var_mean_mean=-1.0,\n",
    "                                  hyper_prior_log_var_mean=-1.74, hyper_prior_nn_std=0.12,\n",
    "                                  hyper_prior_weight=1e-5, lr=1.5e-3, learn_likelihood=True,\n",
    "                                  random_seed=28, num_iter_meta_train=50000)\n",
    "\n",
    "pacoh_model.meta_fit(meta_test_data[:10], eval_period=10000, log_period=1000, plot_period=5000)\n",
    "\n",
    "eval_metrics_mean, eval_metrics_std = pacoh_model.meta_eval_datasets(meta_test_data)\n",
    "for key in eval_metrics_mean:\n",
    "    print(\"%s: %.4f +- %.4f\" % (key, eval_metrics_mean[key], eval_metrics_std[key]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.get_logger().setLevel('ERROR')\n",
    "\n",
    "from pacoh_nn.datasets.regression_datasets import provide_data\n",
    "\n",
    "meta_train_data, meta_test_data, _ = provide_data(dataset='cauchy_20')\n",
    "\n",
    "from pacoh_nn.pacoh_nn_regression import PACOH_NN_Regression\n",
    "pacoh_model = PACOH_NN_Regression(meta_train_data, prior_weight=0.184, bandwidth=480.,\n",
    "                                  hyper_prior_likelihood_log_var_mean_mean=-1.0,\n",
    "                                  hyper_prior_log_var_mean=-1.74, hyper_prior_nn_std=0.12,\n",
    "                                  hyper_prior_weight=1e-5, lr=1.5e-3, learn_likelihood=True,\n",
    "                                  random_seed=28, num_iter_meta_train=100000)\n",
    "\n",
    "pacoh_model.meta_fit(meta_test_data[:10], eval_period=10000, log_period=1000, plot_period=5000)\n",
    "\n",
    "eval_metrics_mean, eval_metrics_std = pacoh_model.meta_eval_datasets(meta_test_data)\n",
    "for key in eval_metrics_mean:\n",
    "    print(\"%s: %.4f +- %.4f\" % (key, eval_metrics_mean[key], eval_metrics_std[key]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Number of context samples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'tf' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[6], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mtf\u001b[49m\u001b[38;5;241m.\u001b[39mget_logger()\u001b[38;5;241m.\u001b[39msetLevel(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mERROR\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[1;32m      3\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mpacoh_nn\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mdatasets\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mregression_datasets\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m provide_data\n\u001b[1;32m      5\u001b[0m meta_train_data, meta_test_data, _ \u001b[38;5;241m=\u001b[39m provide_data(dataset\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mcauchy_20\u001b[39m\u001b[38;5;124m'\u001b[39m, n_samples\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m5\u001b[39m)\n",
      "\u001b[0;31mNameError\u001b[0m: name 'tf' is not defined"
     ]
    }
   ],
   "source": [
    "tf.get_logger().setLevel('ERROR')\n",
    "\n",
    "from pacoh_nn.datasets.regression_datasets import provide_data\n",
    "\n",
    "meta_train_data, meta_test_data, _ = provide_data(dataset='cauchy_20', n_samples=5)\n",
    "\n",
    "from pacoh_nn.pacoh_nn_regression import PACOH_NN_Regression\n",
    "pacoh_model = PACOH_NN_Regression(meta_train_data, prior_weight=0.184, bandwidth=480.,\n",
    "                                  hyper_prior_likelihood_log_var_mean_mean=-1.0,\n",
    "                                  hyper_prior_log_var_mean=-1.74, hyper_prior_nn_std=0.12,\n",
    "                                  hyper_prior_weight=1e-5, lr=1.5e-3, learn_likelihood=True,\n",
    "                                  random_seed=28, num_iter_meta_train=20000)\n",
    "\n",
    "pacoh_model.meta_fit(meta_test_data[:10], eval_period=10000, log_period=1000, plot_period=5000)\n",
    "\n",
    "eval_metrics_mean, eval_metrics_std = pacoh_model.meta_eval_datasets(meta_test_data)\n",
    "for key in eval_metrics_mean:\n",
    "    print(\"%s: %.4f +- %.4f\" % (key, eval_metrics_mean[key], eval_metrics_std[key]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "qam_exp",
   "language": "python",
   "name": "qam_exp"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}