vi_perf_experiments.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "04111ac9",
   "metadata": {},
   "source": [
    "# Variational Inference Performance Experiments\n",
    "This notebook executes performance benchmarks on the BayesVI.jl code. Figures are plotted ands saved for the report. The following experiments are done here:\n",
    "- Increasing number of MC samples vs training time\n",
    "- Increasing epochs vs training time\n",
    "- Increasing nodes in hidden layers vs training time\n",
    "- Increasing depth of neural network vs training time\n",
    "- Increasing number of MC samples vs ELBO gradient approximation\n",
    "\n",
    "! NOTE ! this notebook can take up to 25 minutes to completely execute"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "10850926",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "┌ Info: OMEinsum loaded the CUDA module successfully\n",
      "└ @ OMEinsum C:\\Users\\andre\\.julia\\packages\\OMEinsum\\0C2IK\\src\\cueinsum.jl:117\n"
     ]
    }
   ],
   "source": [
    "begin\n",
    "    using Plots\n",
    "    using Distributions\n",
    "    using Random\n",
    "    include(\"./BayesVI.jl\")\n",
    "    using .BayesVI\n",
    "    include(\"./Datasets.jl\")\n",
    "    using .Datasets\n",
    "end"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8d7be988",
   "metadata": {},
   "source": [
    "## Experiment: increasing MC samples vs training time."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "b94dc4c8",
   "metadata": {},
   "outputs": [],
   "source": [
    "mc_sample_list = 50\n",
    "times = zeros(mc_sample_list)\n",
    "for mc_samples in range(1, mc_sample_list)\n",
    "    num_weights, predictions, logprob, ensemble_prediction = make_neural_network_functions([1, 20, 20, 1], 0.1, 0.01, rbf)\n",
    "    inputs, targets = build_dataset_1()\n",
    "    log_posterior(weights) = logprob(weights, inputs, targets)\n",
    "    objective = black_box_variational_inference(log_posterior, num_weights, mc_samples)\n",
    "    e = @elapsed train_bayesian_neural_network(200, 0.1, num_weights, objective);\n",
    "    times[mc_samples] = e\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "2a1c88a6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Plot results.\n",
    "mcvtime = plot(LinRange(1,mc_sample_list,mc_sample_list), times, title=\"MC Samples vs Training Time (s)\", label=\"\", xlabel=\"MC Samples\", ylabel=\"Train Time (s)\")\n",
    "savefig(mcvtime, \"images/mcvtime.png\")\n",
    "current()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e1f429b6",
   "metadata": {},
   "source": [
    "## Experiment: increasing epochs vs training time."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "4a5b967c",
   "metadata": {},
   "outputs": [],
   "source": [
    "trials = 100\n",
    "repeats = 10\n",
    "times = zeros(trials, repeats)\n",
    "for rep in range(1, repeats)\n",
    "    for epochs in range(1, trials)\n",
    "        num_weights, predictions, logprob, ensemble_prediction = make_neural_network_functions([1, 20, 20, 1], 0.1, 0.01, rbf)\n",
    "        inputs, targets = build_dataset_1()\n",
    "        log_posterior(weights) = logprob(weights, inputs, targets)\n",
    "        objective = black_box_variational_inference(log_posterior, num_weights, 1)\n",
    "        e = @elapsed train_bayesian_neural_network(epochs, 0.1, num_weights, objective);\n",
    "        times[epochs, rep] = e\n",
    "    end\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "69466a45",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Plot results.\n",
    "averaged_times = mean(times, dims=2);\n",
    "epochs_vs_avg_time = plot(LinRange(1,trials,trials), averaged_times, title=\"Epochs vs Training Time (s)\", label=\"\", xlabel=\"Number of Epochs\", ylabel=\"Train Time (s)\")\n",
    "savefig(epochs_vs_avg_time, \"images/epochs_vs_avg_time.png\")\n",
    "current()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1a6a3543",
   "metadata": {},
   "source": [
    "## Experiment: increasing number of nodes in hidden layers vs training time."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "f63228cf",
   "metadata": {},
   "outputs": [],
   "source": [
    "trials = 30\n",
    "repeats = 10\n",
    "times = zeros(trials, repeats)\n",
    "for i in range(1, repeats)\n",
    "    for val in range(1, trials)\n",
    "        num_weights, predictions, logprob, ensemble_prediction = make_neural_network_functions([1, val, val, 1], 0.1, 0.01, rbf)\n",
    "        inputs, targets = build_dataset_1()\n",
    "        log_posterior(weights) = logprob(weights, inputs, targets)\n",
    "        objective = black_box_variational_inference(log_posterior, num_weights, 1)\n",
    "        e = @elapsed train_bayesian_neural_network(200, 0.1, num_weights, objective);\n",
    "        times[val, i] = e\n",
    "    end\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "018c783c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n",
       "<svg xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\" width=\"600\" height=\"400\" viewBox=\"0 0 2400 1600\">\n",
       "<defs>\n",
       "  <clipPath id=\"clip870\">\n",
       "    <rect x=\"0\" y=\"0\" width=\"2400\" height=\"1600\"/>\n",
       "  </clipPath>\n",
       "</defs>\n",
       "<path clip-path=\"url(#clip870)\" d=\"\n",
       "M0 1600 L2400 1600 L2400 0 L0 0  Z\n",
       "  \" fill=\"#ffffff\" fill-rule=\"evenodd\" fill-opacity=\"1\"/>\n",
       "<defs>\n",
       "  <clipPath id=\"clip871\">\n",
       "    <rect x=\"480\" y=\"0\" width=\"1681\" height=\"1600\"/>\n",
       "  </clipPath>\n",
       "</defs>\n",
       "<path clip-path=\"url(#clip870)\" d=\"\n",
       "M249.542 1423.18 L2352.76 1423.18 L2352.76 123.472 L249.542 123.472  Z\n",
       "  \" fill=\"#ffffff\" fill-rule=\"evenodd\" fill-opacity=\"1\"/>\n",
       "<defs>\n",
       "  <clipPath id=\"clip872\">\n",
       "    <rect x=\"249\" y=\"123\" width=\"2104\" height=\"1301\"/>\n",
       "  </clipPath>\n",
       "</defs>\n",
       "<polyline clip-path=\"url(#clip872)\" style=\"stroke:#000000; stroke-linecap:butt; stroke-linejoin:round; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  582.745,1423.18 582.745,123.472 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip872)\" style=\"stroke:#000000; stroke-linecap:butt; stroke-linejoin:round; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  924.842,1423.18 924.842,123.472 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip872)\" style=\"stroke:#000000; stroke-linecap:butt; stroke-linejoin:round; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  1266.94,1423.18 1266.94,123.472 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip872)\" style=\"stroke:#000000; stroke-linecap:butt; stroke-linejoin:round; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  1609.04,1423.18 1609.04,123.472 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip872)\" style=\"stroke:#000000; stroke-linecap:butt; stroke-linejoin:round; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  1951.13,1423.18 1951.13,123.472 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip872)\" style=\"stroke:#000000; stroke-linecap:butt; stroke-linejoin:round; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  2293.23,1423.18 2293.23,123.472 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip870)\" style=\"stroke:#000000; stroke-linecap:butt; stroke-linejoin:round; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  249.542,1423.18 2352.76,1423.18 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip870)\" style=\"stroke:#000000; stroke-linecap:butt; stroke-linejoin:round; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  582.745,1423.18 582.745,1404.28 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip870)\" style=\"stroke:#000000; stroke-linecap:butt; stroke-linejoin:round; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  924.842,1423.18 924.842,1404.28 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip870)\" style=\"stroke:#000000; stroke-linecap:butt; stroke-linejoin:round; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  1266.94,1423.18 1266.94,1404.28 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip870)\" style=\"stroke:#000000; stroke-linecap:butt; stroke-linejoin:round; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  1609.04,1423.18 1609.04,1404.28 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip870)\" style=\"stroke:#000000; stroke-linecap:butt; stroke-linejoin:round; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  1951.13,1423.18 1951.13,1404.28 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip870)\" style=\"stroke:#000000; stroke-linecap:butt; stroke-linejoin:round; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  2293.23,1423.18 2293.23,1404.28 \n",
       "  \"/>\n",
       "<path clip-path=\"url(#clip870)\" d=\"M573.023 1451.02 L591.379 1451.02 L591.379 1454.96 L577.305 1454.96 L577.305 1463.43 Q578.323 1463.08 579.342 1462.92 Q580.361 1462.73 581.379 1462.73 Q587.166 1462.73 590.546 1465.9 Q593.925 1469.08 593.925 1474.49 Q593.925 1480.07 590.453 1483.17 Q586.981 1486.25 580.661 1486.25 Q578.486 1486.25 576.217 1485.88 Q573.972 1485.51 571.564 1484.77 L571.564 1480.07 Q573.648 1481.2 575.87 1481.76 Q578.092 1482.32 580.569 1482.32 Q584.573 1482.32 586.911 1480.21 Q589.249 1478.1 589.249 1474.49 Q589.249 1470.88 586.911 1468.77 Q584.573 1466.67 580.569 1466.67 Q578.694 1466.67 576.819 1467.08 Q574.967 1467.5 573.023 1468.38 L573.023 1451.02 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip870)\" d=\"M899.53 1481.64 L907.168 1481.64 L907.168 1455.28 L898.858 1456.95 L898.858 1452.69 L907.122 1451.02 L911.798 1451.02 L911.798 1481.64 L919.437 1481.64 L919.437 1485.58 L899.53 1485.58 L899.53 1481.64 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip870)\" d=\"M938.881 1454.1 Q935.27 1454.1 933.441 1457.66 Q931.636 1461.2 931.636 1468.33 Q931.636 1475.44 933.441 1479.01 Q935.27 1482.55 938.881 1482.55 Q942.516 1482.55 944.321 1479.01 Q946.15 1475.44 946.15 1468.33 Q946.15 1461.2 944.321 1457.66 Q942.516 1454.1 938.881 1454.1 M938.881 1450.39 Q944.691 1450.39 947.747 1455 Q950.826 1459.58 950.826 1468.33 Q950.826 1477.06 947.747 1481.67 Q944.691 1486.25 938.881 1486.25 Q933.071 1486.25 929.992 1481.67 Q926.937 1477.06 926.937 1468.33 Q926.937 1459.58 929.992 1455 Q933.071 1450.39 938.881 1450.39 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip870)\" d=\"M1242.12 1481.64 L1249.76 1481.64 L1249.76 1455.28 L1241.45 1456.95 L1241.45 1452.69 L1249.72 1451.02 L1254.39 1451.02 L1254.39 1481.64 L1262.03 1481.64 L1262.03 1485.58 L1242.12 1485.58 L1242.12 1481.64 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip870)\" d=\"M1271.52 1451.02 L1289.88 1451.02 L1289.88 1454.96 L1275.8 1454.96 L1275.8 1463.43 Q1276.82 1463.08 1277.84 1462.92 Q1278.86 1462.73 1279.88 1462.73 Q1285.67 1462.73 1289.05 1465.9 Q1292.43 1469.08 1292.43 1474.49 Q1292.43 1480.07 1288.95 1483.17 Q1285.48 1486.25 1279.16 1486.25 Q1276.99 1486.25 1274.72 1485.88 Q1272.47 1485.51 1270.06 1484.77 L1270.06 1480.07 Q1272.15 1481.2 1274.37 1481.76 Q1276.59 1482.32 1279.07 1482.32 Q1283.07 1482.32 1285.41 1480.21 Q1287.75 1478.1 1287.75 1474.49 Q1287.75 1470.88 1285.41 1468.77 Q1283.07 1466.67 1279.07 1466.67 Q1277.19 1466.67 1275.32 1467.08 Q1273.47 1467.5 1271.52 1468.38 L1271.52 1451.02 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip870)\" d=\"M1587.81 1481.64 L1604.13 1481.64 L1604.13 1485.58 L1582.18 1485.58 L1582.18 1481.64 Q1584.85 1478.89 1589.43 1474.26 Q1594.04 1469.61 1595.22 1468.27 Q1597.46 1465.74 1598.34 1464.01 Q1599.24 1462.25 1599.24 1460.56 Q1599.24 1457.8 1597.3 1456.07 Q1595.38 1454.33 1592.28 1454.33 Q1590.08 1454.33 1587.62 1455.09 Q1585.19 1455.86 1582.42 1457.41 L1582.42 1452.69 Q1585.24 1451.55 1587.69 1450.97 Q1590.15 1450.39 1592.18 1450.39 Q1597.56 1450.39 1600.75 1453.08 Q1603.94 1455.77 1603.94 1460.26 Q1603.94 1462.39 1603.13 1464.31 Q1602.35 1466.2 1600.24 1468.8 Q1599.66 1469.47 1596.56 1472.69 Q1593.46 1475.88 1587.81 1481.64 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip870)\" d=\"M1623.94 1454.1 Q1620.33 1454.1 1618.5 1457.66 Q1616.7 1461.2 1616.7 1468.33 Q1616.7 1475.44 1618.5 1479.01 Q1620.33 1482.55 1623.94 1482.55 Q1627.58 1482.55 1629.38 1479.01 Q1631.21 1475.44 1631.21 1468.33 Q1631.21 1461.2 1629.38 1457.66 Q1627.58 1454.1 1623.94 1454.1 M1623.94 1450.39 Q1629.75 1450.39 1632.81 1455 Q1635.89 1459.58 1635.89 1468.33 Q1635.89 1477.06 1632.81 1481.67 Q1629.75 1486.25 1623.94 1486.25 Q1618.13 1486.25 1615.05 1481.67 Q1612 1477.06 1612 1468.33 Q1612 1459.58 1615.05 1455 Q1618.13 1450.39 1623.94 1450.39 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip870)\" d=\"M1930.4 1481.64 L1946.72 1481.64 L1946.72 1485.58 L1924.78 1485.58 L1924.78 1481.64 Q1927.44 1478.89 1932.03 1474.26 Q1936.63 1469.61 1937.81 1468.27 Q1940.06 1465.74 1940.94 1464.01 Q1941.84 1462.25 1941.84 1460.56 Q1941.84 1457.8 1939.9 1456.07 Q1937.97 1454.33 1934.87 1454.33 Q1932.67 1454.33 1930.22 1455.09 Q1927.79 1455.86 1925.01 1457.41 L1925.01 1452.69 Q1927.84 1451.55 1930.29 1450.97 Q1932.74 1450.39 1934.78 1450.39 Q1940.15 1450.39 1943.34 1453.08 Q1946.54 1455.77 1946.54 1460.26 Q1946.54 1462.39 1945.73 1464.31 Q1944.94 1466.2 1942.84 1468.8 Q1942.26 1469.47 1939.15 1472.69 Q1936.05 1475.88 1930.4 1481.64 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip870)\" d=\"M1956.59 1451.02 L1974.94 1451.02 L1974.94 1454.96 L1960.87 1454.96 L1960.87 1463.43 Q1961.89 1463.08 1962.9 1462.92 Q1963.92 1462.73 1964.94 1462.73 Q1970.73 1462.73 1974.11 1465.9 Q1977.49 1469.08 1977.49 1474.49 Q1977.49 1480.07 1974.02 1483.17 Q1970.54 1486.25 1964.22 1486.25 Q1962.05 1486.25 1959.78 1485.88 Q1957.53 1485.51 1955.13 1484.77 L1955.13 1480.07 Q1957.21 1481.2 1959.43 1481.76 Q1961.65 1482.32 1964.13 1482.32 Q1968.14 1482.32 1970.47 1480.21 Q1972.81 1478.1 1972.81 1474.49 Q1972.81 1470.88 1970.47 1468.77 Q1968.14 1466.67 1964.13 1466.67 Q1962.26 1466.67 1960.38 1467.08 Q1958.53 1467.5 1956.59 1468.38 L1956.59 1451.02 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip870)\" d=\"M2282.07 1466.95 Q2285.43 1467.66 2287.31 1469.93 Q2289.2 1472.2 2289.2 1475.53 Q2289.2 1480.65 2285.68 1483.45 Q2282.17 1486.25 2275.68 1486.25 Q2273.51 1486.25 2271.19 1485.81 Q2268.9 1485.39 2266.45 1484.54 L2266.45 1480.02 Q2268.39 1481.16 2270.71 1481.74 Q2273.02 1482.32 2275.55 1482.32 Q2279.94 1482.32 2282.24 1480.58 Q2284.55 1478.84 2284.55 1475.53 Q2284.55 1472.48 2282.4 1470.77 Q2280.27 1469.03 2276.45 1469.03 L2272.42 1469.03 L2272.42 1465.19 L2276.63 1465.19 Q2280.08 1465.19 2281.91 1463.82 Q2283.74 1462.43 2283.74 1459.84 Q2283.74 1457.18 2281.84 1455.77 Q2279.97 1454.33 2276.45 1454.33 Q2274.53 1454.33 2272.33 1454.75 Q2270.13 1455.16 2267.49 1456.04 L2267.49 1451.88 Q2270.15 1451.14 2272.47 1450.77 Q2274.81 1450.39 2276.87 1450.39 Q2282.19 1450.39 2285.29 1452.83 Q2288.39 1455.23 2288.39 1459.35 Q2288.39 1462.22 2286.75 1464.21 Q2285.11 1466.18 2282.07 1466.95 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip870)\" d=\"M2308.07 1454.1 Q2304.46 1454.1 2302.63 1457.66 Q2300.82 1461.2 2300.82 1468.33 Q2300.82 1475.44 2302.63 1479.01 Q2304.46 1482.55 2308.07 1482.55 Q2311.7 1482.55 2313.51 1479.01 Q2315.34 1475.44 2315.34 1468.33 Q2315.34 1461.2 2313.51 1457.66 Q2311.7 1454.1 2308.07 1454.1 M2308.07 1450.39 Q2313.88 1450.39 2316.93 1455 Q2320.01 1459.58 2320.01 1468.33 Q2320.01 1477.06 2316.93 1481.67 Q2313.88 1486.25 2308.07 1486.25 Q2302.26 1486.25 2299.18 1481.67 Q2296.12 1477.06 2296.12 1468.33 Q2296.12 1459.58 2299.18 1455 Q2302.26 1450.39 2308.07 1450.39 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip870)\" d=\"M1020.84 1520.52 L1029.49 1520.52 L1050.56 1560.28 L1050.56 1520.52 L1056.8 1520.52 L1056.8 1568.04 L1048.14 1568.04 L1027.07 1528.29 L1027.07 1568.04 L1020.84 1568.04 L1020.84 1520.52 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip870)\" d=\"M1068.74 1553.98 L1068.74 1532.4 L1074.59 1532.4 L1074.59 1553.75 Q1074.59 1558.81 1076.57 1561.36 Q1078.54 1563.87 1082.49 1563.87 Q1087.23 1563.87 1089.97 1560.85 Q1092.74 1557.83 1092.74 1552.61 L1092.74 1532.4 L1098.59 1532.4 L1098.59 1568.04 L1092.74 1568.04 L1092.74 1562.57 Q1090.6 1565.82 1087.77 1567.41 Q1084.97 1568.97 1081.25 1568.97 Q1075.1 1568.97 1071.92 1565.15 Q1068.74 1561.33 1068.74 1553.98 M1083.47 1531.54 L1083.47 1531.54 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip870)\" d=\"M1138.41 1539.24 Q1140.61 1535.29 1143.66 1533.41 Q1146.72 1531.54 1150.85 1531.54 Q1156.42 1531.54 1159.45 1535.45 Q1162.47 1539.33 1162.47 1546.53 L1162.47 1568.04 L1156.58 1568.04 L1156.58 1546.72 Q1156.58 1541.59 1154.77 1539.11 Q1152.96 1536.63 1149.23 1536.63 Q1144.68 1536.63 1142.04 1539.65 Q1139.4 1542.68 1139.4 1547.9 L1139.4 1568.04 L1133.51 1568.04 L1133.51 1546.72 Q1133.51 1541.56 1131.69 1539.11 Q1129.88 1536.63 1126.09 1536.63 Q1121.6 1536.63 1118.96 1539.68 Q1116.32 1542.71 1116.32 1547.9 L1116.32 1568.04 L1110.43 1568.04 L1110.43 1532.4 L1116.32 1532.4 L1116.32 1537.93 Q1118.33 1534.66 1121.13 1533.1 Q1123.93 1531.54 1127.78 1531.54 Q1131.66 1531.54 1134.37 1533.51 Q1137.1 1535.48 1138.41 1539.24 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip870)\" d=\"M1199.74 1550.25 Q1199.74 1543.79 1197.07 1540.13 Q1194.43 1536.44 1189.78 1536.44 Q1185.13 1536.44 1182.46 1540.13 Q1179.82 1543.79 1179.82 1550.25 Q1179.82 1556.71 1182.46 1560.4 Q1185.13 1564.07 1189.78 1564.07 Q1194.43 1564.07 1197.07 1560.4 Q1199.74 1556.71 1199.74 1550.25 M1179.82 1537.81 Q1181.66 1534.62 1184.47 1533.1 Q1187.3 1531.54 1191.21 1531.54 Q1197.71 1531.54 1201.75 1536.69 Q1205.82 1541.85 1205.82 1550.25 Q1205.82 1558.65 1201.75 1563.81 Q1197.71 1568.97 1191.21 1568.97 Q1187.3 1568.97 1184.47 1567.44 Q1181.66 1565.88 1179.82 1562.7 L1179.82 1568.04 L1173.93 1568.04 L1173.93 1518.52 L1179.82 1518.52 L1179.82 1537.81 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip870)\" d=\"M1246.02 1548.76 L1246.02 1551.62 L1219.1 1551.62 Q1219.48 1557.67 1222.72 1560.85 Q1226 1564 1231.83 1564 Q1235.2 1564 1238.35 1563.17 Q1241.53 1562.35 1244.65 1560.69 L1244.65 1566.23 Q1241.5 1567.57 1238.19 1568.27 Q1234.88 1568.97 1231.48 1568.97 Q1222.95 1568.97 1217.95 1564 Q1212.98 1559.04 1212.98 1550.57 Q1212.98 1541.82 1217.69 1536.69 Q1222.44 1531.54 1230.46 1531.54 Q1237.65 1531.54 1241.82 1536.18 Q1246.02 1540.8 1246.02 1548.76 M1240.17 1547.04 Q1240.1 1542.23 1237.46 1539.37 Q1234.85 1536.5 1230.52 1536.5 Q1225.62 1536.5 1222.66 1539.27 Q1219.73 1542.04 1219.29 1547.07 L1240.17 1547.04 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip870)\" d=\"M1276.29 1537.87 Q1275.3 1537.3 1274.13 1537.04 Q1272.98 1536.76 1271.58 1536.76 Q1266.62 1536.76 1263.94 1540 Q1261.3 1543.22 1261.3 1549.27 L1261.3 1568.04 L1255.41 1568.04 L1255.41 1532.4 L1261.3 1532.4 L1261.3 1537.93 Q1263.15 1534.69 1266.11 1533.13 Q1269.07 1531.54 1273.3 1531.54 Q1273.9 1531.54 1274.64 1531.63 Q1275.37 1531.7 1276.26 1531.85 L1276.29 1537.87 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip870)\" d=\"M1316.97 1536.5 Q1312.26 1536.5 1309.52 1540.19 Q1306.78 1543.85 1306.78 1550.25 Q1306.78 1556.65 1309.49 1560.34 Q1312.23 1564 1316.97 1564 Q1321.65 1564 1324.38 1560.31 Q1327.12 1556.62 1327.12 1550.25 Q1327.12 1543.92 1324.38 1540.23 Q1321.65 1536.5 1316.97 1536.5 M1316.97 1531.54 Q1324.61 1531.54 1328.97 1536.5 Q1333.33 1541.47 1333.33 1550.25 Q1333.33 1559 1328.97 1564 Q1324.61 1568.97 1316.97 1568.97 Q1309.3 1568.97 1304.94 1564 Q1300.61 1559 1300.61 1550.25 Q1300.61 1541.47 1304.94 1536.5 Q1309.3 1531.54 1316.97 1531.54 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip870)\" d=\"M1361.08 1518.52 L1361.08 1523.39 L1355.48 1523.39 Q1352.33 1523.39 1351.09 1524.66 Q1349.88 1525.93 1349.88 1529.24 L1349.88 1532.4 L1359.52 1532.4 L1359.52 1536.95 L1349.88 1536.95 L1349.88 1568.04 L1343.99 1568.04 L1343.99 1536.95 L1338.39 1536.95 L1338.39 1532.4 L1343.99 1532.4 L1343.99 1529.91 Q1343.99 1523.96 1346.76 1521.26 Q1349.53 1518.52 1355.54 1518.52 L1361.08 1518.52 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip870)\" d=\"M1386.96 1520.52 L1395.62 1520.52 L1416.69 1560.28 L1416.69 1520.52 L1422.92 1520.52 L1422.92 1568.04 L1414.27 1568.04 L1393.2 1528.29 L1393.2 1568.04 L1386.96 1568.04 L1386.96 1520.52 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip870)\" d=\"M1449.28 1536.5 Q1444.57 1536.5 1441.83 1540.19 Q1439.09 1543.85 1439.09 1550.25 Q1439.09 1556.65 1441.8 1560.34 Q1444.54 1564 1449.28 1564 Q1453.96 1564 1456.7 1560.31 Q1459.43 1556.62 1459.43 1550.25 Q1459.43 1543.92 1456.7 1540.23 Q1453.96 1536.5 1449.28 1536.5 M1449.28 1531.54 Q1456.92 1531.54 1461.28 1536.5 Q1465.64 1541.47 1465.64 1550.25 Q1465.64 1559 1461.28 1564 Q1456.92 1568.97 1449.28 1568.97 Q1441.61 1568.97 1437.25 1564 Q1432.92 1559 1432.92 1550.25 Q1432.92 1541.47 1437.25 1536.5 Q1441.61 1531.54 1449.28 1531.54 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip870)\" d=\"M1498.8 1537.81 L1498.8 1518.52 L1504.66 1518.52 L1504.66 1568.04 L1498.8 1568.04 L1498.8 1562.7 Q1496.96 1565.88 1494.13 1567.44 Q1491.32 1568.97 1487.38 1568.97 Q1480.92 1568.97 1476.84 1563.81 Q1472.8 1558.65 1472.8 1550.25 Q1472.8 1541.85 1476.84 1536.69 Q1480.92 1531.54 1487.38 1531.54 Q1491.32 1531.54 1494.13 1533.1 Q1496.96 1534.62 1498.8 1537.81 M1478.85 1550.25 Q1478.85 1556.71 1481.49 1560.4 Q1484.16 1564.07 1488.81 1564.07 Q1493.46 1564.07 1496.13 1560.4 Q1498.8 1556.71 1498.8 1550.25 Q1498.8 1543.79 1496.13 1540.13 Q1493.46 1536.44 1488.81 1536.44 Q1484.16 1536.44 1481.49 1540.13 Q1478.85 1543.79 1478.85 1550.25 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip870)\" d=\"M1547.22 1548.76 L1547.22 1551.62 L1520.29 1551.62 Q1520.67 1557.67 1523.92 1560.85 Q1527.2 1564 1533.02 1564 Q1536.39 1564 1539.54 1563.17 Q1542.73 1562.35 1545.85 1560.69 L1545.85 1566.23 Q1542.7 1567.57 1539.39 1568.27 Q1536.08 1568.97 1532.67 1568.97 Q1524.14 1568.97 1519.14 1564 Q1514.18 1559.04 1514.18 1550.57 Q1514.18 1541.82 1518.89 1536.69 Q1523.63 1531.54 1531.65 1531.54 Q1538.84 1531.54 1543.01 1536.18 Q1547.22 1540.8 1547.22 1548.76 M1541.36 1547.04 Q1541.3 1542.23 1538.65 1539.37 Q1536.04 1536.5 1531.71 1536.5 Q1526.81 1536.5 1523.85 1539.27 Q1520.92 1542.04 1520.48 1547.07 L1541.36 1547.04 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip870)\" d=\"M1579.55 1533.45 L1579.55 1538.98 Q1577.07 1537.71 1574.4 1537.07 Q1571.72 1536.44 1568.86 1536.44 Q1564.5 1536.44 1562.3 1537.77 Q1560.14 1539.11 1560.14 1541.79 Q1560.14 1543.82 1561.7 1545 Q1563.26 1546.15 1567.97 1547.2 L1569.97 1547.64 Q1576.21 1548.98 1578.82 1551.43 Q1581.46 1553.85 1581.46 1558.21 Q1581.46 1563.17 1577.52 1566.07 Q1573.6 1568.97 1566.73 1568.97 Q1563.86 1568.97 1560.74 1568.39 Q1557.66 1567.85 1554.22 1566.74 L1554.22 1560.69 Q1557.46 1562.38 1560.62 1563.24 Q1563.77 1564.07 1566.85 1564.07 Q1570.99 1564.07 1573.22 1562.66 Q1575.45 1561.23 1575.45 1558.65 Q1575.45 1556.27 1573.82 1554.99 Q1572.23 1553.72 1566.79 1552.54 L1564.75 1552.07 Q1559.31 1550.92 1556.89 1548.56 Q1554.47 1546.18 1554.47 1542.04 Q1554.47 1537.01 1558.04 1534.27 Q1561.6 1531.54 1568.16 1531.54 Q1571.41 1531.54 1574.27 1532.01 Q1577.13 1532.49 1579.55 1533.45 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><polyline clip-path=\"url(#clip872)\" style=\"stroke:#000000; stroke-linecap:butt; stroke-linejoin:round; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  249.542,1185.15 2352.76,1185.15 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip872)\" style=\"stroke:#000000; stroke-linecap:butt; stroke-linejoin:round; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  249.542,930.556 2352.76,930.556 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip872)\" style=\"stroke:#000000; stroke-linecap:butt; stroke-linejoin:round; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  249.542,675.961 2352.76,675.961 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip872)\" style=\"stroke:#000000; stroke-linecap:butt; stroke-linejoin:round; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  249.542,421.367 2352.76,421.367 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip872)\" style=\"stroke:#000000; stroke-linecap:butt; stroke-linejoin:round; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  249.542,166.772 2352.76,166.772 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip870)\" style=\"stroke:#000000; stroke-linecap:butt; stroke-linejoin:round; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  249.542,1423.18 249.542,123.472 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip870)\" style=\"stroke:#000000; stroke-linecap:butt; stroke-linejoin:round; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  249.542,1185.15 268.44,1185.15 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip870)\" style=\"stroke:#000000; stroke-linecap:butt; stroke-linejoin:round; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  249.542,930.556 268.44,930.556 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip870)\" style=\"stroke:#000000; stroke-linecap:butt; stroke-linejoin:round; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  249.542,675.961 268.44,675.961 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip870)\" style=\"stroke:#000000; stroke-linecap:butt; stroke-linejoin:round; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  249.542,421.367 268.44,421.367 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip870)\" style=\"stroke:#000000; stroke-linecap:butt; stroke-linejoin:round; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  249.542,166.772 268.44,166.772 \n",
       "  \"/>\n",
       "<path clip-path=\"url(#clip870)\" d=\"M127.2 1170.95 Q123.589 1170.95 121.76 1174.51 Q119.955 1178.06 119.955 1185.19 Q119.955 1192.29 121.76 1195.86 Q123.589 1199.4 127.2 1199.4 Q130.834 1199.4 132.64 1195.86 Q134.468 1192.29 134.468 1185.19 Q134.468 1178.06 132.64 1174.51 Q130.834 1170.95 127.2 1170.95 M127.2 1167.25 Q133.01 1167.25 136.066 1171.85 Q139.144 1176.44 139.144 1185.19 Q139.144 1193.91 136.066 1198.52 Q133.01 1203.1 127.2 1203.1 Q121.39 1203.1 118.311 1198.52 Q115.256 1193.91 115.256 1185.19 Q115.256 1176.44 118.311 1171.85 Q121.39 1167.25 127.2 1167.25 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip870)\" d=\"M147.362 1196.55 L152.246 1196.55 L152.246 1202.43 L147.362 1202.43 L147.362 1196.55 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip870)\" d=\"M163.241 1198.5 L170.88 1198.5 L170.88 1172.13 L162.57 1173.8 L162.57 1169.54 L170.834 1167.87 L175.51 1167.87 L175.51 1198.5 L183.149 1198.5 L183.149 1202.43 L163.241 1202.43 L163.241 1198.5 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip870)\" d=\"M192.639 1167.87 L210.996 1167.87 L210.996 1171.81 L196.922 1171.81 L196.922 1180.28 Q197.94 1179.93 198.959 1179.77 Q199.977 1179.58 200.996 1179.58 Q206.783 1179.58 210.162 1182.75 Q213.542 1185.93 213.542 1191.34 Q213.542 1196.92 210.07 1200.02 Q206.598 1203.1 200.278 1203.1 Q198.102 1203.1 195.834 1202.73 Q193.588 1202.36 191.181 1201.62 L191.181 1196.92 Q193.264 1198.06 195.487 1198.61 Q197.709 1199.17 200.186 1199.17 Q204.19 1199.17 206.528 1197.06 Q208.866 1194.95 208.866 1191.34 Q208.866 1187.73 206.528 1185.63 Q204.19 1183.52 200.186 1183.52 Q198.311 1183.52 196.436 1183.94 Q194.584 1184.35 192.639 1185.23 L192.639 1167.87 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip870)\" d=\"M126.205 916.355 Q122.593 916.355 120.765 919.919 Q118.959 923.461 118.959 930.591 Q118.959 937.697 120.765 941.262 Q122.593 944.804 126.205 944.804 Q129.839 944.804 131.644 941.262 Q133.473 937.697 133.473 930.591 Q133.473 923.461 131.644 919.919 Q129.839 916.355 126.205 916.355 M126.205 912.651 Q132.015 912.651 135.07 917.257 Q138.149 921.841 138.149 930.591 Q138.149 939.317 135.07 943.924 Q132.015 948.507 126.205 948.507 Q120.394 948.507 117.316 943.924 Q114.26 939.317 114.26 930.591 Q114.26 921.841 117.316 917.257 Q120.394 912.651 126.205 912.651 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip870)\" d=\"M146.366 941.956 L151.251 941.956 L151.251 947.836 L146.366 947.836 L146.366 941.956 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip870)\" d=\"M165.464 943.901 L181.783 943.901 L181.783 947.836 L159.839 947.836 L159.839 943.901 Q162.501 941.146 167.084 936.517 Q171.69 931.864 172.871 930.521 Q175.116 927.998 175.996 926.262 Q176.899 924.503 176.899 922.813 Q176.899 920.058 174.954 918.322 Q173.033 916.586 169.931 916.586 Q167.732 916.586 165.278 917.35 Q162.848 918.114 160.07 919.665 L160.07 914.943 Q162.894 913.808 165.348 913.23 Q167.802 912.651 169.839 912.651 Q175.209 912.651 178.403 915.336 Q181.598 918.021 181.598 922.512 Q181.598 924.642 180.788 926.563 Q180.001 928.461 177.894 931.054 Q177.315 931.725 174.214 934.942 Q171.112 938.137 165.464 943.901 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip870)\" d=\"M201.598 916.355 Q197.987 916.355 196.158 919.919 Q194.352 923.461 194.352 930.591 Q194.352 937.697 196.158 941.262 Q197.987 944.804 201.598 944.804 Q205.232 944.804 207.037 941.262 Q208.866 937.697 208.866 930.591 Q208.866 923.461 207.037 919.919 Q205.232 916.355 201.598 916.355 M201.598 912.651 Q207.408 912.651 210.463 917.257 Q213.542 921.841 213.542 930.591 Q213.542 939.317 210.463 943.924 Q207.408 948.507 201.598 948.507 Q195.787 948.507 192.709 943.924 Q189.653 939.317 189.653 930.591 Q189.653 921.841 192.709 917.257 Q195.787 912.651 201.598 912.651 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip870)\" d=\"M127.2 661.76 Q123.589 661.76 121.76 665.325 Q119.955 668.866 119.955 675.996 Q119.955 683.102 121.76 686.667 Q123.589 690.209 127.2 690.209 Q130.834 690.209 132.64 686.667 Q134.468 683.102 134.468 675.996 Q134.468 668.866 132.64 665.325 Q130.834 661.76 127.2 661.76 M127.2 658.056 Q133.01 658.056 136.066 662.663 Q139.144 667.246 139.144 675.996 Q139.144 684.723 136.066 689.329 Q133.01 693.913 127.2 693.913 Q121.39 693.913 118.311 689.329 Q115.256 684.723 115.256 675.996 Q115.256 667.246 118.311 662.663 Q121.39 658.056 127.2 658.056 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip870)\" d=\"M147.362 687.362 L152.246 687.362 L152.246 693.241 L147.362 693.241 L147.362 687.362 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip870)\" d=\"M166.459 689.306 L182.778 689.306 L182.778 693.241 L160.834 693.241 L160.834 689.306 Q163.496 686.551 168.079 681.922 Q172.686 677.269 173.866 675.927 Q176.112 673.403 176.991 671.667 Q177.894 669.908 177.894 668.218 Q177.894 665.464 175.95 663.728 Q174.028 661.991 170.927 661.991 Q168.727 661.991 166.274 662.755 Q163.843 663.519 161.065 665.07 L161.065 660.348 Q163.89 659.214 166.343 658.635 Q168.797 658.056 170.834 658.056 Q176.204 658.056 179.399 660.741 Q182.593 663.427 182.593 667.917 Q182.593 670.047 181.783 671.968 Q180.996 673.866 178.889 676.459 Q178.311 677.13 175.209 680.348 Q172.107 683.542 166.459 689.306 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip870)\" d=\"M192.639 658.681 L210.996 658.681 L210.996 662.616 L196.922 662.616 L196.922 671.089 Q197.94 670.741 198.959 670.579 Q199.977 670.394 200.996 670.394 Q206.783 670.394 210.162 673.565 Q213.542 676.737 213.542 682.153 Q213.542 687.732 210.07 690.834 Q206.598 693.913 200.278 693.913 Q198.102 693.913 195.834 693.542 Q193.588 693.172 191.181 692.431 L191.181 687.732 Q193.264 688.866 195.487 689.422 Q197.709 689.977 200.186 689.977 Q204.19 689.977 206.528 687.871 Q208.866 685.764 208.866 682.153 Q208.866 678.542 206.528 676.436 Q204.19 674.329 200.186 674.329 Q198.311 674.329 196.436 674.746 Q194.584 675.163 192.639 676.042 L192.639 658.681 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip870)\" d=\"M126.205 407.165 Q122.593 407.165 120.765 410.73 Q118.959 414.272 118.959 421.401 Q118.959 428.508 120.765 432.073 Q122.593 435.614 126.205 435.614 Q129.839 435.614 131.644 432.073 Q133.473 428.508 133.473 421.401 Q133.473 414.272 131.644 410.73 Q129.839 407.165 126.205 407.165 M126.205 403.462 Q132.015 403.462 135.07 408.068 Q138.149 412.651 138.149 421.401 Q138.149 430.128 135.07 434.735 Q132.015 439.318 126.205 439.318 Q120.394 439.318 117.316 434.735 Q114.26 430.128 114.26 421.401 Q114.26 412.651 117.316 408.068 Q120.394 403.462 126.205 403.462 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip870)\" d=\"M146.366 432.767 L151.251 432.767 L151.251 438.647 L146.366 438.647 L146.366 432.767 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip870)\" d=\"M175.602 420.012 Q178.959 420.73 180.834 422.999 Q182.732 425.267 182.732 428.6 Q182.732 433.716 179.214 436.517 Q175.695 439.318 169.214 439.318 Q167.038 439.318 164.723 438.878 Q162.431 438.461 159.978 437.605 L159.978 433.091 Q161.922 434.225 164.237 434.804 Q166.552 435.383 169.075 435.383 Q173.473 435.383 175.764 433.647 Q178.079 431.91 178.079 428.6 Q178.079 425.545 175.926 423.832 Q173.797 422.096 169.977 422.096 L165.95 422.096 L165.95 418.253 L170.163 418.253 Q173.612 418.253 175.44 416.887 Q177.269 415.499 177.269 412.906 Q177.269 410.244 175.371 408.832 Q173.496 407.397 169.977 407.397 Q168.056 407.397 165.857 407.813 Q163.658 408.23 161.019 409.11 L161.019 404.943 Q163.681 404.202 165.996 403.832 Q168.334 403.462 170.394 403.462 Q175.718 403.462 178.82 405.892 Q181.922 408.3 181.922 412.42 Q181.922 415.29 180.278 417.281 Q178.635 419.249 175.602 420.012 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip870)\" d=\"M201.598 407.165 Q197.987 407.165 196.158 410.73 Q194.352 414.272 194.352 421.401 Q194.352 428.508 196.158 432.073 Q197.987 435.614 201.598 435.614 Q205.232 435.614 207.037 432.073 Q208.866 428.508 208.866 421.401 Q208.866 414.272 207.037 410.73 Q205.232 407.165 201.598 407.165 M201.598 403.462 Q207.408 403.462 210.463 408.068 Q213.542 412.651 213.542 421.401 Q213.542 430.128 210.463 434.735 Q207.408 439.318 201.598 439.318 Q195.787 439.318 192.709 434.735 Q189.653 430.128 189.653 421.401 Q189.653 412.651 192.709 408.068 Q195.787 403.462 201.598 403.462 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip870)\" d=\"M127.2 152.571 Q123.589 152.571 121.76 156.135 Q119.955 159.677 119.955 166.807 Q119.955 173.913 121.76 177.478 Q123.589 181.02 127.2 181.02 Q130.834 181.02 132.64 177.478 Q134.468 173.913 134.468 166.807 Q134.468 159.677 132.64 156.135 Q130.834 152.571 127.2 152.571 M127.2 148.867 Q133.01 148.867 136.066 153.473 Q139.144 158.057 139.144 166.807 Q139.144 175.533 136.066 180.14 Q133.01 184.723 127.2 184.723 Q121.39 184.723 118.311 180.14 Q115.256 175.533 115.256 166.807 Q115.256 158.057 118.311 153.473 Q121.39 148.867 127.2 148.867 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip870)\" d=\"M147.362 178.172 L152.246 178.172 L152.246 184.052 L147.362 184.052 L147.362 178.172 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip870)\" d=\"M176.598 165.418 Q179.954 166.135 181.829 168.404 Q183.727 170.672 183.727 174.006 Q183.727 179.121 180.209 181.922 Q176.69 184.723 170.209 184.723 Q168.033 184.723 165.718 184.283 Q163.427 183.867 160.973 183.01 L160.973 178.496 Q162.917 179.631 165.232 180.209 Q167.547 180.788 170.07 180.788 Q174.468 180.788 176.76 179.052 Q179.075 177.316 179.075 174.006 Q179.075 170.95 176.922 169.237 Q174.792 167.501 170.973 167.501 L166.945 167.501 L166.945 163.658 L171.158 163.658 Q174.607 163.658 176.436 162.293 Q178.264 160.904 178.264 158.311 Q178.264 155.649 176.366 154.237 Q174.491 152.802 170.973 152.802 Q169.052 152.802 166.852 153.219 Q164.653 153.635 162.015 154.515 L162.015 150.348 Q164.677 149.608 166.991 149.237 Q169.329 148.867 171.389 148.867 Q176.714 148.867 179.815 151.297 Q182.917 153.705 182.917 157.825 Q182.917 160.696 181.274 162.686 Q179.63 164.654 176.598 165.418 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip870)\" d=\"M192.639 149.492 L210.996 149.492 L210.996 153.427 L196.922 153.427 L196.922 161.899 Q197.94 161.552 198.959 161.39 Q199.977 161.205 200.996 161.205 Q206.783 161.205 210.162 164.376 Q213.542 167.547 213.542 172.964 Q213.542 178.543 210.07 181.644 Q206.598 184.723 200.278 184.723 Q198.102 184.723 195.834 184.353 Q193.588 183.982 191.181 183.242 L191.181 178.543 Q193.264 179.677 195.487 180.232 Q197.709 180.788 200.186 180.788 Q204.19 180.788 206.528 178.682 Q208.866 176.575 208.866 172.964 Q208.866 169.353 206.528 167.246 Q204.19 165.14 200.186 165.14 Q198.311 165.14 196.436 165.557 Q194.584 165.973 192.639 166.853 L192.639 149.492 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip870)\" d=\"M16.4842 991.717 L16.4842 951.518 L21.895 951.518 L21.895 968.387 L64.0042 968.387 L64.0042 974.848 L21.895 974.848 L21.895 991.717 L16.4842 991.717 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip870)\" d=\"M33.8307 934.49 Q33.2578 935.476 33.0032 936.654 Q32.7167 937.8 32.7167 939.2 Q32.7167 944.166 35.9632 946.839 Q39.1779 949.481 45.2253 949.481 L64.0042 949.481 L64.0042 955.369 L28.3562 955.369 L28.3562 949.481 L33.8944 949.481 Q30.6479 947.635 29.0883 944.675 Q27.4968 941.715 27.4968 937.482 Q27.4968 936.877 27.5923 936.145 Q27.656 935.413 27.8151 934.522 L33.8307 934.49 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip870)\" d=\"M46.0847 912.146 Q46.0847 919.244 47.7079 921.981 Q49.3312 924.718 53.2461 924.718 Q56.3653 924.718 58.2114 922.681 Q60.0256 920.613 60.0256 917.08 Q60.0256 912.21 56.5881 909.282 Q53.1188 906.322 47.3897 906.322 L46.0847 906.322 L46.0847 912.146 M43.6657 900.465 L64.0042 900.465 L64.0042 906.322 L58.5933 906.322 Q61.8398 908.327 63.3994 911.319 Q64.9272 914.31 64.9272 918.639 Q64.9272 924.114 61.8716 927.36 Q58.7843 930.575 53.6281 930.575 Q47.6125 930.575 44.5569 926.564 Q41.5014 922.522 41.5014 914.533 L41.5014 906.322 L40.9285 906.322 Q36.8862 906.322 34.6901 908.995 Q32.4621 911.637 32.4621 916.443 Q32.4621 919.499 33.1941 922.395 Q33.9262 925.291 35.3903 927.965 L29.9795 927.965 Q28.7381 924.75 28.1334 921.727 Q27.4968 918.703 27.4968 915.838 Q27.4968 908.104 31.5072 904.284 Q35.5176 900.465 43.6657 900.465 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip870)\" d=\"M28.3562 888.402 L28.3562 882.546 L64.0042 882.546 L64.0042 888.402 L28.3562 888.402 M14.479 888.402 L14.479 882.546 L21.895 882.546 L21.895 888.402 L14.479 888.402 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip870)\" d=\"M42.4881 840.659 L64.0042 840.659 L64.0042 846.516 L42.679 846.516 Q37.6183 846.516 35.1038 848.489 Q32.5894 850.462 32.5894 854.409 Q32.5894 859.152 35.6131 861.889 Q38.6368 864.626 43.8567 864.626 L64.0042 864.626 L64.0042 870.514 L28.3562 870.514 L28.3562 864.626 L33.8944 864.626 Q30.6797 862.525 29.0883 859.693 Q27.4968 856.828 27.4968 853.104 Q27.4968 846.961 31.3163 843.81 Q35.1038 840.659 42.4881 840.659 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip870)\" d=\"M16.4842 814.592 L16.4842 774.392 L21.895 774.392 L21.895 791.261 L64.0042 791.261 L64.0042 797.723 L21.895 797.723 L21.895 814.592 L16.4842 814.592 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip870)\" d=\"M28.3562 770.446 L28.3562 764.589 L64.0042 764.589 L64.0042 770.446 L28.3562 770.446 M14.479 770.446 L14.479 764.589 L21.895 764.589 L21.895 770.446 L14.479 770.446 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip870)\" d=\"M35.1993 724.581 Q31.2526 722.384 29.3747 719.329 Q27.4968 716.273 27.4968 712.136 Q27.4968 706.566 31.4117 703.542 Q35.2948 700.518 42.4881 700.518 L64.0042 700.518 L64.0042 706.406 L42.679 706.406 Q37.5546 706.406 35.072 708.221 Q32.5894 710.035 32.5894 713.759 Q32.5894 718.31 35.6131 720.952 Q38.6368 723.594 43.8567 723.594 L64.0042 723.594 L64.0042 729.482 L42.679 729.482 Q37.5228 729.482 35.072 731.296 Q32.5894 733.111 32.5894 736.898 Q32.5894 741.386 35.6449 744.028 Q38.6686 746.67 43.8567 746.67 L64.0042 746.67 L64.0042 752.558 L28.3562 752.558 L28.3562 746.67 L33.8944 746.67 Q30.616 744.664 29.0564 741.863 Q27.4968 739.063 27.4968 735.211 Q27.4968 731.328 29.4702 728.623 Q31.4436 725.886 35.1993 724.581 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip870)\" d=\"M44.7161 658.345 L47.5806 658.345 L47.5806 685.272 Q53.6281 684.89 56.8109 681.644 Q59.9619 678.366 59.9619 672.541 Q59.9619 669.167 59.1344 666.016 Q58.3069 662.833 56.6518 659.714 L62.1899 659.714 Q63.5267 662.865 64.227 666.175 Q64.9272 669.485 64.9272 672.891 Q64.9272 681.421 59.9619 686.418 Q54.9967 691.383 46.5303 691.383 Q37.7774 691.383 32.6531 686.673 Q27.4968 681.93 27.4968 673.91 Q27.4968 666.716 32.1438 662.547 Q36.7589 658.345 44.7161 658.345 M42.9973 664.202 Q38.1912 664.266 35.3266 666.907 Q32.4621 669.517 32.4621 673.846 Q32.4621 678.747 35.2312 681.708 Q38.0002 684.636 43.0292 685.081 L42.9973 664.202 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip870)\" d=\"M14.5426 613.945 Q21.8632 618.21 29.0246 620.278 Q36.186 622.347 43.5384 622.347 Q50.8908 622.347 58.1159 620.278 Q65.3091 618.178 72.5979 613.945 L72.5979 619.037 Q65.1182 623.811 57.8931 626.199 Q50.668 628.554 43.5384 628.554 Q36.4406 628.554 29.2474 626.199 Q22.0542 623.843 14.5426 619.037 L14.5426 613.945 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip870)\" d=\"M29.4065 579.856 L34.9447 579.856 Q33.6716 582.339 33.035 585.012 Q32.3984 587.686 32.3984 590.551 Q32.3984 594.911 33.7352 597.107 Q35.072 599.272 37.7456 599.272 Q39.7826 599.272 40.9603 597.712 Q42.1061 596.152 43.1565 591.442 L43.6021 589.437 Q44.9389 583.198 47.3897 580.588 Q49.8086 577.947 54.1691 577.947 Q59.1344 577.947 62.0308 581.893 Q64.9272 585.808 64.9272 592.683 Q64.9272 595.548 64.3543 598.667 Q63.8132 601.754 62.6992 605.192 L56.6518 605.192 Q58.3387 601.945 59.198 598.794 Q60.0256 595.643 60.0256 592.556 Q60.0256 588.418 58.6251 586.19 Q57.1929 583.962 54.6147 583.962 Q52.2276 583.962 50.9545 585.585 Q49.6813 587.177 48.5037 592.619 L48.0262 594.657 Q46.8804 600.099 44.5251 602.518 Q42.138 604.937 38.0002 604.937 Q32.9713 604.937 30.2341 601.372 Q27.4968 597.808 27.4968 591.251 Q27.4968 588.004 27.9743 585.14 Q28.4517 582.275 29.4065 579.856 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip870)\" d=\"M14.5426 569.544 L14.5426 564.451 Q22.0542 559.677 29.2474 557.322 Q36.4406 554.935 43.5384 554.935 Q50.668 554.935 57.8931 557.322 Q65.1182 559.677 72.5979 564.451 L72.5979 569.544 Q65.3091 565.311 58.1159 563.242 Q50.8908 561.141 43.5384 561.141 Q36.186 561.141 29.0246 563.242 Q21.8632 565.311 14.5426 569.544 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip870)\" d=\"M527.852 12.096 L538.87 12.096 L565.687 62.6918 L565.687 12.096 L573.627 12.096 L573.627 72.576 L562.608 72.576 L535.791 21.9802 L535.791 72.576 L527.852 72.576 L527.852 12.096 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip870)\" d=\"M588.818 54.671 L588.818 27.2059 L596.271 27.2059 L596.271 54.3874 Q596.271 60.8284 598.783 64.0691 Q601.295 67.2693 606.318 67.2693 Q612.353 67.2693 615.837 63.421 Q619.362 59.5726 619.362 52.9291 L619.362 27.2059 L626.815 27.2059 L626.815 72.576 L619.362 72.576 L619.362 65.6084 Q616.647 69.7404 613.042 71.7658 Q609.477 73.7508 604.738 73.7508 Q596.92 73.7508 592.869 68.8897 Q588.818 64.0286 588.818 54.671 M607.573 26.1121 L607.573 26.1121 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip870)\" d=\"M677.492 35.9153 Q680.287 30.8922 684.176 28.5022 Q688.065 26.1121 693.331 26.1121 Q700.42 26.1121 704.268 31.0947 Q708.117 36.0368 708.117 45.1919 L708.117 72.576 L700.623 72.576 L700.623 45.4349 Q700.623 38.913 698.314 35.7533 Q696.005 32.5936 691.265 32.5936 Q685.472 32.5936 682.11 36.4419 Q678.748 40.2903 678.748 46.9338 L678.748 72.576 L671.254 72.576 L671.254 45.4349 Q671.254 38.8725 668.945 35.7533 Q666.636 32.5936 661.815 32.5936 Q656.103 32.5936 652.741 36.4824 Q649.379 40.3308 649.379 46.9338 L649.379 72.576 L641.885 72.576 L641.885 27.2059 L649.379 27.2059 L649.379 34.2544 Q651.931 30.082 655.496 28.0971 Q659.06 26.1121 663.962 26.1121 Q668.904 26.1121 672.347 28.6237 Q675.831 31.1352 677.492 35.9153 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip870)\" d=\"M755.553 49.9314 Q755.553 41.7081 752.15 37.0496 Q748.788 32.3505 742.874 32.3505 Q736.959 32.3505 733.557 37.0496 Q730.194 41.7081 730.194 49.9314 Q730.194 58.1548 733.557 62.8538 Q736.959 67.5124 742.874 67.5124 Q748.788 67.5124 752.15 62.8538 Q755.553 58.1548 755.553 49.9314 M730.194 34.0924 Q732.544 30.0415 736.109 28.0971 Q739.714 26.1121 744.697 26.1121 Q752.96 26.1121 758.105 32.6746 Q763.29 39.2371 763.29 49.9314 Q763.29 60.6258 758.105 67.1883 Q752.96 73.7508 744.697 73.7508 Q739.714 73.7508 736.109 71.8063 Q732.544 69.8214 730.194 65.7705 L730.194 72.576 L722.7 72.576 L722.7 9.54393 L730.194 9.54393 L730.194 34.0924 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip870)\" d=\"M814.453 48.0275 L814.453 51.6733 L780.182 51.6733 Q780.669 59.3701 784.8 63.421 Q788.973 67.4314 796.386 67.4314 Q800.68 67.4314 804.69 66.3781 Q808.741 65.3249 812.711 63.2184 L812.711 70.267 Q808.701 71.9684 804.488 72.8596 Q800.275 73.7508 795.94 73.7508 Q785.084 73.7508 778.724 67.4314 Q772.405 61.1119 772.405 50.3365 Q772.405 39.1965 778.4 32.6746 Q784.436 26.1121 794.644 26.1121 Q803.799 26.1121 809.106 32.0264 Q814.453 37.9003 814.453 48.0275 M806.999 45.84 Q806.918 39.7232 803.556 36.0774 Q800.234 32.4315 794.725 32.4315 Q788.487 32.4315 784.719 35.9558 Q780.993 39.4801 780.425 45.8805 L806.999 45.84 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip870)\" d=\"M852.977 34.1734 Q851.721 33.4443 850.223 33.1202 Q848.764 32.7556 846.982 32.7556 Q840.662 32.7556 837.26 36.8875 Q833.897 40.9789 833.897 48.6757 L833.897 72.576 L826.403 72.576 L826.403 27.2059 L833.897 27.2059 L833.897 34.2544 Q836.247 30.1225 840.014 28.1376 Q843.782 26.1121 849.169 26.1121 Q849.939 26.1121 850.871 26.2337 Q851.802 26.3147 852.937 26.5172 L852.977 34.1734 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip870)\" d=\"M904.748 32.4315 Q898.752 32.4315 895.269 37.1306 Q891.785 41.7891 891.785 49.9314 Q891.785 58.0738 895.228 62.7728 Q898.712 67.4314 904.748 67.4314 Q910.703 67.4314 914.186 62.7323 Q917.67 58.0333 917.67 49.9314 Q917.67 41.8701 914.186 37.1711 Q910.703 32.4315 904.748 32.4315 M904.748 26.1121 Q914.47 26.1121 920.02 32.4315 Q925.569 38.7509 925.569 49.9314 Q925.569 61.0714 920.02 67.4314 Q914.47 73.7508 904.748 73.7508 Q894.985 73.7508 889.435 67.4314 Q883.926 61.0714 883.926 49.9314 Q883.926 38.7509 889.435 32.4315 Q894.985 26.1121 904.748 26.1121 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip870)\" d=\"M960.893 9.54393 L960.893 15.7418 L953.764 15.7418 Q949.753 15.7418 948.173 17.3622 Q946.634 18.9825 946.634 23.1955 L946.634 27.2059 L958.908 27.2059 L958.908 32.9987 L946.634 32.9987 L946.634 72.576 L939.14 72.576 L939.14 32.9987 L932.01 32.9987 L932.01 27.2059 L939.14 27.2059 L939.14 24.0462 Q939.14 16.471 942.664 13.0277 Q946.188 9.54393 953.845 9.54393 L960.893 9.54393 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip870)\" d=\"M993.827 12.096 L1004.85 12.096 L1031.66 62.6918 L1031.66 12.096 L1039.6 12.096 L1039.6 72.576 L1028.58 72.576 L1001.77 21.9802 L1001.77 72.576 L993.827 72.576 L993.827 12.096 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip870)\" d=\"M1073.14 32.4315 Q1067.15 32.4315 1063.66 37.1306 Q1060.18 41.7891 1060.18 49.9314 Q1060.18 58.0738 1063.62 62.7728 Q1067.11 67.4314 1073.14 67.4314 Q1079.1 67.4314 1082.58 62.7323 Q1086.07 58.0333 1086.07 49.9314 Q1086.07 41.8701 1082.58 37.1711 Q1079.1 32.4315 1073.14 32.4315 M1073.14 26.1121 Q1082.87 26.1121 1088.42 32.4315 Q1093.97 38.7509 1093.97 49.9314 Q1093.97 61.0714 1088.42 67.4314 Q1082.87 73.7508 1073.14 73.7508 Q1063.38 73.7508 1057.83 67.4314 Q1052.32 61.0714 1052.32 49.9314 Q1052.32 38.7509 1057.83 32.4315 Q1063.38 26.1121 1073.14 26.1121 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip870)\" d=\"M1136.18 34.0924 L1136.18 9.54393 L1143.63 9.54393 L1143.63 72.576 L1136.18 72.576 L1136.18 65.7705 Q1133.83 69.8214 1130.22 71.8063 Q1126.66 73.7508 1121.63 73.7508 Q1113.41 73.7508 1108.22 67.1883 Q1103.08 60.6258 1103.08 49.9314 Q1103.08 39.2371 1108.22 32.6746 Q1113.41 26.1121 1121.63 26.1121 Q1126.66 26.1121 1130.22 28.0971 Q1133.83 30.0415 1136.18 34.0924 M1110.78 49.9314 Q1110.78 58.1548 1114.14 62.8538 Q1117.54 67.5124 1123.46 67.5124 Q1129.37 67.5124 1132.77 62.8538 Q1136.18 58.1548 1136.18 49.9314 Q1136.18 41.7081 1132.77 37.0496 Q1129.37 32.3505 1123.46 32.3505 Q1117.54 32.3505 1114.14 37.0496 Q1110.78 41.7081 1110.78 49.9314 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip870)\" d=\"M1197.79 48.0275 L1197.79 51.6733 L1163.52 51.6733 Q1164.01 59.3701 1168.14 63.421 Q1172.31 67.4314 1179.72 67.4314 Q1184.02 67.4314 1188.03 66.3781 Q1192.08 65.3249 1196.05 63.2184 L1196.05 70.267 Q1192.04 71.9684 1187.82 72.8596 Q1183.61 73.7508 1179.28 73.7508 Q1168.42 73.7508 1162.06 67.4314 Q1155.74 61.1119 1155.74 50.3365 Q1155.74 39.1965 1161.74 32.6746 Q1167.77 26.1121 1177.98 26.1121 Q1187.14 26.1121 1192.44 32.0264 Q1197.79 37.9003 1197.79 48.0275 M1190.34 45.84 Q1190.26 39.7232 1186.89 36.0774 Q1183.57 32.4315 1178.06 32.4315 Q1171.82 32.4315 1168.06 35.9558 Q1164.33 39.4801 1163.76 45.8805 L1190.34 45.84 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip870)\" d=\"M1238.95 28.5427 L1238.95 35.5912 Q1235.79 33.9709 1232.38 33.1607 Q1228.98 32.3505 1225.34 32.3505 Q1219.79 32.3505 1216.99 34.0519 Q1214.24 35.7533 1214.24 39.156 Q1214.24 41.7486 1216.22 43.2475 Q1218.21 44.7058 1224.2 46.0426 L1226.75 46.6097 Q1234.69 48.3111 1238.02 51.4303 Q1241.38 54.509 1241.38 60.0587 Q1241.38 66.3781 1236.35 70.0644 Q1231.37 73.7508 1222.62 73.7508 Q1218.98 73.7508 1215.01 73.0216 Q1211.08 72.3329 1206.7 70.9151 L1206.7 63.2184 Q1210.83 65.3654 1214.84 66.4591 Q1218.85 67.5124 1222.78 67.5124 Q1228.05 67.5124 1230.89 65.73 Q1233.72 63.9071 1233.72 60.6258 Q1233.72 57.5877 1231.66 55.9673 Q1229.63 54.3469 1222.7 52.8481 L1220.11 52.2405 Q1213.18 50.7821 1210.1 47.7845 Q1207.03 44.7463 1207.03 39.4801 Q1207.03 33.0797 1211.56 29.5959 Q1216.1 26.1121 1224.45 26.1121 Q1228.58 26.1121 1232.22 26.7198 Q1235.87 27.3274 1238.95 28.5427 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip870)\" d=\"M1274.27 27.2059 L1282.17 27.2059 L1296.35 65.2844 L1310.53 27.2059 L1318.43 27.2059 L1301.41 72.576 L1291.29 72.576 L1274.27 27.2059 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip870)\" d=\"M1357.64 28.5427 L1357.64 35.5912 Q1354.48 33.9709 1351.08 33.1607 Q1347.67 32.3505 1344.03 32.3505 Q1338.48 32.3505 1335.68 34.0519 Q1332.93 35.7533 1332.93 39.156 Q1332.93 41.7486 1334.91 43.2475 Q1336.9 44.7058 1342.89 46.0426 L1345.45 46.6097 Q1353.39 48.3111 1356.71 51.4303 Q1360.07 54.509 1360.07 60.0587 Q1360.07 66.3781 1355.05 70.0644 Q1350.06 73.7508 1341.31 73.7508 Q1337.67 73.7508 1333.7 73.0216 Q1329.77 72.3329 1325.39 70.9151 L1325.39 63.2184 Q1329.53 65.3654 1333.54 66.4591 Q1337.55 67.5124 1341.48 67.5124 Q1346.74 67.5124 1349.58 65.73 Q1352.41 63.9071 1352.41 60.6258 Q1352.41 57.5877 1350.35 55.9673 Q1348.32 54.3469 1341.39 52.8481 L1338.8 52.2405 Q1331.88 50.7821 1328.8 47.7845 Q1325.72 44.7463 1325.72 39.4801 Q1325.72 33.0797 1330.25 29.5959 Q1334.79 26.1121 1343.14 26.1121 Q1347.27 26.1121 1350.91 26.7198 Q1354.56 27.3274 1357.64 28.5427 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip870)\" d=\"M1390.25 12.096 L1441.41 12.096 L1441.41 18.9825 L1419.94 18.9825 L1419.94 72.576 L1411.72 72.576 L1411.72 18.9825 L1390.25 18.9825 L1390.25 12.096 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip870)\" d=\"M1463.08 34.1734 Q1461.83 33.4443 1460.33 33.1202 Q1458.87 32.7556 1457.09 32.7556 Q1450.77 32.7556 1447.37 36.8875 Q1444 40.9789 1444 48.6757 L1444 72.576 L1436.51 72.576 L1436.51 27.2059 L1444 27.2059 L1444 34.2544 Q1446.35 30.1225 1450.12 28.1376 Q1453.89 26.1121 1459.28 26.1121 Q1460.05 26.1121 1460.98 26.2337 Q1461.91 26.3147 1463.04 26.5172 L1463.08 34.1734 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip870)\" d=\"M1491.52 49.7694 Q1482.49 49.7694 1479 51.8354 Q1475.52 53.9013 1475.52 58.8839 Q1475.52 62.8538 1478.11 65.2034 Q1480.75 67.5124 1485.24 67.5124 Q1491.44 67.5124 1495.17 63.1374 Q1498.93 58.7219 1498.93 51.4303 L1498.93 49.7694 L1491.52 49.7694 M1506.39 46.6907 L1506.39 72.576 L1498.93 72.576 L1498.93 65.6895 Q1496.38 69.8214 1492.57 71.8063 Q1488.77 73.7508 1483.26 73.7508 Q1476.29 73.7508 1472.16 69.8619 Q1468.07 65.9325 1468.07 59.3701 Q1468.07 51.7138 1473.17 47.825 Q1478.32 43.9361 1488.48 43.9361 L1498.93 43.9361 L1498.93 43.2069 Q1498.93 38.0623 1495.53 35.2672 Q1492.17 32.4315 1486.05 32.4315 Q1482.16 32.4315 1478.48 33.3632 Q1474.79 34.295 1471.39 36.1584 L1471.39 29.2718 Q1475.48 27.692 1479.33 26.9223 Q1483.18 26.1121 1486.82 26.1121 Q1496.67 26.1121 1501.53 31.2163 Q1506.39 36.3204 1506.39 46.6907 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip870)\" d=\"M1521.74 27.2059 L1529.19 27.2059 L1529.19 72.576 L1521.74 72.576 L1521.74 27.2059 M1521.74 9.54393 L1529.19 9.54393 L1529.19 18.9825 L1521.74 18.9825 L1521.74 9.54393 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip870)\" d=\"M1582.5 45.1919 L1582.5 72.576 L1575.05 72.576 L1575.05 45.4349 Q1575.05 38.994 1572.54 35.7938 Q1570.03 32.5936 1565 32.5936 Q1558.97 32.5936 1555.48 36.4419 Q1552 40.2903 1552 46.9338 L1552 72.576 L1544.51 72.576 L1544.51 27.2059 L1552 27.2059 L1552 34.2544 Q1554.67 30.163 1558.28 28.1376 Q1561.93 26.1121 1566.67 26.1121 Q1574.48 26.1121 1578.49 30.9732 Q1582.5 35.7938 1582.5 45.1919 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip870)\" d=\"M1597.37 27.2059 L1604.83 27.2059 L1604.83 72.576 L1597.37 72.576 L1597.37 27.2059 M1597.37 9.54393 L1604.83 9.54393 L1604.83 18.9825 L1597.37 18.9825 L1597.37 9.54393 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip870)\" d=\"M1658.13 45.1919 L1658.13 72.576 L1650.68 72.576 L1650.68 45.4349 Q1650.68 38.994 1648.17 35.7938 Q1645.66 32.5936 1640.63 32.5936 Q1634.6 32.5936 1631.12 36.4419 Q1627.63 40.2903 1627.63 46.9338 L1627.63 72.576 L1620.14 72.576 L1620.14 27.2059 L1627.63 27.2059 L1627.63 34.2544 Q1630.31 30.163 1633.91 28.1376 Q1637.56 26.1121 1642.3 26.1121 Q1650.11 26.1121 1654.12 30.9732 Q1658.13 35.7938 1658.13 45.1919 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip870)\" d=\"M1702.86 49.3643 Q1702.86 41.2625 1699.49 36.8065 Q1696.17 32.3505 1690.14 32.3505 Q1684.14 32.3505 1680.78 36.8065 Q1677.46 41.2625 1677.46 49.3643 Q1677.46 57.4256 1680.78 61.8816 Q1684.14 66.3376 1690.14 66.3376 Q1696.17 66.3376 1699.49 61.8816 Q1702.86 57.4256 1702.86 49.3643 M1710.31 66.9452 Q1710.31 78.5308 1705.17 84.1616 Q1700.02 89.8329 1689.41 89.8329 Q1685.48 89.8329 1681.99 89.2252 Q1678.51 88.6581 1675.23 87.4428 L1675.23 80.1917 Q1678.51 81.9741 1681.71 82.8248 Q1684.91 83.6755 1688.23 83.6755 Q1695.57 83.6755 1699.21 79.8271 Q1702.86 76.0193 1702.86 68.282 L1702.86 64.5957 Q1700.55 68.6061 1696.94 70.5911 Q1693.34 72.576 1688.31 72.576 Q1679.97 72.576 1674.87 66.2161 Q1669.76 59.8562 1669.76 49.3643 Q1669.76 38.832 1674.87 32.472 Q1679.97 26.1121 1688.31 26.1121 Q1693.34 26.1121 1696.94 28.0971 Q1700.55 30.082 1702.86 34.0924 L1702.86 27.2059 L1710.31 27.2059 L1710.31 66.9452 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip870)\" d=\"M1743.97 12.096 L1795.14 12.096 L1795.14 18.9825 L1773.67 18.9825 L1773.67 72.576 L1765.44 72.576 L1765.44 18.9825 L1743.97 18.9825 L1743.97 12.096 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip870)\" d=\"M1800.16 27.2059 L1807.61 27.2059 L1807.61 72.576 L1800.16 72.576 L1800.16 27.2059 M1800.16 9.54393 L1807.61 9.54393 L1807.61 18.9825 L1800.16 18.9825 L1800.16 9.54393 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip870)\" d=\"M1858.53 35.9153 Q1861.33 30.8922 1865.22 28.5022 Q1869.11 26.1121 1874.37 26.1121 Q1881.46 26.1121 1885.31 31.0947 Q1889.16 36.0368 1889.16 45.1919 L1889.16 72.576 L1881.66 72.576 L1881.66 45.4349 Q1881.66 38.913 1879.35 35.7533 Q1877.05 32.5936 1872.31 32.5936 Q1866.51 32.5936 1863.15 36.4419 Q1859.79 40.2903 1859.79 46.9338 L1859.79 72.576 L1852.29 72.576 L1852.29 45.4349 Q1852.29 38.8725 1849.99 35.7533 Q1847.68 32.5936 1842.86 32.5936 Q1837.14 32.5936 1833.78 36.4824 Q1830.42 40.3308 1830.42 46.9338 L1830.42 72.576 L1822.93 72.576 L1822.93 27.2059 L1830.42 27.2059 L1830.42 34.2544 Q1832.97 30.082 1836.54 28.0971 Q1840.1 26.1121 1845 26.1121 Q1849.95 26.1121 1853.39 28.6237 Q1856.87 31.1352 1858.53 35.9153 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip870)\" d=\"M1942.83 48.0275 L1942.83 51.6733 L1908.56 51.6733 Q1909.05 59.3701 1913.18 63.421 Q1917.35 67.4314 1924.77 67.4314 Q1929.06 67.4314 1933.07 66.3781 Q1937.12 65.3249 1941.09 63.2184 L1941.09 70.267 Q1937.08 71.9684 1932.87 72.8596 Q1928.65 73.7508 1924.32 73.7508 Q1913.46 73.7508 1907.1 67.4314 Q1900.78 61.1119 1900.78 50.3365 Q1900.78 39.1965 1906.78 32.6746 Q1912.82 26.1121 1923.02 26.1121 Q1932.18 26.1121 1937.49 32.0264 Q1942.83 37.9003 1942.83 48.0275 M1935.38 45.84 Q1935.3 39.7232 1931.94 36.0774 Q1928.61 32.4315 1923.1 32.4315 Q1916.87 32.4315 1913.1 35.9558 Q1909.37 39.4801 1908.8 45.8805 L1935.38 45.84 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip870)\" d=\"M1999.34 9.62495 Q1993.91 18.942 1991.28 28.0566 Q1988.65 37.1711 1988.65 46.5287 Q1988.65 55.8863 1991.28 65.0818 Q1993.95 74.2369 1999.34 83.5134 L1992.86 83.5134 Q1986.78 73.9938 1983.75 64.7983 Q1980.75 55.6027 1980.75 46.5287 Q1980.75 37.4952 1983.75 28.3401 Q1986.74 19.1851 1992.86 9.62495 L1999.34 9.62495 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip870)\" d=\"M2042.73 28.5427 L2042.73 35.5912 Q2039.57 33.9709 2036.17 33.1607 Q2032.76 32.3505 2029.12 32.3505 Q2023.57 32.3505 2020.77 34.0519 Q2018.02 35.7533 2018.02 39.156 Q2018.02 41.7486 2020 43.2475 Q2021.99 44.7058 2027.98 46.0426 L2030.53 46.6097 Q2038.47 48.3111 2041.8 51.4303 Q2045.16 54.509 2045.16 60.0587 Q2045.16 66.3781 2040.14 70.0644 Q2035.15 73.7508 2026.4 73.7508 Q2022.76 73.7508 2018.79 73.0216 Q2014.86 72.3329 2010.48 70.9151 L2010.48 63.2184 Q2014.61 65.3654 2018.62 66.4591 Q2022.64 67.5124 2026.56 67.5124 Q2031.83 67.5124 2034.67 65.73 Q2037.5 63.9071 2037.5 60.6258 Q2037.5 57.5877 2035.44 55.9673 Q2033.41 54.3469 2026.48 52.8481 L2023.89 52.2405 Q2016.96 50.7821 2013.89 47.7845 Q2010.81 44.7463 2010.81 39.4801 Q2010.81 33.0797 2015.34 29.5959 Q2019.88 26.1121 2028.23 26.1121 Q2032.36 26.1121 2036 26.7198 Q2039.65 27.3274 2042.73 28.5427 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip870)\" d=\"M2055.85 9.62495 L2062.33 9.62495 Q2068.41 19.1851 2071.41 28.3401 Q2074.45 37.4952 2074.45 46.5287 Q2074.45 55.6027 2071.41 64.7983 Q2068.41 73.9938 2062.33 83.5134 L2055.85 83.5134 Q2061.24 74.2369 2063.87 65.0818 Q2066.55 55.8863 2066.55 46.5287 Q2066.55 37.1711 2063.87 28.0566 Q2061.24 18.942 2055.85 9.62495 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><polyline clip-path=\"url(#clip872)\" style=\"stroke:#009af9; stroke-linecap:butt; stroke-linejoin:round; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  309.067,1220.39 377.486,1386.4 445.906,1346.5 514.325,1259.33 582.745,1257.98 651.164,1163.94 719.584,1157.74 788.003,1103.41 856.423,1062.17 924.842,1044.05 \n",
       "  993.261,1005.04 1061.68,957.562 1130.1,910.042 1198.52,881.9 1266.94,832.196 1335.36,796.119 1403.78,737.187 1472.2,696.603 1540.62,666.949 1609.04,596.48 \n",
       "  1677.46,571.043 1745.88,521.48 1814.29,483.033 1882.71,430.155 1951.13,259.746 2019.55,193.827 2087.97,195.312 2156.39,244.455 2224.81,206.313 2293.23,160.256 \n",
       "  \n",
       "  \"/>\n",
       "</svg>\n"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Plot results.\n",
    "averaged_times = mean(times, dims=2);\n",
    "num_nodes_vs_train_time = plot(LinRange(1,trials,trials), averaged_times, title=\"Number of Nodes vs Training Time (s)\", label=\"\", xlabel=\"Number of Nodes\", ylabel=\"Train Time (s)\")\n",
    "savefig(num_nodes_vs_train_time, \"images/num_nodes_vs_train_time.png\")\n",
    "current()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "78b025ea",
   "metadata": {},
   "source": [
    "## Experiment: increasing depth of neural network vs training time."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "ab760040",
   "metadata": {},
   "outputs": [],
   "source": [
    "depth = 50\n",
    "repeats = 10\n",
    "times = zeros(depth, repeats)\n",
    "for i in range(1, repeats)\n",
    "    for val in range(1, depth)\n",
    "        architecture = convert(Array{Int64}, vcat(1, ones(val) .* 20, 1))\n",
    "        num_weights, predictions, logprob, ensemble_prediction = make_neural_network_functions(architecture, 0.1, 0.01, rbf)\n",
    "        inputs, targets = build_dataset_1()\n",
    "        log_posterior(weights) = logprob(weights, inputs, targets)\n",
    "        objective = black_box_variational_inference(log_posterior, num_weights, 1)\n",
    "        e = @elapsed train_bayesian_neural_network(200, 0.1, num_weights, objective);\n",
    "        times[val, i] = e\n",
    "    end\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "08ac5fa2",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Plot results.\n",
    "averaged_times = mean(times, dims=2);\n",
    "architecture_vs_train_time = plot(LinRange(1,depth,depth), averaged_times, title=\"Network Depth vs Training Time (s)\", label=\"\", xlabel=\"Depth of Neural Network\", ylabel=\"Train Time (s)\")\n",
    "savefig(architecture_vs_train_time, \"images/architecture_vs_train_time.png\")\n",
    "current()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bc63ec8b",
   "metadata": {},
   "source": [
    "## Experiment: increasing MC samples vs ELBO"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "22d55a09",
   "metadata": {},
   "outputs": [],
   "source": [
    "Random.seed!(100)\n",
    "mc_samples = [1,10,20,30,40]\n",
    "times = zeros(depth, repeats)\n",
    "elbos_list = Array{}[]\n",
    "for mc_sample in mc_samples\n",
    "    num_weights, predictions, logprob, ensemble_prediction = make_neural_network_functions([1, 20, 20, 1], 0.1, 0.01, rbf)\n",
    "    inputs, targets = build_dataset_1()\n",
    "    log_posterior(weights) = logprob(weights, inputs, targets)\n",
    "    objective = black_box_variational_inference(log_posterior, num_weights, mc_sample)\n",
    "    _, elbos = train_bayesian_neural_network(200, 0.1, num_weights, objective)\n",
    "    push!(elbos_list, elbos)\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "bafe4094",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Plot results\n",
    "for i in range(1,5)\n",
    "    mc_sample = mc_samples[i]\n",
    "    mc_vs_elbo = plot(elbos_list[i], title=\"Epoch vs ELBO ($mc_sample MC Sample)\", xlabel=\"Epochs\", ylabel=\"ELBO\", label=\"\")\n",
    "    savefig(mc_vs_elbo, \"images/mc_vs_elbo_$mc_sample.png\")\n",
    "end"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Julia 1.7.1",
   "language": "julia",
   "name": "julia-1.7"
  },
  "language_info": {
   "file_extension": ".jl",
   "mimetype": "application/julia",
   "name": "julia",
   "version": "1.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}