Merge pull request #41 from berkeley-dsep-infra/add_test_nb

Add a test notebook to trigger github action
berkeley-dsep-infra · May 10, 2024 · 5d1acd1 · 5d1acd1
2 parents 7c3bb4e + 1de1525
commit 5d1acd1
Showing 1 changed file with 283 additions and 0 deletions.
diff --git a/test_notebooks/query_logs_bigquery.ipynb b/test_notebooks/query_logs_bigquery.ipynb
@@ -0,0 +1,283 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "9283674e-7f91-4843-a172-ba53acb8c47e",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Requirement already satisfied: google-cloud-bigquery in /srv/conda/lib/python3.10/site-packages (3.21.0)\n",
+      "Requirement already satisfied: google-api-core!=2.0.*,!=2.1.*,!=2.10.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,<3.0.0dev,>=1.34.1 in /srv/conda/lib/python3.10/site-packages (from google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.10.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,<3.0.0dev,>=1.34.1->google-cloud-bigquery) (2.18.0)\n",
+      "Requirement already satisfied: google-auth<3.0.0dev,>=2.14.1 in /srv/conda/lib/python3.10/site-packages (from google-cloud-bigquery) (2.29.0)\n",
+      "Requirement already satisfied: google-cloud-core<3.0.0dev,>=1.6.0 in /srv/conda/lib/python3.10/site-packages (from google-cloud-bigquery) (2.4.1)\n",
+      "Requirement already satisfied: google-resumable-media<3.0dev,>=0.6.0 in /srv/conda/lib/python3.10/site-packages (from google-cloud-bigquery) (2.7.0)\n",
+      "Requirement already satisfied: packaging>=20.0.0 in /srv/conda/lib/python3.10/site-packages (from google-cloud-bigquery) (24.0)\n",
+      "Requirement already satisfied: python-dateutil<3.0dev,>=2.7.2 in /srv/conda/lib/python3.10/site-packages (from google-cloud-bigquery) (2.9.0)\n",
+      "Requirement already satisfied: requests<3.0.0dev,>=2.21.0 in /srv/conda/lib/python3.10/site-packages (from google-cloud-bigquery) (2.31.0)\n",
+      "Requirement already satisfied: googleapis-common-protos<2.0.dev0,>=1.56.2 in /srv/conda/lib/python3.10/site-packages (from google-api-core!=2.0.*,!=2.1.*,!=2.10.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,<3.0.0dev,>=1.34.1->google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.10.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,<3.0.0dev,>=1.34.1->google-cloud-bigquery) (1.63.0)\n",
+      "Requirement already satisfied: protobuf!=3.20.0,!=3.20.1,!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5,<5.0.0.dev0,>=3.19.5 in /srv/conda/lib/python3.10/site-packages (from google-api-core!=2.0.*,!=2.1.*,!=2.10.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,<3.0.0dev,>=1.34.1->google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.10.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,<3.0.0dev,>=1.34.1->google-cloud-bigquery) (4.25.3)\n",
+      "Requirement already satisfied: proto-plus<2.0.0dev,>=1.22.3 in /srv/conda/lib/python3.10/site-packages (from google-api-core!=2.0.*,!=2.1.*,!=2.10.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,<3.0.0dev,>=1.34.1->google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.10.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,<3.0.0dev,>=1.34.1->google-cloud-bigquery) (1.23.0)\n",
+      "Requirement already satisfied: grpcio<2.0dev,>=1.33.2 in /srv/conda/lib/python3.10/site-packages (from google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.10.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,<3.0.0dev,>=1.34.1->google-cloud-bigquery) (1.62.2)\n",
+      "Requirement already satisfied: grpcio-status<2.0.dev0,>=1.33.2 in /srv/conda/lib/python3.10/site-packages (from google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.10.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,<3.0.0dev,>=1.34.1->google-cloud-bigquery) (1.62.2)\n",
+      "Requirement already satisfied: cachetools<6.0,>=2.0.0 in /srv/conda/lib/python3.10/site-packages (from google-auth<3.0.0dev,>=2.14.1->google-cloud-bigquery) (5.3.3)\n",
+      "Requirement already satisfied: pyasn1-modules>=0.2.1 in /srv/conda/lib/python3.10/site-packages (from google-auth<3.0.0dev,>=2.14.1->google-cloud-bigquery) (0.4.0)\n",
+      "Requirement already satisfied: rsa<5,>=3.1.4 in /srv/conda/lib/python3.10/site-packages (from google-auth<3.0.0dev,>=2.14.1->google-cloud-bigquery) (4.9)\n",
+      "Requirement already satisfied: google-crc32c<2.0dev,>=1.0 in /srv/conda/lib/python3.10/site-packages (from google-resumable-media<3.0dev,>=0.6.0->google-cloud-bigquery) (1.5.0)\n",
+      "Requirement already satisfied: six>=1.5 in /srv/conda/lib/python3.10/site-packages (from python-dateutil<3.0dev,>=2.7.2->google-cloud-bigquery) (1.16.0)\n",
+      "Requirement already satisfied: charset-normalizer<4,>=2 in /srv/conda/lib/python3.10/site-packages (from requests<3.0.0dev,>=2.21.0->google-cloud-bigquery) (2.1.1)\n",
+      "Requirement already satisfied: idna<4,>=2.5 in /srv/conda/lib/python3.10/site-packages (from requests<3.0.0dev,>=2.21.0->google-cloud-bigquery) (3.4)\n",
+      "Requirement already satisfied: urllib3<3,>=1.21.1 in /srv/conda/lib/python3.10/site-packages (from requests<3.0.0dev,>=2.21.0->google-cloud-bigquery) (1.26.11)\n",
+      "Requirement already satisfied: certifi>=2017.4.17 in /srv/conda/lib/python3.10/site-packages (from requests<3.0.0dev,>=2.21.0->google-cloud-bigquery) (2024.2.2)\n",
+      "Requirement already satisfied: pyasn1<0.7.0,>=0.4.6 in /srv/conda/lib/python3.10/site-packages (from pyasn1-modules>=0.2.1->google-auth<3.0.0dev,>=2.14.1->google-cloud-bigquery) (0.6.0)\n"
+     ]
+    }
+   ],
+   "source": [
+    "#import logs data from bigquery\n",
+    "!pip install google-cloud-bigquery"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "d3e54884-3c0d-4adf-b580-cdd886484519",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Requirement already satisfied: db-dtypes in /srv/conda/lib/python3.10/site-packages (1.2.0)\n",
+      "Requirement already satisfied: packaging>=17.0 in /srv/conda/lib/python3.10/site-packages (from db-dtypes) (24.0)\n",
+      "Requirement already satisfied: pandas>=0.24.2 in /srv/conda/lib/python3.10/site-packages (from db-dtypes) (2.2.1)\n",
+      "Requirement already satisfied: pyarrow>=3.0.0 in /srv/conda/lib/python3.10/site-packages (from db-dtypes) (16.0.0)\n",
+      "Requirement already satisfied: numpy>=1.16.6 in /srv/conda/lib/python3.10/site-packages (from db-dtypes) (1.26.4)\n",
+      "Requirement already satisfied: python-dateutil>=2.8.2 in /srv/conda/lib/python3.10/site-packages (from pandas>=0.24.2->db-dtypes) (2.9.0)\n",
+      "Requirement already satisfied: pytz>=2020.1 in /srv/conda/lib/python3.10/site-packages (from pandas>=0.24.2->db-dtypes) (2024.1)\n",
+      "Requirement already satisfied: tzdata>=2022.7 in /srv/conda/lib/python3.10/site-packages (from pandas>=0.24.2->db-dtypes) (2024.1)\n",
+      "Requirement already satisfied: six>=1.5 in /srv/conda/lib/python3.10/site-packages (from python-dateutil>=2.8.2->pandas>=0.24.2->db-dtypes) (1.16.0)\n"
+     ]
+    }
+   ],
+   "source": [
+    "!pip install db-dtypes"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "fe89cd5d-2f24-4b66-8924-d016ec245234",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Requirement already satisfied: pandas_gbq in /srv/conda/lib/python3.10/site-packages (0.22.0)\n",
+      "Requirement already satisfied: setuptools in /srv/conda/lib/python3.10/site-packages (from pandas_gbq) (65.5.1)\n",
+      "Requirement already satisfied: db-dtypes<2.0.0,>=1.0.4 in /srv/conda/lib/python3.10/site-packages (from pandas_gbq) (1.2.0)\n",
+      "Requirement already satisfied: numpy>=1.16.6 in /srv/conda/lib/python3.10/site-packages (from pandas_gbq) (1.26.4)\n",
+      "Requirement already satisfied: pandas>=1.1.4 in /srv/conda/lib/python3.10/site-packages (from pandas_gbq) (2.2.1)\n",
+      "Requirement already satisfied: pyarrow>=3.0.0 in /srv/conda/lib/python3.10/site-packages (from pandas_gbq) (16.0.0)\n",
+      "Requirement already satisfied: pydata-google-auth>=1.5.0 in /srv/conda/lib/python3.10/site-packages (from pandas_gbq) (1.8.2)\n",
+      "Requirement already satisfied: google-api-core<3.0.0dev,>=2.10.2 in /srv/conda/lib/python3.10/site-packages (from pandas_gbq) (2.18.0)\n",
+      "Requirement already satisfied: google-auth>=2.13.0 in /srv/conda/lib/python3.10/site-packages (from pandas_gbq) (2.29.0)\n",
+      "Requirement already satisfied: google-auth-oauthlib>=0.7.0 in /srv/conda/lib/python3.10/site-packages (from pandas_gbq) (1.2.0)\n",
+      "Requirement already satisfied: google-cloud-bigquery<4.0.0dev,>=3.3.5 in /srv/conda/lib/python3.10/site-packages (from pandas_gbq) (3.21.0)\n",
+      "Requirement already satisfied: packaging>=20.0.0 in /srv/conda/lib/python3.10/site-packages (from pandas_gbq) (24.0)\n",
+      "Requirement already satisfied: googleapis-common-protos<2.0.dev0,>=1.56.2 in /srv/conda/lib/python3.10/site-packages (from google-api-core<3.0.0dev,>=2.10.2->pandas_gbq) (1.63.0)\n",
+      "Requirement already satisfied: protobuf!=3.20.0,!=3.20.1,!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5,<5.0.0.dev0,>=3.19.5 in /srv/conda/lib/python3.10/site-packages (from google-api-core<3.0.0dev,>=2.10.2->pandas_gbq) (4.25.3)\n",
+      "Requirement already satisfied: proto-plus<2.0.0dev,>=1.22.3 in /srv/conda/lib/python3.10/site-packages (from google-api-core<3.0.0dev,>=2.10.2->pandas_gbq) (1.23.0)\n",
+      "Requirement already satisfied: requests<3.0.0.dev0,>=2.18.0 in /srv/conda/lib/python3.10/site-packages (from google-api-core<3.0.0dev,>=2.10.2->pandas_gbq) (2.31.0)\n",
+      "Requirement already satisfied: cachetools<6.0,>=2.0.0 in /srv/conda/lib/python3.10/site-packages (from google-auth>=2.13.0->pandas_gbq) (5.3.3)\n",
+      "Requirement already satisfied: pyasn1-modules>=0.2.1 in /srv/conda/lib/python3.10/site-packages (from google-auth>=2.13.0->pandas_gbq) (0.4.0)\n",
+      "Requirement already satisfied: rsa<5,>=3.1.4 in /srv/conda/lib/python3.10/site-packages (from google-auth>=2.13.0->pandas_gbq) (4.9)\n",
+      "Requirement already satisfied: requests-oauthlib>=0.7.0 in /srv/conda/lib/python3.10/site-packages (from google-auth-oauthlib>=0.7.0->pandas_gbq) (2.0.0)\n",
+      "Requirement already satisfied: google-cloud-core<3.0.0dev,>=1.6.0 in /srv/conda/lib/python3.10/site-packages (from google-cloud-bigquery<4.0.0dev,>=3.3.5->pandas_gbq) (2.4.1)\n",
+      "Requirement already satisfied: google-resumable-media<3.0dev,>=0.6.0 in /srv/conda/lib/python3.10/site-packages (from google-cloud-bigquery<4.0.0dev,>=3.3.5->pandas_gbq) (2.7.0)\n",
+      "Requirement already satisfied: python-dateutil<3.0dev,>=2.7.2 in /srv/conda/lib/python3.10/site-packages (from google-cloud-bigquery<4.0.0dev,>=3.3.5->pandas_gbq) (2.9.0)\n",
+      "Requirement already satisfied: pytz>=2020.1 in /srv/conda/lib/python3.10/site-packages (from pandas>=1.1.4->pandas_gbq) (2024.1)\n",
+      "Requirement already satisfied: tzdata>=2022.7 in /srv/conda/lib/python3.10/site-packages (from pandas>=1.1.4->pandas_gbq) (2024.1)\n",
+      "Requirement already satisfied: grpcio<2.0dev,>=1.33.2 in /srv/conda/lib/python3.10/site-packages (from google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.10.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,<3.0.0dev,>=1.34.1->google-cloud-bigquery<4.0.0dev,>=3.3.5->pandas_gbq) (1.62.2)\n",
+      "Requirement already satisfied: grpcio-status<2.0.dev0,>=1.33.2 in /srv/conda/lib/python3.10/site-packages (from google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.10.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,<3.0.0dev,>=1.34.1->google-cloud-bigquery<4.0.0dev,>=3.3.5->pandas_gbq) (1.62.2)\n",
+      "Requirement already satisfied: google-crc32c<2.0dev,>=1.0 in /srv/conda/lib/python3.10/site-packages (from google-resumable-media<3.0dev,>=0.6.0->google-cloud-bigquery<4.0.0dev,>=3.3.5->pandas_gbq) (1.5.0)\n",
+      "Requirement already satisfied: pyasn1<0.7.0,>=0.4.6 in /srv/conda/lib/python3.10/site-packages (from pyasn1-modules>=0.2.1->google-auth>=2.13.0->pandas_gbq) (0.6.0)\n",
+      "Requirement already satisfied: six>=1.5 in /srv/conda/lib/python3.10/site-packages (from python-dateutil<3.0dev,>=2.7.2->google-cloud-bigquery<4.0.0dev,>=3.3.5->pandas_gbq) (1.16.0)\n",
+      "Requirement already satisfied: charset-normalizer<4,>=2 in /srv/conda/lib/python3.10/site-packages (from requests<3.0.0.dev0,>=2.18.0->google-api-core<3.0.0dev,>=2.10.2->pandas_gbq) (2.1.1)\n",
+      "Requirement already satisfied: idna<4,>=2.5 in /srv/conda/lib/python3.10/site-packages (from requests<3.0.0.dev0,>=2.18.0->google-api-core<3.0.0dev,>=2.10.2->pandas_gbq) (3.4)\n",
+      "Requirement already satisfied: urllib3<3,>=1.21.1 in /srv/conda/lib/python3.10/site-packages (from requests<3.0.0.dev0,>=2.18.0->google-api-core<3.0.0dev,>=2.10.2->pandas_gbq) (1.26.11)\n",
+      "Requirement already satisfied: certifi>=2017.4.17 in /srv/conda/lib/python3.10/site-packages (from requests<3.0.0.dev0,>=2.18.0->google-api-core<3.0.0dev,>=2.10.2->pandas_gbq) (2024.2.2)\n",
+      "Requirement already satisfied: oauthlib>=3.0.0 in /srv/conda/lib/python3.10/site-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib>=0.7.0->pandas_gbq) (3.2.2)\n"
+     ]
+    }
+   ],
+   "source": [
+    "!pip install pandas_gbq"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "479c71d9-1b55-4b07-96e4-003140de7423",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas_gbq"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "614fc9fb-885b-454c-aa4d-9b79849334d8",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from google.cloud import bigquery"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "1490d4ea-df53-4322-8e88-5b42cbe752d8",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "/home/rstudio/Datahub Semester by Semester Course Analysis/Python Scripts for querying course info\n"
+     ]
+    }
+   ],
+   "source": [
+    "import os\n",
+    "print(os.getcwd())\n",
+    "os.environ[\"GOOGLE_APPLICATION_CREDENTIALS\"]=\"ucb-datahub-2018-530ec2c2b5f0.json\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "5be40bfd-4c6c-4053-a91d-707bf99c0b93",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "client = bigquery.Client(project='ucb-datahub-2018')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "5df63ca0-4e53-453c-9f00-1106dd42bfb5",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "<google.cloud.bigquery.client.Client object at 0x7e049bf840a0>\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(client)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "a41a7124-db66-49b5-9410-9ac1199f8798",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "sql = \"\"\"\n",
+    "SELECT * FROM `ucb-datahub-2018.datahub_demo_log`\n",
+    "\"\"\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "9eb9260a-5f06-4e4b-8202-51037ca07d98",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "sql = \"\"\"\n",
+    "SELECT * FROM `ucb-datahub-2018.costs.gke_cluster_resource_usage`\n",
+    "\"\"\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "72521569-d1d3-4397-82f1-e55504d7fe52",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df = client.query(sql).to_dataframe()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "d1b56589-4a40-44b2-8f15-744c00d34e7f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df.head(20)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "8d363091-c077-4054-b2f6-ad0b3e19c61b",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "UsageError: Cell magic `%%bigquery` not found.\n"
+     ]
+    }
+   ],
+   "source": [
+    "%%bigquery results --project ucb-datahub-2018\n",
+    "SELECT * FROM `ucb-datahub-2018.costs.gke_cluster_resource_usage` #this t"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "50aca4b7-8fcb-4ae0-bc02-102e45c1fc6d",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}