From edfc87bfac672a0c3516877228cf5a3294bcae22 Mon Sep 17 00:00:00 2001 From: Balaji Alwar Date: Fri, 10 May 2024 14:15:22 -0700 Subject: [PATCH] Add test notebook! --- test_notebooks/query_logs_bigquery.ipynb | 1073 +++++++++++++++++++++- 1 file changed, 1026 insertions(+), 47 deletions(-) diff --git a/test_notebooks/query_logs_bigquery.ipynb b/test_notebooks/query_logs_bigquery.ipynb index 802a14e..e5f77f1 100644 --- a/test_notebooks/query_logs_bigquery.ipynb +++ b/test_notebooks/query_logs_bigquery.ipynb @@ -10,29 +10,61 @@ "name": "stdout", "output_type": "stream", "text": [ - "Requirement already satisfied: google-cloud-bigquery in /srv/conda/lib/python3.10/site-packages (3.21.0)\n", - "Requirement already satisfied: google-api-core!=2.0.*,!=2.1.*,!=2.10.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,<3.0.0dev,>=1.34.1 in /srv/conda/lib/python3.10/site-packages (from google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.10.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,<3.0.0dev,>=1.34.1->google-cloud-bigquery) (2.18.0)\n", - "Requirement already satisfied: google-auth<3.0.0dev,>=2.14.1 in /srv/conda/lib/python3.10/site-packages (from google-cloud-bigquery) (2.29.0)\n", - "Requirement already satisfied: google-cloud-core<3.0.0dev,>=1.6.0 in /srv/conda/lib/python3.10/site-packages (from google-cloud-bigquery) (2.4.1)\n", - "Requirement already satisfied: google-resumable-media<3.0dev,>=0.6.0 in /srv/conda/lib/python3.10/site-packages (from google-cloud-bigquery) (2.7.0)\n", + "Collecting google-cloud-bigquery\n", + " Using cached google_cloud_bigquery-3.21.0-py2.py3-none-any.whl.metadata (8.9 kB)\n", + "Collecting google-api-core!=2.0.*,!=2.1.*,!=2.10.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,<3.0.0dev,>=1.34.1 (from google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.10.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,<3.0.0dev,>=1.34.1->google-cloud-bigquery)\n", + " Using cached google_api_core-2.18.0-py3-none-any.whl.metadata (2.7 kB)\n", + "Collecting google-auth<3.0.0dev,>=2.14.1 (from google-cloud-bigquery)\n", + " Using cached google_auth-2.29.0-py2.py3-none-any.whl.metadata (4.7 kB)\n", + "Collecting google-cloud-core<3.0.0dev,>=1.6.0 (from google-cloud-bigquery)\n", + " Using cached google_cloud_core-2.4.1-py2.py3-none-any.whl.metadata (2.7 kB)\n", + "Collecting google-resumable-media<3.0dev,>=0.6.0 (from google-cloud-bigquery)\n", + " Using cached google_resumable_media-2.7.0-py2.py3-none-any.whl.metadata (2.2 kB)\n", "Requirement already satisfied: packaging>=20.0.0 in /srv/conda/lib/python3.10/site-packages (from google-cloud-bigquery) (24.0)\n", "Requirement already satisfied: python-dateutil<3.0dev,>=2.7.2 in /srv/conda/lib/python3.10/site-packages (from google-cloud-bigquery) (2.9.0)\n", "Requirement already satisfied: requests<3.0.0dev,>=2.21.0 in /srv/conda/lib/python3.10/site-packages (from google-cloud-bigquery) (2.31.0)\n", - "Requirement already satisfied: googleapis-common-protos<2.0.dev0,>=1.56.2 in /srv/conda/lib/python3.10/site-packages (from google-api-core!=2.0.*,!=2.1.*,!=2.10.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,<3.0.0dev,>=1.34.1->google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.10.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,<3.0.0dev,>=1.34.1->google-cloud-bigquery) (1.63.0)\n", - "Requirement already satisfied: protobuf!=3.20.0,!=3.20.1,!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5,<5.0.0.dev0,>=3.19.5 in /srv/conda/lib/python3.10/site-packages (from google-api-core!=2.0.*,!=2.1.*,!=2.10.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,<3.0.0dev,>=1.34.1->google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.10.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,<3.0.0dev,>=1.34.1->google-cloud-bigquery) (4.25.3)\n", - "Requirement already satisfied: proto-plus<2.0.0dev,>=1.22.3 in /srv/conda/lib/python3.10/site-packages (from google-api-core!=2.0.*,!=2.1.*,!=2.10.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,<3.0.0dev,>=1.34.1->google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.10.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,<3.0.0dev,>=1.34.1->google-cloud-bigquery) (1.23.0)\n", - "Requirement already satisfied: grpcio<2.0dev,>=1.33.2 in /srv/conda/lib/python3.10/site-packages (from google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.10.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,<3.0.0dev,>=1.34.1->google-cloud-bigquery) (1.62.2)\n", - "Requirement already satisfied: grpcio-status<2.0.dev0,>=1.33.2 in /srv/conda/lib/python3.10/site-packages (from google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.10.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,<3.0.0dev,>=1.34.1->google-cloud-bigquery) (1.62.2)\n", - "Requirement already satisfied: cachetools<6.0,>=2.0.0 in /srv/conda/lib/python3.10/site-packages (from google-auth<3.0.0dev,>=2.14.1->google-cloud-bigquery) (5.3.3)\n", - "Requirement already satisfied: pyasn1-modules>=0.2.1 in /srv/conda/lib/python3.10/site-packages (from google-auth<3.0.0dev,>=2.14.1->google-cloud-bigquery) (0.4.0)\n", - "Requirement already satisfied: rsa<5,>=3.1.4 in /srv/conda/lib/python3.10/site-packages (from google-auth<3.0.0dev,>=2.14.1->google-cloud-bigquery) (4.9)\n", - "Requirement already satisfied: google-crc32c<2.0dev,>=1.0 in /srv/conda/lib/python3.10/site-packages (from google-resumable-media<3.0dev,>=0.6.0->google-cloud-bigquery) (1.5.0)\n", + "Collecting googleapis-common-protos<2.0.dev0,>=1.56.2 (from google-api-core!=2.0.*,!=2.1.*,!=2.10.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,<3.0.0dev,>=1.34.1->google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.10.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,<3.0.0dev,>=1.34.1->google-cloud-bigquery)\n", + " Using cached googleapis_common_protos-1.63.0-py2.py3-none-any.whl.metadata (1.5 kB)\n", + "Collecting protobuf!=3.20.0,!=3.20.1,!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5,<5.0.0.dev0,>=3.19.5 (from google-api-core!=2.0.*,!=2.1.*,!=2.10.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,<3.0.0dev,>=1.34.1->google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.10.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,<3.0.0dev,>=1.34.1->google-cloud-bigquery)\n", + " Using cached protobuf-4.25.3-cp37-abi3-manylinux2014_x86_64.whl.metadata (541 bytes)\n", + "Collecting proto-plus<2.0.0dev,>=1.22.3 (from google-api-core!=2.0.*,!=2.1.*,!=2.10.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,<3.0.0dev,>=1.34.1->google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.10.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,<3.0.0dev,>=1.34.1->google-cloud-bigquery)\n", + " Using cached proto_plus-1.23.0-py3-none-any.whl.metadata (2.2 kB)\n", + "Collecting grpcio<2.0dev,>=1.33.2 (from google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.10.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,<3.0.0dev,>=1.34.1->google-cloud-bigquery)\n", + " Using cached grpcio-1.62.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (4.0 kB)\n", + "Collecting grpcio-status<2.0.dev0,>=1.33.2 (from google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.10.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,<3.0.0dev,>=1.34.1->google-cloud-bigquery)\n", + " Using cached grpcio_status-1.62.2-py3-none-any.whl.metadata (1.3 kB)\n", + "Collecting cachetools<6.0,>=2.0.0 (from google-auth<3.0.0dev,>=2.14.1->google-cloud-bigquery)\n", + " Using cached cachetools-5.3.3-py3-none-any.whl.metadata (5.3 kB)\n", + "Collecting pyasn1-modules>=0.2.1 (from google-auth<3.0.0dev,>=2.14.1->google-cloud-bigquery)\n", + " Using cached pyasn1_modules-0.4.0-py3-none-any.whl.metadata (3.4 kB)\n", + "Collecting rsa<5,>=3.1.4 (from google-auth<3.0.0dev,>=2.14.1->google-cloud-bigquery)\n", + " Using cached rsa-4.9-py3-none-any.whl.metadata (4.2 kB)\n", + "Collecting google-crc32c<2.0dev,>=1.0 (from google-resumable-media<3.0dev,>=0.6.0->google-cloud-bigquery)\n", + " Using cached google_crc32c-1.5.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (2.3 kB)\n", "Requirement already satisfied: six>=1.5 in /srv/conda/lib/python3.10/site-packages (from python-dateutil<3.0dev,>=2.7.2->google-cloud-bigquery) (1.16.0)\n", "Requirement already satisfied: charset-normalizer<4,>=2 in /srv/conda/lib/python3.10/site-packages (from requests<3.0.0dev,>=2.21.0->google-cloud-bigquery) (2.1.1)\n", "Requirement already satisfied: idna<4,>=2.5 in /srv/conda/lib/python3.10/site-packages (from requests<3.0.0dev,>=2.21.0->google-cloud-bigquery) (3.4)\n", "Requirement already satisfied: urllib3<3,>=1.21.1 in /srv/conda/lib/python3.10/site-packages (from requests<3.0.0dev,>=2.21.0->google-cloud-bigquery) (1.26.11)\n", "Requirement already satisfied: certifi>=2017.4.17 in /srv/conda/lib/python3.10/site-packages (from requests<3.0.0dev,>=2.21.0->google-cloud-bigquery) (2024.2.2)\n", - "Requirement already satisfied: pyasn1<0.7.0,>=0.4.6 in /srv/conda/lib/python3.10/site-packages (from pyasn1-modules>=0.2.1->google-auth<3.0.0dev,>=2.14.1->google-cloud-bigquery) (0.6.0)\n" + "Collecting pyasn1<0.7.0,>=0.4.6 (from pyasn1-modules>=0.2.1->google-auth<3.0.0dev,>=2.14.1->google-cloud-bigquery)\n", + " Using cached pyasn1-0.6.0-py2.py3-none-any.whl.metadata (8.3 kB)\n", + "Using cached google_cloud_bigquery-3.21.0-py2.py3-none-any.whl (235 kB)\n", + "Using cached google_api_core-2.18.0-py3-none-any.whl (138 kB)\n", + "Using cached google_auth-2.29.0-py2.py3-none-any.whl (189 kB)\n", + "Using cached google_cloud_core-2.4.1-py2.py3-none-any.whl (29 kB)\n", + "Using cached google_resumable_media-2.7.0-py2.py3-none-any.whl (80 kB)\n", + "Using cached cachetools-5.3.3-py3-none-any.whl (9.3 kB)\n", + "Using cached google_crc32c-1.5.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (32 kB)\n", + "Using cached googleapis_common_protos-1.63.0-py2.py3-none-any.whl (229 kB)\n", + "Using cached grpcio-1.62.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (5.5 MB)\n", + "Using cached grpcio_status-1.62.2-py3-none-any.whl (14 kB)\n", + "Using cached proto_plus-1.23.0-py3-none-any.whl (48 kB)\n", + "Using cached protobuf-4.25.3-cp37-abi3-manylinux2014_x86_64.whl (294 kB)\n", + "Using cached pyasn1_modules-0.4.0-py3-none-any.whl (181 kB)\n", + "Using cached rsa-4.9-py3-none-any.whl (34 kB)\n", + "Using cached pyasn1-0.6.0-py2.py3-none-any.whl (85 kB)\n", + "Installing collected packages: pyasn1, protobuf, grpcio, google-crc32c, cachetools, rsa, pyasn1-modules, proto-plus, googleapis-common-protos, google-resumable-media, grpcio-status, google-auth, google-api-core, google-cloud-core, google-cloud-bigquery\n", + "Successfully installed cachetools-5.3.3 google-api-core-2.18.0 google-auth-2.29.0 google-cloud-bigquery-3.21.0 google-cloud-core-2.4.1 google-crc32c-1.5.0 google-resumable-media-2.7.0 googleapis-common-protos-1.63.0 grpcio-1.62.2 grpcio-status-1.62.2 proto-plus-1.23.0 protobuf-4.25.3 pyasn1-0.6.0 pyasn1-modules-0.4.0 rsa-4.9\n" ] } ], @@ -51,15 +83,21 @@ "name": "stdout", "output_type": "stream", "text": [ - "Requirement already satisfied: db-dtypes in /srv/conda/lib/python3.10/site-packages (1.2.0)\n", + "Collecting db-dtypes\n", + " Using cached db_dtypes-1.2.0-py2.py3-none-any.whl.metadata (3.0 kB)\n", "Requirement already satisfied: packaging>=17.0 in /srv/conda/lib/python3.10/site-packages (from db-dtypes) (24.0)\n", "Requirement already satisfied: pandas>=0.24.2 in /srv/conda/lib/python3.10/site-packages (from db-dtypes) (2.2.1)\n", - "Requirement already satisfied: pyarrow>=3.0.0 in /srv/conda/lib/python3.10/site-packages (from db-dtypes) (16.0.0)\n", + "Collecting pyarrow>=3.0.0 (from db-dtypes)\n", + " Using cached pyarrow-16.0.0-cp310-cp310-manylinux_2_28_x86_64.whl.metadata (3.0 kB)\n", "Requirement already satisfied: numpy>=1.16.6 in /srv/conda/lib/python3.10/site-packages (from db-dtypes) (1.26.4)\n", "Requirement already satisfied: python-dateutil>=2.8.2 in /srv/conda/lib/python3.10/site-packages (from pandas>=0.24.2->db-dtypes) (2.9.0)\n", "Requirement already satisfied: pytz>=2020.1 in /srv/conda/lib/python3.10/site-packages (from pandas>=0.24.2->db-dtypes) (2024.1)\n", "Requirement already satisfied: tzdata>=2022.7 in /srv/conda/lib/python3.10/site-packages (from pandas>=0.24.2->db-dtypes) (2024.1)\n", - "Requirement already satisfied: six>=1.5 in /srv/conda/lib/python3.10/site-packages (from python-dateutil>=2.8.2->pandas>=0.24.2->db-dtypes) (1.16.0)\n" + "Requirement already satisfied: six>=1.5 in /srv/conda/lib/python3.10/site-packages (from python-dateutil>=2.8.2->pandas>=0.24.2->db-dtypes) (1.16.0)\n", + "Using cached db_dtypes-1.2.0-py2.py3-none-any.whl (14 kB)\n", + "Using cached pyarrow-16.0.0-cp310-cp310-manylinux_2_28_x86_64.whl (40.8 MB)\n", + "Installing collected packages: pyarrow, db-dtypes\n", + "Successfully installed db-dtypes-1.2.0 pyarrow-16.0.0\n" ] } ], @@ -77,16 +115,19 @@ "name": "stdout", "output_type": "stream", "text": [ - "Requirement already satisfied: pandas_gbq in /srv/conda/lib/python3.10/site-packages (0.22.0)\n", + "Collecting pandas_gbq\n", + " Using cached pandas_gbq-0.22.0-py2.py3-none-any.whl.metadata (3.4 kB)\n", "Requirement already satisfied: setuptools in /srv/conda/lib/python3.10/site-packages (from pandas_gbq) (65.5.1)\n", "Requirement already satisfied: db-dtypes<2.0.0,>=1.0.4 in /srv/conda/lib/python3.10/site-packages (from pandas_gbq) (1.2.0)\n", "Requirement already satisfied: numpy>=1.16.6 in /srv/conda/lib/python3.10/site-packages (from pandas_gbq) (1.26.4)\n", "Requirement already satisfied: pandas>=1.1.4 in /srv/conda/lib/python3.10/site-packages (from pandas_gbq) (2.2.1)\n", "Requirement already satisfied: pyarrow>=3.0.0 in /srv/conda/lib/python3.10/site-packages (from pandas_gbq) (16.0.0)\n", - "Requirement already satisfied: pydata-google-auth>=1.5.0 in /srv/conda/lib/python3.10/site-packages (from pandas_gbq) (1.8.2)\n", + "Collecting pydata-google-auth>=1.5.0 (from pandas_gbq)\n", + " Using cached pydata_google_auth-1.8.2-py2.py3-none-any.whl.metadata (3.2 kB)\n", "Requirement already satisfied: google-api-core<3.0.0dev,>=2.10.2 in /srv/conda/lib/python3.10/site-packages (from pandas_gbq) (2.18.0)\n", "Requirement already satisfied: google-auth>=2.13.0 in /srv/conda/lib/python3.10/site-packages (from pandas_gbq) (2.29.0)\n", - "Requirement already satisfied: google-auth-oauthlib>=0.7.0 in /srv/conda/lib/python3.10/site-packages (from pandas_gbq) (1.2.0)\n", + "Collecting google-auth-oauthlib>=0.7.0 (from pandas_gbq)\n", + " Using cached google_auth_oauthlib-1.2.0-py2.py3-none-any.whl.metadata (2.7 kB)\n", "Requirement already satisfied: google-cloud-bigquery<4.0.0dev,>=3.3.5 in /srv/conda/lib/python3.10/site-packages (from pandas_gbq) (3.21.0)\n", "Requirement already satisfied: packaging>=20.0.0 in /srv/conda/lib/python3.10/site-packages (from pandas_gbq) (24.0)\n", "Requirement already satisfied: googleapis-common-protos<2.0.dev0,>=1.56.2 in /srv/conda/lib/python3.10/site-packages (from google-api-core<3.0.0dev,>=2.10.2->pandas_gbq) (1.63.0)\n", @@ -96,7 +137,8 @@ "Requirement already satisfied: cachetools<6.0,>=2.0.0 in /srv/conda/lib/python3.10/site-packages (from google-auth>=2.13.0->pandas_gbq) (5.3.3)\n", "Requirement already satisfied: pyasn1-modules>=0.2.1 in /srv/conda/lib/python3.10/site-packages (from google-auth>=2.13.0->pandas_gbq) (0.4.0)\n", "Requirement already satisfied: rsa<5,>=3.1.4 in /srv/conda/lib/python3.10/site-packages (from google-auth>=2.13.0->pandas_gbq) (4.9)\n", - "Requirement already satisfied: requests-oauthlib>=0.7.0 in /srv/conda/lib/python3.10/site-packages (from google-auth-oauthlib>=0.7.0->pandas_gbq) (2.0.0)\n", + "Collecting requests-oauthlib>=0.7.0 (from google-auth-oauthlib>=0.7.0->pandas_gbq)\n", + " Using cached requests_oauthlib-2.0.0-py2.py3-none-any.whl.metadata (11 kB)\n", "Requirement already satisfied: google-cloud-core<3.0.0dev,>=1.6.0 in /srv/conda/lib/python3.10/site-packages (from google-cloud-bigquery<4.0.0dev,>=3.3.5->pandas_gbq) (2.4.1)\n", "Requirement already satisfied: google-resumable-media<3.0dev,>=0.6.0 in /srv/conda/lib/python3.10/site-packages (from google-cloud-bigquery<4.0.0dev,>=3.3.5->pandas_gbq) (2.7.0)\n", "Requirement already satisfied: python-dateutil<3.0dev,>=2.7.2 in /srv/conda/lib/python3.10/site-packages (from google-cloud-bigquery<4.0.0dev,>=3.3.5->pandas_gbq) (2.9.0)\n", @@ -111,7 +153,13 @@ "Requirement already satisfied: idna<4,>=2.5 in /srv/conda/lib/python3.10/site-packages (from requests<3.0.0.dev0,>=2.18.0->google-api-core<3.0.0dev,>=2.10.2->pandas_gbq) (3.4)\n", "Requirement already satisfied: urllib3<3,>=1.21.1 in /srv/conda/lib/python3.10/site-packages (from requests<3.0.0.dev0,>=2.18.0->google-api-core<3.0.0dev,>=2.10.2->pandas_gbq) (1.26.11)\n", "Requirement already satisfied: certifi>=2017.4.17 in /srv/conda/lib/python3.10/site-packages (from requests<3.0.0.dev0,>=2.18.0->google-api-core<3.0.0dev,>=2.10.2->pandas_gbq) (2024.2.2)\n", - "Requirement already satisfied: oauthlib>=3.0.0 in /srv/conda/lib/python3.10/site-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib>=0.7.0->pandas_gbq) (3.2.2)\n" + "Requirement already satisfied: oauthlib>=3.0.0 in /srv/conda/lib/python3.10/site-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib>=0.7.0->pandas_gbq) (3.2.2)\n", + "Using cached pandas_gbq-0.22.0-py2.py3-none-any.whl (26 kB)\n", + "Using cached google_auth_oauthlib-1.2.0-py2.py3-none-any.whl (24 kB)\n", + "Using cached pydata_google_auth-1.8.2-py2.py3-none-any.whl (15 kB)\n", + "Using cached requests_oauthlib-2.0.0-py2.py3-none-any.whl (24 kB)\n", + "Installing collected packages: requests-oauthlib, google-auth-oauthlib, pydata-google-auth, pandas_gbq\n", + "Successfully installed google-auth-oauthlib-1.2.0 pandas_gbq-0.22.0 pydata-google-auth-1.8.2 requests-oauthlib-2.0.0\n" ] } ], @@ -179,7 +227,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "\n" + "\n" ] } ], @@ -189,71 +237,1002 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 38, "id": "a41a7124-db66-49b5-9410-9ac1199f8798", "metadata": {}, "outputs": [], "source": [ - "sql = \"\"\"\n", - "SELECT * FROM `ucb-datahub-2018.datahub_demo_log`\n", + "sql_gceguest = \"\"\"\n", + "SELECT * FROM `ucb-datahub-2018.datahub_spring24.GCEGuestAgent_20240423`\n", "\"\"\"" ] }, { "cell_type": "code", - "execution_count": 11, - "id": "9eb9260a-5f06-4e4b-8202-51037ca07d98", + "execution_count": 39, + "id": "72521569-d1d3-4397-82f1-e55504d7fe52", "metadata": {}, "outputs": [], "source": [ - "sql = \"\"\"\n", - "SELECT * FROM `ucb-datahub-2018.costs.gke_cluster_resource_usage`\n", - "\"\"\"" + "df_gceguest = client.query(sql_gceguest).to_dataframe()" ] }, { "cell_type": "code", - "execution_count": null, - "id": "72521569-d1d3-4397-82f1-e55504d7fe52", + "execution_count": 40, + "id": "d1b56589-4a40-44b2-8f15-744c00d34e7f", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "(351, 17)" + ] + }, + "execution_count": 40, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "df = client.query(sql).to_dataframe()" + "df_gceguest.shape" ] }, { "cell_type": "code", - "execution_count": null, - "id": "d1b56589-4a40-44b2-8f15-744c00d34e7f", + "execution_count": 62, + "id": "c20a1b8c-d5d7-4e36-9a03-659f11615e8d", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
logNameresourcetextPayloadjsonPayloadtimestampreceiveTimestampseverityinsertIdhttpRequestlabelsoperationtracespanIdtraceSampledsourceLocationspliterrorGroups
0projects/ucb-datahub-2018/logs/GCEGuestAgent{'type': 'gce_instance', 'labels': {'instance_...None{'localtimestamp': '2024-04-23T22:10:58.1077Z'...2024-04-23 22:10:58.107794+00:002024-04-23 22:10:59.056146+00:00INFOy56q1qf5rdac1None{'instance_name': 'gke-fall-2019-user-data8-20...NoneNoneNone<NA>{'file': 'non_windows_accounts.go', 'line': 15...None[]
1projects/ucb-datahub-2018/logs/GCEGuestAgent{'type': 'gce_instance', 'labels': {'instance_...None{'localtimestamp': '2024-04-23T22:10:58.2791Z'...2024-04-23 22:10:58.279183+00:002024-04-23 22:10:59.056146+00:00INFOy56q1qf5rdacgNone{'instance_name': 'gke-fall-2019-user-data8-20...NoneNoneNone<NA>{'file': 'non_windows_accounts.go', 'line': 14...None[]
2projects/ucb-datahub-2018/logs/GCEGuestAgent{'type': 'gce_instance', 'labels': {'instance_...None{'localtimestamp': '2024-04-23T22:10:58.2566Z'...2024-04-23 22:10:58.256725+00:002024-04-23 22:10:59.056146+00:00INFOy56q1qf5rdacdNone{'instance_name': 'gke-fall-2019-user-data8-20...NoneNoneNone<NA>{'file': 'non_windows_accounts.go', 'line': 15...None[]
3projects/ucb-datahub-2018/logs/GCEGuestAgent{'type': 'gce_instance', 'labels': {'instance_...None{'localtimestamp': '2024-04-23T22:10:58.2789Z'...2024-04-23 22:10:58.278964+00:002024-04-23 22:10:59.056146+00:00INFOy56q1qf5rdacfNone{'instance_name': 'gke-fall-2019-user-data8-20...NoneNoneNone<NA>{'file': 'non_windows_accounts.go', 'line': 15...None[]
4projects/ucb-datahub-2018/logs/GCEGuestAgent{'type': 'gce_instance', 'labels': {'instance_...None{'localtimestamp': '2024-04-23T22:10:58.2076Z'...2024-04-23 22:10:58.207703+00:002024-04-23 22:10:59.056146+00:00INFOy56q1qf5rdac9None{'instance_name': 'gke-fall-2019-user-data8-20...NoneNoneNone<NA>{'file': 'non_windows_accounts.go', 'line': 15...None[]
......................................................
95projects/ucb-datahub-2018/logs/GCEGuestAgent{'type': 'gce_instance', 'labels': {'instance_...None{'localtimestamp': '2024-04-23T21:26:38.8015Z'...2024-04-23 21:26:38.801591+00:002024-04-23 21:26:39.427606+00:00INFO1ygpnrfa9ye7kNone{'instance_name': 'gke-fall-2019-user-stat20-2...NoneNoneNone<NA>{'file': 'non_windows_accounts.go', 'line': 14...None[]
96projects/ucb-datahub-2018/logs/GCEGuestAgent{'type': 'gce_instance', 'labels': {'instance_...None{'localtimestamp': '2024-04-23T21:26:38.2349Z'...2024-04-23 21:26:38.235307+00:002024-04-23 21:26:39.427606+00:00INFO1ygpnrfa9ye6zNone{'instance_name': 'gke-fall-2019-user-stat20-2...NoneNoneNone<NA>{'file': 'main.go', 'line': 144, 'function': '...None[]
97projects/ucb-datahub-2018/logs/GCEGuestAgent{'type': 'gce_instance', 'labels': {'instance_...None{'localtimestamp': '2024-04-23T21:26:38.5919Z'...2024-04-23 21:26:38.591951+00:002024-04-23 21:26:39.427606+00:00INFO1ygpnrfa9ye73None{'instance_name': 'gke-fall-2019-user-stat20-2...NoneNoneNone<NA>{'file': 'non_windows_accounts.go', 'line': 39...None[]
98projects/ucb-datahub-2018/logs/GCEGuestAgent{'type': 'gce_instance', 'labels': {'instance_...None{'localtimestamp': '2024-04-23T21:26:38.6495Z'...2024-04-23 21:26:38.649562+00:002024-04-23 21:26:39.427606+00:00INFO1ygpnrfa9ye77None{'instance_name': 'gke-fall-2019-user-stat20-2...NoneNoneNone<NA>{'file': 'non_windows_accounts.go', 'line': 15...None[]
99projects/ucb-datahub-2018/logs/GCEGuestAgent{'type': 'gce_instance', 'labels': {'instance_...None{'localtimestamp': '2024-04-23T21:26:38.8244Z'...2024-04-23 21:26:38.824443+00:002024-04-23 21:26:39.427606+00:00INFO1ygpnrfa9ye7lNone{'instance_name': 'gke-fall-2019-user-stat20-2...NoneNoneNone<NA>{'file': 'non_windows_accounts.go', 'line': 15...None[]
\n", + "

100 rows × 17 columns

\n", + "
" + ], + "text/plain": [ + " logName \\\n", + "0 projects/ucb-datahub-2018/logs/GCEGuestAgent \n", + "1 projects/ucb-datahub-2018/logs/GCEGuestAgent \n", + "2 projects/ucb-datahub-2018/logs/GCEGuestAgent \n", + "3 projects/ucb-datahub-2018/logs/GCEGuestAgent \n", + "4 projects/ucb-datahub-2018/logs/GCEGuestAgent \n", + ".. ... \n", + "95 projects/ucb-datahub-2018/logs/GCEGuestAgent \n", + "96 projects/ucb-datahub-2018/logs/GCEGuestAgent \n", + "97 projects/ucb-datahub-2018/logs/GCEGuestAgent \n", + "98 projects/ucb-datahub-2018/logs/GCEGuestAgent \n", + "99 projects/ucb-datahub-2018/logs/GCEGuestAgent \n", + "\n", + " resource textPayload \\\n", + "0 {'type': 'gce_instance', 'labels': {'instance_... None \n", + "1 {'type': 'gce_instance', 'labels': {'instance_... None \n", + "2 {'type': 'gce_instance', 'labels': {'instance_... None \n", + "3 {'type': 'gce_instance', 'labels': {'instance_... None \n", + "4 {'type': 'gce_instance', 'labels': {'instance_... None \n", + ".. ... ... \n", + "95 {'type': 'gce_instance', 'labels': {'instance_... None \n", + "96 {'type': 'gce_instance', 'labels': {'instance_... None \n", + "97 {'type': 'gce_instance', 'labels': {'instance_... None \n", + "98 {'type': 'gce_instance', 'labels': {'instance_... None \n", + "99 {'type': 'gce_instance', 'labels': {'instance_... None \n", + "\n", + " jsonPayload \\\n", + "0 {'localtimestamp': '2024-04-23T22:10:58.1077Z'... \n", + "1 {'localtimestamp': '2024-04-23T22:10:58.2791Z'... \n", + "2 {'localtimestamp': '2024-04-23T22:10:58.2566Z'... \n", + "3 {'localtimestamp': '2024-04-23T22:10:58.2789Z'... \n", + "4 {'localtimestamp': '2024-04-23T22:10:58.2076Z'... \n", + ".. ... \n", + "95 {'localtimestamp': '2024-04-23T21:26:38.8015Z'... \n", + "96 {'localtimestamp': '2024-04-23T21:26:38.2349Z'... \n", + "97 {'localtimestamp': '2024-04-23T21:26:38.5919Z'... \n", + "98 {'localtimestamp': '2024-04-23T21:26:38.6495Z'... \n", + "99 {'localtimestamp': '2024-04-23T21:26:38.8244Z'... \n", + "\n", + " timestamp receiveTimestamp severity \\\n", + "0 2024-04-23 22:10:58.107794+00:00 2024-04-23 22:10:59.056146+00:00 INFO \n", + "1 2024-04-23 22:10:58.279183+00:00 2024-04-23 22:10:59.056146+00:00 INFO \n", + "2 2024-04-23 22:10:58.256725+00:00 2024-04-23 22:10:59.056146+00:00 INFO \n", + "3 2024-04-23 22:10:58.278964+00:00 2024-04-23 22:10:59.056146+00:00 INFO \n", + "4 2024-04-23 22:10:58.207703+00:00 2024-04-23 22:10:59.056146+00:00 INFO \n", + ".. ... ... ... \n", + "95 2024-04-23 21:26:38.801591+00:00 2024-04-23 21:26:39.427606+00:00 INFO \n", + "96 2024-04-23 21:26:38.235307+00:00 2024-04-23 21:26:39.427606+00:00 INFO \n", + "97 2024-04-23 21:26:38.591951+00:00 2024-04-23 21:26:39.427606+00:00 INFO \n", + "98 2024-04-23 21:26:38.649562+00:00 2024-04-23 21:26:39.427606+00:00 INFO \n", + "99 2024-04-23 21:26:38.824443+00:00 2024-04-23 21:26:39.427606+00:00 INFO \n", + "\n", + " insertId httpRequest \\\n", + "0 y56q1qf5rdac1 None \n", + "1 y56q1qf5rdacg None \n", + "2 y56q1qf5rdacd None \n", + "3 y56q1qf5rdacf None \n", + "4 y56q1qf5rdac9 None \n", + ".. ... ... \n", + "95 1ygpnrfa9ye7k None \n", + "96 1ygpnrfa9ye6z None \n", + "97 1ygpnrfa9ye73 None \n", + "98 1ygpnrfa9ye77 None \n", + "99 1ygpnrfa9ye7l None \n", + "\n", + " labels operation trace spanId \\\n", + "0 {'instance_name': 'gke-fall-2019-user-data8-20... None None None \n", + "1 {'instance_name': 'gke-fall-2019-user-data8-20... None None None \n", + "2 {'instance_name': 'gke-fall-2019-user-data8-20... None None None \n", + "3 {'instance_name': 'gke-fall-2019-user-data8-20... None None None \n", + "4 {'instance_name': 'gke-fall-2019-user-data8-20... None None None \n", + ".. ... ... ... ... \n", + "95 {'instance_name': 'gke-fall-2019-user-stat20-2... None None None \n", + "96 {'instance_name': 'gke-fall-2019-user-stat20-2... None None None \n", + "97 {'instance_name': 'gke-fall-2019-user-stat20-2... None None None \n", + "98 {'instance_name': 'gke-fall-2019-user-stat20-2... None None None \n", + "99 {'instance_name': 'gke-fall-2019-user-stat20-2... None None None \n", + "\n", + " traceSampled sourceLocation split \\\n", + "0 {'file': 'non_windows_accounts.go', 'line': 15... None \n", + "1 {'file': 'non_windows_accounts.go', 'line': 14... None \n", + "2 {'file': 'non_windows_accounts.go', 'line': 15... None \n", + "3 {'file': 'non_windows_accounts.go', 'line': 15... None \n", + "4 {'file': 'non_windows_accounts.go', 'line': 15... None \n", + ".. ... ... ... \n", + "95 {'file': 'non_windows_accounts.go', 'line': 14... None \n", + "96 {'file': 'main.go', 'line': 144, 'function': '... None \n", + "97 {'file': 'non_windows_accounts.go', 'line': 39... None \n", + "98 {'file': 'non_windows_accounts.go', 'line': 15... None \n", + "99 {'file': 'non_windows_accounts.go', 'line': 15... None \n", + "\n", + " errorGroups \n", + "0 [] \n", + "1 [] \n", + "2 [] \n", + "3 [] \n", + "4 [] \n", + ".. ... \n", + "95 [] \n", + "96 [] \n", + "97 [] \n", + "98 [] \n", + "99 [] \n", + "\n", + "[100 rows x 17 columns]" + ] + }, + "execution_count": 62, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "df.head(20)" + "df_gceguest.head(100)" ] }, { "cell_type": "code", - "execution_count": 11, - "id": "8d363091-c077-4054-b2f6-ad0b3e19c61b", + "execution_count": 42, + "id": "40afc376-ded5-46de-a020-357632c70f52", "metadata": {}, "outputs": [ { - "name": "stderr", + "name": "stdout", "output_type": "stream", "text": [ - "UsageError: Cell magic `%%bigquery` not found.\n" + "{'file': 'non_windows_accounts.go', 'line': 142, 'function': 'main.(*accountsMgr).set'}\n" ] } ], "source": [ - "%%bigquery results --project ucb-datahub-2018\n", - "SELECT * FROM `ucb-datahub-2018.costs.gke_cluster_resource_usage` #this t" + "print(df_gceguest.sourceLocation[1])" + ] + }, + { + "cell_type": "code", + "execution_count": 54, + "id": "a50476ea-995f-4350-ba07-cbda51dfda09", + "metadata": {}, + "outputs": [], + "source": [ + "sql_stderr = \"\"\"\n", + "SELECT * FROM `ucb-datahub-2018.datahub_spring24.stderr_20240424`\n", + "\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 55, + "id": "66b30147-5e76-4eb9-88be-8d69ab61e683", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
logNameresourcetextPayloadtimestampreceiveTimestampseverityinsertIdhttpRequestlabelsoperationtracespanIdtraceSampledsourceLocationspliterrorGroupsjsonPayload
\n", + "
" + ], + "text/plain": [ + "Empty DataFrame\n", + "Columns: [logName, resource, textPayload, timestamp, receiveTimestamp, severity, insertId, httpRequest, labels, operation, trace, spanId, traceSampled, sourceLocation, split, errorGroups, jsonPayload]\n", + "Index: []" + ] + }, + "execution_count": 55, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df_stderr = client.query(sql_stderr).to_dataframe()\n", + "df_stderr.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 56, + "id": "28975f08-4d7f-4cb2-8831-888fd08d5a82", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(0, 17)" + ] + }, + "execution_count": 56, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df_stderr.shape" + ] + }, + { + "cell_type": "code", + "execution_count": 57, + "id": "f8b00ab3-6c9f-4aec-abd1-d8b16b3e264a", + "metadata": {}, + "outputs": [], + "source": [ + "# Getting stderr logs\n", + "sql_OSConfigAgent = \"\"\"\n", + "SELECT * FROM `ucb-datahub-2018.datahub_spring24.OSConfigAgent_20240423`\n", + "\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 58, + "id": "f25c1eef-fc37-40f8-a13e-a89b18a614f0", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
logNameresourcetextPayloadjsonPayloadtimestampreceiveTimestampseverityinsertIdhttpRequestlabelsoperationtracespanIdtraceSampledsourceLocationspliterrorGroups
0projects/ucb-datahub-2018/logs/OSConfigAgent{'type': 'k8s_container', 'labels': {'location...None{'localtimestamp': '2024-04-23T21:23:34.4536Z'...2024-04-23 21:23:34.453695+00:002024-04-23 21:23:35.641507+00:00INFOad3aqffwqqo8None{'agent_version': '20230403.00', 'instance_nam...NoneNoneNone<NA>{'file': 'main.go', 'line': 151, 'function': '...None[]
1projects/ucb-datahub-2018/logs/OSConfigAgent{'type': 'k8s_container', 'labels': {'location...None{'localtimestamp': '2024-04-23T21:26:38.3250Z'...2024-04-23 21:26:38.325492+00:002024-04-23 21:26:39.502759+00:00INFOx1226heyjxs6None{'agent_version': '20230403.00', 'instance_nam...NoneNoneNone<NA>{'file': 'main.go', 'line': 151, 'function': '...None[]
2projects/ucb-datahub-2018/logs/OSConfigAgent{'type': 'k8s_container', 'labels': {'location...None{'localtimestamp': '2024-04-23T21:35:06.7813Z'...2024-04-23 21:35:06.781393+00:002024-04-23 21:35:07.981791+00:00INFO1lq1gg7fc49lawNone{'agent_version': '20230403.00', 'instance_nam...NoneNoneNone<NA>{'file': 'main.go', 'line': 151, 'function': '...None[]
3projects/ucb-datahub-2018/logs/OSConfigAgent{'type': 'k8s_container', 'labels': {'location...None{'localtimestamp': '2024-04-23T21:48:39.6748Z'...2024-04-23 21:48:39.674923+00:002024-04-23 21:48:41.024838+00:00INFO1jhrlxlf9y8wgmNone{'agent_version': '20230403.00', 'instance_nam...NoneNoneNone<NA>{'file': 'main.go', 'line': 151, 'function': '...None[]
4projects/ucb-datahub-2018/logs/OSConfigAgent{'type': 'k8s_container', 'labels': {'location...None{'localtimestamp': '2024-04-23T22:10:57.9018Z'...2024-04-23 22:10:57.902641+00:002024-04-23 22:10:59.142872+00:00INFO1xn7eq9fd6mufzNone{'agent_version': '20230403.00', 'instance_nam...NoneNoneNone<NA>{'file': 'main.go', 'line': 151, 'function': '...None[]
\n", + "
" + ], + "text/plain": [ + " logName \\\n", + "0 projects/ucb-datahub-2018/logs/OSConfigAgent \n", + "1 projects/ucb-datahub-2018/logs/OSConfigAgent \n", + "2 projects/ucb-datahub-2018/logs/OSConfigAgent \n", + "3 projects/ucb-datahub-2018/logs/OSConfigAgent \n", + "4 projects/ucb-datahub-2018/logs/OSConfigAgent \n", + "\n", + " resource textPayload \\\n", + "0 {'type': 'k8s_container', 'labels': {'location... None \n", + "1 {'type': 'k8s_container', 'labels': {'location... None \n", + "2 {'type': 'k8s_container', 'labels': {'location... None \n", + "3 {'type': 'k8s_container', 'labels': {'location... None \n", + "4 {'type': 'k8s_container', 'labels': {'location... None \n", + "\n", + " jsonPayload \\\n", + "0 {'localtimestamp': '2024-04-23T21:23:34.4536Z'... \n", + "1 {'localtimestamp': '2024-04-23T21:26:38.3250Z'... \n", + "2 {'localtimestamp': '2024-04-23T21:35:06.7813Z'... \n", + "3 {'localtimestamp': '2024-04-23T21:48:39.6748Z'... \n", + "4 {'localtimestamp': '2024-04-23T22:10:57.9018Z'... \n", + "\n", + " timestamp receiveTimestamp severity \\\n", + "0 2024-04-23 21:23:34.453695+00:00 2024-04-23 21:23:35.641507+00:00 INFO \n", + "1 2024-04-23 21:26:38.325492+00:00 2024-04-23 21:26:39.502759+00:00 INFO \n", + "2 2024-04-23 21:35:06.781393+00:00 2024-04-23 21:35:07.981791+00:00 INFO \n", + "3 2024-04-23 21:48:39.674923+00:00 2024-04-23 21:48:41.024838+00:00 INFO \n", + "4 2024-04-23 22:10:57.902641+00:00 2024-04-23 22:10:59.142872+00:00 INFO \n", + "\n", + " insertId httpRequest \\\n", + "0 ad3aqffwqqo8 None \n", + "1 x1226heyjxs6 None \n", + "2 1lq1gg7fc49law None \n", + "3 1jhrlxlf9y8wgm None \n", + "4 1xn7eq9fd6mufz None \n", + "\n", + " labels operation trace spanId \\\n", + "0 {'agent_version': '20230403.00', 'instance_nam... None None None \n", + "1 {'agent_version': '20230403.00', 'instance_nam... None None None \n", + "2 {'agent_version': '20230403.00', 'instance_nam... None None None \n", + "3 {'agent_version': '20230403.00', 'instance_nam... None None None \n", + "4 {'agent_version': '20230403.00', 'instance_nam... None None None \n", + "\n", + " traceSampled sourceLocation split \\\n", + "0 {'file': 'main.go', 'line': 151, 'function': '... None \n", + "1 {'file': 'main.go', 'line': 151, 'function': '... None \n", + "2 {'file': 'main.go', 'line': 151, 'function': '... None \n", + "3 {'file': 'main.go', 'line': 151, 'function': '... None \n", + "4 {'file': 'main.go', 'line': 151, 'function': '... None \n", + "\n", + " errorGroups \n", + "0 [] \n", + "1 [] \n", + "2 [] \n", + "3 [] \n", + "4 [] " + ] + }, + "execution_count": 58, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df_OSConfigAgent = client.query(sql_OSConfigAgent).to_dataframe()" + ] + }, + { + "cell_type": "code", + "execution_count": 64, + "id": "00655f03-9147-4511-bb20-9169e146263d", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
logNameresourcetextPayloadjsonPayloadtimestampreceiveTimestampseverityinsertIdhttpRequestlabelsoperationtracespanIdtraceSampledsourceLocationspliterrorGroups
\n", + "
" + ], + "text/plain": [ + "Empty DataFrame\n", + "Columns: [logName, resource, textPayload, jsonPayload, timestamp, receiveTimestamp, severity, insertId, httpRequest, labels, operation, trace, spanId, traceSampled, sourceLocation, split, errorGroups]\n", + "Index: []" + ] + }, + "execution_count": 64, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df_OSConfigAgent = df_OSConfigAgent[df_OSConfigAgent['textPayload'].notna()]\n", + "df_OSConfigAgent.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 59, + "id": "d4990eb5-5c47-4186-af9a-6549f197ed7e", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(13, 17)" + ] + }, + "execution_count": 59, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df_OSConfigAgent.shape" + ] + }, + { + "cell_type": "code", + "execution_count": 60, + "id": "c574d26a-aa4b-4a89-a4b2-98a182cee0db", + "metadata": {}, + "outputs": [], + "source": [ + "# Getting stdout logs\n", + "\n", + "sql_stdout = \"\"\"\n", + "SELECT * FROM `ucb-datahub-2018.datahub_spring24.stdout_20240424`\n", + "\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 61, + "id": "8bb62af3-c9d0-4ef5-a30e-d66419bf2259", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
logNameresourcetextPayloadtimestampreceiveTimestampseverityinsertIdhttpRequestlabelsoperationtracespanIdtraceSampledsourceLocationspliterrorGroupsjsonPayload
\n", + "
" + ], + "text/plain": [ + "Empty DataFrame\n", + "Columns: [logName, resource, textPayload, timestamp, receiveTimestamp, severity, insertId, httpRequest, labels, operation, trace, spanId, traceSampled, sourceLocation, split, errorGroups, jsonPayload]\n", + "Index: []" + ] + }, + "execution_count": 61, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df_stdout = client.query(sql_stdout).to_dataframe()\n", + "df_stdout.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 53, + "id": "b5cdacd2-d151-4843-9bfd-904a56d4cb80", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(0, 17)" + ] + }, + "execution_count": 53, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df_stdout.shape" + ] + }, + { + "cell_type": "code", + "execution_count": 65, + "id": "68940a42-2b7b-4544-883a-fa08e505f585", + "metadata": {}, + "outputs": [], + "source": [ + "sql_query = \"\"\"\n", + "SELECT * FROM `ucb-datahub-2018.datahub_spring24.*`\n", + "\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4b7ed1a7-9ac2-404c-996f-53b794fa287c", + "metadata": {}, + "outputs": [], + "source": [ + "df_sql_query = client.query(sql_query).to_dataframe()\n", + "df_sql_query.head()" ] }, { "cell_type": "code", "execution_count": null, - "id": "50aca4b7-8fcb-4ae0-bc02-102e45c1fc6d", + "id": "1f8093ee-0446-410b-a972-ca1c4e63c9fe", "metadata": {}, "outputs": [], "source": []