UBA_liveparser.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests, json\n",
    "from datetime import datetime\n",
    "import pandas as pd\n",
    "import hashlib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "# function that takes two lists of same length and makes a dict of them, identifying element i with element i\n",
    "def stitch_to_dict(a,b):\n",
    "    r={}\n",
    "    for i in range(len(a)):\n",
    "        r[a[i]]=b[i]\n",
    "    return r\n",
    "\n",
    "# everything lowercase, replaces slashes and spaces with underscores, ...\n",
    "def idstr(idinput):\n",
    "    \"\"\"returns strings in the conventional format for iot.ids\"\"\"\n",
    "    return (str(idinput).lower().replace(\" \", \"_\").replace(\"/\", \"_\").replace(\n",
    "        \"ä\", \"ae\").replace(\"ö\", \"oe\").replace(\"ü\", \"ue\").replace(\"ß\", \"ss\"))\n",
    "\n",
    "# returns the first 7 digits of the sha1 hash of the input string\n",
    "def hashfunc(inputstring):\n",
    "    return hashlib.sha1(bytes(str(inputstring), 'utf-8')).hexdigest()[0:7]\n",
    "\n",
    "# returns the full 40 digits of the sha1 hash of the input string\n",
    "def hashfuncfull(inputstring):\n",
    "    return hashlib.sha1(bytes(str(inputstring), 'utf-8')).hexdigest()\n",
    "\n",
    "# UBA has phenomenonTimes end with hour 24 instead of 0 the next day. datetime cant deal with that, have to replace\n",
    "def todatetimeUTCstring(string):\n",
    "    if(string[-8:-6] == '24'):\n",
    "        res=pd.to_datetime(string.replace(\" 24:\",\" 23:\"))+pd.Timedelta('1 hour')\n",
    "    else: \n",
    "        res=pd.to_datetime(string)\n",
    "    return (res - pd.Timedelta('1 hour')).strftime(\"%Y-%m-%d\" + \"T\" + \"%H\" + \":00:00.000Z\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "url = \"https://api.smartaq.net/v1.0\"\n",
    "\n",
    "uba_stations = json.loads(requests.get(url + \"/Things?$filter=properties/operator.domain%20eq%20%27umweltbundesamt.de%27&$expand=datastreams($expand=sensor,observedProperty)\").text)[\"value\"]\n",
    "\n",
    "current_component = \"1\" #PM10\n",
    "current_scope = \"2\" #hourly averages\n",
    "\n",
    "for thing in uba_stations:\n",
    "    station_no = thing[\"properties\"][\"station_no\"]\n",
    "    current_thing_idparts = thing[\"properties\"][\"operator.domain\"] + \":\" + thing[\"properties\"][\"shortname\"] + \":\" + thing[\"properties\"][\"hardware.id\"]\n",
    "    \n",
    "    for stream in thing[\"Datastreams\"]:\n",
    "        current_sensor_idparts = stream[\"Sensor\"][\"properties\"][\"manufacturer.domain\"] + \":\" + stream[\"Sensor\"][\"properties\"][\"shortname\"]\n",
    "        current_datastream_idparts = current_thing_idparts + \":\" + current_sensor_idparts + \":\" + stream[\"properties\"][\"hardware.serial_number\"] + \":\" + stream[\"ObservedProperty\"][\"properties\"][\"shortname\"]\n",
    "        \n",
    "        timestamp_from = stream[\"phenomenonTime\"].split(\"/\")[1]\n",
    "        \n",
    "        start_date=timestamp_from.split(\"T\")[0]\n",
    "        start_result_time=str(int(timestamp_from.split(\"T\")[1].split(\":\")[0]) + 1)\n",
    "        end_date=datetime.strftime(datetime.now(),\"%Y-%m-%d\")\n",
    "        end_result_time=datetime.strftime(datetime.now(),\"%H\")\n",
    "\n",
    "        data=json.loads(requests.get(\"https://www.umweltbundesamt.de/api/air_data/v2/measures/json?date_from=\" + start_date + \"&time_from=\" + start_result_time + \"&date_to=\" + end_date + \"&time_to=\" + end_result_time + \"&station=\" + str(station_no) + \"&component=\" + current_component + \"&scope=\" + current_scope).text)\n",
    "\n",
    "        val_index=data[\"indices\"][\"data\"][\"station id\"][\"date start\"].index(\"value\")\n",
    "        end_index=data[\"indices\"][\"data\"][\"station id\"][\"date start\"].index(\"date end\")\n",
    "\n",
    "        #function todatetimeUTCstring converts to pandas datetime, converts CET to UTC, then outputs ISO string\n",
    "        list_of_results=list(map(lambda x: {\"result\":data['data'][str(station_no)][x][val_index],\"resultTime\":todatetimeUTCstring(data['data'][str(station_no)][x][end_index]) ,\"phenomenonTime\": todatetimeUTCstring(x) + \"/\" + todatetimeUTCstring(data['data'][str(station_no)][x][end_index])},data['data'][str(station_no)].keys()))\n",
    "        \n",
    "        totalobs = len(list_of_results)\n",
    "\n",
    "        for thisresult in list_of_results:\n",
    "            thisobs = thisresult\n",
    "            thisobs[\"@iot.id\"] = \"saqn:o:\" + hashfuncfull(current_datastream_idparts + \":\" + idstr(thisresult[\"phenomenonTime\"]))\n",
    "            p = requests.post(url + \"/Datastreams('\" + stream[\"@iot.id\"]  + \"')/Observations\",json.dumps(thisobs))        "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}