direct copy BigData

VU-IVM · Aug 6, 2024 · 948ab3a · 948ab3a
commit 948ab3a
Show file tree

Hide file tree

Showing 143 changed files with 1,222,323 additions and 0 deletions.
diff --git a/.github/workflows/deploy-book.yml b/.github/workflows/deploy-book.yml
@@ -0,0 +1,52 @@
+name: deploy-book
+
+# Only run this when the master branch changes!
+on:
+  push:
+    branches:
+    - master
+    - main
+    # If your git repository has the Jupyter Book within some-subfolder next to
+    # unrelated files, you can make this run only if a file within that specific
+    # folder has been modified.
+    #
+    # paths:
+    # - some-subfolder/**
+
+# This job installs dependencies, builds the book, and pushes it to `gh-pages`
+jobs:
+
+
+
+  deploy-book:
+    runs-on: ubuntu-latest
+
+    permissions:
+      contents: write # We generate static HTML, JS, CSS so we need write access to the repo
+      pages: write
+      id-token: write
+
+    steps:
+    - uses: actions/checkout@v2
+
+    # Install dependencies
+    - name: Set up Python 3.8
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.8
+
+    - name: Install dependencies
+      run: |
+        pip install -r requirements.txt
+
+    # Build the book
+    - name: Build the book
+      run: |
+        jupyter-book build .
+
+    # Push the book's HTML to github-pages
+    - name: GitHub Pages action
+      uses: peaceiris/[email protected]
+      with:
+        github_token: ${{ secrets.GITHUB_TOKEN }}
+        publish_dir: ./_build/html
diff --git a/.ipynb_checkpoints/README-checkpoint.md b/.ipynb_checkpoints/README-checkpoint.md
@@ -0,0 +1,3 @@
+# Big Data for Sustainability Sciences
+
+This repository contains the course manual for the course Big Data for Sustainability Sciences for the bachelor programme 'Aarde, Economie & Duurzaamheid'.
diff --git a/_config.yml b/_config.yml
@@ -0,0 +1,51 @@
+# Book settings
+# Learn more at https://jupyterbook.org/customize/config.html
+
+title           : Big Data for Sustainability Sciences
+author          : Dr Elco Koks & Dr Thales Pupo-West
+logo            : _static/images/logo.png
+announcement    : "This course is work in progress!" # A banner announcement at the top of the site.
+
+# Force re-execution of notebooks on each build.
+# See https://jupyterbook.org/content/execute.html
+execute:
+  execute_notebooks: 'off'
+
+# Define the name of the latex output file for PDF builds
+latex:
+  latex_documents:
+    targetname: book.tex
+
+# Add a bibtex file so that we can create citations
+bibtex_bibfiles:
+  - references.bib
+
+# print execution errors directly
+sphinx:
+  config:
+    execution_show_tb: True
+
+# Launch button settings
+launch_buttons:
+  notebook_interface        : classic  # The interface interactive links will activate ["classic", "jupyterlab"]
+  colab_url                 : "https://colab.research.google.com" # The URL of Google Colab ()
+  binderhub_url             : ""  # Binder seems to be broken often during link checks?
+
+# Information about where the book exists on the web
+repository:
+  url: https://github.com/ElcoK/BigData_AED  # Online location of your book
+  branch: main  # Which branch of the repository should be used when creating links (optional)
+
+# Add GitHub buttons to your book
+# See https://jupyterbook.org/customize/config.html#add-a-link-to-your-repository
+html:
+  use_issues_button: true
+  use_repository_button: true
+
+# add parsing
+parse:
+  myst_enable_extensions:
+    # don't forget to list any other extensions you want enabled,
+    # including those that are enabled by default!
+    - html_image
+    - colon_fence
diff --git a/_static/images/API_Keys.PNG b/_static/images/API_Keys.PNG
diff --git a/_static/images/CDS_registration.gif b/_static/images/CDS_registration.gif
diff --git a/_static/images/CLMS_registration.gif b/_static/images/CLMS_registration.gif
diff --git a/_static/images/Elco.jpg b/_static/images/Elco.jpg
diff --git a/_static/images/Maria.jpg b/_static/images/Maria.jpg
diff --git a/_static/images/Marijn.jpg b/_static/images/Marijn.jpg
diff --git a/_static/images/Rhoda.jpg b/_static/images/Rhoda.jpg
diff --git a/_static/images/Thales.jpg b/_static/images/Thales.jpg
diff --git a/_static/images/Token.PNG b/_static/images/Token.PNG
diff --git a/_static/images/broadcasting.png b/_static/images/broadcasting.png
diff --git a/_static/images/computer_panda.gif b/_static/images/computer_panda.gif
diff --git a/_static/images/dataframe.png b/_static/images/dataframe.png
diff --git a/_static/images/decision-tree-example-2-1.jpg b/_static/images/decision-tree-example-2-1.jpg
diff --git a/_static/images/figure_overview.jpg b/_static/images/figure_overview.jpg
diff --git a/_static/images/flickr_registration.gif b/_static/images/flickr_registration.gif
diff --git a/_static/images/groupby_1.png b/_static/images/groupby_1.png
diff --git a/_static/images/groupby_2.png b/_static/images/groupby_2.png
diff --git a/_static/images/inner_join.png b/_static/images/inner_join.png
diff --git a/_static/images/join.png b/_static/images/join.png
diff --git a/_static/images/left_join.png b/_static/images/left_join.png
diff --git a/_static/images/logo.jpg b/_static/images/logo.jpg
diff --git a/_static/images/logo.png b/_static/images/logo.png
diff --git a/_static/images/machine-learning-infographic_2.jpg b/_static/images/machine-learning-infographic_2.jpg
diff --git a/_static/images/melt_pivot.gif b/_static/images/melt_pivot.gif
diff --git a/_static/images/numpy.png b/_static/images/numpy.png
diff --git a/_static/images/numpy_arrays.png b/_static/images/numpy_arrays.png
diff --git a/_static/images/numpy_paper.png b/_static/images/numpy_paper.png
diff --git a/_static/images/outer_join.png b/_static/images/outer_join.png
diff --git a/_static/images/pandas.png b/_static/images/pandas.png
diff --git a/_static/images/pies.png b/_static/images/pies.png
diff --git a/_static/images/pies_broadcast.png b/_static/images/pies_broadcast.png
diff --git a/_static/images/pies_loop.png b/_static/images/pies_loop.png
diff --git a/_static/images/series.png b/_static/images/series.png
diff --git a/_static/images/series_addition.png b/_static/images/series_addition.png
diff --git a/_static/images/tidy.png b/_static/images/tidy.png
diff --git a/_static/images/triangle.png b/_static/images/triangle.png
diff --git a/_static/images/twitter_elevated.gif b/_static/images/twitter_elevated.gif
diff --git a/_static/images/twitter_registration.gif b/_static/images/twitter_registration.gif
diff --git a/_static/images/twitter_token.gif b/_static/images/twitter_token.gif
diff --git a/_static/images/twtter_api.gif b/_static/images/twtter_api.gif
diff --git a/_static/images/vu_logo.jpg b/_static/images/vu_logo.jpg
diff --git a/_static/images/vu_logo.png b/_static/images/vu_logo.png
diff --git a/_toc.yml b/_toc.yml
@@ -0,0 +1,51 @@
+# Table of contents
+# Learn more at https://jupyterbook.org/customize/toc.html
+
+format: jb-book
+root: intro
+
+parts:
+  - caption: Course overview
+    chapters:
+    - file: course_basics/course_intro.md
+    - file: course_basics/teachers.md
+    - file: course_basics/schedule.md    
+  - caption: Week 1
+    chapters:
+    - file: week1/lecture
+    - file: week1/tutorial1
+    - file: week1/tutorial2
+    - file: week1/tutorial3
+  - caption: Week 2
+    chapters:
+    - file: week2/lecture
+    - file: week2/tutorial1
+    - file: week2/tutorial2
+  - caption: Week 3
+    chapters:
+    - file: week3/lecture
+    - file: week3/tutorial1
+    - file: week3/tutorial2
+  - caption: Week 4
+    chapters:
+    - file: week4/lecture
+    - file: week4/tutorial1
+    - file: week4/tutorial2
+  - caption: Week 5
+    chapters:
+    - file: week5/lecture
+    - file: week5/tutorial1
+    - file: week5/tutorial2
+  - caption: Week 6
+    chapters:
+    - file: week6/lecture
+    - file: week6/tutorial1
+    - file: week6/tutorial2
+  - caption: Week 7
+    chapters:
+    - file: week7/lecture
+    - file: week7/tutorial1
+  - caption: Week 8
+    chapters:
+    - file: week8/exam
+
diff --git a/course_basics/course_intro.md b/course_basics/course_intro.md
@@ -0,0 +1,27 @@
+# Introduction
+
+In the course Big Data for Sustainability Sciences, you will learn the tools and knowledge to work with large datasets that are widely used within sustainability sciences. This ranges from filling gaps in census data through machine learning approaches, to working with earth observation data to map wildfires. 
+
+The course is structured around six main data types and related methods. Specifically, we will:
+
+1. explore census data and use machine-learning methods to fill data gaps and derive new information and knowledge;
+2. better understand how you can use Earth Observation data to classify land cover.
+3. assess how droughts affect us through using remote sensing data;
+4. understand how to use open-source and Volunteered geographic information (VGI);
+5. apply nature language processing techniques to identify the occurrence of floods through social media data;
+6. learn how the right visualization can help us to extract the right messages from our data. 
+
+In the first week, we will provide a crash course in Python that will be used throughout the remainder of the course. You are not expected to have any prior knowledge in Python before this course. 
+
+## Objectives
+The key objectives of this course are:
+* to know how and when big data can be used to solve sustainability problems.
+* to gain a better understanding of methods and tools to analyze big data.
+
+## Teaching methods
+This course will be a combination of lectures and tutorials. Each week starts with an introductory lecture to the method and/or data type that will be applied and/or analyzed in that particular week. During the lecture, students will gain the required theoretical knowledge to apply the methods during the two tutorials. The tutorials are each a 90 minute computer practical in which you will develop the skills and knowledge to work with the method and/or data-type through a hands-on assignment. 
+
+## Methods of assessment
+There will be four graded assignments and a multiple-choice exam in the final week. The weekly assignments will account for 60% of the grade, and the final exam will account for 40% of the grade. The weekly assignments will be made in groups of two, whereas the final exam will be individual.
+
+You must pass both elements (5.5 or higher). 
diff --git a/course_basics/images/Elco.jpg b/course_basics/images/Elco.jpg
diff --git a/course_basics/images/Marijn.jpg b/course_basics/images/Marijn.jpg
diff --git a/course_basics/images/Rhoda.jpg b/course_basics/images/Rhoda.jpg
diff --git a/course_basics/images/Thales.jpg b/course_basics/images/Thales.jpg
diff --git a/course_basics/images/logo.jpg b/course_basics/images/logo.jpg
diff --git a/course_basics/images/logo.png b/course_basics/images/logo.png
diff --git a/course_basics/images/vu_logo.jpg b/course_basics/images/vu_logo.jpg
diff --git a/course_basics/images/vu_logo.png b/course_basics/images/vu_logo.png
diff --git a/course_basics/schedule.md b/course_basics/schedule.md
@@ -0,0 +1,14 @@
+# Course schedule
+
+````{div} full-width
+|Week|Data type|Lecture|Tutorial #1|Tutorial #2|Grading|
+|:---:|---|---|---|---|---|
+| 1  ||Course Introduction|Introduction to Python|Introduction to Data Analysis|pass/fail|
+| 2  |Census data|Introduction to Random Forests|Data Exploration & Regression|Random Forests|12% of grade|
+| 3  |Census data|Introduction to Neural Networks|Neural Networks|Model Validation|12% of grade|
+| 4  |Satellite data|Earth Observation & Google Earth Engine|Land-use classification|Drought detection|12% of grade|
+| 5  |Open-Source and VGI data|Big Data in the public domain|Working with OpenStreetMap|Natural Hazard Risk Assessment|12% of grade|
+| 6  |Social media data|Social Media and Natural Language Processing (NLP)|Social Media & Natural Hazards|Social Media & Valuation of Landscapes|12% of grade|
+| 7  |How to: Visualization|Visualizing your results|   |pass/fail|
+| 8  |   |   |Exam (40%)|   |   |
+````
diff --git a/course_basics/teachers.md b/course_basics/teachers.md
@@ -0,0 +1,46 @@
+# Teachers
+
+<style>
+table th:first-of-type {
+    width: 30%;
+}
+table th:nth-of-type(2) {
+    width: 70%;
+}
+
+</style>
+
+&nbsp;
+&nbsp;
+
+|Dr Elco Koks|Course coordinator|
+|:-:|:--|
+| <img src="../_static/images/Elco.jpg" class="bg-primary" width="150px"> |Elco Koks is an Associate Professor within the department of Water and Climate Risk group at the Institute for Environmental Studies (IVM) and Honorary Research Fellow at the Oxford Programme for Sustainable Infrastructure Systems (OPSIS) at the University of Oxford, UK. His research combines knowledge from disaster impact modelling, critical infrastructure, network analysis and macroeconomics.|
+
+&nbsp;
+&nbsp;  
+
+|Dr Thales Pupo-West||
+|:-:|:--|
+|<img src="../_static/images/Thales.jpg"  class="bg-primary" width="150px">|Thales A.P. West is an Assistant Professor at the department of Environmental Geography and the Institute for Environmental Studies (IVM) at Vrije Universiteit Amsterdam and Fellow of the Centre for Environment, Energy and Natural Resource Governance (C-EENRG) at the University of Cambridge, UK. His research is focused on the sustainability of coupled human-natural systems and topics like land-use/cover change, resource management, environmental economics, impact evaluation (econometrics), climate governance, and payments for environmental services, e.g., REDD+ schemes.|
+
+&nbsp;
+&nbsp;  
+
+|Marijn Ton, MSc||
+|:-:|:--|
+|<img src="../_static/images/Marijn.jpg" class="bg-primary" width="150px">|Marijn Ton is a PhD candidate in the department of Water and Climate Risk of the Institute for Environmental Studies (IVM) at Vrije Universiteit Amsterdam (VU). She obtained a bachelor’s degree in Economics & Governance at Wageningen University and then continued her studies at VU Amsterdam, where she completed a master’s degree in Econometrics & Operations Research. In her PhD research, she studies coastal migration caused by climate change and sea level rise. She develops a global agent based model to simulate migration flows between 2020-2100 integrating flood projections and human migration decisions.|
+
+&nbsp;
+&nbsp;  
+
+|Rhoda Odongo, MSc||
+|:-:|:--|
+|<img src="../_static/images/Rhoda.jpg" class="bg-primary" width="150px">|Rhoda Odongo is a PhD candidate in the department of Water and Climate Risk of the Institute for Environmental Studies (IVM) at Vrije Universiteit Amsterdam. She has a background in Water Resources Engineering from the Katholieke Universiteit Leuven and Biosystems Engineering from the University of Nairobi. She works on data-driven assessment of drought risk to understand existing and possible future interactions between climate, hydrology, land and water use, and management focusing mainly on the drought and subsurface water storage.
+
+&nbsp;
+&nbsp;  
+
+|Maria Fonseca, MSc||
+|:-:|:--|
+|<img src="../_static/images/Maria.jpg" class="bg-primary" width="150px">|Maria Fonseca is a PhD candidate in the department of Water and Climate Risk of the Institute for Environmental Studies (IVM) at Vrije Universiteit Amsterdam (VU). She obtained a bachelor’s degree in Civil Engineering at Western Institute of Technology and Higher Education in Mexico, and a master’s degree in Water Management at TU Delft. In her PhD research, she assesses climate risk of current and future wind and hail storm events, how the affects urban areas, communities and investments, and the available adaptation options to reduce the risks. |
diff --git a/intro.md b/intro.md
@@ -0,0 +1,12 @@
+![](_static/images/vu_logo.jpg?h=750&w=1260)
+
+# 
+## Big Data for Sustainability Sciences
+
+Data is everywhere around us. But how can we make use of all this data to answer questions in sustainability sciences? How can twitter data help us to understand where and how severe people are affected by floods? Or how can we use earth observation data to classify land use? 
+
+In the course Big Data in Sustainability Sciences, you will learn the tools and knowledge to work with large datasets that are widely used within sustainability sciences. This ranges from filling gaps in census data through machine learning approaches, to working with earth observation data to map droughts.
+
+
+```{tableofcontents}
+```
diff --git a/readme.md b/readme.md
@@ -0,0 +1,3 @@
+# Big Data for Sustainability Sciences
+
+This repository contains the course manual for the course Big Data for Sustainability Sciences for the bachelor programme 'Aarde, Economie & Duurzaamheid'.
diff --git a/references.bib b/references.bib
@@ -0,0 +1,56 @@
+---
+---
+
+@inproceedings{holdgraf_evidence_2014,
+	address = {Brisbane, Australia, Australia},
+	title = {Evidence for {Predictive} {Coding} in {Human} {Auditory} {Cortex}},
+	booktitle = {International {Conference} on {Cognitive} {Neuroscience}},
+	publisher = {Frontiers in Neuroscience},
+	author = {Holdgraf, Christopher Ramsay and de Heer, Wendy and Pasley, Brian N. and Knight, Robert T.},
+	year = {2014}
+}
+
+@article{holdgraf_rapid_2016,
+	title = {Rapid tuning shifts in human auditory cortex enhance speech intelligibility},
+	volume = {7},
+	issn = {2041-1723},
+	url = {http://www.nature.com/doifinder/10.1038/ncomms13654},
+	doi = {10.1038/ncomms13654},
+	number = {May},
+	journal = {Nature Communications},
+	author = {Holdgraf, Christopher Ramsay and de Heer, Wendy and Pasley, Brian N. and Rieger, Jochem W. and Crone, Nathan and Lin, Jack J. and Knight, Robert T. and Theunissen, Frédéric E.},
+	year = {2016},
+	pages = {13654},
+	file = {Holdgraf et al. - 2016 - Rapid tuning shifts in human auditory cortex enhance speech intelligibility.pdf:C\:\\Users\\chold\\Zotero\\storage\\MDQP3JWE\\Holdgraf et al. - 2016 - Rapid tuning shifts in human auditory cortex enhance speech intelligibility.pdf:application/pdf}
+}
+
+@inproceedings{holdgraf_portable_2017,
+	title = {Portable learning environments for hands-on computational instruction using container-and cloud-based technology to teach data science},
+	volume = {Part F1287},
+	isbn = {978-1-4503-5272-7},
+	doi = {10.1145/3093338.3093370},
+	abstract = {© 2017 ACM. There is an increasing interest in learning outside of the traditional classroom setting. This is especially true for topics covering computational tools and data science, as both are challenging to incorporate in the standard curriculum. These atypical learning environments offer new opportunities for teaching, particularly when it comes to combining conceptual knowledge with hands-on experience/expertise with methods and skills. Advances in cloud computing and containerized environments provide an attractive opportunity to improve the effciency and ease with which students can learn. This manuscript details recent advances towards using commonly-Available cloud computing services and advanced cyberinfrastructure support for improving the learning experience in bootcamp-style events. We cover the benets (and challenges) of using a server hosted remotely instead of relying on student laptops, discuss the technology that was used in order to make this possible, and give suggestions for how others could implement and improve upon this model for pedagogy and reproducibility.},
+	booktitle = {{ACM} {International} {Conference} {Proceeding} {Series}},
+	author = {Holdgraf, Christopher Ramsay and Culich, A. and Rokem, A. and Deniz, F. and Alegro, M. and Ushizima, D.},
+	year = {2017},
+	keywords = {Teaching, Bootcamps, Cloud computing, Data science, Docker, Pedagogy}
+}
+
+@article{holdgraf_encoding_2017,
+	title = {Encoding and decoding models in cognitive electrophysiology},
+	volume = {11},
+	issn = {16625137},
+	doi = {10.3389/fnsys.2017.00061},
+	abstract = {© 2017 Holdgraf, Rieger, Micheli, Martin, Knight and Theunissen. Cognitive neuroscience has seen rapid growth in the size and complexity of data recorded from the human brain as well as in the computational tools available to analyze this data. This data explosion has resulted in an increased use of multivariate, model-based methods for asking neuroscience questions, allowing scientists to investigate multiple hypotheses with a single dataset, to use complex, time-varying stimuli, and to study the human brain under more naturalistic conditions. These tools come in the form of “Encoding” models, in which stimulus features are used to model brain activity, and “Decoding” models, in which neural features are used to generated a stimulus output. Here we review the current state of encoding and decoding models in cognitive electrophysiology and provide a practical guide toward conducting experiments and analyses in this emerging field. Our examples focus on using linear models in the study of human language and audition. We show how to calculate auditory receptive fields from natural sounds as well as how to decode neural recordings to predict speech. The paper aims to be a useful tutorial to these approaches, and a practical introduction to using machine learning and applied statistics to build models of neural activity. The data analytic approaches we discuss may also be applied to other sensory modalities, motor systems, and cognitive systems, and we cover some examples in these areas. In addition, a collection of Jupyter notebooks is publicly available as a complement to the material covered in this paper, providing code examples and tutorials for predictive modeling in python. The aimis to provide a practical understanding of predictivemodeling of human brain data and to propose best-practices in conducting these analyses.},
+	journal = {Frontiers in Systems Neuroscience},
+	author = {Holdgraf, Christopher Ramsay and Rieger, J.W. and Micheli, C. and Martin, S. and Knight, R.T. and Theunissen, F.E.},
+	year = {2017},
+	keywords = {Decoding models, Encoding models, Electrocorticography (ECoG), Electrophysiology/evoked potentials, Machine learning applied to neuroscience, Natural stimuli, Predictive modeling, Tutorials}
+}
+
+@book{ruby,
+  title     = {The Ruby Programming Language},
+  author    = {Flanagan, David and Matsumoto, Yukihiro},
+  year      = {2008},
+  publisher = {O'Reilly Media}
+}
diff --git a/requirements.txt b/requirements.txt
@@ -0,0 +1,10 @@
+jupyter-book
+matplotlib
+numpy
+pandas
+geopandas
+osmnx
+geocube
+contextily
+ee
+geemap
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,3 @@
		# Big Data for Sustainability Sciences

		This repository contains the course manual for the course Big Data for Sustainability Sciences for the bachelor programme 'Aarde, Economie & Duurzaamheid'.