diff --git a/.github/workflows/deployment.yml b/.github/workflows/deployment.yml index 4aa902fae2..4661830b10 100644 --- a/.github/workflows/deployment.yml +++ b/.github/workflows/deployment.yml @@ -10,43 +10,26 @@ jobs: name: Deploy to GitHub Pages runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - with: - path: ./repo + - uses: actions/checkout@v3 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + - name: Build Jekyll site + run: | + # GitHub-specific config + echo "url: https://${{ github.repository_owner }}.github.io" >> _config.yaml + echo "baseurl: /operating-systems" >> _config.yaml - - name: Build and push - uses: docker/build-push-action@v2 - with: - context: ./repo - file: ./repo/Dockerfile - push: false - load: true - tags: operating-systems-oer/docusaurus:latest - cache-from: type=gha - cache-to: type=gha + # Build Docker image for Jekyll + docker build -t jekyll-image . - - name: Load image - run: | - mkdir output - docker image list - docker run -v $GITHUB_WORKSPACE/repo:/content -v $GITHUB_WORKSPACE/output:/output operating-systems-oer/docusaurus:latest + # Build Jekyll site + docker run --rm \ + -v ${{ github.workspace }}:/usr/src/app \ + jekyll-image bundle exec jekyll build - # Popular action to deploy to GitHub Pages: - # Docs: https://github.com/peaceiris/actions-gh-pages#%EF%B8%8F-docusaurus - name: Deploy to GitHub Pages uses: peaceiris/actions-gh-pages@v3 with: - personal_token: ${{ secrets.ACCESS_TOKEN }} - # Build output to publish to the `gh-pages` branch: - publish_dir: ./output - # The following lines assign commit authorship to the official - # GH-Actions bot for deploys to `gh-pages` branch: - # https://github.com/actions/checkout/issues/13#issuecomment-724415212 - # The GH actions bot is used by default if you didn't specify the two fields. - # You can swap them out with your own user credentials. - user_name: 'github-actions[bot]' - user_email: 'github-actions[bot]@users.noreply.github.com' + github_token: ${{ secrets.GITHUB_TOKEN }} + publish_dir: _site publish_branch: gh-pages + enable_jekyll: true diff --git a/.github/workflows/os-runner.yml b/.github/workflows/os-runner.yml index bacebafa0c..86c7b074cd 100644 --- a/.github/workflows/os-runner.yml +++ b/.github/workflows/os-runner.yml @@ -17,7 +17,7 @@ jobs: contents: read steps: - - name: Checkout + - name: Checkout uses: actions/checkout@v3 - name: Set up QEMU diff --git a/.github/workflows/pr-deployment.yml b/.github/workflows/pr-deployment.yml index 1699179349..01c20fbbf3 100644 --- a/.github/workflows/pr-deployment.yml +++ b/.github/workflows/pr-deployment.yml @@ -1,67 +1,65 @@ name: OpenEduHub - PR Deployment on: + workflow_dispatch: pull_request_target: types: [labeled] jobs: deploy: - if: ${{ github.event.label.name == 'needs-rendering' }} + if: ${{ github.event.label.name == 'needs-rendering' || github.event_name == 'workflow_dispatch' }} name: Deploy to GitHub Pages runs-on: ubuntu-latest permissions: contents: write pull-requests: write steps: + - name: Set Deployment ID + run: | + # Use PR number for PR trigger and commit SHA for manual trigger + if [[ -z "${{ github.event.pull_request.number }}" ]]; then + echo "DEPLOYMENT_ID=trigger-${GITHUB_SHA::7}" >> $GITHUB_ENV + else + echo "DEPLOYMENT_ID=${{ github.event.pull_request.number }}" >> $GITHUB_ENV + fi + echo "DEPLOYMENT_ID=${{ env.DEPLOYMENT_ID }}" + - uses: actions/checkout@v3 with: - path: ./repo repository: ${{ github.event.pull_request.head.repo.full_name }} ref: ${{ github.head_ref }} - - run: | - cd repo - REF=$(echo ${{ github.event.number }} | sed 's/\//\\\//g') - sed -i "s/baseUrl: \/operating-systems\//baseUrl: \/operating-systems\/$REF\//" config.yaml - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + - name: Build Jekyll site + run: | + # GitHub-specific config + echo "url: https://${{ github.repository_owner }}.github.io" >> _config.yaml + echo "baseurl: /operating-systems/${{ env.DEPLOYMENT_ID }}" >> _config.yaml - - name: Build and push - uses: docker/build-push-action@v3 - with: - context: ./repo - file: ./repo/Dockerfile - push: false - load: true - tags: operating-systems/docusaurus:latest - cache-from: type=gha - cache-to: type=gha + # Build Docker image for Jekyll + docker build -t jekyll-site . - - name: Load Image - run: | - mkdir -p ${{ github.event.number }} - docker image list - docker run -v $GITHUB_WORKSPACE/repo:/content -v $GITHUB_WORKSPACE/${{ github.event.number }}:/output operating-systems/docusaurus:latest + # Build Jekyll site + docker run --rm \ + -v ${{ github.workspace }}:/usr/src/app/ \ + jekyll-site bundle exec jekyll build - # Popular action to deploy to GitHub Pages: - # Docs: https://github.com/peaceiris/actions-gh-pages#%EF%B8%8F-docusaurus - name: Deploy to GitHub Pages uses: peaceiris/actions-gh-pages@v3 with: github_token: ${{ secrets.GITHUB_TOKEN }} - # Build output to publish to the `gh-pages-pr` branch: - publish_dir: ./${{ github.event.number }} - destination_dir: ${{ github.event.number }} - # The following lines assign commit authorship to the official - # GH-Actions bot for deploys to `gh-pages` branch: - # https://github.com/actions/checkout/issues/13#issuecomment-724415212 - # The GH actions bot is used by default if you didn't specify the two fields. - # You can swap them out with your own user credentials. + publish_dir: _site + destination_dir: ${{ env.DEPLOYMENT_ID }} publish_branch: gh-pages + enable_jekyll: true - name: Add Comment to PR + if: ${{ !startsWith(env.DEPLOYMENT_ID, 'trigger-') }} uses: thollander/actions-comment-pull-request@v2 with: message: | - Published at https://${{ github.repository_owner }}.github.io/operating-systems/${{ github.event.number }}/ + Published at https://${{ github.repository_owner }}.github.io/operating-systems/${{ env.DEPLOYMENT_ID }}/ + + - name: Output Deployment URL + if: ${{ startsWith(env.DEPLOYMENT_ID, 'trigger-') }} + run: | + echo "The deployment is available at https://${{ github.repository_owner }}.github.io/operating-systems/${{ env.DEPLOYMENT_ID }}/" diff --git a/.gitignore b/.gitignore index bbb7ac1930..47e538303d 100644 --- a/.gitignore +++ b/.gitignore @@ -76,109 +76,15 @@ dkms.conf # reveal-md slides output folder _site/ -# Temporary files -*.swp -*.swo -*~ -slides.md - -# .gif files generated with ffmpeg -*-generated.gif - -# JavaScript files - -# compiled output -/dist -/tmp -/out-tsc - -# Runtime data -pids -*.pid -*.seed -*.pid.lock - -# Directory for instrumented libs generated by jscoverage/JSCover -lib-cov - -# Coverage directory used by tools like istanbul -coverage - -# nyc test coverage -.nyc_output - -# Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) -.grunt - -# Bower dependency directory (https://bower.io/) -bower_components - -# node-waf configuration -.lock-wscript - -# IDEs and editors -.idea -.project -.classpath -.c9/ -*.launch -.settings/ -*.sublime-workspace - -# IDE - VSCode -.vscode/* -!.vscode/settings.json -!.vscode/tasks.json -!.vscode/launch.json -!.vscode/extensions.json - -# misc -.sass-cache -connect.lock -typings - -# Logs -logs -*.log -npm-debug.log* -yarn-debug.log* -yarn-error.log* - -# Dependency directories -node_modules/ -jspm_packages/ - -# Optional npm cache directory -.npm - -# Optional eslint cache -.eslintcache - -# Optional REPL history -.node_repl_history - -# Output of 'npm pack' -*.tgz - -# Yarn Integrity file -.yarn-integrity - -# dotenv environment variables file -.env - -# next.js build output -.next - -# Lerna -lerna-debug.log - -# System Files -.DS_Store -Thumbs.db - -# Docusaurus build files -.output/ -.view/ - -# VSCode -.vscode/ +# Builder output folder +/.output/ + +# Ruby files for the local Jekyll server +.sass-cache/ +.jekyll-cache/ +.jekyll-metadata + +# Ignore folders generated by Bundler +.bundle/ +vendor/ +*.lock diff --git a/Dockerfile b/Dockerfile index 5deb5a82f2..c7b7a85158 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,22 +1,15 @@ -FROM ghcr.io/open-education-hub/openedu-builder:0.6.1 +FROM ruby:3.0-slim -# Install ffmpeg -RUN apt-get update && \ - apt-get install -y ffmpeg curl make +RUN apt-get update && apt-get install -y \ + build-essential \ + zlib1g-dev \ + && rm -rf /var/lib/apt/lists/* -# Install markdown-pp -RUN pip install MarkdownPP +WORKDIR /usr/src/app -# Install node LTS (16) -RUN curl -fsSL https://deb.nodesource.com/setup_lts.x | bash - && \ - apt-get install -y nodejs +COPY Gemfile ./ +RUN gem install bundler:2.5.23 && bundle install -# Install reveal md -RUN npm install -g reveal-md +EXPOSE 4000 -# Install docusaurus -RUN npm install create-docusaurus@2.1.0 - -WORKDIR /content - -ENTRYPOINT ["oe_builder"] +CMD ["bundle", "exec", "jekyll", "serve", "--host", "0.0.0.0"] diff --git a/Gemfile b/Gemfile new file mode 100644 index 0000000000..0d0761459f --- /dev/null +++ b/Gemfile @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +source "https://rubygems.org" + +gem 'jekyll-titles-from-headings' +gem 'jekyll-seo-tag' +gem 'jekyll-remote-theme' +gem 'github-pages', group: :jekyll_plugins diff --git a/Makefile b/Makefile index 682fea2d77..bd6e69db22 100644 --- a/Makefile +++ b/Makefile @@ -1,46 +1,20 @@ -REPO_NAME = operating-systems -IMAGE_NAME = $(REPO_NAME)/docusaurus:latest -CONTAINER_NAME = open-edu-hub-$(REPO_NAME)-bash -OUTPUT_DIR = $$PWD/.output/$(REPO_NAME) +IMG_NAME=$(shell basename $(CURDIR)) +CONT_NAME=$(IMG_NAME) -.PHONY: all buildimg build serve run_bash enter_bash stop_bash clean cleanall - -all: build - -buildimg: - docker build -f ./Dockerfile --tag $(IMAGE_NAME) . - -build: buildimg - @echo "Building content. This will take a while (several minutes) ..." - @echo "After the build, run" - @echo "" - @echo " make serve" - @echo "" - @mkdir -p $(OUTPUT_DIR) - docker run --rm -v $$PWD/:/content -v $(OUTPUT_DIR):/output $(IMAGE_NAME) +build: + docker build -t $(IMG_NAME) . serve: - @echo "Point your browser to http://localhost:8080/$(REPO_NAME)" - @cd $(OUTPUT_DIR)/.. && python3 -m http.server 8080 - -run_bash: buildimg - @mkdir -p $(OUTPUT_DIR) - docker run -d -it --entrypoint /bin/bash --name $(CONTAINER_NAME) -v $$PWD/:/content -v $(OUTPUT_DIR):/output $(IMAGE_NAME) - -enter_bash: - docker exec -it $(CONTAINER_NAME) /bin/bash - -stop_bash: - -test "$(shell docker container inspect -f '{{.State.Running}}' $(CONTAINER_NAME) 2> /dev/null)" = "true" && docker stop $(CONTAINER_NAME) + docker run --rm -p 4000:4000 -v $$PWD:/usr/src/app $(IMG_NAME) -clean: stop_bash - -docker container inspect $(CONTAINER_NAME) > /dev/null 2>&1 && docker rm $(CONTAINER_NAME) - -sudo rm -fr $(OUTPUT_DIR) +stop: + @docker ps -q --filter "name=$(CONT_NAME)" | grep -q . && docker stop $(CONT_NAME) \ + || echo "No running container to stop." -cleanall: clean - -docker inspect --type=image $(IMAGE_NAME) > /dev/null 2>&1 && docker image rm $(IMAGE_NAME) +clean: stop + docker rmi $(IMG_NAME) + rm -rf _site -# Linters .PHONY: lint typos lint: typos diff --git a/README.md b/README.md index 3d1c7d10e6..6d587b4a85 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,7 @@ +--- +nav_order: 1 +--- + # Operating Systems (OER) [![copying.md](https://img.shields.io/static/v1?label=license&message=CCBY-SA4.0&color=%23385177)](https://github.com/open-education-hub/operating-systems/blob/master/COPYING.md) @@ -13,36 +17,28 @@ They are to be used by teachers, trainers, students and hobbyists who want to le ## Using the Content -Content is located in the `content/` folder. -It currently consists of 5 chapters: - -* [Software Stack](content/chapters/software-stack/) -* [Data](content/chapters/data/) -* [Compute](content/chapters/compute/) -* [Input/Output](content/chapters/io/) -* [Application Interaction](content/chapters/app-interact/) - -Each chapter has its own folder. -Content for each chapter is split in two subfolders: - -* `lecture/`: content to be presented and discussed as part of lectures -* `lab/`: content to be worked on as practical activities during labs / seminars - -Lecture content is expected to be presented and followed. -Lab content is expected to be used as practice work. - -## Chapter Contents - -Lecture content consists of slides and demos. -Slides are written in [GitHub Markdown](https://guides.github.com/features/mastering-markdown/) and use [reveal-md](https://github.com/webpro/reveal-md) and [reveal.js](https://revealjs.com/) to render HTML output. -Lecture slides are built from the `slides.md` file using the `make` command (and the `Makefile`). -Demos are snippets of code and support files that showcase concepts and ideas related to the lecture. -Demos are located in the `demo/` folder. -Each demo has its own folder with source code, `Makefile` or other build files (if required) and support files. - -Lab content consists of lab text and lab activities. -Lab text is placed in the `README.md` file. -Each lab activity has its own folder with source code, `Makefile` or other build files (if required) and support files. +The content is built using [Jekyll](https://jekyllrb.com/) and is hosted on [GitHub Pages](https://pages.github.com/). +It consists of 5 main sections: + +* Software Stack + * [Lab 1 - Operating System Perspective](labs/lab-01/) + * [Lab 2 - Library Perspective](labs/lab-02/) +* Data + * [Lab 3 - Memory](labs/lab-03/) + * [Lab 4 - Investigate Memory](labs/lab-04/) + * [Lab 5 - Memory Security](labs/lab-05/) +* Compute + * [Lab 6 - Multiprocess and Multithread](labs/lab-06/) + * [Lab 7 - Copy-on-Write](labs/lab-07/) + * [Lab 8 - Synchronization](labs/lab-08/) +* Input/Output + * [Lab 9 - File Descriptors](labs/lab-09/) + * [Lab 10 - Inter-Process Communication](labs/lab-10/) + * [Lab 11 - IO Optimizations](labs/lab-11/) +* Application Interaction + * [Lab 12 - Application Interaction](labs/lab-12/) + +Each chapter has a dedicated assignment in the [`assignments/`](assignments/) folder. ## Contributing diff --git a/_config.yaml b/_config.yaml new file mode 100644 index 0000000000..5279de06fd --- /dev/null +++ b/_config.yaml @@ -0,0 +1,31 @@ +# SPDX-License-Identifier: CC BY-SA 4.0 + +title: Operating Systems +remote_theme: just-the-docs/just-the-docs +repository: cs-pub-ro/operating-systems + +exclude: + - README.md + - REVIEWING.md + - config.yaml + - chapters/ + - .git/ + - .vscode/ + - .github/ + - content/ + - util/ + - landing-page/ + - .view/ + - .output/ + +plugins: + - jekyll-titles-from-headings + - jekyll-seo-tag + +callouts: + warning: + title: warning + color: yellow + tip: + title: tip + color: green diff --git a/assignments/README.md b/assignments/README.md new file mode 100644 index 0000000000..00274e458f --- /dev/null +++ b/assignments/README.md @@ -0,0 +1,24 @@ +--- +title: Assignments +nav_order: 3 +has_children: true +--- + +# Assignments + +The assignments are a way to test your understanding of the course material. +Each assignment is a small project that encompasses the concepts you have learned in the labs. + +Each assignment compreises of: + +- a description of the assignment in the `README.md` file +- a `src/` directory that contains the code skeleton for the assignment +- a `tests/` directory that contains the test cases for the assignment +- optionally a `utils/` directory that contains utility functions that should not be changed + +You can find the assignments in the `assignments` directory. + +## Building and Testing the Assignments + +Each assignment has its own Makefile to automate the building and testing process. +For more details on how to use it, check the `README.md` file in the assignment directory. diff --git a/assignments/async-web-server/README.md b/assignments/async-web-server/README.md new file mode 100644 index 0000000000..30f3d5309f --- /dev/null +++ b/assignments/async-web-server/README.md @@ -0,0 +1,203 @@ +--- +parent: Assignments +title: Asynchronous Web Server +nav_order: 5 +--- + +# Asynchronous Web Server + +## Objectives + +- Deepening the concepts related to working with sockets. +- Developing skills in implementing and designing applications that use asynchronous operations and other advanced I/O operations. +- Deepening the use of the API for advanced I/O operations in the Linux operating system. + +## Statement + +Implement a web server that uses the following advanced I/O operations: + +- Asynchronous operations on files +- Non-blocking operations on sockets +- Zero-copying +- Multiplexing I/O operations + +The server implements a limited functionality of the HTTP protocol: passing files to clients. + +The web server will use the multiplexing API to wait for connections from clients - [epoll](https://man7.org/linux/man-pages/man7/epoll.7.html). +On the established connections, requests from clients will be received and then responses will be distributed to them. + + +The server will serve files from the `AWS_DOCUMENT_ROOT` directory, defined within the assignments' [header](./skel/aws.h). +Files are only found in subdirectories `AWS_DOCUMENT_ROOT/static/` and `AWS_DOCUMENT_ROOT/dynamic/`. +The corresponding request paths will be, for example, `AWS_DOCUMENT_ROOT/static/test.dat` and `AWS_DOCUMENT_ROOT/dynamic/test.dat`. +The file processing will be: + +- The files in the `AWS_DOCUMENT_ROOT/static/` directory are static files that will be transmitted to clients using the zero-copying API - [sendfile](https://man7.org/linux/man-pages/man2/sendfile.2.html)] +- Files in the `AWS_DOCUMENT_ROOT/dynamic/` directory are files that are supposed to require a server-side post-processing phase. These files will be read from disk using the asynchronous API and then pushed to the clients. Streaming will use non-blocking sockets (Linux) +- An [HTTP 404](https://en.wikipedia.org/wiki/HTTP_404) message will be sent for invalid request paths + +After transmitting a file, according to the HTTP protocol, the connection is closed. + +### Details and recommendations for the implementation + +- Implementing the assignment requires having a state machine for each connection, which you periodically query and update as the transfer proceeds. +Check the `connection_state` data structure defined in the [assignment header](./skel/awh.h). +- Find the `connection` data structure defined in the [assignment header](./skel/awh.h). +This can be used to keep track of an open connection. +- Definitions of other useful macros and data structures can be found in the assignment header. +- HTTP responses will have the code `200` for existing files and `404` for not existing files. + - A valid response consists of the HTTP header, containing the related directives, two newlines (`\r\n\r\n`), followed by the actual content (the file). + - Sample answers can be found in the parser test file or in the provided sample. + - You can use predefined request directives such as `Date`, `Last-Modified`, etc. + - The `Content-Length` directive **must** specify the size of the HTTP content (actual data) in bytes. + - The `Connection` directive **must** be initialized to `close`. +- The port on which the web server listens for connections is defined within the assignment header: the `AWS_LISTEN_PORT` macro. +- The root directory relative to which the resources/files are searched is defined within the assignment header as the `AWS_DOCUMENT_ROOT` macro. + +## Support Code + +### HTTP Parser + +The clients and server will communicate using the HTTP protocol. +For parsing HTTP requests from clients we recommend using [this HTTP parser](https://github.com/nodejs/http-parser), also available in the assignments' [http-parser](./skel/http-parser). +You will need to use a callback to get the path to the local resource requested by the client. +Find a simplified example of using the parser in the [samples directory](./skel/http-parser/samples/). + +### API and Implementation Tasks + +The [skel/aws.c](./skel/aws.c) file contains the code skelethon with several functions that have to be implemented. +Follow the `TODO` areas in the file to start your implementation. + +> It can be reorganized as desired, as long as all the requirements of the assignment are implemented. + +## Testing and Grading + +The testing is automated. +Tests are located in the `tests/` directory. + +To test your implementation, do the following steps: + +- Run the `make` command inside the `skel/` directory and make sure it compiles with no errors and that the `aws` executable is generated. +- Run the `make check` command in the `tests/` directory. + +There are 35 tests for this assignment, of which 13 are doubled by a memory leak check test. +A successful run looks as the following: + +``` +student@so:~/operating-systems/content/assignments/async-web-server/tests$ make check +make -C _test +make[1]: Entering directory '/home/student/operating-systems/content/assignments/async-web-server/tests/_test' +make[1]: Nothing to be done for 'all'. +make[1]: Leaving directory '/home/student/operating-systems/content/assignments/async-web-server/tests/_test' + + = Testing - Asynchronous Web Server = + +01) Test executable exists.............................................passed [01/90] +02) Test executable runs...............................................passed [01/90] +03) Test listening.....................................................passed [01/90] +04) Test listening on port.............................................passed [01/90] +05) Test accepts connections...........................................passed [01/90] +06) Test accepts multiple connections..................................passed [01/90] +07) Test epoll usage...................................................passed [01/90] +08) Test disconnect....................................................passed [01/90] +09) Test multiple disconnect...........................................passed [01/90] +10) Test connect disconnect connect....................................passed [01/90] +11) Test multiple connect disconnect connect...........................passed [01/90] +12) Test unordered connect disconnect connect..........................passed [01/90] +13) Test replies http request..........................................passed [02/90] +13) Test replies http request - memcheck...............................passed [01/90] +14) Test second replies http request...................................passed [01/90] +15) Test sendfile usage................................................passed [02/90] +16) Test small static file wget........................................passed [02/90] +17) Test small static file wget cmp....................................passed [04/90] +17) Test small static file wget cmp - memcheck.........................passed [01/90] +18) Test large static file wget........................................passed [02/90] +19) Test large static file wget cmp....................................passed [04/90] +19) Test large static file wget cmp - memcheck.........................passed [01/90] +20) Test bad static file 404...........................................passed [02/90] +21) Test bad path 404..................................................passed [02/90] +22) Test get one static file then another..............................passed [02/90] +22) Test get one static file then another - memcheck...................passed [01/90] +23) Test get two simultaneous static files.............................passed [03/90] +23) Test get two simultaneous static files - memcheck..................passed [01/90] +24) Test get multiple simultaneous static files........................passed [04/90] +24) Test get multiple simultaneous static files - memcheck.............passed [01/90] +25) Test io submit uses................................................passed [02/90] +26) Test small dynamic file wget.......................................passed [02/90] +27) Test small dynamic file wget cmp...................................passed [04/90] +27) Test small dynamic file wget cmp - memcheck........................passed [01/90] +28) Test large dynamic file wget.......................................passed [02/90] +29) Test large dynamic file wget cmp...................................passed [04/90] +29) Test large dynamic file wget cmp - memcheck........................passed [01/90] +30) Test bad dynamic file 404..........................................passed [02/90] +31) Test get one dynamic file then another.............................passed [03/90] +31) Test get one dynamic file then another - memcheck..................passed [01/90] +32) Test get two simultaneous dynamic files............................passed [04/90] +32) Test get two simultaneous dynamic files - memcheck.................passed [01/90] +33) Test get multiple simultaneous dynamic files.......................passed [05/90] +33) Test get multiple simultaneous dynamic files - memcheck............passed [01/90] +34) Test get two simultaneous static and dynamic files.................passed [03/90] +34) Test get two simultaneous static and dynamic files - memcheck......passed [01/90] +35) Test get multiple simultaneous static and dynamic files............passed [04/90] +35) Test get multiple simultaneous static and dynamic files - memcheck.passed [01/90] + + Total: [90/100] +``` + +Individual tests can be run using the `./run_test.sh` bash script as the following: + +``` +student@so:~/operating-systems/content/assignments/async-web-server/tests$ ./_test/run_test.sh 3 +03) Test listening.....................................................passed [01/90] + +``` + +Where `3` is the test you want to run. + +Some tests are doubled by a memory check test. +This will only run if the regular test passed. +For example, test 31 will output the following in case of success: + +``` +student@so:~/operating-systems/content/assignments/async-web-server/tests$ ./_test/run_test.sh 31 +31) Test get one dynamic file then another.............................passed [03/90] +31) Test get one dynamic file then another - memcheck..................passed [01/90] +``` + +and one of the following in case of error: + +``` +# if the regular tests failed, the memory check tests is not performed +student@so:~/operating-systems/content/assignments/async-web-server/tests$ ./_test/run_test.sh 31 +31) Test get one dynamic file then another.............................failed [ 0/90] +31) Test get one dynamic file then another - memcheck..................passed [01/90] +``` + +> Note: The memcheck test for failed regular tests will not be taken into consideration for the final score. + This output will be fixed in the next commit. + + +Tests use the `static/` and `dynamic/` folders. +These folders are created and removed using the `init` and `cleanup` arguments to `_test/run_test.sh`. + +### Behind the Scenes + +Tests are basically unit tests. + +Each test function follows the unit test patter: initialization, action, +evaluation. + +Each test starts the server, creates a given context, checks for validity and +then terminates the server process. + +### Debugging + +Logs are collected in `test.log` and `wget.log` files. + +## Resources + +- [sendfile](https://man7.org/linux/man-pages/man2/sendfile.2.html) + +- [io_setup & friends](https://man7.org/linux/man-pages/man2/io_setup.2.html) + +- [epoll](https://man7.org/linux/man-pages/man7/epoll.7.html) diff --git a/content/assignments/async-web-server/src/.gitignore b/assignments/async-web-server/src/.gitignore similarity index 100% rename from content/assignments/async-web-server/src/.gitignore rename to assignments/async-web-server/src/.gitignore diff --git a/content/assignments/async-web-server/src/Makefile b/assignments/async-web-server/src/Makefile similarity index 100% rename from content/assignments/async-web-server/src/Makefile rename to assignments/async-web-server/src/Makefile diff --git a/content/assignments/async-web-server/src/aws.c b/assignments/async-web-server/src/aws.c similarity index 100% rename from content/assignments/async-web-server/src/aws.c rename to assignments/async-web-server/src/aws.c diff --git a/content/assignments/async-web-server/src/aws.h b/assignments/async-web-server/src/aws.h similarity index 100% rename from content/assignments/async-web-server/src/aws.h rename to assignments/async-web-server/src/aws.h diff --git a/content/assignments/async-web-server/src/http-parser/.gitignore b/assignments/async-web-server/src/http-parser/.gitignore similarity index 100% rename from content/assignments/async-web-server/src/http-parser/.gitignore rename to assignments/async-web-server/src/http-parser/.gitignore diff --git a/content/assignments/async-web-server/src/http-parser/CONTRIBUTIONS b/assignments/async-web-server/src/http-parser/CONTRIBUTIONS similarity index 100% rename from content/assignments/async-web-server/src/http-parser/CONTRIBUTIONS rename to assignments/async-web-server/src/http-parser/CONTRIBUTIONS diff --git a/content/assignments/async-web-server/src/http-parser/CPPLINT.cfg b/assignments/async-web-server/src/http-parser/CPPLINT.cfg similarity index 100% rename from content/assignments/async-web-server/src/http-parser/CPPLINT.cfg rename to assignments/async-web-server/src/http-parser/CPPLINT.cfg diff --git a/content/assignments/async-web-server/src/http-parser/LICENSE-MIT b/assignments/async-web-server/src/http-parser/LICENSE-MIT similarity index 98% rename from content/assignments/async-web-server/src/http-parser/LICENSE-MIT rename to assignments/async-web-server/src/http-parser/LICENSE-MIT index 40ebce2a9d..a3187cc100 100644 --- a/content/assignments/async-web-server/src/http-parser/LICENSE-MIT +++ b/assignments/async-web-server/src/http-parser/LICENSE-MIT @@ -16,4 +16,4 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -IN THE SOFTWARE. +IN THE SOFTWARE. diff --git a/content/assignments/async-web-server/src/http-parser/Makefile b/assignments/async-web-server/src/http-parser/Makefile similarity index 100% rename from content/assignments/async-web-server/src/http-parser/Makefile rename to assignments/async-web-server/src/http-parser/Makefile diff --git a/content/assignments/async-web-server/src/http-parser/README.md b/assignments/async-web-server/src/http-parser/README.md similarity index 100% rename from content/assignments/async-web-server/src/http-parser/README.md rename to assignments/async-web-server/src/http-parser/README.md diff --git a/content/assignments/async-web-server/src/http-parser/http_parser.c b/assignments/async-web-server/src/http-parser/http_parser.c similarity index 100% rename from content/assignments/async-web-server/src/http-parser/http_parser.c rename to assignments/async-web-server/src/http-parser/http_parser.c diff --git a/content/assignments/async-web-server/src/http-parser/http_parser.h b/assignments/async-web-server/src/http-parser/http_parser.h similarity index 100% rename from content/assignments/async-web-server/src/http-parser/http_parser.h rename to assignments/async-web-server/src/http-parser/http_parser.h diff --git a/content/assignments/async-web-server/src/http-parser/samples/.gitignore b/assignments/async-web-server/src/http-parser/samples/.gitignore similarity index 100% rename from content/assignments/async-web-server/src/http-parser/samples/.gitignore rename to assignments/async-web-server/src/http-parser/samples/.gitignore diff --git a/content/assignments/async-web-server/src/http-parser/samples/CPPLINT.cfg b/assignments/async-web-server/src/http-parser/samples/CPPLINT.cfg similarity index 100% rename from content/assignments/async-web-server/src/http-parser/samples/CPPLINT.cfg rename to assignments/async-web-server/src/http-parser/samples/CPPLINT.cfg diff --git a/content/assignments/async-web-server/src/http-parser/samples/GNUmakefile b/assignments/async-web-server/src/http-parser/samples/GNUmakefile similarity index 100% rename from content/assignments/async-web-server/src/http-parser/samples/GNUmakefile rename to assignments/async-web-server/src/http-parser/samples/GNUmakefile diff --git a/content/assignments/async-web-server/src/http-parser/samples/Makefile b/assignments/async-web-server/src/http-parser/samples/Makefile similarity index 100% rename from content/assignments/async-web-server/src/http-parser/samples/Makefile rename to assignments/async-web-server/src/http-parser/samples/Makefile diff --git a/content/assignments/async-web-server/src/http-parser/samples/README b/assignments/async-web-server/src/http-parser/samples/README similarity index 100% rename from content/assignments/async-web-server/src/http-parser/samples/README rename to assignments/async-web-server/src/http-parser/samples/README diff --git a/content/assignments/async-web-server/src/http-parser/samples/test_get_request_path.c b/assignments/async-web-server/src/http-parser/samples/test_get_request_path.c similarity index 100% rename from content/assignments/async-web-server/src/http-parser/samples/test_get_request_path.c rename to assignments/async-web-server/src/http-parser/samples/test_get_request_path.c diff --git a/content/assignments/async-web-server/src/http-parser/test.c b/assignments/async-web-server/src/http-parser/test.c similarity index 100% rename from content/assignments/async-web-server/src/http-parser/test.c rename to assignments/async-web-server/src/http-parser/test.c diff --git a/content/assignments/async-web-server/src/samples/.gitignore b/assignments/async-web-server/src/samples/.gitignore similarity index 100% rename from content/assignments/async-web-server/src/samples/.gitignore rename to assignments/async-web-server/src/samples/.gitignore diff --git a/content/assignments/async-web-server/src/samples/CPPLINT.cfg b/assignments/async-web-server/src/samples/CPPLINT.cfg similarity index 100% rename from content/assignments/async-web-server/src/samples/CPPLINT.cfg rename to assignments/async-web-server/src/samples/CPPLINT.cfg diff --git a/content/assignments/async-web-server/src/samples/Makefile b/assignments/async-web-server/src/samples/Makefile similarity index 100% rename from content/assignments/async-web-server/src/samples/Makefile rename to assignments/async-web-server/src/samples/Makefile diff --git a/content/assignments/async-web-server/src/samples/README b/assignments/async-web-server/src/samples/README similarity index 100% rename from content/assignments/async-web-server/src/samples/README rename to assignments/async-web-server/src/samples/README diff --git a/content/assignments/async-web-server/src/samples/epoll_echo_server.c b/assignments/async-web-server/src/samples/epoll_echo_server.c similarity index 100% rename from content/assignments/async-web-server/src/samples/epoll_echo_server.c rename to assignments/async-web-server/src/samples/epoll_echo_server.c diff --git a/content/assignments/async-web-server/src/samples/http_reply_once.c b/assignments/async-web-server/src/samples/http_reply_once.c similarity index 100% rename from content/assignments/async-web-server/src/samples/http_reply_once.c rename to assignments/async-web-server/src/samples/http_reply_once.c diff --git a/content/assignments/async-web-server/src/utils/debug.h b/assignments/async-web-server/src/utils/debug.h similarity index 100% rename from content/assignments/async-web-server/src/utils/debug.h rename to assignments/async-web-server/src/utils/debug.h diff --git a/content/assignments/async-web-server/src/utils/sock_util.c b/assignments/async-web-server/src/utils/sock_util.c similarity index 100% rename from content/assignments/async-web-server/src/utils/sock_util.c rename to assignments/async-web-server/src/utils/sock_util.c diff --git a/chapters/app-interact/overview/guides/comm-channels/utils/sock/sock_util.h b/assignments/async-web-server/src/utils/sock_util.h similarity index 100% rename from chapters/app-interact/overview/guides/comm-channels/utils/sock/sock_util.h rename to assignments/async-web-server/src/utils/sock_util.h diff --git a/content/assignments/async-web-server/src/utils/util.h b/assignments/async-web-server/src/utils/util.h similarity index 100% rename from content/assignments/async-web-server/src/utils/util.h rename to assignments/async-web-server/src/utils/util.h diff --git a/content/assignments/async-web-server/src/utils/w_epoll.h b/assignments/async-web-server/src/utils/w_epoll.h similarity index 100% rename from content/assignments/async-web-server/src/utils/w_epoll.h rename to assignments/async-web-server/src/utils/w_epoll.h diff --git a/content/assignments/async-web-server/tests/.gitignore b/assignments/async-web-server/tests/.gitignore similarity index 100% rename from content/assignments/async-web-server/tests/.gitignore rename to assignments/async-web-server/tests/.gitignore diff --git a/content/assignments/async-web-server/tests/CPPLINT.cfg b/assignments/async-web-server/tests/CPPLINT.cfg similarity index 100% rename from content/assignments/async-web-server/tests/CPPLINT.cfg rename to assignments/async-web-server/tests/CPPLINT.cfg diff --git a/content/assignments/async-web-server/tests/Makefile b/assignments/async-web-server/tests/Makefile similarity index 100% rename from content/assignments/async-web-server/tests/Makefile rename to assignments/async-web-server/tests/Makefile diff --git a/content/assignments/async-web-server/tests/_test/CPPLINT.cfg b/assignments/async-web-server/tests/_test/CPPLINT.cfg similarity index 100% rename from content/assignments/async-web-server/tests/_test/CPPLINT.cfg rename to assignments/async-web-server/tests/_test/CPPLINT.cfg diff --git a/content/assignments/async-web-server/tests/_test/Makefile b/assignments/async-web-server/tests/_test/Makefile similarity index 100% rename from content/assignments/async-web-server/tests/_test/Makefile rename to assignments/async-web-server/tests/_test/Makefile diff --git a/content/assignments/async-web-server/tests/_test/run_test.sh b/assignments/async-web-server/tests/_test/run_test.sh similarity index 100% rename from content/assignments/async-web-server/tests/_test/run_test.sh rename to assignments/async-web-server/tests/_test/run_test.sh diff --git a/content/assignments/async-web-server/tests/_test/sockop_preload.c b/assignments/async-web-server/tests/_test/sockop_preload.c similarity index 100% rename from content/assignments/async-web-server/tests/_test/sockop_preload.c rename to assignments/async-web-server/tests/_test/sockop_preload.c diff --git a/content/assignments/parallel-firewall/tests/grade.sh b/assignments/async-web-server/tests/grade.sh old mode 100755 new mode 100644 similarity index 100% rename from content/assignments/parallel-firewall/tests/grade.sh rename to assignments/async-web-server/tests/grade.sh diff --git a/content/assignments/async-web-server/tests/run_all.sh b/assignments/async-web-server/tests/run_all.sh similarity index 100% rename from content/assignments/async-web-server/tests/run_all.sh rename to assignments/async-web-server/tests/run_all.sh diff --git a/assignments/lambda-function-loader/README.md b/assignments/lambda-function-loader/README.md new file mode 100644 index 0000000000..a4efe5a83c --- /dev/null +++ b/assignments/lambda-function-loader/README.md @@ -0,0 +1,197 @@ +--- +parent: Assignments +title: Lambda Function Loader (hackathon) +nav_order: 6 +--- + +# Lambda Function Loader + +## Application Development + +All development work should be carried out exclusively within the [virtual machines dedicated to Operating Systems](https://cs-pub-ro.github.io/operating-systems/resources#virtual-machine). + +**Note:** +It's crucial to avoid running tests locally, either on personal computers or within your allocated virtual machines. +Discrepancies might arise between local environments and the dedicated OS virtual machines. +Grading will consider results obtained solely in the OS virtual machine environment. + +The core of this project revolves around the construction of a system capable of dynamic library loading and subsequent function execution. +The project architecture embraces a [client-server model](https://cs-pub-ro.github.io/operating-systems/IO/lab10#client-server-model), where the server is responsible for receiving and fulfilling requests to execute functions sourced from specific dynamic libraries present within the system. + +This functionality marks the initial phase toward realizing capabilities akin to [AWS Lambda](https://aws.amazon.com/lambda/). +In this context, users can load and execute functions across distinct servers upon request. +In our envisioned project, these functions are pre-implemented within libraries, and the client instigates their execution by issuing specific commands to the server. +Detailed below are the primary requisites and potential enhancements. + +### Implementation Details and Notes + +The implementation involves the capability to receive commands utilizing [UNIX sockets](https://cs-pub-ro.github.io/operating-systems/IO/lab10#unix-sockets), encompassing the designation of a library (the path to the library file) and optionally specifying a function. +A command transmitted from the client to the server conforms to the format: ` [ []]` where: + +- `libname` represents the path to the intended library. +- `funcname` is an optional parameter denoting the function's name within the library. +By default, if unspecified, the function used is named `run`. +- `filename` is an optional parameter indicating the filename containing input data, provided as an argument to the `funcname` function. + +The resulting output data generated from execution will be written to a file, and the file's name is returned as a response to the client. +This returned file exclusively contains messages printed to the standard output by the invoked library function. + +The communication sequence between the client and server follows this structure: + +```plaintext +Client Server + listen() +connect() accept() +send() -—---libname [funcname [filename]]-----> recv() +recv() <--------------outputfile--------------- send() +``` + +Post receiving and parsing a message from a client, the server systematically triggers a series of functions, each serving distinct purposes: + +- Prepare/pre-hooks: Initial operations independent of loading the library or executing functions. +- Library loading: Loading the library and other related management tasks. +- Execution of requested functionality: +Execution of the specified function from the library. +If `funcname` is absent, the `run` function is invoked. +The `filename` parameter is permissible only when `funcname` is specified. +- Unloading the library: Procedures associated with unloading the library from memory. +- Post-hooks: +Operations conducted after execution, regardless of the library's presence in memory. + +**Tip:** +The mentioned five functions serve as guidelines. +Depending on the implementation, some functions might remain as stubs if not fully implemented. + +**Important:** +The server's implementation must be **parallelized** to expedite the handling of client requests. +Each team is responsible for selecting the parallelization model (via processes, threads, hybrid methods, utilizing work pools, etc.) based on the project's objectives. + +Fundamental functionality necessitates implementation in C, designed for the Linux operating system, employing the POSIX API: + +- Establishment of client-server connection. + - The connection will utilize UNIX Sockets through the `libipc.so` library. + - Necessary functions for creating, connecting, and sending/receiving data in this library (`libipc.so`) will be included. + - The library can be modified as long as the provided test client (`checker/client.c`) compiles and connects to the server successfully. + - The test client (`checker/client.c`) remains unmodifiable. + Any changes required for additional functionalities should be implemented in a new test client. +- Communication between client and server: The client transmits requests following the specified format and awaits the response, which contains the file path in the system storing the results. +The server receives and processes the client's request. +- Server request handling: Parsing the received command, loading the library, and executing the requested function from the library. + - In case of an error, an error message, `Error: could not be executed.`, followed by relevant error messages (Hint: utilize `strerror()`), will be written to the output file. + - The output file will be created with a randomly generated name employing the `mkstemp()` function, utilizing `OUTPUTFILE_TEMPLATE` as a filename template. +- Returning a response to the client: +The response entails the filename generated in the preceding step based on `OUTPUTFILE_TEMPLATE`. +- Parallelizing the client handling mode. + - Ensuring synchronization of output data and library loading/unloading does not hinder the functionality of other threads/processes. + +**Warning:** +It is crucial to ensure proper server functionality in your implementation. +Executing functions from a user-requested library can result in unexpected behavior (e.g. invalid accesses, forced closures, etc.). +The server's implementation must be robust, enabling continuous execution even in such cases. + +### Necessary Theoretical Concepts + +What is needed for the implementation of the proposed project: + +- Understanding inter-process communication - using Unix sockets and read/write or send/receive operations ([I/O Chapter](https://cs-pub-ro.github.io/operating-systems/IO/)) +- Understanding the API for loading/unloading libraries and executing functions from dynamic libraries: + - Hint: `man dlopen` +- Working with shared data between processes or threads ([Compute Chapter](https://cs-pub-ro.github.io/operating-systems/Compute/)) +- Working with memory +- Working with files + +### Testing + +#### Example Run + +```bash +## Starting the server +student@os:.../lambda-function-loader/src$ make +student@os:.../lambda-function-loader/src$ ./server + +## Client +# +# Open another terminal and instruct the client to send a library to the server. +# Please remember the calling convention for client +# ./client [ []] +# +# Note: When is not provided, the server must execute the `run` function. +student@os:.../lambda-function-loader/$ cd tests +student@os:.../lambda-function-loader/tests$ make +student@os:.../lambda-function-loader/tests$ ./client $(realpath libbasic.so) +Output file: /home/student/operating-systems/content/assignments/lambda-function-loader/tests/output/out-bSJdTv + +student@os:.../lambda-function-loader/tests$ cat /home/student/operating-systems/content/assignments/lambda-function-loader/tests/output/out-bSJdTv +run + +# Execute the "function" function from libbasic.so. +# The function does not exist, so an error is printed. +student@os:.../lambda-function-loader/tests$ ./client $(realpath libbasic.so) function +Output file: /home/student/operating-systems/content/assignments/lambda-function-loader/tests/output/out-qOcoAA + +student@os:.../lambda-function-loader/tests$ cat /home/so/output/out-qOcoAA +Error: /home/student/operating-systems/content/assignments/lambda-function-loader/tests/libbasic.so function could not be executed. + +# Execute the "cat" function from libbasic.so with the argument being the full path of file "Makefile" +student@os:.../lambda-function-loader/tests$ ./client $(realpath libbasic.so) cat $(realpath Makefile) +Output file: /home/student/operating-systems/content/assignments/lambda-function-loader/tests/output/out-y732bN + +student@os:.../lambda-function-loader/tests$ cat /home/student/operating-systems/content/assignments/lambda-function-loader/tests/output/out-y732bN +CC=gcc +[...] +``` + +#### Checker Run + +You have a checker available for partial verification of your implementation. + +The checker behaves as follows: + +- it automatically builds the source files then it starts the `server`. +- it runs all the tests without restarting the `server`. +- it then kills the `server`. + +For debugging purposes, you can instruct the checker to restart the server before each test. +Please see below. + +```bash +student@os:.../lambda-function-loader$ cd tests + +# Run the full test suite +student@os:.../lambda-function-loader/tests$ make check +[...] + +# Run a specific test be setting `TEST=` variable +student@os:.../lambda-function-loader/tests$ make check TEST=3 +./checker.sh 3 +Running test 3: Basic file content [ 5/ 5] Test passed + +# Run the checker with server debugging on +student@os:.../lambda-function-loader/tests$ SERVER_DEBUG=1 make check +./checker.sh +Running the checker with server restart per test +[...] +``` + +### Additional Tasks / Tie-breakers + +All teams must implement the fundamental features as a base requirement. +Additionally, for differentiation purposes, teams have the option to introduce supplementary functionalities to the application. +Each added functionality should be thoroughly documented in a README file, providing details on its purpose and the testing methodologies employed. + +Here are some potential additional functionalities to consider (or propose other relevant ones): + +- Developing an interface for client-server connectivity: +Depending on runtime parameters, utilize either network or UNIX sockets. +- Expanding the client handling mode: +Allow the server to process multiple commands until encountering an "exit"/"quit" command. +- Enhancing server security: +Implement measures to prevent data leaks. +- Conducting performance optimizations. +- Establishing application logging. +- Managing configuration files/options: +Configure the server during startup based on specified options/configuration files such as maximum client count, socket type preferences, etc. +- Real-time server monitoring and statistical extraction (e.g. client count, loaded libraries, memory usage, etc.). +- Adapting the implementation to another programming language (the initial skeleton is in C, but the communication's nature allows implementation in any language meeting project requirements). +- Generating a suitable response in cases where the requested function exceeds a `TIMEOUT` or performs actions leading to server shutdown (e.g. invalid memory access, exit/abort calls). +The solution's complexity may vary based on how the client's connection termination is handled (whether an error message is conveyed) and how the cause of execution termination is identified. diff --git a/content/assignments/lambda-function-loader/src/Makefile b/assignments/lambda-function-loader/src/Makefile similarity index 100% rename from content/assignments/lambda-function-loader/src/Makefile rename to assignments/lambda-function-loader/src/Makefile diff --git a/content/assignments/lambda-function-loader/src/ipc.c b/assignments/lambda-function-loader/src/ipc.c similarity index 100% rename from content/assignments/lambda-function-loader/src/ipc.c rename to assignments/lambda-function-loader/src/ipc.c diff --git a/content/assignments/lambda-function-loader/src/ipc.h b/assignments/lambda-function-loader/src/ipc.h similarity index 100% rename from content/assignments/lambda-function-loader/src/ipc.h rename to assignments/lambda-function-loader/src/ipc.h diff --git a/content/assignments/lambda-function-loader/src/server.c b/assignments/lambda-function-loader/src/server.c similarity index 100% rename from content/assignments/lambda-function-loader/src/server.c rename to assignments/lambda-function-loader/src/server.c diff --git a/content/assignments/lambda-function-loader/src/server.h b/assignments/lambda-function-loader/src/server.h similarity index 100% rename from content/assignments/lambda-function-loader/src/server.h rename to assignments/lambda-function-loader/src/server.h diff --git a/content/assignments/lambda-function-loader/tests/Makefile b/assignments/lambda-function-loader/tests/Makefile similarity index 100% rename from content/assignments/lambda-function-loader/tests/Makefile rename to assignments/lambda-function-loader/tests/Makefile diff --git a/content/assignments/lambda-function-loader/tests/advanced.c b/assignments/lambda-function-loader/tests/advanced.c similarity index 100% rename from content/assignments/lambda-function-loader/tests/advanced.c rename to assignments/lambda-function-loader/tests/advanced.c diff --git a/content/assignments/lambda-function-loader/tests/basic.c b/assignments/lambda-function-loader/tests/basic.c similarity index 100% rename from content/assignments/lambda-function-loader/tests/basic.c rename to assignments/lambda-function-loader/tests/basic.c diff --git a/content/assignments/lambda-function-loader/tests/checker.sh b/assignments/lambda-function-loader/tests/checker.sh similarity index 100% rename from content/assignments/lambda-function-loader/tests/checker.sh rename to assignments/lambda-function-loader/tests/checker.sh diff --git a/content/assignments/lambda-function-loader/tests/client.c b/assignments/lambda-function-loader/tests/client.c similarity index 100% rename from content/assignments/lambda-function-loader/tests/client.c rename to assignments/lambda-function-loader/tests/client.c diff --git a/content/assignments/lambda-function-loader/tests/ref/ref10 b/assignments/lambda-function-loader/tests/ref/ref10 similarity index 100% rename from content/assignments/lambda-function-loader/tests/ref/ref10 rename to assignments/lambda-function-loader/tests/ref/ref10 diff --git a/content/assignments/lambda-function-loader/tests/ref/ref11 b/assignments/lambda-function-loader/tests/ref/ref11 similarity index 100% rename from content/assignments/lambda-function-loader/tests/ref/ref11 rename to assignments/lambda-function-loader/tests/ref/ref11 diff --git a/content/assignments/lambda-function-loader/tests/ref/test3 b/assignments/lambda-function-loader/tests/ref/test3 similarity index 100% rename from content/assignments/lambda-function-loader/tests/ref/test3 rename to assignments/lambda-function-loader/tests/ref/test3 diff --git a/content/assignments/lambda-function-loader/tests/ref/test4 b/assignments/lambda-function-loader/tests/ref/test4 similarity index 100% rename from content/assignments/lambda-function-loader/tests/ref/test4 rename to assignments/lambda-function-loader/tests/ref/test4 diff --git a/content/assignments/lambda-function-loader/tests/ref/test7 b/assignments/lambda-function-loader/tests/ref/test7 similarity index 100% rename from content/assignments/lambda-function-loader/tests/ref/test7 rename to assignments/lambda-function-loader/tests/ref/test7 diff --git a/content/assignments/lambda-function-loader/tests/ref/test8 b/assignments/lambda-function-loader/tests/ref/test8 similarity index 100% rename from content/assignments/lambda-function-loader/tests/ref/test8 rename to assignments/lambda-function-loader/tests/ref/test8 diff --git a/content/assignments/lambda-function-loader/tests/special.c b/assignments/lambda-function-loader/tests/special.c similarity index 100% rename from content/assignments/lambda-function-loader/tests/special.c rename to assignments/lambda-function-loader/tests/special.c diff --git a/content/assignments/memory-allocator/.vscode/launch.json b/assignments/memory-allocator/.vscode/launch.json similarity index 100% rename from content/assignments/memory-allocator/.vscode/launch.json rename to assignments/memory-allocator/.vscode/launch.json diff --git a/assignments/memory-allocator/README.md b/assignments/memory-allocator/README.md new file mode 100644 index 0000000000..410fa8f822 --- /dev/null +++ b/assignments/memory-allocator/README.md @@ -0,0 +1,338 @@ +--- +parent: Assignments +title: Memory Allocator +nav_order: 2 +--- + +# Memory Allocator + +## Objectives + +- Learn the basics of memory management by implementing minimal versions of `malloc()`, `calloc()`, `realloc()`, and `free()`. +- Accommodate with the memory management syscalls in Linux: `brk()`, `mmap()`, and `munmap()`. +- Understand the bottlenecks of memory allocation and how to reduce them. + +## Statement + +Build a minimalistic memory allocator that can be used to manually manage virtual memory. +The goal is to have a reliable library that accounts for explicit allocation, reallocation, and initialization of memory. + +## Support Code + +The support code consists of three directories: + +- `src/` will contain your solution +- `tests/` contains the test suite and a Python script to verify your work +- `utils/` contains `osmem.h` that describes your library interface, `block_meta.h` which contains details of `struct block_meta`, and an implementation for `printf()` function that does **NOT** use the heap + +The test suite consists of `.c` files that will be dynamically linked to your library, `libosmem.so`. +You can find the sources in the `tests/snippets/` directory. +The results of the previous will also be stored in `tests/snippets/` and the reference files are in the `tests/ref/` directory. + +The automated checking is performed using `run_tests.py`. +It runs each test and compares the syscalls made by the `os_*` functions with the reference file, providing a diff if the test failed. + +## API + +1. `void *os_malloc(size_t size)` + + Allocates `size` bytes and returns a pointer to the allocated memory. + + Chunks of memory smaller than `MMAP_THRESHOLD` are allocated with `brk()`. + Bigger chunks are allocated using `mmap()`. + The memory is uninitialized. + + - Passing `0` as `size` will return `NULL`. + +1. `void *os_calloc(size_t nmemb, size_t size)` + + Allocates memory for an array of `nmemb` elements of `size` bytes each and returns a pointer to the allocated memory. + + Chunks of memory smaller than [`page_size`](https://man7.org/linux/man-pages/man2/getpagesize.2.html) are allocated with `brk()`. + Bigger chunks are allocated using `mmap()`. + The memory is set to zero. + + - Passing `0` as `nmemb` or `size` will return `NULL`. + +1. `void *os_realloc(void *ptr, size_t size)` + + Changes the size of the memory block pointed to by `ptr` to `size` bytes. + If the size is smaller than the previously allocated size, the memory block will be truncated. + + If `ptr` points to a block on heap, `os_realloc()` will first try to expand the block, rather than moving it. + Otherwise, the block will be reallocated and its contents copied. + + When attempting to expand a block followed by multiple free blocks, `os_realloc()` will coalesce them one at a time and verify the condition for each. + Blocks will remain coalesced even if the resulting block will not be big enough for the new size. + + Calling `os_realloc()` on a block that has `STATUS_FREE` should return `NULL`. + This is a measure to prevent undefined behavior and make the implementation robust, it should not be considered a valid use case of `os_realloc()`. + + - Passing `NULL` as `ptr` will have the same effect as `os_malloc(size)`. + - Passing `0` as `size` will have the same effect as `os_free(ptr)`. + +1. `void os_free(void *ptr)` + + Frees memory previously allocated by `os_malloc()`, `os_calloc()` or `os_realloc()`. + + `os_free()` will not return memory from the heap to the OS by calling `brk()`, but rather mark it as free and reuse it in future allocations. + In the case of mapped memory blocks, `os_free()` will call `munmap()`. + +1. General + + - Allocations that increase the heap size will only expand the last block if it is free. + - You are allowed to use `sbrk()` instead of `brk()`, in view of the fact that [on Linux](https://man7.org/linux/man-pages/man2/brk.2.html#NOTES) `sbrk()` is implemented using the `brk()`. + - Do **NOT** use [`mremap()`](https://man7.org/linux/man-pages/man2/mremap.2.html) + - You must check the error code returned by every syscall. + You can use the `DIE()` macro for this. + +## Implementation + +An efficient implementation must keep data aligned, keep track of memory blocks and reuse freed blocks. +This can be further improved by reducing the number of syscalls and block operations. + +### [Memory Alignment](https://stackoverflow.com/a/381368) + +Allocated memory should be aligned (i.e. all addresses are multiple of a given size). +This is a space-time trade-off because memory blocks are padded so each can be read in one transaction. +It also allows for atomicity when interacting with a block of memory. + +All memory allocations should be aligned to **8 bytes** as required by 64 bit systems. + +### Block Reuse + +#### `struct block_meta` + +We will consider a **block** to be a continuous zone of memory, allocated and managed by our implementation. +The structure `block_meta` will be used to manage the metadata of a block. +Each allocated zone will comprise of a `block_meta` structure placed at the start, followed by data (**payload**). +For all functions, the returned address will be that of the **payload** (not of the `block_meta` structure). + +```c +struct block_meta { + size_t size; + int status; + struct block_meta *prev; + struct block_meta *next; +}; +``` + +_Note_: Both the `struct block_meta` and the **payload** of a block should be aligned to **8 bytes**. + +_Note_: Most compilers will automatically pad the structure, but you should still align it for portability. + +![memory-block](./img/memory-block.svg) + +#### Split Block + +Reusing memory blocks improves the allocator's performance, but might lead to [Internal Memory Fragmentation](https://www.tutorialspoint.com/difference-between-internal-fragmentation-and-external-fragmentation#:~:text=What%20is%20Internal%20Fragmentation%3F). +This happens when we allocate a size smaller than all available free blocks. +If we use one larger block the remaining size of that block will be wasted since it cannot be used for another allocation. + +To avoid this, a block should be truncated to the required size and the remaining bytes should be used to create a new free block. + +![Split Block](./img/split-block.svg) + +The resulting free block should be reusable. +The split will not be performed if the remaining size (after reserving space for `block_meta` structure and payload) is not big enough to fit another block (`block_meta` structure and at least **1 byte** of usable memory). + +_Note_: Do not forget the alignment! + +#### Coalesce Blocks + +There are cases when there is enough free memory for an allocation, but it is spread across multiple blocks that cannot be used. +This is called [External Memory Fragmentation](https://www.tutorialspoint.com/difference-between-internal-fragmentation-and-external-fragmentation#:~:text=What%20is%20External%20Fragmentation%3F). + +One technique to reduce external memory fragmentation is **block coalescing** which implies merging adjacent free blocks to form a contiguous chunk. + +![Coalesce Block Image](./img/coalesce-blocks.svg) + +Coalescing will be used before searching for a block and in `os_realloc()` to expand the current block when possible. + +_Note_: You might still need to split the block after coalesce. + +#### Find Best Block + +Our aim is to reuse a free block with a size closer to what we need in order to reduce the number of future operations on it. +This strategy is called **find best**. +On every allocation we need to search the whole list of blocks and choose the best fitting free block. + +In practice, it also uses a list of free blocks to avoid parsing all blocks, but this is out of the scope of the assignment. + +_Note_: For consistent results, coalesce all adjacent free blocks before searching. + +### Heap Preallocation + +Heap is used in most modern programs. +This hints at the possibility of preallocating a relatively big chunk of memory (i.e. **128 kilobytes**) when the heap is used for the first time. +This reduces the number of future `brk()` syscalls. + +For example, if we try to allocate 1000 bytes we should first allocate a block of 128 kilobytes and then split it. +On future small allocations, we should proceed to split the preallocated chunk. + +_Note_: Heap preallocation happens only once. + +## Building Memory Allocator + +To build `libosmem.so`, run `make` in the `src/` directory: + +```console +student@os:~/.../mem-alloc$ cd src/ +student@os:~/.../mem-alloc/src$ make +gcc -fPIC -Wall -Wextra -g -I../utils -c -o osmem.o osmem.c +gcc -fPIC -Wall -Wextra -g -I../utils -c -o ../utils/printf.o ../utils/printf.c +gcc -shared -o libosmem.so osmem.o helpers.o ../utils/printf.o +``` + +## Testing and Grading + +Testing is automated. +Tests are located in the `tests/` directory: + +```console +student@so:~/.../mem-alloc/tests$ ls -F +Makefile grade.sh@ ref/ run_tests.py snippets/ +``` + +To test and grade your assignment solution, enter the `tests/` directory and run `grade.sh`. +Note that this requires linters being available. +The easiest is to use a Docker-based setup with everything installed, as shown in the section ["Running the Linters"](#running-the-linters). +When using `grade.sh` you will get grades for correctness (maximum `90` points) and for coding style (maximum `10` points). +A successful run will provide you an output ending with: + +```console +### GRADE + + +Checker: 90/ 90 +Style: 10/ 10 +Total: 100/100 + + +### STYLE SUMMARY + + +``` + +### Running the Checker + +To run only the checker, use the `run_tests.py` script from the `tests/` directory. + +Before running `run_tests.py`, you first have to build `libosmem.so` in the `src/` directory and generate the test binaries in `tests/snippets`. +You can do so using the all-in-one `Makefile` rule from `tests/`: `make check`. + +```console +student@os:~/.../mem-alloc$ cd tests/ +student@os:~/.../mem-alloc/tests$ make check +gcc -fPIC -Wall -Wextra -g -I../utils -c -o osmem.o osmem.c +gcc -fPIC -Wall -Wextra -g -I../utils -c -o helpers.o helpers.c +gcc -fPIC -Wall -Wextra -g -I../utils -c -o ../utils/printf.o ../utils/printf.c +[...] +gcc -I../utils -fPIC -Wall -Wextra -g -o snippets/test-all snippets/test-all.c -L../src -losmem +gcc -I../utils -fPIC -Wall -Wextra -g -o snippets/test-calloc-arrays snippets/test-calloc-arrays.c -L../src -losmem +gcc -I../utils -fPIC -Wall -Wextra -g -o snippets/test-calloc-block-reuse snippets/test-calloc-block-reuse.c -L../src -losmem +gcc -I../utils -fPIC -Wall -Wextra -g -o snippets/test-calloc-coalesce-big snippets/test-calloc-coalesce-big.c -L../src -losmem +gcc -I../utils -fPIC -Wall -Wextra -g -o snippets/test-calloc-coalesce snippets/test-calloc-coalesce.c -L../src -losmem +gcc -I../utils -fPIC -Wall -Wextra -g -o snippets/test-calloc-expand-block snippets/test-calloc-expand-block.c -L../src -losmem +[...] +test-malloc-no-preallocate ........................ passed ... 2 +test-malloc-preallocate ........................ passed ... 3 +test-malloc-arrays ........................ passed ... 5 +test-malloc-block-reuse ........................ passed ... 3 +test-malloc-expand-block ........................ passed ... 2 +test-malloc-no-split ........................ passed ... 2 +test-malloc-split-one-block ........................ passed ... 3 +test-malloc-split-first ........................ passed ... 2 +test-malloc-split-last ........................ passed ... 2 +test-malloc-split-middle ........................ passed ... 3 +test-malloc-split-vector ........................ passed ... 2 +test-malloc-coalesce ........................ passed ... 3 +test-malloc-coalesce-big ........................ passed ... 3 +test-calloc-no-preallocate ........................ passed ... 1 +test-calloc-preallocate ........................ passed ... 1 +test-calloc-arrays ........................ passed ... 5 +test-calloc-block-reuse ........................ passed ... 1 +test-calloc-expand-block ........................ passed ... 1 +test-calloc-no-split ........................ passed ... 1 +test-calloc-split-one-block ........................ passed ... 1 +test-calloc-split-first ........................ passed ... 1 +test-calloc-split-last ........................ passed ... 1 +test-calloc-split-middle ........................ passed ... 1 +test-calloc-split-vector ........................ passed ... 2 +test-calloc-coalesce ........................ passed ... 2 +test-calloc-coalesce-big ........................ passed ... 2 +test-realloc-no-preallocate ........................ passed ... 1 +test-realloc-preallocate ........................ passed ... 1 +test-realloc-arrays ........................ passed ... 3 +test-realloc-block-reuse ........................ passed ... 3 +test-realloc-expand-block ........................ passed ... 2 +test-realloc-no-split ........................ passed ... 3 +test-realloc-split-one-block ........................ passed ... 3 +test-realloc-split-first ........................ passed ... 3 +test-realloc-split-last ........................ passed ... 3 +test-realloc-split-middle ........................ passed ... 2 +test-realloc-split-vector ........................ passed ... 2 +test-realloc-coalesce ........................ passed ... 3 +test-realloc-coalesce-big ........................ passed ... 1 +test-all ........................ passed ... 5 + +Total: 90/100 +``` + +**NOTE:** By default, `run_tests.py` checks for memory leaks, which can be time-consuming. +To speed up testing, use the `-d` flag or `make check-fast` to skip memory leak checks. + +### Running the Linters + +To run the linters, use the `make lint` command in the `tests/` directory. +Note that the linters have to be installed on your system: [`checkpatch.pl`](https://github.com/torvalds/linux/blob/master/scripts/checkpatch.pl), [`cpplint`](https://github.com/cpplint/cpplint), [`shellcheck`](https://www.shellcheck.net/) with certain configuration options. +It's easiest to run them in a Docker-based setup with everything configured: + +```console +student@so:~/.../mem-alloc/tests$ make lint +[...] +cd .. && checkpatch.pl -f checker/*.sh tests/*.sh +[...] +cd .. && cpplint --recursive src/ tests/ checker/ +[...] +cd .. && shellcheck checker/*.sh tests/*.sh +``` + +### Debugging + +`run_tests.py` uses `ltrace` to capture all the libcalls and syscalls performed. + +The output of `ltrace` is formatted to show only top level library calls and nested system calls. +For consistency, the heap start and addresses returned by `mmap()` are replaced with labels. +Every other address is displayed as `