add porting helper and documentation

+ update dockerfile

Signed-off-by: Hugo Lefeuvre <[email protected]>
Signed-off-by: Vlad-Andrei Badoiu <[email protected]>

README.md

@@ -335,7 +335,19 @@ The `fcalls` debugging backend will replace all gates with simple function calls
 
 ## Porting Tips & Tricks
 
-### Gate insertion with MPK (and other intra-AS isolation technologies)
+### Automatic gate insertion
+
+We provide users with a simple tool to automatically insert gate placeholders
+in their code. The tool uses `cscope` to determine calls performed by a file
+outside of the current library, and
+[Coccinelle](https://github.com/coccinelle/coccinelle) to automatically insert
+a gate at this position. The tool is available under
+`flexos-support/porthelper` and comes preinstalled in the FlexOS docker
+container.
+
+*TODO: add more documentation here on how to use the tool!*
+
+### Checking gate insertions with MPK (and other intra-AS isolation technologies)
 
 The first part of porting a library to run as isolated component is to insert gates.
 

docker/flexos.dockerfile

@@ -1,80 +1,58 @@
 # You can easily build it with the following command:
-# $ MY_KEY=$(cat ~/.ssh/id_rsa)
-# $ docker build --build-arg SSH_KEY="$MY_KEY" -f flexos.dockerfile --tag flexos-dev .
+# $ docker build --tag flexos-dev -f flexos-dev.dockerfile .
 #
-# IMPORTANT NOTE: YOUR SSH KEY MUST NOT HAVE A PASSWORD
-# if so, remove it with ssh-keygen -p
-#
-# Once built, remove all intermediate containers with:
-# $ docker rmi -f $(docker images -q --filter label=stage=intermediate)
+# If the build fails because you are rate-limited by GitHub, generate an app
+# token (https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token)
+# and run instead:
+# $ docker build --build-arg UK_KRAFT_GITHUB_TOKEN="<YOUR TOKEN>" --tag flexos-dev
 #
 # and run with:
-# $ docker run -ti -v $(dirname $SSH_AUTH_SOCK):$(dirname $SSH_AUTH_SOCK) -e SSH_AUTH_SOCK=$SSH_AUTH_SOCK flexos-dev bash
+# $ docker run --privileged -ti flexos-dev bash
 
-# Choose and name our temporary image.
-FROM debian:10 as intermediate
-# Add metadata identifying these images as our build containers (this will be useful later!)
-LABEL stage=intermediate
+FROM debian:10
 
-# Take an SSH key as a build argument.
-ARG SSH_KEY
+ARG UK_KRAFT_GITHUB_TOKEN=
+ENV UK_KRAFT_GITHUB_TOKEN=${UK_KRAFT_GITHUB_TOKEN}
 
-COPY kraftrc.default /root/.kraftrc
+##############
+# Dependencies
 
+RUN echo "deb-src http://deb.debian.org/debian buster main contrib non-free" >> /etc/apt/sources.list
+RUN echo "deb-src http://security.debian.org/ buster/updates main contrib non-free" >> /etc/apt/sources.list
+RUN echo "deb-src http://deb.debian.org/debian/ buster-updates main contrib non-free" >> /etc/apt/sources.list
 RUN apt update
-RUN apt install -y python3-pip git
+RUN apt build-dep -y coccinelle
+RUN apt install -y build-essential libncurses-dev python3 expect-dev moreutils \
+	flex unzip bison wget libxml2-utils tclsh python python-tempita python-six \
+	python-future python-ply xorriso qemu-system-x86 qemu qemu-kvm vim qemu-system \
+	qemu-utils curl gawk git procps socat uuid-runtime python3-pip libsqlite3-dev \
+	bc libiscsi-dev librbd1 libnfs-dev libgfapi0 libffi-dev libiperf-dev net-tools \
+	bridge-utils iperf dnsmasq ninja-build gdb cscope
 
-# 1. Create the SSH directory.
-# 2. Populate the private key file.
-# 3. Set the required permissions.
-# 4. Add github to our list of known hosts for ssh.
-RUN mkdir -p /root/.ssh/ && \
-    echo "$SSH_KEY" > /root/.ssh/id_rsa && \
-    chmod -R 600 /root/.ssh/ && \
-    ssh-keyscan -t rsa github.com >> ~/.ssh/known_hosts
+##############
+# Toolchain
 
 WORKDIR /root
 
-RUN cat /root/.ssh/id_rsa
-RUN git clone [email protected]:ukflexos/kraft.git
+RUN git clone [email protected]:project-flexos/kraft.git
 
 WORKDIR /root/kraft
 
 RUN pip3 install -e .
 
-RUN kraft list update
-RUN kraft -v list pull flexos-microbenchmarks@staging iperf@staging \
-		  newlib@staging tlsf@staging flexos-example@staging \
-		  lwip@staging redis@staging unikraft@staging \
-		  pthread-embedded@staging nginx@staging
-
-# Choose the base image for our final image
-FROM debian:10
-
 COPY kraftcleanup /usr/local/bin/kraftcleanup
 COPY kraftrc.default /root/.kraftrc
 
-# Copy across the files from our `intermediate` container
-COPY --from=intermediate /root/kraft /root/kraft
-COPY --from=intermediate /root/.unikraft /root/.unikraft
-
-RUN echo "deb-src http://deb.debian.org/debian buster main contrib non-free" >> /etc/apt/sources.list
-RUN echo "deb-src http://security.debian.org/ buster/updates main contrib non-free" >> /etc/apt/sources.list
-RUN echo "deb-src http://deb.debian.org/debian/ buster-updates main contrib non-free" >> /etc/apt/sources.list
-RUN apt update
-RUN apt build-dep -y coccinelle
-RUN apt install -y python3-pip git flex bison wget unzip
-
-WORKDIR /root/kraft
-
-RUN pip3 install -e .
-
 WORKDIR /root
 
+RUN wget https://raw.githubusercontent.com/unikraft/kraft/6217d48668cbdf0847c7864bc6368a6adb94f6a6/scripts/qemu-guest
+RUN chmod a+x /root/qemu-guest
+
 RUN git clone https://github.com/coccinelle/coccinelle
 
 WORKDIR /root/coccinelle
 
+RUN git checkout ae337fce1512ff15aabc3ad5b6d2e537f97ab62a
 RUN ./autogen
 RUN ./configure
 RUN make
@@ -84,4 +62,32 @@ RUN make install
 RUN mkdir /usr/local/bin/lib
 RUN ln -s /usr/local/lib/coccinelle /usr/local/bin/lib/coccinelle
 
+##############
+# FlexOS EPT QEMU
+
+RUN git clone https://github.com/qemu/qemu.git
+
+WORKDIR /root/qemu
+
+RUN apt install -y ninja-build
+RUN git checkout 9ad4c7c9b63f89c308fd988d509bed1389953c8b
+COPY resources/0001-Myshmem.patch /root/0001-Myshmem.patch
+RUN git apply /root/0001-Myshmem.patch
+RUN ./configure --target-list=x86_64-softmmu
+RUN sed -i -e 's/-lstdc++ -Wl,--end-group/-lrt -lstdc++ -Wl,--end-group/g' build/build.ninja
+RUN make -j8
+RUN cp build/qemu-system-x86_64 /root/qemu-system-ept
+RUN cp -r build/pc-bios /root/pc-bios
+RUN rm /root/0001-Myshmem.patch
+
+##############
+# FlexOS
+
+RUN kraft list update
+RUN kraft -v list pull flexos-microbenchmarks@staging iperf@staging \
+		  newlib@staging tlsf@staging flexos-example@staging \
+		  lwip@staging redis@staging unikraft@staging \
+		  pthread-embedded@staging nginx@staging
+RUN cp /root/.unikraft/unikraft/flexos-support/porthelper/* /root/.unikraft/
+
 WORKDIR /root/.unikraft

docker/kraftcleanup

@@ -2,7 +2,7 @@
 
 make properclean
 # git checkout . && git clean -xdf in all repositories would be fine too
-rm -rf /root/.kraftcache /root/.unikraft/libs /root/.unikraft/unikraft
+rm -rf /.kraftcache /.unikraft/libs /.unikraft/unikraft
 kraft list update
 kraft list pull flexos-microbenchmarks@staging iperf@staging newlib@staging \
 		tlsf@staging flexos-example@staging lwip@staging redis@staging \

docker/kraftrc.default

@@ -10,19 +10,19 @@ architecture = "x86_64"
 
 [list]
 origins = [
-  "[email protected]:ukflexos/lib-newlib.git",
-  "[email protected]:ukflexos/lib-tlsf.git",
-  "[email protected]:ukflexos/app-flexos-example.git",
-  "[email protected]:ukflexos/app-flexos-microbenchmarks.git",
-  "[email protected]:ukflexos/lib-flexos-example.git",
-  "[email protected]:ukflexos/lib-flexos-microbenchmarks.git",
-  "[email protected]:ukflexos/lib-lwip.git",
-  "[email protected]:ukflexos/lib-nginx.git",
-  "[email protected]:ukflexos/app-nginx.git",
-  "[email protected]:ukflexos/lib-redis.git",
-  "[email protected]:ukflexos/app-redis.git",
-  "[email protected]:ukflexos/lib-iperf.git",
-  "[email protected]:ukflexos/app-iperf.git",
-  "[email protected]:ukflexos/unikraft.git",
-  "[email protected]:ukflexos/lib-pthread-embedded.git"
+  "[email protected]:project-flexos/lib-newlib.git",
+  "[email protected]:project-flexos/lib-tlsf.git",
+  "[email protected]:project-flexos/app-flexos-example.git",
+  "[email protected]:project-flexos/app-flexos-microbenchmarks.git",
+  "[email protected]:project-flexos/lib-flexos-example.git",
+  "[email protected]:project-flexos/lib-flexos-microbenchmarks.git",
+  "[email protected]:project-flexos/lib-lwip.git",
+  "[email protected]:project-flexos/lib-nginx.git",
+  "[email protected]:project-flexos/app-nginx.git",
+  "[email protected]:project-flexos/lib-redis.git",
+  "[email protected]:project-flexos/app-redis.git",
+  "[email protected]:project-flexos/lib-iperf.git",
+  "[email protected]:project-flexos/app-iperf.git",
+  "[email protected]:project-flexos/unikraft.git",
+  "[email protected]:project-flexos/lib-pthread-embedded.git"
 ]

flexos-support/porthelper/parse_results.py

@@ -0,0 +1,32 @@
+import glob, os
+import csv
+
+def write_csv(outfile, lines):
+    f = open(outfile, 'w+', newline='')
+    writer = csv.writer(f, delimiter=',',
+                            quotechar='|', quoting=csv.QUOTE_MINIMAL)
+
+    for line in lines:
+        # definition is in unikraft/lib
+        if "unikraft/lib" in line:
+            lib_name = line.split("/")[2]
+            if lib_name == "nolibc" or lib_name == "newlib":
+                lib_name = "libc"
+            else:
+                lib_name = "lib" + lib_name
+            function_name = line.split(" ")[1]
+            writer.writerow([function_name, lib_name])
+
+        # definition is in external library libs/
+        if "libs/" in line:
+            lib_name = line.split("/")[1]
+            if lib_name == "nolibc" or lib_name == "newlib":
+                lib_name = "libc"
+            else:
+                lib_name = "lib" + lib_name
+            function_name = line.split(" ")[1]
+            writer.writerow([function_name, lib_name])
+
+f = open("res.deps", "r")
+lines = [line.rstrip() for line in f]
+write_csv("callfile.csv", lines)

flexos-support/porthelper/porthelper.sh

@@ -0,0 +1,59 @@
+#!/bin/bash
+
+# argument 1: file path
+filetoport=$1
+
+# format of the call file:
+# function_name,libname
+# e.g.,
+# accept,liblwip
+callfile="/root/.unikraft/callfile.csv"
+
+rulefile="/tmp/porthelper.cocci"
+
+templaterule="./rule.cocci.in"
+
+# 1. generate call file (CSV)
+
+# Find the symbols used by the target file
+rm -rf cscope*
+cscope -L -2 ".*" $filetoport | grep -v "extern" | grep -v "/usr/"  | awk -F ' ' '{print $2}' | sort | uniq > res.symb
+rm -rf cscope*
+
+# Create a the DB with all the symbols exported by internal and external libraries
+find . -name '*.c' > cscope.files
+cscope -b -q -k
+
+USE_NEWLIB=True
+
+# For each symbol we found, we search for its definition, if the definition is not in the lib that the target files is part of, then we add it to a temporary dependncy file. We add the symbol, and the path to its definition.
+echo "" > res.deps
+while read in; do
+	if [[ $(cscope -d -L1 $in | grep ".*(.*)" | grep -v "newlib" | grep -v "define" | grep -v "lwip-2.1.2/src/apps/" | sort | uniq | wc -l) -eq 1 ]]; then
+		cscope -d -L1 $in | grep ".*(.*)" | grep -v "newlib" | grep -v "define" | grep -v "lwip-2.1.2/src/apps/" | sort | uniq >> res.deps
+	fi
+done < res.symb
+
+# We create a csv file from the dependency file
+python3 parse_results.py
+
+# Cleanup
+rm -rf cscope*
+rm res.deps
+rm res.symb
+
+# 2. generate coccinelle rules
+
+i=0
+rm -f $rulefile && touch $rulefile
+while IFS=, read -r fname lname; do
+  cat $templaterule >> $rulefile
+  sed -i "s/{{ rule_nr }}/${i}/g" $rulefile
+  sed -i "s/{{ lname }}/${lname}/g" $rulefile
+  sed -i "s/{{ fname }}/${fname}/g" $rulefile
+  i=$(( i + 1 ))
+done < $callfile
+
+# 3. apply coccinelle rules
+
+spatch -sp_file $rulefile $filetoport

flexos-support/porthelper/rule.cocci.in

@@ -0,0 +1,13 @@
+@return{{ rule_nr }}@
+expression list EL;
+expression ret, var;
+@@
+- var = {{ fname }}(EL);
++ flexos_gate({{ lname }}, var, {{ fname }}, EL);
+
+@noreturn{{ rule_nr }}@
+expression list EL;
+expression ret, var;
+@@
+- {{ fname }}(EL);
++ flexos_gate({{ lname }}, {{ fname }}, EL);