diff --git a/.github/DISCUSSION_TEMPLATE/bugs.yml b/.github/DISCUSSION_TEMPLATE/bugs.yml deleted file mode 100644 index 0e61641..0000000 --- a/.github/DISCUSSION_TEMPLATE/bugs.yml +++ /dev/null @@ -1,14 +0,0 @@ -body: - - type: textarea - attributes: - label: 概要 - description: 操作手順やソースコードなど。 - validations: - required: true - - type: textarea - attributes: - label: 開発環境 - description: "`make doctor` の実行結果を貼付すると楽です。" - render: bash - validations: - required: true diff --git a/.github/DISCUSSION_TEMPLATE/corrections.yml b/.github/DISCUSSION_TEMPLATE/corrections.yml deleted file mode 100644 index 4b666de..0000000 --- a/.github/DISCUSSION_TEMPLATE/corrections.yml +++ /dev/null @@ -1,14 +0,0 @@ -body: - - type: textarea - attributes: - label: ページ番号 (図・表番号) - description: "例: P12 図3.4, P103-P105" - validations: - required: true - - type: textarea - attributes: - label: 概要 - description: 誤字・脱字など修正すべき事項。 - render: bash - validations: - required: true diff --git a/.github/DISCUSSION_TEMPLATE/help.yml b/.github/DISCUSSION_TEMPLATE/help.yml deleted file mode 100644 index 0ce610c..0000000 --- a/.github/DISCUSSION_TEMPLATE/help.yml +++ /dev/null @@ -1,20 +0,0 @@ -body: - - type: textarea - attributes: - label: 概要 - description: 困っていること・質問したいことをここに記述してください。 - validations: - required: true - - type: textarea - attributes: - label: 開発環境 - description: HinaOSのデバッグに困っている場合のみ。`make doctor` の実行結果を貼付すると楽です。 - render: bash - validations: - required: false - - type: input - attributes: - label: ソースコードのURL - description: HinaOSのデバッグに困っている場合のみ。自身のGitHubリポジトリを共有してもらえると助かります。 - validations: - required: false diff --git a/.github/workflows/auto-approve.yml b/.github/workflows/auto-approve.yml new file mode 100644 index 0000000..98f9ec1 --- /dev/null +++ b/.github/workflows/auto-approve.yml @@ -0,0 +1,16 @@ +name: Auto approve + +on: + pull_request: + types: [opened, synchronize, ready_for_review] + +jobs: + auto-approve: + if: | + github.event.pull_request.user.login == github.repository_owner + && ! github.event.pull_request.draft + runs-on: ubuntu-latest + permissions: + pull-requests: write + steps: + - uses: hmarr/auto-approve-action@v4 \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 68a146c..9085447 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,13 +5,16 @@ on: branches: [main] pull_request: types: [opened, synchronize] - + jobs: linux_build: runs-on: ubuntu-latest timeout-minutes: 15 steps: - - uses: actions/checkout@v2 + - name: Checkout repository and submodules + uses: actions/checkout@v2 + with: + submodules: recursive - name: Install packages run: sudo apt-get update && sudo apt-get install -y llvm clang lld python3-pip qemu-system tshark - name: Install pip packages @@ -20,114 +23,15 @@ jobs: run: make doctor - name: Debug build run: make build -j2 - - name: Release build - run: make build -j2 RELEASE=1 - name: Run tests (debug build, 1 CPU) run: make test -j2 env: TEST_DEFAULT_TIMEOUT: 10 FLAKE_RUNS: 4 CPUS: 1 - - name: Run tests (release build, 1 CPU) - run: make test -j2 RELEASE=1 - env: - TEST_DEFAULT_TIMEOUT: 10 - FLAKE_RUNS: 4 - CPUS: 1 - name: Run tests (debug build, 4 CPUs) run: make test -j2 env: TEST_DEFAULT_TIMEOUT: 10 FLAKE_RUNS: 4 - CPUS: 4 - - name: Run tests (release build, 4 CPUs) - run: make test -j2 RELEASE=1 - env: - TEST_DEFAULT_TIMEOUT: 10 - FLAKE_RUNS: 4 - CPUS: 4 - - macos_build: - runs-on: macos-latest - timeout-minutes: 15 - steps: - - uses: actions/checkout@v2 - - name: Install packages - run: brew install llvm python3 qemu - - name: Install pip packages - run: pip3 install --user -r tools/requirements.txt - - name: make doctor - run: make doctor - - name: Debug build - run: make build -j3 - - name: Release build - run: make build -j3 RELEASE=1 - - name: Run tests (debug build, 1 CPU) - run: make test -j3 - env: - TEST_DEFAULT_TIMEOUT: 10 - FLAKE_RUNS: 4 - CPUS: 1 - - name: Run tests (release build, 1 CPU) - run: make test -j3 RELEASE=1 - env: - TEST_DEFAULT_TIMEOUT: 10 - FLAKE_RUNS: 4 - CPUS: 1 - - name: Run tests (debug build, 4 CPUs) - run: make test -j3 - env: - TEST_DEFAULT_TIMEOUT: 10 - FLAKE_RUNS: 4 - CPUS: 4 - - name: Run tests (release build, 4 CPUs) - run: make test -j3 RELEASE=1 - env: - TEST_DEFAULT_TIMEOUT: 10 - FLAKE_RUNS: 4 - CPUS: 4 - - windows_build: - runs-on: windows-latest - timeout-minutes: 15 - steps: - - uses: actions/checkout@v2 - - name: Install Chocolatey packages - run: choco install qemu --version=2023.6.29 - - name: Install pip packages - run: py -m pip install --user -r tools/requirements.txt - - name: make doctor - run: make doctor - - name: Debug build - run: make build -j2 - - name: Release build - run: make build -j2 RELEASE=1 - - name: Run tests (debug build, 1 CPU) - run: make test -j2 - env: - TEST_DEFAULT_TIMEOUT: 30 - FLAKE_RUNS: 1 - CPUS: 1 - QEMU: C:\Program Files\qemu\qemu-system-riscv32 - - name: Run tests (release build, 1 CPU) - run: make test -j2 RELEASE=1 - env: - TEST_DEFAULT_TIMEOUT: 30 - FLAKE_RUNS: 1 - CPUS: 1 - QEMU: C:\Program Files\qemu\qemu-system-riscv32 - - name: Run tests (debug build, 2 CPUs) - run: make test -j2 - env: - TEST_DEFAULT_TIMEOUT: 30 - FLAKE_RUNS: 1 - CPUS: 2 - QEMU: C:\Program Files\qemu\qemu-system-riscv32 - - name: Run tests (release build, 2 CPUs) - run: make test -j2 RELEASE=1 - env: - TEST_DEFAULT_TIMEOUT: 30 - FLAKE_RUNS: 1 - CPUS: 2 - QEMU: C:\Program Files\qemu\qemu-system-riscv32 - + CPUS: 4 \ No newline at end of file diff --git a/Makefile b/Makefile index aec8b9b..235f509 100644 --- a/Makefile +++ b/Makefile @@ -164,7 +164,7 @@ QEMUFLAGS += -drive file=$(hinafs_img),if=none,format=raw,id=drive0 QEMUFLAGS += -device virtio-blk-device,drive=drive0,bus=virtio-mmio-bus.0 QEMUFLAGS += -device virtio-net-device,netdev=net0,bus=virtio-mmio-bus.1 QEMUFLAGS += -object filter-dump,id=fiter0,netdev=net0,file=virtio-net.pcap -QEMUFLAGS += -netdev user,id=net0,hostfwd=tcp:127.0.0.1:1234-:80 +QEMUFLAGS += -netdev user,id=net0 wasmos_elf := $(BUILD_DIR)/wasmos.elf boot_elf := $(BUILD_DIR)/servers/vm.elf @@ -197,16 +197,15 @@ gdb: $(PROGRESS) GDB $(BUILD_DIR)/gdbinit $(GDB) -q -ex "source $(BUILD_DIR)/gdbinit" -# 自動テスト +# Automated test export QEMUFLAGS .PHONY: test test: - $(PYTHON3) \ - -m pytest tests.py -p no:cacheprovider \ - --qemu "$(QEMU)" --make "$(MAKE)" \ - $(if $(FLAKE_RUNS),--flake-finder --flake-runs=$(FLAKE_RUNS)) \ - $(if $(RELEASE),--release,) - + $(PYTHON3) \ + -m pytest tests -p no:cacheprovider \ + --qemu $(QEMU) \ + $(if $(FLAKE_RUNS),--flake-finder --flake-runs=$(FLAKE_RUNS)) + # トラブルシューティングに役立つ情報を表示するコマンド .PHONY: doctor doctor: diff --git a/conftest.py b/conftest.py deleted file mode 100644 index 8e9bb83..0000000 --- a/conftest.py +++ /dev/null @@ -1,103 +0,0 @@ -""" - -tests.py で使うユーティリティを定義 (pytestが自動で読み込む) - -""" -import pytest -import os -import subprocess -import struct -import tempfile -import shlex -import sys -import re - -AUTORUN_PLACEHOLDER = "__REPLACE_ME_AUTORUN__" + "a" * 512 - -cached_runner = None - -@pytest.fixture -def run_hinaos(request): - global cached_runner - if cached_runner is None: - build_argv = [request.config.getoption("--make"), "build", f"AUTORUN={AUTORUN_PLACEHOLDER}", f"-j{os.cpu_count()}"] - if request.config.getoption("--release"): - build_argv.append("RELEASE=1") - subprocess.run(build_argv, check=True) - - cached_runner = Runner( - os.environ.get("QEMU", request.config.getoption("--qemu")), - shlex.split(os.environ["QEMUFLAGS"]), - "build/hinaos.elf" - ) - return do_run_hinaos - -def do_run_hinaos(script, timeout=None, ignore_timeout=False, qemu_net0_options=None): - if timeout is None: - timeout = int(os.environ.get("TEST_DEFAULT_TIMEOUT", 3)) - return cached_runner.run(script + "; shutdown", timeout=timeout, ignore_timeout=ignore_timeout, qemu_net0_options=qemu_net0_options) - -class Result: - def __init__(self, log, raw_log): - self.log = log - self.raw_log = raw_log - -ANSI_ESCAPE_SEQ_REGEX = re.compile(r'\x1B\[[^m]+m') - -class Runner: - def __init__(self, qemu, qemu_flags, image_path): - self.qemu = qemu - self.qemu_args = qemu_flags - self.image = open(image_path, "rb").read() - self.placeholder_bytes = AUTORUN_PLACEHOLDER.encode('ascii') - - def run(self, script, timeout=None, ignore_timeout=False, qemu_net0_options=None): - if len(script.encode('ascii')) > len(self.placeholder_bytes): - raise ValueError("script is too long") - - qemu_args = [] - for arg in self.qemu_args: - if qemu_net0_options is not None and "-netdev user,id=net0" in arg: - qemu_args.append(arg + "," + ",".join(qemu_net0_options)) - else: - qemu_args.append(arg) - - script_bytes = struct.pack(f"{len(AUTORUN_PLACEHOLDER)}s", script.encode('ascii')) - - # make run AUTORUN="..." でも実行できるが、この方法だとAUTORUNが変わるたびにビルド - # し直してしまう。そこで、AUTORUN_PLACEHOLDERを使ってビルドした後に、それを単純に置換 - # する。 - with tempfile.NamedTemporaryFile(mode="wb+", delete=False) as f: - f.write(self.image.replace(self.placeholder_bytes, script_bytes)) - # Windowsではファイルをいったん閉じないと使えない。 - f.close() - - qemu_argv = [self.qemu, "-kernel", f.name, "-snapshot"] + qemu_args - try: - raw_log = subprocess.check_output(qemu_argv, timeout=timeout) - except subprocess.TimeoutExpired as e: - raw_log = e.stdout - pcap_dump = "" - try: - pcap_dump = subprocess.check_output(["tshark", "-r", "virtio-net.pcap"], text=True) - except Exception as e: - print(f"Failed to run tshark ({e}). Skipping pcap dump...") - if pcap_dump: - print("\n\npcap dump:\n\n" + pcap_dump) - if not ignore_timeout: - sys.stdout.buffer.write(raw_log) - raise Exception("QEMU timed out") - finally: - os.remove(f.name) - - sys.stdout.buffer.write(raw_log) - - log = raw_log.decode('utf-8', errors='backslashreplace') - log = ANSI_ESCAPE_SEQ_REGEX.sub('', log) # 色を抜く - - return Result(log, raw_log) - -def pytest_addoption(parser): - parser.addoption("--qemu", required=True) - parser.addoption("--make", required=True) - parser.addoption("--release", action="store_true") diff --git a/servers/hello_wasmvm/app.c b/servers/hello_wasmvm/app.c deleted file mode 100644 index 849b875..0000000 --- a/servers/hello_wasmvm/app.c +++ /dev/null @@ -1,27 +0,0 @@ -#include - -#define ASSERT_OK(expr) \ - do { \ - __typeof__(expr) __expr = (expr); \ - if (__expr < 0) { \ - return -1; \ - } \ - } while(0) - -__attribute__((export_name("main"))) -int main(void) { - while(1) { - struct message m; - ASSERT_OK(ipc_recv(IPC_ANY, &m)); - switch(m.type) { - case PING_MSG: { - m.type = PING_REPLY_MSG; - m.ping_reply.value = 42; - ipc_reply(m.src, &m); - break; - } - default: - break; - } - } -} \ No newline at end of file diff --git a/servers/hello_wasmvm/build.mk b/servers/hello_wasmvm/build.mk deleted file mode 100644 index 6dc05a9..0000000 --- a/servers/hello_wasmvm/build.mk +++ /dev/null @@ -1,12 +0,0 @@ -objs-y += main.o wasm.o -cflags-y += -D__wasm_path__=\"$(build_dir)/app.wasm\" - -# build wasm first -$(build_dir)/wasm.o: $(build_dir)/app.wasm - -# Remove compiler flags specific to riscv and add ones for WASM -$(build_dir)/app.wasm: WASMCFLAGS += -Wl,--strip-all,--no-entry,--allow-undefined - -$(build_dir)/app.wasm: $(dir)/app.c - $(PROGRESS) CC app.wasm - $(CC) $(WASMCFLAGS) -o $@ $^ \ No newline at end of file diff --git a/servers/hello_wasmvm/main.c b/servers/hello_wasmvm/main.c deleted file mode 100644 index e26eac8..0000000 --- a/servers/hello_wasmvm/main.c +++ /dev/null @@ -1,26 +0,0 @@ -#include -#include -#include -#include - -// defined in wasm.S -extern uint8_t wasm_start[]; -extern uint32_t wasm_size[]; - -void main(void) { - task_t server = sys_wasmvm("wasm_pong", wasm_start, wasm_size[0], task_self()); - ASSERT_OK(server); - - // send message to WASMVM task - struct message m; - m.type = PING_MSG; - m.ping.value = 123; - ASSERT_OK(ipc_call(server, &m)); - - // display message from WASMVM task - INFO("reply type: %s", msgtype2str(m.type)); - INFO("reply value: %d", m.ping_reply.value); - - // destroy WASMVM task - ASSERT_OK(sys_task_destroy(server)); -} \ No newline at end of file diff --git a/servers/hello_wasmvm/wasm.S b/servers/hello_wasmvm/wasm.S deleted file mode 100644 index 47d69d1..0000000 --- a/servers/hello_wasmvm/wasm.S +++ /dev/null @@ -1,8 +0,0 @@ - .section .data - .global wasm_start -wasm_start: - .incbin __wasm_path__ -wasm_end: - .global wasm_size -wasm_size: - .int wasm_end - wasm_start \ No newline at end of file diff --git a/tests.py b/tests.py deleted file mode 100644 index e4155b3..0000000 --- a/tests.py +++ /dev/null @@ -1,78 +0,0 @@ -""" - -HinaOSの自動テスト (pytest) - -使い方: - - $ make test - $ make test RELEASE=1 - $ make test CPUS=<プロセッサ数> - -テストの書き方: - - - test_ で始まる関数を定義する。 - - run_hinaos関数を呼ぶとHinaOSを自動でビルド・起動し、指定されたスクリプトを実行する。 - 成功するとログが入ったResultオブジェクトが返る。 - -""" -import http -import http.server -import threading - -def test_hello_world(run_hinaos): - r = run_hinaos("echo howdy") - assert "howdy" in r.log - -def test_start_hello(run_hinaos): - r = run_hinaos("start hello; echo howdy") - assert "Hello World!" in r.log - assert "howdy" in r.log - -def test_read_file(run_hinaos): - r = run_hinaos("cat hello.txt") - assert "Hello World from HinaFS" in r.log - -def test_write_file(run_hinaos): - r = run_hinaos("write lfg.txt LFG; cat lfg.txt; ls") - assert '[FILE] "lfg.txt"' in r.log - assert '[shell] LFG' in r.log - -def test_ls(run_hinaos): - r = run_hinaos("ls") - assert "hello.txt" in r.log - -def test_mkdir(run_hinaos): - r = run_hinaos("mkdir new_dir; ls") - assert '[DIR ] "new_dir"' in r.log - -def test_hinavm(run_hinaos): - r = run_hinaos("start hello_hinavm") - assert "hinavm_server: pc=7: 123" in r.log - assert "reply value: 42" in r.log - -def test_crack(run_hinaos): - # crackに成功するまでタイムアウトを伸ばしていく - for i in range(1, 5): - r = run_hinaos("start crack", timeout=2**i, ignore_timeout=True) - if "exploited!" in r.log: - break - else: - assert False, "all attempts failed" - -def test_http(run_hinaos): - class TeapotServer(http.server.BaseHTTPRequestHandler): - def do_GET(self): - self.send_response(418) - self.end_headers() - self.wfile.write("🫖".encode("utf-8")) - httpd = http.server.HTTPServer(("", 1234), TeapotServer) - httpd_thread = threading.Thread(target=lambda: httpd.serve_forever(), daemon=True) - httpd_thread.start() - - r = run_hinaos("http http://10.0.2.100:1234/teapot", - qemu_net0_options=["guestfwd=tcp:10.0.2.100:1234-tcp:127.0.0.1:1234"]) - assert "🫖" in r.log - - httpd.shutdown() - httpd.server_close() - httpd_thread.join() diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..6222c25 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,101 @@ +""" + +Utility functions used in test.py (pytest loads this file automatically). + +""" + +import pytest +import os +import subprocess +import shlex +import struct +import tempfile +import re +import sys + +# To avoid building WasmOS every time, use placefolder and replace this with the specified script. +AUTORUN_PLACEHOLDER = "__REPLACE_ME_AUTORUN__" + "a" * 512 +ANSI_ESCAPE_SEQ_REGEX = re.compile(r'\x1B\[[^m]+m') + +runner = None + +# Build WasmOS and create a runner. +@pytest.fixture +def run_wasmos(request): + # Build WasmOS if not built. + global runner + if runner is None: + build_argv = [ + "make", + "build", + f"AUTORUN={AUTORUN_PLACEHOLDER}", + f"-j{os.cpu_count()}" + ] + subprocess.run(build_argv, check=True) + + # Create Runner. + runner = Runner( + request.config.getoption("--qemu"), + shlex.split(os.environ["QEMUFLAGS"]), + "build/wasmos.elf" + ) + + return do_run_wasmos + +def do_run_wasmos(script, qemu_net0_options=None): + return runner.run(script + "; shutdown", qemu_net0_options) + +class Result: + def __init__(self, log, raw_log): + self.log = log + self.raw_log = raw_log + +class Runner: + def __init__(self, qemu, default_qemu_flags, image_path): + self.qemu = qemu + self.default_qemu_flags = default_qemu_flags + self.image = open(image_path, "rb").read() + self.placeholder_bytes = AUTORUN_PLACEHOLDER.encode("ascii") + + def run(self, script, qemu_net0_options): + if len(script.encode("ascii")) > len(self.placeholder_bytes): + raise ValueError("script is too long") + + qemu_args = [] + + for arg in self.default_qemu_flags: + if qemu_net0_options is not None and "-netdev user, id=net0" in arg: + qemu_args.append(arg + "," + ",".join(qemu_net0_options)) + else: + qemu_args.append(arg) + + script_bytes = struct.pack( + f"{len(AUTORUN_PLACEHOLDER)}s", + script.encode("ascii") + ) + + with tempfile.NamedTemporaryFile(mode="wb+", delete=False) as f: + # Replace the placefolder with the specified script. + f.write(self.image.replace(self.placeholder_bytes, script_bytes)) + f.close() + + # Execute the specified script on Qemu. + qemu_argv = [self.qemu, "-kernel", f.name, "-snapshot"] + qemu_args + + try: + raw_log = subprocess.check_output(qemu_argv) + finally: + os.remove(f.name) + + # Write outputs to stdout (for debugging). + sys.stdout.buffer.write(raw_log) + + log = raw_log.decode('utf-8', errors='backslashreplace') + # Remove color. + log = re.sub(ANSI_ESCAPE_SEQ_REGEX, '', log) + + return Result(log, raw_log) + +# Define command line options +def pytest_addoption(parser): + parser.addoption("--qemu", required=True) \ No newline at end of file diff --git a/tests/test_commands.py b/tests/test_commands.py new file mode 100644 index 0000000..80c141a --- /dev/null +++ b/tests/test_commands.py @@ -0,0 +1,77 @@ +""" + +Test shell server commands. + +""" + +import http +import http.server +import threading + +def test_echo(run_wasmos): + r = run_wasmos("echo howdy") + assert "howdy" in r.log + +def test_start(run_wasmos): + r = run_wasmos("start hello") + assert "Hello World!" in r.log + +def test_hinavm(run_wasmos): + r = run_wasmos("start hello_hinavm") + assert "hinavm_server: pc=7: 123" in r.log + assert "reply value: 42" in r.log + +def test_ls(run_wasmos): + r = run_wasmos("ls") + assert "hello.txt" in r.log + +def test_cat(run_wasmos): + r = run_wasmos("cat hello.txt") + assert "Hello World from HinaFS" in r.log + +def test_write(run_wasmos): + r = run_wasmos("write lfg.txt LFG; ls; cat lfg.txt") + assert '[FILE] "lfg.txt"' in r.log + assert '[shell] LFG' in r.log + +def test_mkdir(run_wasmos): + r = run_wasmos("mkdir new_dir; ls") + assert '[DIR ] "new_dir"' in r.log + +def test_delete_file(run_wasmos): + r = run_wasmos("write lfg.txt LFG; delete lfg.txt; ls") + assert '[FILE] "lfg.txt"' not in r.log + +def test_delete_dir(run_wasmos): + r = run_wasmos("mkdir new_dir; delete new_dir; ls") + assert '[DIR ] "new_dir"' not in r.log + +def test_shutdow(run_wasmos): + r = run_wasmos("shutdown") + assert "[shell] shutting down..." in r.log + +def test_hinavm(run_wasmos): + r = run_wasmos("start hello_hinavm") + assert "hinavm_server: pc=7: 123" in r.log + assert "reply value: 42" in r.log + +def test_http(run_wasmos): + class TeapotServer(http.server.BaseHTTPRequestHandler): + def do_GET(self): + self.send_response(418) + self.end_headers() + self.wfile.write("Hello HTTP!".encode("utf-8")) + + httpd = http.server.HTTPServer(("", 1234), TeapotServer) + httpd_thread = threading.Thread(target=lambda: httpd.serve_forever(), daemon=True) + httpd_thread.start() + + r = run_wasmos( + "http http://10.0.2.100:1234/teapot", + qemu_net0_options=["guestfwd=tcp:10.0.2.100:1234-tcp:127.0.0.1:1234"] + ) + assert "Hello HTTP!" in r.log + + httpd.shutdown() + httpd.server_close() + httpd_thread.join() \ No newline at end of file