diff --git a/compose.yaml b/compose.yaml index 401e34178..b4974a50f 100644 --- a/compose.yaml +++ b/compose.yaml @@ -28,6 +28,9 @@ services: build: dockerfile: docker/godbolt.Dockerfile network_mode: host + volumes: + - ./target/scala-3.1.0/:/target/scala-3.1.0/ + # - ./docker/compiler-explorer:/compiler-explorer ports: - "10240:10240" diff --git a/docker/godbolt.Dockerfile b/docker/godbolt.Dockerfile index f4d08310b..5b049dc5a 100644 --- a/docker/godbolt.Dockerfile +++ b/docker/godbolt.Dockerfile @@ -2,7 +2,7 @@ FROM ghcr.io/uq-pac/basil:latest as compiler-explorer # https://github.com/madduci/docker-compiler-explorer/tree/master RUN DEBIAN_FRONTEND=noninteractive apt-get update \ && apt-get install -y curl \ - && curl -sL https://deb.nodesource.com/setup_18.x | bash - \ + && curl -sL https://deb.nodesource.com/setup_18.x | bash - \ # TODO: update install method && apt-get install -y \ wget \ ca-certificates \ @@ -10,26 +10,20 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get update \ make \ git \ dotnet6 \ + && dotnet tool install --global boogie || true \ + && apt-get install clang gcc gcc-13-aarch64-linux-gnu binutils-aarch64-linux-gnu gcc-13-cross-base libc6-dev-arm64-cross libc6-dev-armel-cross libc6-dev-armhf-cross libc6-dev-i386 -y \ && apt-get autoremove --purge -y \ && apt-get autoclean -y \ - && dotnet tool install --global boogie || true + && rm -rf /var/cache/apt/* /tmp/* # to force a new clone after a new commit +WORKDIR /compiler-explorer ADD https://api.github.com/repos/ailrst/compiler-explorer/branches/main /tmp/head -RUN rm -rf /var/cache/apt/* /tmp/* \ - && git clone https://github.com/ailrst/compiler-explorer.git /compiler-explorer \ +RUN \ + git clone https://github.com/ailrst/compiler-explorer.git /compiler-explorer \ && cd /compiler-explorer \ - && echo "Add missing dependencies" \ && npm i @sentry/node \ && npm run webpack -WORKDIR /compiler-explorer -RUN DEBIAN_FRONTEND=noninteractive apt-get update \ - && apt-get install clang gcc gcc-13-aarch64-linux-gnu binutils-aarch64-linux-gnu gcc-13-cross-base libc6-dev-arm64-cross libc6-dev-armel-cross libc6-dev-armhf-cross libc6-dev-i386 -y \ - && apt-get autoclean -y ENTRYPOINT [ "make" ] CMD ["run"] FROM compiler-explorer AS ghcr.io/uq-pac/basil-compiler-explorer:latest -ADD docker/godbolt/basil-tool.py /compiler-explorer/basil-tool.py -RUN chmod +x /compiler-explorer/basil-tool.py -ADD docker/godbolt/basil.local.properties /compiler-explorer/etc/config/c.defaults.properties -ADD docker/godbolt/compiler-explorer.local.properties /compiler-explorer/etc/config/compiler-explorer.local.properties diff --git a/docker/godbolt/basil-tool.py b/docker/godbolt/basil-tool.py deleted file mode 100644 index 456b82eba..000000000 --- a/docker/godbolt/basil-tool.py +++ /dev/null @@ -1,252 +0,0 @@ -#!/usr/bin/python3 -import subprocess -import argparse -import tempfile -import hashlib -import os -import sys -import logging - -READELF_BIN="/usr/bin/readelf" -BASIL_JAR="/home/am/Documents/programming/2023/bil-to-boogie-translator/target/scala-3.1.0/wptool-boogie-assembly-0.0.1.jar $WORKDIR/out.adt" -DEFAULT_LOGGER_NAME = 'default_logger' - - -def get_tempdir(seed: str): - m = hashlib.md5() - m.update(seed.encode('utf8')) - dir_name = os.path.join(tempfile.gettempdir(), "basil-tool", m.hexdigest()) - if not os.path.exists(dir_name): - logging.info("Created Dir %s", dir_name) - os.makedirs(dir_name) - else: - logging.info("Dir Exists %s", dir_name ) - return dir_name - -def make_tempdir(ignored: str): - return tempfile.mkdtemp() - -def bin_name(tmp_dir) -> str: - bin_file = os.path.join(tmp_dir, "a.out") - return bin_file - -def read_write_binary(tmp_dir:str, filename: str) -> str: - """ - Save binary from stdin. - """ - - bin_hash = hashlib.sha3_256() - content = sys.stdin.buffer.read() - bin_hash.update(content) - bin_file = bin_name(tmp_dir) - - hash_file = os.path.join(tmp_dir, "bin_hash.sha256") - hex_hash = bin_hash.hexdigest() - - if (os.path.exists(hash_file)): - logging.info("Binary Exists") - with open(hash_file, 'r') as f: - old_hash = f.read() - if old_hash != hex_hash: - logging.info("Error: binary mismatch, got the binary from a different compilation?") - else: - logging.info("Writing binary %s", bin_file) - with open(hash_file, 'w') as f: - f.write(hex_hash) - with open(bin_file, 'wb') as f: - f.write(content) - - logging.info("Loaded binary: %s %s", hex_hash, bin_file) - print("binary", content) - return bin_file - - -def run_bap_lift(tmp_dir: str, use_asli: bool): - logging.info("Bap") - adtfile = f"{tmp_dir}/out.adt" - birfile = f"{tmp_dir}/out.bir" - - binary = bin_name(tmp_dir) - - command = (f"bap {binary}").split(" ") - args = [ "-d", f"adt:{adtfile}", "-d", f"bir:{birfile}"] - #if use_asli: - # args += ["--primus-lisp-semantics=disable"] - #else: - # args += ["--primus-lisp-semantics=enable"] - - command += args - logging.info("command: %s", command) - if not (os.path.exists(adtfile) and os.path.exists(birfile)): - res = subprocess.run(command, check=True) - logging.info(res.stdout) - logging.info(res.stderr) - - return {"adt": adtfile, "bir": birfile, "default": birfile} - -def run_readelf(tmp_dir): - logging.info("Readelf") - command = [READELF_BIN, "-s", "-r", "-W", bin_name(tmp_dir)] - res = subprocess.run(command, capture_output=True, check=True) - logging.info(res.stdout) - logging.info(res.stderr) - - readelf_file = f"{tmp_dir}/out.relf" - - with open(readelf_file, "w") as f: - f.write(res.stdout.decode('utf-8')) - - return {"relf": readelf_file, "default": readelf_file} - -def run_basil(tmp_dir: str, spec: str | None =None): - logging.info("Basil") - boogie_file = f"{tmp_dir}/boogie_out.bpl" - outputs = {"boogie": boogie_file} - - # dependencies - outputs.update(run_bap_lift(tmp_dir, False)) - outputs.update(run_readelf(tmp_dir)) - outputs["default"] = boogie_file - - adtfile = outputs['adt'] - birfile = outputs['bir'] - readelf_file = outputs['relf'] - os.chdir(tmp_dir) # so the output file is in the right dir - command = f"java -jar /target/scala-3.1.0/wptool-boogie-assembly-0.0.1.jar".split(" ") - files = ["-a", adtfile, "-r", readelf_file, "-o", boogie_file] - if spec: - files += ["-s", spec] - outputs["spec"] = spec - command += files - logging.info(command) - res = subprocess.run(command, capture_output=True, check=False) - logging.info(res.stdout.decode('utf-8')) - logging.info(res.stderr.decode('utf-8')) - - return outputs - -def run_boogie(tmp_dir: str, args: list = [], spec = None): - outputs = run_basil(tmp_dir, spec) - - boogie_file = outputs['boogie'] - adt_file = outputs['adt'] - bir_file = outputs['bir'] - readelf_file = outputs['relf'] - - command = (f"/root/.dotnet/tools/boogie {boogie_file}").split(" ") - command += args - res = subprocess.run(command, capture_output=True, check=True) - out = res.stdout.decode('utf-8') - err = res.stderr.decode('utf-8') - - - boogie_outbothfile = f"{tmp_dir}/boogie_stdout_stderr" - boogie_out = f"{tmp_dir}/boogie_stdout" - boogie_err = f"{tmp_dir}/boogie_stderr" - - with open(boogie_out, 'w') as f: - f.write(out) - - with open(boogie_err, 'w') as f: - f.write(err) - - with open(boogie_outbothfile, 'w') as f: - f.write(out) - f.write(err) - - outputs.update({ - "boogie_stdout": boogie_out, - "boogie_stderr": boogie_err, - "boogie_stdout_stderr": boogie_outbothfile - }) - - return outputs - - -def cleanup_tempdirs(): - """ - Because temporary directories are shared between invocations we need to cleanup those that are no longer needed. - - """ - return 0 - -def main(tmp_dir): - parser = argparse.ArgumentParser( - prog='BasilTool', - description='Runs Basil and Associated Tools', - epilog='') - parser.add_argument('sourcefile') - parser.add_argument('-d', '--directory', required=False, help="Used to identify the compilation") - parser.add_argument('-t', '--tool', help="Which tool to run, basil/bap/readelf", default="basil") - parser.add_argument('-o', '--output', help="Which output to send to stdout", default="default") - parser.add_argument('-a', '--args', help="Extra args to pass to the tool", default=[]) - parser.add_argument('-s', '--spec', help="Specfile for basil") - parser.add_argument('-v', '--verbose', help="Enable log output", action="store_true") - - - args = parser.parse_args() - - if args.verbose: - logging.basicConfig(stream=sys.stdout, level=logging.DEBUG) - else: - logging.basicConfig(stream=sys.stderr, level=logging.ERROR) - - - logging.info(args) - - - data = None - with open(args.sourcefile, 'rb') as f: - data = f.read() - with open(bin_name(tmp_dir), 'wb') as f: - f.write(data) - - - # Copy spec - spec = None - if (args.spec): - spec = f"{tmp_dir}/in.spec" - specfile = None - with open(os.path.join(args.directory, args.spec), 'r') as f: - specfile = f.read() - #print(specfile) - with open(spec, 'w') as f: - f.write(specfile) - - -# TODO: give basil spec files -# TODO: run boogie -# TODO: primus lifter and asli lifter - - outputs = {} - - if (args.args): - args.args = args.args.split(" ") - - if args.tool == "readelf": - outputs = run_readelf(tmp_dir) - elif args.tool == "bap": - outputs = run_bap_lift(tmp_dir, False) - elif args.tool == "basil": - outputs = run_basil(tmp_dir, spec) - elif args.tool == "boogie": - outputs = run_boogie(tmp_dir, args.args, spec) - else: - print("Allowed tools: [readelf, bap, basil]") - exit(1) - - if args.output not in outputs: - print("Output unavailable, allowed are:", ", ".join(outputs.keys())) - exit(1) - - with open(outputs[args.output], 'r') as f: - logging.info("Printinng output: %s", outputs[args.output]) - print(f.read()) - exit(0) - - - - -if __name__ == "__main__": - with tempfile.TemporaryDirectory() as tmp_dir: - main(tmp_dir) diff --git a/docker/godbolt/basil.local.properties b/docker/godbolt/basil.local.properties deleted file mode 100644 index c6a4d9ba9..000000000 --- a/docker/godbolt/basil.local.properties +++ /dev/null @@ -1,145 +0,0 @@ -# Default settings for C -compilers=&gcc:&clang:&aarch64 -defaultCompiler=gcc13aarch -demangler=c++filt -objdumper=objdump -postProcess= -supportsBinary=true -supportsBinaryObject=true -binaryHideFuncRe=^(__.*|_(init|start|fini)|(de)?register_tm_clones|call_gmon_start|frame_dummy|\.plt.*)$ -stubRe=\bmain\b -stubText=int main(void){return 0;/*stub provided by Compiler Explorer*/} -supportsLibraryCodeFilter=true - -group.aarch64.compilers=gcc13aarch:clang14aarch -compiler.gcc13aarch.exe=/usr/bin/aarch64-linux-gnu-gcc -compiler.gcc13aarch.name=gcc 13 (aarch64) -compiler.gcc13aarch.objdumper=/usr/bin/aarch64-linux-gnu-objdump -compiler.gcc13aarch.supportsBinaryObject=true -compiler.clang14aarch.exe=/usr/bin/clang-15 -compiler.clang14aarch.objdumper=/usr/bin/aarch64-linux-gnu-objdump -compiler.clang14aarch.name=clang 15 (aarch64) -compiler.clang14aarch.options=--target=aarch64-linux-gnu -compiler.clang14aarch.supportsBinaryObject=true - -group.gcc.compilers=cg44:cg45:cg46:cg47:cg48:cg5:cg6x:cg7:cg8:cg9:cg10:cg11:cgdefault -compiler.cg44.exe=/usr/bin/gcc-4.4 -compiler.cg44.name=gcc 4.4 -compiler.cg45.exe=/usr/bin/gcc-4.5 -compiler.cg45.name=gcc 4.5 -compiler.cg46.exe=/usr/bin/gcc-4.6 -compiler.cg46.name=gcc 4.6 -compiler.cg47.exe=/usr/bin/gcc-4.7 -compiler.cg47.name=gcc 4.7 -compiler.cg48.exe=/usr/bin/gcc-4.8 -compiler.cg48.name=gcc 4.8 -compiler.cg5.exe=/usr/bin/gcc-5 -compiler.cg5.name=gcc 5.x -compiler.cg6x.exe=/usr/bin/gcc-6 -compiler.cg6x.name=gcc 6.x -compiler.cg6x.alias=cg6 -compiler.cg7.exe=/usr/bin/gcc-7 -compiler.cg7.name=gcc 7.x -compiler.cg8.exe=/usr/bin/gcc-8 -compiler.cg8.name=gcc 8.x -compiler.cg9.exe=/usr/bin/gcc-9 -compiler.cg9.name=gcc 9.x -compiler.cg10.exe=/usr/bin/gcc-10 -compiler.cg10.name=gcc 10.x -compiler.cg11.exe=/usr/bin/gcc-11 -compiler.cg11.name=gcc 11.x -compiler.cgdefault.exe=/usr/bin/gcc -compiler.cgdefault.name=gcc default - -group.clang.compilers=cclang7:cclang8:cclang9:cclang10:cclang11:cclang12:cclangdefault:cclangccdefault -group.clang.intelAsm=-mllvm --x86-asm-syntax=intel -compiler.cclang7.exe=/usr/bin/clang-7 -compiler.cclang7.name=clang 7 -compiler.cclang8.exe=/usr/bin/clang-8 -compiler.cclang8.name=clang 8 -compiler.cclang9.exe=/usr/bin/clang-9 -compiler.cclang9.name=clang 9 -compiler.cclang10.exe=/usr/bin/clang-10 -compiler.cclang10.name=clang 10 -compiler.cclang11.exe=/usr/bin/clang-11 -compiler.cclang11.name=clang 11 -compiler.cclang12.exe=/usr/bin/clang-12 -compiler.cclang12.name=clang 12 -compiler.cclangdefault.exe=/usr/bin/clang -compiler.cclangdefault.name=clang default -compiler.cclangccdefault.exe=/usr/bin/clangcc -compiler.cclangccdefault.name=clang default - -tools=clangquerydefault:clangtidydefault:readelf:nm:objdump:basil:bapbir:bapadt:basil-readelf:boogie - -tools.clangquerydefault.exe=/usr/bin/clang-query -tools.clangquerydefault.name=clang-query (default) -tools.clangquerydefault.type=independent -tools.clangquerydefault.class=clang-query-tool -tools.clangquerydefault.stdinHint=Query commands -tools.clangquerydefault.monacoStdin=true - -tools.clangtidydefault.exe=/usr/bin/clang-tidy -tools.clangtidydefault.name=clang-tidy (default) -tools.clangtidydefault.type=independent -tools.clangtidydefault.class=clang-tidy-tool -tools.clangtidydefault.stdinHint=disabled - -tools.readelf.name=readelf (default) -tools.readelf.exe=/usr/bin/readelf -tools.readelf.type=postcompilation -tools.readelf.class=readelf-tool -tools.readelf.exclude=djggp -tools.readelf.stdinHint=disabled - -tools.nm.name=nm (default) -tools.nm.exe=/usr/bin/nm -tools.nm.type=postcompilation -tools.nm.class=nm-tool -tools.nm.exclude=djggp -tools.nm.stdinHint=disabled - -tools.objdump.exe=/usr/bin/objdump -tools.objdump.name=objdump -tools.objdump.type=postcompilation -tools.objdump.class=pahole-tool -tools.objdump.stdinHint=disabled - -tools.basil-readelf.exe=/compiler-explorer/basil-tool.py -tools.basil-readelf.name=readelf (aarch64) -tools.basil-readelf.exclude=&gcc:&clang -tools.basil-readelf.type=postcompilation -tools.basil-readelf.options=-t readelf -tools.basil-readelf.class=basil-tool - -tools.bapadt.exe=/compiler-explorer/basil-tool.py -tools.bapadt.name=BAP (ADT) -tools.bapadt.exclude=&gcc:&clang -tools.bapadt.type=postcompilation -tools.bapadt.options=-t bap -o adt -tools.bapadt.class=basil-tool -tools.bapadt.languageId=asm - -tools.bapbir.exe=/compiler-explorer/basil-tool.py -tools.bapbir.name=BAP (BIR) -tools.bapbir.exclude=&gcc:&clang -tools.bapbir.type=postcompilation -tools.bapbir.options=-t bap -o bir -tools.bapbir.class=basil-tool -tools.bapbir.languageId=cpp - -tools.basil.exe=/compiler-explorer/basil-tool.py -tools.basil.name=basil -tools.basil.exclude=&gcc:&clang -tools.basil.type=postcompilation -tools.basil.options=-t basil -o boogie -tools.basil.languageId=scala -tools.basil.class=basil-tool - - -tools.boogie.exe=/compiler-explorer/basil-tool.py -tools.boogie.name=boogie -tools.boogie.exclude=&gcc:&clang -tools.boogie.type=postcompilation -tools.boogie.options=-t boogie -o boogie_stdout -tools.boogie.class=basil-tool diff --git a/docker/godbolt/compiler-explorer.local.properties b/docker/godbolt/compiler-explorer.local.properties deleted file mode 100644 index c328f928c..000000000 --- a/docker/godbolt/compiler-explorer.local.properties +++ /dev/null @@ -1,59 +0,0 @@ -# Default settings for GCC Explorer. -# Make this large due to boogie -compileTimeoutMs=300000 -binaryExecTimeoutMs=10000 -defaultSource=builtin -cacheConfig=InMemory(50) -executableCacheConfig=InMemory(50) -apiMaxAgeSecs=600 -maxConcurrentCompiles=4 -staticMaxAgeSecs=1 -maxUploadSize=16mb -supportsExecute=true -optionsAllowedRe=.* -optionsForbiddenRe=^(-W[alp],)?((--?(wrapper|fplugin.*|specs|load|plugin|include|fmodule-mapper)|(@.*)|-I|-i)(=.*)?|--)$ -allowedShortUrlHostRe=^([-a-z.]+\.)?(xania|godbolt)\.org$ -googleShortLinkRewrite=^https?://goo.gl/(.*)$|https://godbolt.org/g/$1 -urlShortenService=default -storageSolution=local -localStorageFolder=/storage/data/ -showSponsors=false -compilerCacheConfig=OnDisk(out/compiler-cache,1024) - -demanglerType=default -objdumperType=default - -python3=/usr/bin/python3 - -cvCompilerCountMax=15 - -textBanner=Compilation provided by Compiler Explorer at https://godbolt.org/ - -# If you run your own public instance of Compiler Explorer you are encouraged to write your own cookie policy and -# privacy policy, and then to enable both these below. -cookiePolicyEnabled=false -privacyPolicyEnabled=false - -supportsLibraryCodeFilter=false -ldPath=${exePath}/../lib|${exePath}/../lib32|${exePath}/../lib64 -ceToolsPath=../compiler-explorer-tools -remoteStorageServer=https://godbolt.org - -# this timeout exists only to work around a bug (#1875) -# it is set to 5 minutes so that it will never be triggered -# under any normal circumstances -compilationEnvTimeoutMs=300000 -# this timeout determines when a compilation in the queue is abandoned because it has been waiting too long and the -# client will have given up on it by now -compilationStaleAfterMs=300000 - -cmake=cmake -useninja=false -ld=ld -readelf=readelf - -# set this true to keep temporary folders for a while for debugging purposes -delayCleanupTemp=false - -thirdPartyIntegrationEnabled=true -statusTrackingEnabled=true diff --git a/docker/nix.Dockerfile b/docker/nix.Dockerfile new file mode 100644 index 000000000..47c9f9a99 --- /dev/null +++ b/docker/nix.Dockerfile @@ -0,0 +1,52 @@ +# Not working +FROM ubuntu:23.04 as nix-container +RUN apt-get update && apt-get install --yes default-jre-headless python3 libgmp-dev yasm m4 \ + libcurl4-gnutls-dev pkg-config zlib1g-dev cmake ninja-build g++-10 \ + radare2 z3 libz3-dev llvm-14-dev \ + re2c \ + libpcre3-dev \ + xz-utils \ + clang-14 clang-15 gcc-aarch64-linux-gnu \ + wget \ + git \ + && apt-get autoremove --purge -y \ + && apt-get autoclean -y +RUN useradd -ms /bin/bash godbolt +RUN mkdir -m 0755 /nix && chown godbolt /nix +USER godbolt +ENV USER godbolt +WORKDIR /home/godbolt +RUN wget --output-document=/dev/stdout https://nixos.org/nix/install | sh +RUN . .nix-profile/etc/profile.d/nix.sh && nix-channel --update +RUN . .nix-profile/etc/profile.d/nix.sh \ + && nix-env -iA cachix -f https://cachix.org/api/v1/install \ + && echo "trusted-users = root godbolt" | tee -a /nix/nix.conf \ + && cachix use pac-nix +RUN . .nix-profile/etc/profile.d/nix.sh && nix-env -iA nixpkgs.boogie \ + && nix-channel --add https://github.com/katrinafyi/pac-nix/archive/refs/heads/main.tar.gz pac \ + && nix-channel --update \ + && nix-env -iA pac.bap-aslp + +FROM nix-container AS compiler-explorer-tools +USER root +RUN DEBIAN_FRONTEND=noninteractive apt-get update && apt-get install -y ca-certificates gnupg \ + && mkdir -p /etc/apt/keyrings \ + && wget -q -S -O - https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg \ + && echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_16.x nodistro main" > /etc/apt/sources.list.d/nodesource.list \ + && apt-get update \ + && apt-get install npm nodejs clang gcc gcc-13-aarch64-linux-gnu binutils-aarch64-linux-gnu gcc-13-cross-base libc6-dev-arm64-cross libc6-dev-armel-cross libc6-dev-armhf-cross libc6-dev-i386 -y \ + && apt-get autoclean -y +ADD https://api.github.com/repos/ailrst/compiler-explorer/branches/main /tmp/head +WORKDIR /compiler-explorer +USER root +RUN git clone https://github.com/ailrst/compiler-explorer.git /compiler-explorer \ + && chown -R godbolt:godbolt /compiler-explorer +USER godbolt +RUN cd /compiler-explorer \ + && echo "Add missing dependencies" \ + && npm i @sentry/node +ENV PATH "$PATH:/home/godbolt/.nix-profile/bin" +ENTRYPOINT ["make"] +CMD ["run"] + + diff --git a/docker/readme.md b/docker/readme.md index b3048cb27..298c58883 100644 --- a/docker/readme.md +++ b/docker/readme.md @@ -23,3 +23,7 @@ $ podman-compose push basil $ podman-compose push basil-dev ``` +# Compiler-Explorer + +The configuration used to run add the tool to compiler explorer lives here: https://github.com/ailrst/compiler-explorer +