add neurips 2023 folder (FALCONN-LIB#107)

* add neurips 2023 folder

* docker stuff

* files

* more files added

neurips-2023/ood/install/Dockerfile

@@ -0,0 +1,13 @@
+FROM ubuntu:jammy
+
+RUN apt-get update && apt-get install -y python3-numpy python3-scipy python3-pip build-essential git axel wget
+RUN wget https://aka.ms/downloadazcopy-v10-linux && mv downloadazcopy-v10-linux azcopy.tgz && tar xzf azcopy.tgz --transform 's!^[^/]\+\($\|/\)!azcopy_folder\1!' 
+RUN cp azcopy_folder/azcopy /usr/bin
+
+RUN pip3 install -U pip
+
+WORKDIR /home/app
+COPY requirements_py3.10.txt run_algorithm.py ./
+RUN pip3 install -r requirements_py3.10.txt
+
+ENTRYPOINT ["python3", "-u", "run_algorithm.py"]

neurips-2023/ood/install/Dockerfile.diskann

@@ -0,0 +1,16 @@
+FROM big-ann-benchmark-v2-lighweight
+
+RUN apt update
+RUN apt install -y software-properties-common
+RUN add-apt-repository -y ppa:git-core/ppa
+RUN apt update
+RUN DEBIAN_FRONTEND=noninteractive apt install -y git make cmake g++ libaio-dev libgoogle-perftools-dev libunwind-dev clang-format libboost-dev libboost-program-options-dev libmkl-full-dev libcpprest-dev python3.10-dev python3.10-venv python3-pip
+
+WORKDIR /app
+RUN git clone https://github.com/microsoft/DiskANN.git
+WORKDIR /app/DiskANN
+RUN git checkout harshasi/python-mem-index
+RUN pip install build scikit-learn
+RUN python3.10 -m build
+RUN pip install dist/diskannpy-0.5.0-cp310-cp310-linux_x86_64.whl
+ENV LD_PRELOAD="/lib/x86_64-linux-gnu/libmkl_intel_thread.so:/lib/x86_64-linux-gnu/libmkl_intel_ilp64.so:/lib/x86_64-linux-gnu/libmkl_core.so:/lib/x86_64-linux-gnu/libiomp5.so:/lib/x86_64-linux-gnu/libmkl_avx2.so:/lib/x86_64-linux-gnu/libmkl_def.so"

neurips-2023/streaming/install.py

@@ -0,0 +1,78 @@
+import json
+import os
+import argparse
+import subprocess
+from multiprocessing import Pool
+
+
+def build(library, args, dockerfile):
+    print('Building %s...' % library)
+    if args is not None and len(args) != 0:
+        q = " ".join(["--build-arg " + x.replace(" ", "\\ ") for x in args])
+    else:
+        q = ""
+
+    try:
+        command = 'docker build %s --rm -t big-ann-benchmark-v2-lighweight-%s -f' \
+                    % (q, library )
+        command += ' neurips-2023/install/Dockerfile.%s .' % (library)  \
+                    if not dockerfile else ' %s .' % dockerfile
+        subprocess.check_call(command, shell=True)
+        return {library: 'success'}
+    except subprocess.CalledProcessError:
+        return {library: 'fail'}
+
+
+def build_multiprocess(args):
+    return build(*args)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+    parser.add_argument(
+        "--proc",
+        default=1,
+        type=int,
+        help="the number of process to build docker images")
+    parser.add_argument(
+        '--algorithm',
+        metavar='NAME',
+        help='build only the named algorithm image',
+        default=None)
+    parser.add_argument(
+        '--dockerfile',
+        metavar='PATH',
+        help='build only the image from a Dockerfile path',
+        default=None)
+    parser.add_argument(
+        '--build-arg',
+        help='pass given args to all docker builds',
+        nargs="+")
+    args = parser.parse_args()
+
+    print('Building base image...')
+    subprocess.check_call(
+        'docker build \
+        --rm -t big-ann-benchmark-v2-lighweight -f neurips-2023/install/Dockerfile .', shell=True)
+
+    if args.dockerfile:
+        tags = [os.path.basename(os.path.dirname(args.dockerfile))]
+    elif args.algorithm:
+        tags = [args.algorithm]
+    elif os.getenv('LIBRARY'):
+        tags = [os.getenv('LIBRARY')]
+    else:
+        tags = [fn.split('.')[-1] for fn in os.listdir('neurips-2023/install') if fn.startswith('Dockerfile.') and not 'faissgpu' in fn]
+
+    print('Building algorithm images... with (%d) processes' % args.proc)
+
+    if args.proc == 1:
+        install_status = [build(tag, args.build_arg, args.dockerfile) for tag in tags ]
+    else:
+        pool = Pool(processes=args.proc)
+        install_status = pool.map(build_multiprocess, [(tag, args.build_arg, args.dockerfile) for tag in tags ])
+        pool.close()
+        pool.join()
+
+    print('\n\nInstall Status:\n' + '\n'.join(str(algo) for algo in install_status))

neurips-2023/streaming/install/Dockerfile

@@ -0,0 +1,13 @@
+FROM ubuntu:jammy
+
+RUN apt-get update && apt-get install -y python3-numpy python3-scipy python3-pip build-essential git axel wget
+RUN wget https://aka.ms/downloadazcopy-v10-linux && mv downloadazcopy-v10-linux azcopy.tgz && tar xzf azcopy.tgz --transform 's!^[^/]\+\($\|/\)!azcopy_folder\1!' 
+RUN cp azcopy_folder/azcopy /usr/bin
+
+RUN pip3 install -U pip
+
+WORKDIR /home/app
+COPY requirements_py3.10.txt run_algorithm.py ./
+RUN pip3 install -r requirements_py3.10.txt
+
+ENTRYPOINT ["python3", "-u", "run_algorithm.py"]

neurips-2023/streaming/install/Dockerfile.diskann

@@ -0,0 +1,16 @@
+FROM big-ann-benchmark-v2-lighweight
+
+RUN apt update
+RUN apt install -y software-properties-common
+RUN add-apt-repository -y ppa:git-core/ppa
+RUN apt update
+RUN DEBIAN_FRONTEND=noninteractive apt install -y git make cmake g++ libaio-dev libgoogle-perftools-dev libunwind-dev clang-format libboost-dev libboost-program-options-dev libmkl-full-dev libcpprest-dev python3.10-dev python3.10-venv python3-pip
+
+WORKDIR /app
+RUN git clone https://github.com/microsoft/DiskANN.git
+WORKDIR /app/DiskANN
+RUN git checkout harshasi/python-mem-index
+RUN pip install build scikit-learn
+RUN python3.10 -m build
+RUN pip install dist/diskannpy-0.5.0-cp310-cp310-linux_x86_64.whl
+ENV LD_PRELOAD="/lib/x86_64-linux-gnu/libmkl_intel_thread.so:/lib/x86_64-linux-gnu/libmkl_intel_ilp64.so:/lib/x86_64-linux-gnu/libmkl_core.so:/lib/x86_64-linux-gnu/libiomp5.so:/lib/x86_64-linux-gnu/libmkl_avx2.so:/lib/x86_64-linux-gnu/libmkl_def.so"