bench-closest.py

#!/usr/bin/env python3

"""
Benchmark the /v1/city/closest endpoint.

Usage:
  bench-closest.py [--runs=<n>]

Options:
  --runs=<n>    Number of runs to perform [default: 2].
"""

from pathlib import Path
import subprocess

from docopt import docopt


def main(runs):
    endpoint_paths = {
        'praha-by-geometry': '/city/v1/closest?language=pl&lat=50.107&lon=14.574',
        'otice-by-geometry': '/city/v1/closest?language=pl&lat=49.915&lon=17.870',
        'otice-by-centroid': '/city/v1/closest?language=pl&lat=49.914&lon=17.877',
    }
    images = {
        '1rt-rs': 'https://gitlab.com/GoOutNet/Developers/Backend/locations-rs/-/blob/1rt-rs/src/services/locations_repo.rs#L176-220',
        '2rt-rs': 'https://gitlab.com/GoOutNet/Developers/Backend/locations-rs/-/blob/2rt-rs/src/services/locations_repo.rs#L176-228',
        '2rt-kt': 'https://gitlab.com/GoOutNet/Backend/-/blob/archive/locations-2rt/Locations/src/main/kotlin/repository/ElasticCityAndRegionRepository.kt#L136-173',
        '1rt-kt': 'https://gitlab.com/GoOutNet/Backend/-/blob/archive/locations-1rt/Locations/src/main/kotlin/repository/ElasticCityAndRegionRepository.kt#L136-166',
    }

    script_path = Path(__file__)
    script_dir = script_path.parent.absolute()
    test_image = str(script_dir / 'test-image.py')
    render_tests = str(script_dir / 'render-tests.py')
    script_commit = run(['git', 'rev-parse', 'HEAD'], cwd=script_dir, check=True, capture_output=True, text=True) \
        .stdout.strip()

    for i in range(1, runs + 1):
        for ep_type, path in endpoint_paths.items():
            Path(ep_type).mkdir(exist_ok=True)
            for image in images:
                outfile = f'{ep_type}/{image}-{i}.checks.bench.json'
                if Path(outfile).exists():
                    print(f'{outfile} exists, skipping this benchmark.')
                    continue

                try:
                    run([test_image, image, '--bench-url', path, '--bench-out', outfile], check=True)
                except subprocess.CalledProcessError:
                    print('Called process exited with non-zero exit status, but continuing.')

    html_lines = []
    for ep_type, path in endpoint_paths.items():
        dev_endpoint_url = f'https://dev.goout.net/services/locations{path}'
        with open(f'{ep_type}/notes.md', 'w') as f:
            print(f'* Tested endpoint: [`{path}`]({dev_endpoint_url}) ({ep_type}).', file=f)
            print(f'* Each round (for 1, 2, 4 connections) lasts **20 seconds** (increase from past).', file=f)
            print(f'* Implementations:', file=f)
            for image, link in images.items():
                print(f'  * `{image}`: {link}', file=f)
            print(f'* This benchmark was generated by [`{script_path.name}` commit {script_commit[:7]}](https://'
                  f'gitlab.com/GoOutNet/Developers/Backend/locations-rs/-/blob/{script_commit}/bench-closest.py).',
                  file=f)

        run([render_tests], cwd=ep_type)
        html_lines.append(f'<a href="{ep_type}/bench-results.html">{ep_type}</a>: <a href="{dev_endpoint_url}"><code>{path}</code></a>')

    html_lines.append('')
    html_lines.append('<i>Note that different cities take different time to serialize on ES (probably due to excluding '
                      'geometry fields), so compare the same cities.</i>')
    html = '<br>\n'.join(html_lines)
    with open('index.html', 'w') as f:
        f.write(html)


# TODO: deduplicate
def run(program_args, **kwargs):
    print(f"$ {' '.join(program_args)}")
    try:
        return subprocess.run(program_args, **kwargs)
    except subprocess.CalledProcessError as e:
        if e.stdout:
            print(f"stdout: {e.stdout}")
        if e.stderr:
            print(f"stderr: {e.stderr}")
        raise


if __name__ == '__main__':
    args = docopt(__doc__)
    main(int(args['--runs']))