Update to FLIP v1.6 (#38)

- Flipped the ꟻ in ꟻLIP. The entire name (FLIP) should now be readable on all devices. - Published Python version of FLIP to PyPI (URL: https://pypi.org/project/flip-evaluator/). - The Python version of FLIP (tool and API) is now installed by pip install flip-evaluator. - The distribution has been tested on Windows, Linux (Ubuntu 24.04), and OS X (≥ 10.15). Wheels are built for each (and various CPython versions ≥ 3.8) using cibuildwheel. Note that FLIP's output might differ slightly between the different operative systems. The references used for src/tests/test.py are made for Windows. While the mean tests (means compared up to six decimal points) pass on each mentioned operative system, not all error map pixels are identical. - After installation, the tool can be run directly in a shell by `flip --reference reference.{png|exr} --test test.{png|exr}`. - After installation, the FLIP API is available in Python by `import flip_evaluator as flip`. - Python version is now built using scikit instead of setuptools, and uses nanobind instead of pybind11. - Directory structure in the FLIP repository has been slightly altered to accomodate the Python version being published to PyPI. - Updated Python/C++/CUDA test script. - Various significant bugfixes. - NOTE: Skipped version 1.5 due to PyPI-related mistake. Version 1.6 is the same as version 1.5 was supposed to be.
NVlabs · Nov 11, 2024 · 7967578 · 7967578
1 parent 615885c
commit 7967578
Show file tree

Hide file tree

Showing 50 changed files with 651 additions and 537 deletions.
diff --git a/.github/workflows/flip-deploy.yml b/.github/workflows/flip-deploy.yml
@@ -0,0 +1,67 @@
+# Much of the code retrieved from Mitsuba3 (https://github.com/mitsuba-renderer/mitsuba3/blob/master/.github/workflows/wheels.yml.)
+
+name: Build wheels
+
+on:
+  workflow_dispatch:
+  pull_request:
+  push:
+    branches:
+      - main
+  release:
+    types:
+      - published
+
+jobs:
+  build_wheels:
+    strategy:
+      matrix:
+        # macos-13 is an intel runner, macos-14 is apple silicon
+        os: [ubuntu-latest, windows-latest, macos-13, macos-14]
+        python: [cp38, cp39, cp310, cp311, cp312, cp312_stable, cp313]
+        exclude:
+          # The first Python version to target Apple arm64 architectures is 3.9.
+          - os: macos-14
+            python: cp38
+    name: >
+      ${{ matrix.python }} wheel for ${{ matrix.os }}
+      ${{ (endsWith(matrix.python, '_stable') && '(stable ABI)') || '' }}
+    runs-on: ${{ matrix.os }}
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v4
+        name: Install Python
+        with:
+          python-version: '3.8'
+
+      - name: Install cibuildwheel
+        run: |
+          python -m pip install cibuildwheel==2.20.0
+
+      #########################
+      # Build and store wheels
+      #########################
+      - name: Build wheel
+        run: |
+          python -m cibuildwheel --output-dir wheelhouse
+
+      - uses: actions/upload-artifact@v3
+        with:
+          name: wheels
+          path: ./wheelhouse/*.whl
+
+  build_sdist:
+    name: Build source distribution
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Build sdist
+        run: pipx run build --sdist
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: cibw-sdist
+          path: dist/*.tar.gz
diff --git a/.github/workflows/flip_ci.yml b/.github/workflows/flip_ci.yml
@@ -24,12 +24,12 @@ jobs:
 
     - name: Configure CMake
       run: >
-        cmake -LA -B ${{github.workspace}}/build
+        cmake -LA -B ${{github.workspace}}/src/build -S ${{github.workspace}}/src/
         -DCMAKE_BUILD_TYPE=${{ matrix.config }}
-        -DCMAKE_INSTALL_PREFIX=${{github.workspace}}/build/install
+        -DCMAKE_INSTALL_PREFIX=${{github.workspace}}/src/build/install
         -DCMAKE_CUDA_ARCHITECTURES=all
         -DCMAKE_CUDA_HOST_COMPILER=g++-10
         -DFLIP_ENABLE_CUDA=${{ matrix.os == 'ubuntu-latest' }}
 
     - name: Build
-      run: cmake --build ${{github.workspace}}/build --config ${{ matrix.config }} --target install
+      run: cmake --build ${{github.workspace}}/src/build --config ${{ matrix.config }} --target install
diff --git a/.gitignore b/.gitignore
@@ -32,8 +32,8 @@ _ReSharper*/
 
 #  Python ignores
 *__pycache__*
-python/build/
-python/*.egg-info/
+dist/
+*.egg-info/
 
 #  LaTeX ignores
 *.synctex.gz

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -30,22 +30,60 @@
 # SPDX-License-Identifier: BSD-3-Clause
 #################################################################################
 
-cmake_minimum_required(VERSION 3.9)
+cmake_minimum_required(VERSION 3.15...3.27)
+project(flip_evaluator LANGUAGES CXX)
 
-set(CMAKE_DISABLE_SOURCE_CHANGES ON)
-set(CMAKE_DISABLE_IN_SOURCE_BUILD ON)
+# Warn if the user invokes CMake directly
+if (NOT SKBUILD)
+  message(WARNING "\
+  This CMake file is meant to be executed using 'scikit-build-core'.
+  Running it directly will almost certainly not produce the desired
+  result. If you are a user trying to install this package, use the
+  command below, which will install all necessary build dependencies,
+  compile the package in an isolated environment, and then install it.
+  =====================================================================
+   $ pip install .
+  =====================================================================
+  If you are a software developer, and this is your own package, then
+  it is usually much more efficient to install the build dependencies
+  in your environment once and use the following command that avoids
+  a costly creation of a new virtual environment at every compilation:
+  =====================================================================
+   $ pip install nanobind scikit-build-core[pyproject]
+   $ pip install --no-build-isolation -ve .
+  =====================================================================
+  You may optionally add -Ceditable.rebuild=true to auto-rebuild when
+  the package is imported. Otherwise, you need to rerun the above
+  after editing C++ files.")
+endif()
 
-set(CMAKE_CXX_STANDARD 17)
-set(CMAKE_CXX_STANDARD_REQUIRED ON)
-set(CMAKE_CXX_EXTENSIONS OFF)
+if (CMAKE_VERSION VERSION_LESS 3.18)
+  set(DEV_MODULE Development)
+else()
+  set(DEV_MODULE Development.Module)
+endif()
 
-set(CMAKE_BUILD_TYPE_INIT "Release")
+if(MSVC)
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -openmp")
+elseif((NOT APPLE) AND UNIX)
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fopenmp")
+    set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -lgomp")
+endif()
 
-project(flip LANGUAGES CXX)
+find_package(Python 3.8
+  REQUIRED COMPONENTS Interpreter Development.Module
+  OPTIONAL_COMPONENTS Development.SABIModule)
 
-set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR})
-include(GNUInstallDirs)
+if (NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES)
+  set(CMAKE_BUILD_TYPE Release CACHE STRING "Choose the type of build." FORCE)
+  set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release" "MinSizeRel" "RelWithDebInfo")
+endif()
 
-option(FLIP_ENABLE_CUDA "Include CUDA version of flip" OFF)
+execute_process(
+  COMMAND "${Python_EXECUTABLE}" -m nanobind --cmake_dir
+  OUTPUT_STRIP_TRAILING_WHITESPACE OUTPUT_VARIABLE nanobind_ROOT)
+find_package(nanobind CONFIG REQUIRED)
 
-add_subdirectory(cpp)
+nanobind_add_module(pbflip STABLE_ABI src/nanobindFLIP.cpp)
+
+install(TARGETS pbflip LIBRARY DESTINATION flip_evaluator)
diff --git a/LICENSE b/LICENSE
@@ -0,0 +1,31 @@
+BSD 3-Clause License
+
+Copyright (c) 2020-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+this list of conditions and the following disclaimer in the documentation
+and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+SPDX-FileCopyrightText: Copyright (c) 2020-2024 NVIDIA CORPORATION & AFFILIATES
+SPDX-License-Identifier: BSD-3-Clause
diff --git a/README.md b/README.md
@@ -1,9 +1,9 @@
 ![Teaser image](images/teaser.png "Teaser image")
 
-# ꟻLIP: A Tool for Visualizing and Communicating Errors in Rendered Images (v1.4)
+# FLIP: A Tool for Visualizing and Communicating Errors in Rendered Images (v1.6)
 
 By
-[Pontus Ebelin](https://research.nvidia.com/person/pontus-ebelin),
+[Pontus Ebelin](https://research.nvidia.com/person/pontus-ebelin)
 and
 [Tomas Akenine-Möller](https://research.nvidia.com/person/tomas-akenine-m%C3%B6ller),
 with
@@ -14,57 +14,68 @@ Jim Nilsson,
 and
 [Peter Shirley](https://research.nvidia.com/person/peter-shirley).
 
-This repository holds implementations of the [LDR-ꟻLIP](https://research.nvidia.com/publication/2020-07_FLIP)
-and [HDR-ꟻLIP](https://research.nvidia.com/publication/2021-05_HDR-FLIP) image error metrics.
-It also holds code for the ꟻLIP tool, presented in [Ray Tracing Gems II](https://www.realtimerendering.com/raytracinggems/rtg2/index.html).
+This repository holds implementations of the [LDR-FLIP](https://research.nvidia.com/publication/2020-07_FLIP)
+and [HDR-FLIP](https://research.nvidia.com/publication/2021-05_HDR-FLIP) image error metrics.
+It also holds code for the FLIP tool, presented in [Ray Tracing Gems II](https://www.realtimerendering.com/raytracinggems/rtg2/index.html).
 
-The changes made for the different versions of ꟻLIP are summarized in the [version list](misc/versionList.md).
+The changes made for the different versions of FLIP are summarized in the [version list](https://github.com/NVlabs/flip/blob/main/misc/versionList.md).
 
-[A list of papers](misc/papersUsingFLIP.md) that use/cite ꟻLIP.
+[A list of papers](https://github.com/NVlabs/flip/blob/main/misc/papersUsingFLIP.md) that use/cite FLIP.
 
-[A note](misc/precision.md) about the precision of ꟻLIP.
+[A note](https://github.com/NVlabs/flip/blob/main/misc/precision.md) about the precision of FLIP.
 
 [An image gallery](https://research.nvidia.com/node/3525) displaying a large quantity of reference/test images and corresponding error maps from
 different metrics.
 
-**Note**: in v1.3, we switched to a *single header* ([FLIP.h](https://github.com/NVlabs/flip/blob/singleheader_WIP/cpp/FLIP.h)) for C++/CUDA for easier integration.
+**Note**: since v1.6, the Python version of FLIP can now be installed via `pip install flip-evaluator`.
+
+**Note**: in v1.3, we switched to a *single header* ([FLIP.h](src/cpp/FLIP.h)) for C++/CUDA for easier integration.
 
 # License
 
 Copyright © 2020-2024, NVIDIA Corporation & Affiliates. All rights reserved.
 
-This work is made available under a [BSD 3-Clause License](misc/LICENSE.md).
+This work is made available under a [BSD 3-Clause License](LICENSE).
 
-The repository distributes code for `tinyexr`, which is subject to a [BSD 3-Clause License](misc/LICENSE-third-party.md#bsd-3-clause-license),<br>
-and `stb_image`, which is subject to an [MIT License](misc/LICENSE-third-party.md#mit-license).
+The repository distributes code for `tinyexr`, which is subject to a [BSD 3-Clause License](https://github.com/NVlabs/flip/blob/main/misc/LICENSE-third-party.md#bsd-3-clause-license),<br>
+and `stb_image`, which is subject to an [MIT License](https://github.com/NVlabs/flip/blob/main/misc/LICENSE-third-party.md#mit-license).
 
-For individual contributions to the project, please confer the [Individual Contributor License Agreement](misc/CLA.md).
+For individual contributions to the project, please confer the [Individual Contributor License Agreement](https://github.com/NVlabs/flip/blob/main/misc/CLA.md).
 
 For business inquiries, please visit our website and submit the form: [NVIDIA Research Licensing](https://www.nvidia.com/en-us/research/inquiries/).
 
+# Simplest Way To Get Started
+The simplest way to run FLIP to compare a test image `testImage.png` to a reference image `referenceImage.png` is as follows:
+```
+pip install flip-evaluator
+flip -r referenceImage.png -t testImage.png
+```
+For more information about the tool's capabilities, try running `flip -h`.
+
+If you wish to use FLIP in your Python or C++ evaluation scripts, please read the next sections.
+
 # Python (API and Tool)
 **Setup** (with pip):
 ```
-cd python
-pip install -r requirements.txt .
+pip install flip-evaluator
 ```
 
 **Usage:**<br>
 
 API:<br>
-See the example script `python/api_example.py`. Note that the script requires `matplotlib`.
+See the example script `src/python/api_example.py`.
 
 Tool:
 ```
-python flip.py --reference reference.{exr|png} --test test.{exr|png} [--options]
+flip --reference reference.{exr|png} --test test.{exr|png} [--options]
 ```
 
-See the [README](python/README.md) in the `python` folder and run `python flip.py -h` for further information and usage instructions.
+See the [README](https://github.com/NVlabs/flip/blob/main/src/python/README.md) in the `python` folder and run `flip -h` for further information and usage instructions.
 
 # C++ and CUDA (API and Tool)
 **Setup:**
 
-The `FLIP.sln` solution contains one CUDA backend project and one pure C++ backend project.
+The `src/cpp/FLIP.sln` solution contains one CUDA backend project and one pure C++ backend project.
 
 Compiling the CUDA project requires a CUDA compatible GPU. Instruction on how to install CUDA can be found [here](https://docs.nvidia.com/cuda/cuda-installation-guide-microsoft-windows/index.html).
 
@@ -84,17 +95,17 @@ CUDA support is enabled via the `FLIP_ENABLE_CUDA`, which can be passed to CMake
 **Usage:**<br>
 
 API:<br>
-See the [README](cpp/README.md).
+See the [README](https://github.com/NVlabs/flip/blob/main/src/cpp/README.md).
 
 Tool:
 ```
 flip[-cuda].exe --reference reference.{exr|png} --test test.{exr|png} [options]
 ```
 
-See the [README](cpp/README.md) in the `cpp` folder and run `flip[-cuda].exe -h` for further information and usage instructions.
+See the [README](https://github.com/NVlabs/flip/blob/main/src/cpp/README.md) in the `src/cpp` folder and run `flip[-cuda].exe -h` for further information and usage instructions.
 
 # PyTorch (Loss Function)
-**Setup** (with Anaconda3):
+**Setup** (with Anaconda3 or Miniconda):
 ```
 conda create -n flip_dl python numpy matplotlib
 conda activate flip_dl
@@ -106,22 +117,22 @@ conda install -c conda-forge openexr-python
 
 *Remember to activate the* `flip_dl` *environment through* `conda activate flip_dl` *before using the loss function.*
 
-LDR- and HDR-ꟻLIP are implemented as loss modules in `flip_loss.py`. An example where the loss function is used to train a simple autoencoder is provided in `train.py`.
+LDR- and HDR-FLIP are implemented as loss modules in `src/pytorch/flip_loss.py`. An example where the loss function is used to train a simple autoencoder is provided in `src/pytorch/train.py`.
 
-See the [README](pytorch/README.md) in the `pytorch` folder for further information and usage instructions.
+See the [README](https://github.com/NVlabs/flip/blob/main/src/pytorch/README.md) in the `pytorch` folder for further information and usage instructions.
 
 # Citation
-If your work uses the ꟻLIP tool to find the errors between *low dynamic range* images,
-please cite the LDR-ꟻLIP paper:<br>
-[Paper](https://research.nvidia.com/publication/2020-07_FLIP) | [BibTeX](misc/LDRFLIP.txt)
+If your work uses the FLIP tool to find the errors between *low dynamic range* images,
+please cite the LDR-FLIP paper:<br>
+[Paper](https://research.nvidia.com/publication/2020-07_FLIP) | [BibTeX](https://github.com/NVlabs/flip/blob/main/misc/LDRFLIP.txt)
 
-If it uses the ꟻLIP tool to find the errors between *high dynamic range* images,
-instead cite the HDR-ꟻLIP paper:<br>
-[Paper](https://research.nvidia.com/publication/2021-05_HDR-FLIP) | [BibTeX](misc/HDRFLIP.txt)
+If it uses the FLIP tool to find the errors between *high dynamic range* images,
+instead cite the HDR-FLIP paper:<br>
+[Paper](https://research.nvidia.com/publication/2021-05_HDR-FLIP) | [BibTeX](https://github.com/NVlabs/flip/blob/main/misc/HDRFLIP.txt)
 
-Should your work use the ꟻLIP tool in a more general fashion, please cite the Ray Tracing Gems II article:<br>
-[Chapter](https://link.springer.com/chapter/10.1007%2F978-1-4842-7185-8_19) | [BibTeX](misc/FLIP.txt)
+Should your work use the FLIP tool in a more general fashion, please cite the Ray Tracing Gems II article:<br>
+[Chapter](https://link.springer.com/chapter/10.1007%2F978-1-4842-7185-8_19) | [BibTeX](https://github.com/NVlabs/flip/blob/main/misc/FLIP.txt)
 
 # Acknowledgements
 We appreciate the following peoples' contributions to this repository:
-Jonathan Granskog, Jacob Munkberg, Jon Hasselgren, Jefferson Amstutz, Alan Wolfe, Killian Herveau, Vinh Truong, Philippe Dagobert, Hannes Hergeth, Matt Pharr, Tizian Zeltner, Jan Honsbrok, and Chris Zhang.
+Jonathan Granskog, Jacob Munkberg, Jon Hasselgren, Jefferson Amstutz, Alan Wolfe, Killian Herveau, Vinh Truong, Philippe Dagobert, Hannes Hergeth, Matt Pharr, Tizian Zeltner, Jan Honsbrok, Chris Zhang, and Wenzel Jakob.
diff --git a/misc/LICENSE.md b/misc/LICENSE.md