mapillary · cojosef96 · Jul 9, 2020 · Jul 13, 2020 · Jul 26, 2020 · Jul 26, 2020
diff --git a/README.md b/README.md
@@ -21,3 +21,90 @@ Checkout this [blog post with more demos](http://blog.mapillary.com/update/2014/
 [Building the library]: https://docs.opensfm.org/building.html (OpenSfM building instructions)
 [Running a reconstruction]: https://docs.opensfm.org/using.html (OpenSfM usage)
 [Documentation]: https://docs.opensfm.org  (OpenSfM documentation)
+
+## Update Sift_GPU version
+
+### Changes:
+* added SIFT_GPU parameters to the config file.
+* added SiftGPU class.
+* added feature detection function in features.py file.
+* added matching functions to the matching.py file.
+* changed the commands to support the sift_gpu option.
+* added functions to of save and load sift_gpu keypoints to the dataset.py file.
+
+
+
+### Requirements:
+This update depends on the silx python package.
+
+To install it you will need to install pyopencl.
+Please see the installation guide [Here](http://www.silx.org/doc/silx/dev/install.html).
+
+### Usage
+In order to test if all the packages were installed correctly. Try to run my test code. 
+Which match two of the images in the Berlin data. The code can be found in the main folder under the name "test_sift_gpu.py".
+
+To run this Code simply write:
+```
+cd OpenSfm_sift_gpu
+python test_sift_gpu.py
+```
+### Run OpenSfm
+
+If all the dependencies were installed correctly.
+To run the opensfm pipeline enter this commands:
+```
+cd OpenSfm_sift_gpu
+./bin/opensfm_run_all data/berlin_gpu
+```
+
+To check the sfm, enter:
+
+`python3 -m http.server`
+
+and click on this [link](http://localhost:8000/viewer/reconstruction.html#file=/data/berlin_gpu/reconstruction.meshed.json) to see the reconstruction. 
+
+### Benchmark
+Here you can see the results of the Sift_GPU implementation.
+
+#### Feature detection
+<p align="center">
+  <img src="misc/img1_features.png" width="400">
+  <img src="misc/img2_features.png" width="400">
+</p>
+
+
+<p align="center">
+  <img src="misc/img1_features_zoom.png" width="400">
+  <img src="misc/img2_features_zoom.png" width="400">
+</p>
+
+#### Feature Matching
+
+<p align="center">
+  <img src="misc/sift_gpu_matching.png" width="800">
+</p>
+
+<p align="center">
+  <img src="misc/sift_matching_2.png" width="800">
+</p>
+
+#### SFM Creation
+
+ <p align="center">
+  <img src="misc/sfm_berlin_gpu.png" width="800">
+</p>
+
+<p align="center">
+  <img src="misc/sfm_berlin_2.png" width="800">
+</p>
+
+#### Speed
+Feature Matching on image (3264x2448) on 1080-TI-GTX took 0.28 sec
+
+Feature Matching took 0.025 sec
+
+#### Contact Me
+
+For more information, you can contact me via [Email](mailto:[email protected])
+
diff --git a/data/berlin_gpu/config.yaml b/data/berlin_gpu/config.yaml
@@ -0,0 +1,11 @@
+
+# OpenSfM will use the default parameters from opensfm/config.py
+# Set here any parameter that you want to override for this dataset
+# For example:
+processes: 1                  # Number of threads to use, SIFT_GPU does not work on parallel
+depthmap_min_consistent_views: 2      # Min number of views that should reconstruct a point for it to be valid
+depthmap_save_debug_files: yes        # Save debug files with partial reconstruction results
+feature_process_size: -1
+bundle_use_gcp: yes
+feature_type: SIFT_GPU         # Feature type (AKAZE, SURF, SIFT, HAHOG, ORB, SIFT_GPU)
+
diff --git a/data/berlin_gpu/ground_control_points.json b/data/berlin_gpu/ground_control_points.json
@@ -0,0 +1,64 @@
+{
+    "points": [
+        {
+            "position": {
+                "latitude": 52.51926834404209,
+                "altitude": 14.946108041331172,
+                "longitude": 13.400703631118825
+            },
+            "id": "0",
+            "observations": [
+                {
+                    "projection": [
+                        -0.015689,
+                        -0.0625362
+                    ],
+                    "shot_id": "02.jpg"
+                },
+                {
+                    "projection": [
+                        -0.0624397,
+                        0.0440716
+                    ],
+                    "shot_id": "03.jpg"
+                }
+            ]
+        },
+        {
+            "id": "1",
+            "observations": [
+                {
+                    "projection": [
+                        0.049848,
+                        0.144291
+                    ],
+                    "shot_id": "02.jpg"
+                },
+                {
+                    "projection": [
+                        0.0105317,
+                        0.244958
+                    ],
+                    "shot_id": "01.jpg"
+                }
+            ]
+        },
+        {
+            "position": {
+                "latitude": 52.5192651808067,
+                "altitude": 12.859175567515194,
+                "longitude": 13.400764257288497
+            },
+            "id": "3",
+            "observations": [
+                {
+                    "projection": [
+                        0.0608681,
+                        -0.00730461
+                    ],
+                    "shot_id": "02.jpg"
+                }
+            ]
+        }
+    ]
+}
diff --git a/data/berlin_gpu/images/01.jpg b/data/berlin_gpu/images/01.jpg
diff --git a/data/berlin_gpu/images/02.jpg b/data/berlin_gpu/images/02.jpg
diff --git a/data/berlin_gpu/images/03.jpg b/data/berlin_gpu/images/03.jpg
diff --git a/data/berlin_gpu/masks/01.jpg.png b/data/berlin_gpu/masks/01.jpg.png
diff --git a/data/berlin_gpu/masks/02.jpg.png b/data/berlin_gpu/masks/02.jpg.png
diff --git a/data/berlin_gpu/masks/03.jpg.png b/data/berlin_gpu/masks/03.jpg.png
diff --git a/misc/img1_features.png b/misc/img1_features.png
diff --git a/misc/img1_features_zoom.png b/misc/img1_features_zoom.png
diff --git a/misc/img2_features.png b/misc/img2_features.png
diff --git a/misc/img2_features_zoom.png b/misc/img2_features_zoom.png
diff --git a/misc/sfm_berlin_2.png b/misc/sfm_berlin_2.png
diff --git a/misc/sfm_berlin_gpu.png b/misc/sfm_berlin_gpu.png
diff --git a/misc/sift_gpu_matching.png b/misc/sift_gpu_matching.png
diff --git a/misc/sift_matching_2.png b/misc/sift_matching_2.png
diff --git a/opensfm/commands/detect_features.py b/opensfm/commands/detect_features.py
@@ -69,12 +69,17 @@ def detect(args):
         data.feature_type().upper(), image))
 
     start = timer()
-
-    p_unmasked, f_unmasked, c_unmasked = features.extract_features(
-        data.load_image(image), data.config)
+    if data.config["feature_type"] == "SIFT_GPU":
+        unmasked, keypoints = features.extract_features(
+            data.load_image(image), data.config)
+        p_unmasked, f_unmasked, c_unmasked = unmasked
+    else:
+        p_unmasked, f_unmasked, c_unmasked = features.extract_features(
+            data.load_image(image), data.config)
 
     fmask = data.load_features_mask(image, p_unmasked)
-
+    if data.config["feature_type"] == "SIFT_GPU":
+        keypoints = keypoints[fmask]
     p_unsorted = p_unmasked[fmask]
     f_unsorted = f_unmasked[fmask]
     c_unsorted = c_unmasked[fmask]
@@ -88,6 +93,9 @@ def detect(args):
     p_sorted = p_unsorted[order, :]
     f_sorted = f_unsorted[order, :]
     c_sorted = c_unsorted[order, :]
+    if data.config["feature_type"] == "SIFT_GPU":
+        keypoints_sorted = keypoints[order]
+        data.save_gpu_features(image, keypoints_sorted)
     data.save_features(image, p_sorted, f_sorted, c_sorted)
 
     if need_words:

diff --git a/opensfm/config.py b/opensfm/config.py
@@ -7,7 +7,7 @@
 default_focal_prior: 0.85
 
 # Params for features
-feature_type: HAHOG           # Feature type (AKAZE, SURF, SIFT, HAHOG, ORB)
+feature_type: HAHOG           # Feature type (AKAZE, SURF, SIFT, HAHOG, ORB, SIFT_GPU)
 feature_root: 1               # If 1, apply square root mapping to features
 feature_min_frames: 4000      # If fewer frames are detected, sift_peak_threshold/surf_hessian_threshold is reduced.
 feature_process_size: 2048    # Resize the image if its size is larger than specified. Set to -1 for original size
@@ -17,6 +17,13 @@
 sift_peak_threshold: 0.1     # Smaller value -> more features
 sift_edge_threshold: 10       # See OpenCV doc
 
+# Params for SIFT_GPU  
+# More information on Feature detection http://www.silx.org/doc/silx/dev/_modules/silx/opencl/sift/plan.html#SiftPlan
+# More information on Feature Matching http://www.silx.org/doc/silx/dev/_modules/silx/opencl/sift/match.html
+sift_gpu_init_sigma: 1.6  # blurring width, you should have good reasons to modify the 1.6 default value 
+pix_per_keypoints: 10  # Number of key-point pre-allocated: 1 for 10 pixel
+sift_gpu_device_type: GPU  # Can be 'CPU' or 'GPU'
+
 # Params for SURF
 surf_hessian_threshold: 3000  # Smaller value -> more features
 surf_n_octaves: 4             # See OpenCV doc
@@ -39,7 +46,7 @@
 
 # Params for general matching
 lowes_ratio: 0.8              # Ratio test for matches
-matcher_type: FLANN           # FLANN, BRUTEFORCE, or WORDS
+matcher_type: FLANN           # FLANN, BRUTEFORCE, SIFT_GPU or WORDS
 symmetric_matching: yes       # Match symmetricly or one-way
 
 # Params for FLANN matching

diff --git a/opensfm/dataset.py b/opensfm/dataset.py
@@ -33,6 +33,7 @@ class DataSet(object):
     It is possible to store data remotely or in different formats
     by subclassing this class and overloading its methods.
     """
+
     def __init__(self, data_path):
         """Init dataset associated to a folder."""
         self.data_path = data_path
@@ -316,8 +317,8 @@ def _save_features(self, filepath, points, descriptors, colors=None):
         features.save_features(filepath, points, descriptors, colors, self.config)
 
     def features_exist(self, image):
-        return os.path.isfile(self._feature_file(image)) or\
-            os.path.isfile(self._feature_file_legacy(image))
+        return os.path.isfile(self._feature_file(image)) or \
+               os.path.isfile(self._feature_file_legacy(image))
 
     def load_features(self, image):
         if os.path.isfile(self._feature_file_legacy(image)):
@@ -327,9 +328,27 @@ def load_features(self, image):
     def save_features(self, image, points, descriptors, colors):
         self._save_features(self._feature_file(image), points, descriptors, colors)
 
+    def save_gpu_features(self, image, keypoints):
+        io.mkdir_p(self._feature_path())
+        path = "./"+self._gpu_feature_file(image)
+        # Store data (serialize)
+        with open(path, 'wb+') as handle:
+            pickle.dump(keypoints, handle, protocol=pickle.HIGHEST_PROTOCOL)
+
+    def load_gpu_features(self, image):
+        path = self._gpu_feature_file(image)
+        if os.path.isfile(path):
+            with open(path, 'rb') as handle:
+                keypoints = pickle.load(handle)
+                return keypoints
+        return None
+
     def _words_file(self, image):
         return os.path.join(self._feature_path(), image + '.words.npz')
 
+    def _gpu_feature_file(self, image):
+        return os.path.join(self._feature_path(), image.rsplit(".")[0] + '_gpu.pkl')
+
     def words_exist(self, image):
         return os.path.isfile(self._words_file(image))
 

diff --git a/opensfm/feature_loading.py b/opensfm/feature_loading.py
@@ -10,7 +10,6 @@
 
 from opensfm import features as ft
 
-
 logger = logging.getLogger(__name__)
 
 
@@ -103,3 +102,24 @@ def _load_features_nocache(self, data, image):
         else:
             points = np.array(points[:, :3], dtype=float)
         return points, features, colors
+
+    def create_gpu_keypoints_from_features(self, p1, f1):
+        ########################################################################
+        # Merge keypoints in central memory
+        ########################################################################
+        if type(f1[0][0]) is not int:
+            f1 = np.uint8(f1 * 512)
+        total_size = len(p1)
+        dtype_kp = np.dtype([('x', np.float32),
+                             ('y', np.float32),
+                             ('scale', np.float32),
+                             ('angle', np.float32),
+                             ('desc', (np.uint8, 128))
+                             ])
+        output = np.recarray(shape=(total_size,), dtype=dtype_kp)
+        output[:total_size].x = p1[:, 0]
+        output[:total_size].y = p1[:, 1]
+        output[:total_size].scale = p1[:, 2]
+        # output[:total_size].angle = p1[:, 3]
+        output[:total_size].desc = f1
+        return output