forked from SensorsINI/v2e
-
Notifications
You must be signed in to change notification settings - Fork 0
/
v2e.py
910 lines (796 loc) · 39.9 KB
/
v2e.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
#!/usr/bin/env python
"""
Python code for extracting frames from video file and synthesizing fake DVS
events from this video after SuperSloMo has generated interpolated
frames from the original video frames.
@author: Tobi Delbruck, Yuhuang Hu, Zhe He
"""
# todo refractory period for pixel
import glob
import argparse
import importlib
import sys
import argcomplete
import cv2
import numpy as np
import os
from tempfile import TemporaryDirectory
from engineering_notation import EngNumber as eng # only from pip
from tqdm import tqdm
import torch
import v2ecore.desktop as desktop
from v2ecore.base_synthetic_input import base_synthetic_input
from v2ecore.v2e_utils import all_images, read_image, \
check_lowpass, v2e_quit
from v2ecore.v2e_utils import set_output_dimension
from v2ecore.v2e_utils import set_output_folder
from v2ecore.v2e_utils import ImageFolderReader
from v2ecore.v2e_args import v2e_args, write_args_info, SmartFormatter
from v2ecore.v2e_args import v2e_check_dvs_exposure_args
from v2ecore.v2e_args import NO_SLOWDOWN
from v2ecore.renderer import EventRenderer, ExposureMode
from v2ecore.slomo import SuperSloMo
from v2ecore.emulator import EventEmulator
from v2ecore.v2e_utils import inputVideoFileDialog
import logging
import time
from typing import Optional, Any
logging.basicConfig()
root = logging.getLogger()
LOGGING_LEVEL=logging.INFO
root.setLevel(LOGGING_LEVEL) # todo move to info for production
# https://stackoverflow.com/questions/384076/how-can-i-color-python-logging-output/7995762#7995762
logging.addLevelName(
logging.DEBUG, "\033[1;36m%s\033[1;0m" % logging.getLevelName(
logging.DEBUG)) # cyan foreground
logging.addLevelName(
logging.INFO, "\033[1;34m%s\033[1;0m" % logging.getLevelName(
logging.INFO)) # blue foreground
logging.addLevelName(
logging.WARNING, "\033[1;31m%s\033[1;0m" % logging.getLevelName(
logging.WARNING)) # red foreground
logging.addLevelName(
logging.ERROR, "\033[38;5;9m%s\033[1;0m" % logging.getLevelName(
logging.ERROR)) # red background
logger = logging.getLogger(__name__)
# torch device
torch_device:str = torch.device('cuda' if torch.cuda.is_available() else 'cpu').type
logger.info(f'torch device is {torch_device}')
if torch_device=='cpu':
logger.warning('CUDA GPU acceleration of pytorch operations is not available; '
'see https://pytorch.org/get-started/locally/ '
'to generate the correct conda install command to enable GPU-accelerated CUDA.')
# may only apply to windows
try:
# from scripts.regsetup import description
from gooey import Gooey # pip install Gooey
except Exception as e:
logger.info(f"{e}: Gooey GUI builder not available, "
f"will use command line arguments.\n"
f"Install with 'pip install Gooey if you want a no-arg GUI to invoke v2e'. See README")
def get_args():
""" proceses input arguments
:returns: (args_namespace,other_args,command_line) """
parser = argparse.ArgumentParser(
description='v2e: generate simulated DVS events from video.',
epilog='Run with no --input to open file dialog', allow_abbrev=True,
formatter_class=SmartFormatter)
parser = v2e_args(parser)
# parser.add_argument(
# "--rotate180", type=bool, default=False,
# help="rotate all output 180 deg.")
# https://kislyuk.github.io/argcomplete/#global-completion
# Shellcode (only necessary if global completion is not activated -
# see Global completion below), to be put in e.g. .bashrc:
# eval "$(register-python-argcomplete v2e.py)"
argcomplete.autocomplete(parser)
(args_namespace,other_args) = parser.parse_known_args() # change to known arguments so that synthetic input module can take arguments
command_line=''
for a in sys.argv:
command_line=command_line+' '+a
return (args_namespace,other_args,command_line)
def main():
try:
ga = Gooey(get_args, program_name="v2e", default_size=(575, 600))
logger.info(
"Use --ignore-gooey to disable GUI and "
"run with command line arguments")
ga()
except Exception as e:
logger.info(
f'{e}: Gooey package GUI not available, using command line arguments. \n'
f'You can try to install with "pip install Gooey"')
(args,other_args,command_line) = get_args()
# set input file
input_file = args.input
synthetic_input:str = args.synthetic_input
if synthetic_input is not None and input_file is not None:
logger.error(f'Both input_file {input_file} and synthetic_input {synthetic_input} are specified - you can only specify one of them')
v2e_quit(1)
if synthetic_input is None and input_file is None:
try:
input_file = inputVideoFileDialog()
if input_file is None:
logger.info('no file selected, quitting')
v2e_quit()
except Exception as e:
logger.error(f'no input file specified and cannot show input file dialog; are you running without graphical display? ({e})')
v2e_quit(1)
# Set output folder
output_folder = set_output_folder(
args.output_folder,
input_file,
args.unique_output_folder if not args.overwrite else False,
args.overwrite,
args.output_in_place if (not synthetic_input) else False,
logger)
# Set output width and height based on the arguments
output_width, output_height = set_output_dimension(
args.output_width, args.output_height,
args.dvs128, args.dvs240, args.dvs346,
args.dvs640, args.dvs1024,
logger)
# Visualization
avi_frame_rate = args.avi_frame_rate
dvs_vid = args.dvs_vid if not args.skip_video_output else None
dvs_vid_full_scale = args.dvs_vid_full_scale
vid_orig = args.vid_orig if not args.skip_video_output else None
vid_slomo = args.vid_slomo if not args.skip_video_output else None
preview = not args.no_preview
# setup synthetic input classes and method
synthetic_input_module = None
synthetic_input_class = None
synthetic_input_instance:Optional[base_synthetic_input] = None
synthetic_input_next_frame_method = None
if synthetic_input is not None:
try:
synthetic_input_module = importlib.import_module(synthetic_input)
if '.' in synthetic_input:
classname=synthetic_input[synthetic_input.rindex('.')+1:]
else:
classname=synthetic_input
synthetic_input_class:synthetic_input = getattr(
synthetic_input_module, classname)
vid_path=os.path.join(output_folder, vid_orig) if not vid_orig is None else None
synthetic_input_instance:base_synthetic_input = synthetic_input_class(
width=output_width, height=output_height,
preview=not args.no_preview, arg_list=other_args, avi_path=vid_path,parent_args=args) #TODO output folder might not be unique, could write to first output folder
if not isinstance(synthetic_input_instance,base_synthetic_input):
logger.error(f'synthetic input instance of {synthetic_input} is of type {type(synthetic_input_instance)}, but it should be a sublass of synthetic_input;'
f'there is no guarentee that it implements the necessary methods')
synthetic_input_next_frame_method = getattr(
synthetic_input_class, 'next_frame')
synthetic_input_total_frames_method = getattr(
synthetic_input_class, 'total_frames')
srcNumFramesToBeProccessed=synthetic_input_instance.total_frames()
logger.info(
f'successfully instanced {synthetic_input_instance} with method {synthetic_input_next_frame_method}:'
'{synthetic_input_module.__doc__}')
except ModuleNotFoundError as e:
logger.error(f'Could not import {synthetic_input}: {e}')
v2e_quit(1)
except AttributeError as e:
logger.error(f'{synthetic_input} method incorrect?: {e}')
v2e_quit(1)
# check to make sure there are no other arguments that might be bogus misspelled arguments in case
# we don't have synthetic input class to pass these to
# if synthetic_input_instance is None and len(other_args)>0:
# logger.error(f'There is no synthetic input class specified but there are extra arguments {other_args} that are probably incorrect')
# v2e_quit(1)
# Writing the info file
infofile = write_args_info(args, output_folder,other_args,command_line)
fh = logging.FileHandler(infofile,mode='a')
fh.setLevel(LOGGING_LEVEL)
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
fh.setFormatter(formatter)
logger.addHandler(fh)
num_frames = 0
srcNumFramesToBeProccessed = 0
srcDurationToBeProcessed=float("NaN")
# input file checking
# if (not input_file or not os.path.isfile(input_file)
# or not os.path.isdir(input_file)) \
# and not base_synthetic_input:
if (not synthetic_input):
if not os.path.isfile(input_file) and not os.path.isdir(input_file):
logger.error('input file {} does not exist'.format(input_file))
v2e_quit(1)
if os.path.isdir(input_file):
if len(os.listdir(input_file))==0:
logger.error(f'input folder {input_file} is empty')
v2e_quit(1)
# define video parameters
# the input start and stop time, may be round to actual
# frame timestamp
input_start_time = args.start_time
input_stop_time = args.stop_time
def is_float(element: Any) -> bool:
try:
float(element)
return True
except ValueError:
return False
if not input_start_time is None and not input_stop_time is None and is_float(input_start_time) and is_float(input_stop_time) and input_stop_time<=input_start_time:
logger.error(f'stop time {input_stop_time} must be later than start time {input_start_time}')
v2e_quit(1)
input_slowmotion_factor: float = args.input_slowmotion_factor
input_frame_rate:Optional[float] =args.input_frame_rate
timestamp_resolution: float = args.timestamp_resolution
auto_timestamp_resolution: bool = args.auto_timestamp_resolution
disable_slomo: bool = args.disable_slomo
slomo = None # make it later on
if not disable_slomo and auto_timestamp_resolution is False \
and timestamp_resolution is None:
logger.error(
'if --auto_timestamp_resolution=False, '
'then --timestamp_resolution must be set to '
'some desired DVS event timestamp resolution in seconds, '
'e.g. 0.01')
v2e_quit()
if auto_timestamp_resolution is True \
and timestamp_resolution is not None:
logger.info(
f'auto_timestamp_resolution=True and '
f'timestamp_resolution={timestamp_resolution}: '
f'Limiting automatic upsampling to maximum timestamp interval.')
# DVS pixel thresholds
pos_thres = args.pos_thres
neg_thres = args.neg_thres
sigma_thres = args.sigma_thres
record_single_pixel_states=args.record_single_pixel_states
# Cutoff and noise frequencies
cutoff_hz = args.cutoff_hz
leak_rate_hz = args.leak_rate_hz
if leak_rate_hz > 0 and sigma_thres == 0:
logger.warning(
'leak_rate_hz>0 but sigma_thres==0, '
'so all leak events will be synchronous')
shot_noise_rate_hz = args.shot_noise_rate_hz
# Event saving options
dvs_h5 = args.dvs_h5
dvs_aedat2 = args.dvs_aedat2
dvs_aedat4 = args.dvs_aedat4
dvs_text = args.dvs_text
# signal noise output CSV file
label_signal_noise=args.label_signal_noise
if label_signal_noise and dvs_text is None and dvs_aedat2 is None and dvs_aedat4 is None:
logger.error('if you specify --label_signal_noise you must specify --dvs_text and/or --dvs_aedat2 and/or --dvs_aedat4')
v2e_quit(1)
if label_signal_noise and args.photoreceptor_noise:
logger.error('if you specify --label_signal_noise you cannot use --photoreceptor_noise option')
v2e_quit(1)
if label_signal_noise and shot_noise_rate_hz==0:
logger.error('You specified --label_signal_noise, but --shot_noise_rate=0 and there will be no noise events')
v2e_quit(1)
# Debug feature: if show slomo stats
slomo_stats_plot = args.slomo_stats_plot
# rotate180 = args.rotate180 # never used, consider removing
batch_size = args.batch_size
# DVS exposure
exposure_mode, exposure_val, area_dimension = \
v2e_check_dvs_exposure_args(args)
if exposure_mode == ExposureMode.DURATION:
dvsFps = 1. / exposure_val
time_run_started = time.time()
slomoTimestampResolutionS = None
if synthetic_input is None:
logger.info("opening video input file " + input_file)
if os.path.isdir(input_file):
if input_frame_rate is None:
logger.error(
"When the video is presented as a folder, "
"The user must set --input_frame_rate manually")
v2e_quit(1)
cap = ImageFolderReader(input_file, args.input_frame_rate)
srcFps = cap.frame_rate
srcNumFrames = cap.num_frames
else:
cap = cv2.VideoCapture(input_file)
srcFps = cap.get(cv2.CAP_PROP_FPS)
srcNumFrames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
if input_frame_rate is not None:
logger.info(f'Input video frame rate {srcFps}Hz is overridden by command line argument --input_frame_rate={args.input_frame_rate}')
srcFps=args.input_frame_rate
if cap is not None:
# set the output width and height from first image in folder, but only if they were not already set
set_size = False
if output_height is None and hasattr(cap,'frame_height'):
set_size = True
output_height = cap.frame_height
if output_width is None and hasattr(cap,'frame_width'):
set_size = True
output_width = cap.frame_width
if set_size:
logger.warning(
f'From input frame automatically set DVS output_width={output_width} and/or output_height={output_height}. '
f'This may not be desired behavior. \nCheck DVS camera sizes arguments.')
time.sleep(5);
elif output_height is None or output_width is None:
logger.warning(
'Could not read video frame size from video input and so could not automatically set DVS output size. \nCheck DVS camera sizes arguments.')
# Check frame rate and number of frames
if srcFps == 0:
logger.error(
'source {} fps is 0; v2e needs to have a timescale '
'for input video'.format(input_file))
v2e_quit()
if srcNumFrames < 2:
logger.warning(
'num frames is less than 2, probably cannot be determined '
'from cv2.CAP_PROP_FRAME_COUNT')
srcTotalDuration = (srcNumFrames-1)/srcFps
# the index of the frames, from 0 to srcNumFrames-1
start_frame = int(srcNumFrames*(input_start_time/srcTotalDuration)) \
if input_start_time else 0
stop_frame = int(srcNumFrames*(input_stop_time/srcTotalDuration)) \
if input_stop_time else srcNumFrames-1
srcNumFramesToBeProccessed = stop_frame-start_frame+1
# the duration to be processed, should subtract 1 frame when
# calculating duration
srcDurationToBeProcessed = (srcNumFramesToBeProccessed-1)/srcFps
# redefining start and end time using the time calculated
# from the frames, the minimum resolution there is
start_time = start_frame/srcFps
stop_time = stop_frame/srcFps
srcFrameIntervalS = (1./srcFps)/input_slowmotion_factor
slowdown_factor = NO_SLOWDOWN # start with factor 1 for upsampling
if disable_slomo:
logger.warning(
'slomo interpolation disabled by command line option; '
'output DVS timestamps will have source frame interval '
'resolution')
# time stamp resolution equals to source frame interval
slomoTimestampResolutionS = srcFrameIntervalS
elif not auto_timestamp_resolution:
slowdown_factor = int(
np.ceil(srcFrameIntervalS/timestamp_resolution))
if slowdown_factor < NO_SLOWDOWN:
slowdown_factor = NO_SLOWDOWN
logger.warning(
'timestamp resolution={}s is >= source '
'frame interval={}s, will not upsample'
.format(timestamp_resolution, srcFrameIntervalS))
elif slowdown_factor > 100 and cutoff_hz == 0:
logger.warning(
f'slowdown_factor={slowdown_factor} is >100 but '
'cutoff_hz={cutoff_hz}. We have observed that '
'numerical errors in SuperSloMo can cause noise '
'that makes fake events at the upsampling rate. '
'Recommend to set physical cutoff_hz, '
'e.g. --cutoff_hz=200 (or leave the default cutoff_hz)')
slomoTimestampResolutionS = srcFrameIntervalS/slowdown_factor
logger.info(
f'--auto_timestamp_resolution is False, '
f'srcFps={srcFps}Hz '
f'input_slowmotion_factor={input_slowmotion_factor}, '
f'real src FPS={srcFps*input_slowmotion_factor}Hz, '
f'srcFrameIntervalS={eng(srcFrameIntervalS)}s, '
f'timestamp_resolution={eng(timestamp_resolution)}s, '
f'so SuperSloMo will use slowdown_factor={slowdown_factor} '
f'and have '
f'slomoTimestampResolutionS={eng(slomoTimestampResolutionS)}s')
if slomoTimestampResolutionS > timestamp_resolution:
logger.warning(
'Upsampled src frame intervals of {}s is larger than\n '
'the desired DVS timestamp resolution of {}s'
.format(slomoTimestampResolutionS, timestamp_resolution))
check_lowpass(cutoff_hz, 1/slomoTimestampResolutionS, logger)
else: # auto_timestamp_resolution
if timestamp_resolution is not None:
slowdown_factor = int(
np.ceil(srcFrameIntervalS/timestamp_resolution))
logger.info(
f'--auto_timestamp_resolution=True and '
f'timestamp_resolution={eng(timestamp_resolution)}s: '
f'source video will be automatically upsampled but '
f'with at least upsampling factor of {slowdown_factor}')
else:
logger.info(
'--auto_timestamp_resolution=True and '
'timestamp_resolution is not set: '
'source video will be automatically upsampled to '
'limit maximum interframe motion to 1 pixel')
# the SloMo model, set no SloMo model if no slowdown
if not disable_slomo and \
(auto_timestamp_resolution or slowdown_factor != NO_SLOWDOWN):
slomo = SuperSloMo(
model=args.slomo_model,
auto_upsample=auto_timestamp_resolution,
upsampling_factor=slowdown_factor,
video_path=None if args.skip_video_output else output_folder,
vid_orig=None if args.skip_video_output else vid_orig,
vid_slomo=None if args.skip_video_output else vid_slomo,
preview=preview, batch_size=batch_size)
if not synthetic_input and not auto_timestamp_resolution:
logger.info(
f'\n events will have timestamp resolution '
f'{eng(slomoTimestampResolutionS)}s,')
if exposure_mode == ExposureMode.DURATION \
and dvsFps > (1 / slomoTimestampResolutionS):
logger.warning(
'DVS video frame rate={}Hz is larger than '
'the effective DVS frame rate of {}Hz; '
'DVS video will have blank frames'.format(
dvsFps, (1 / slomoTimestampResolutionS)))
if not synthetic_input:
logger.info(
'Source video {} has total {} frames with total duration {}s. '
'\nSource video is {}fps with slowmotion_factor {} '
'(frame interval {}s),'
'\nWill convert {} frames {} to {}\n'
'(From {}s to {}s, duration {}s)'
.format(input_file, srcNumFrames, eng(srcTotalDuration),
eng(srcFps), eng(input_slowmotion_factor),
eng(srcFrameIntervalS),
stop_frame-start_frame+1, start_frame, stop_frame,
start_time, stop_time, (stop_time-start_time)))
if exposure_mode == ExposureMode.DURATION:
dvsNumFrames = np.math.floor(
dvsFps*srcDurationToBeProcessed/input_slowmotion_factor)
dvsDuration = dvsNumFrames/dvsFps
dvsPlaybackDuration = dvsNumFrames/avi_frame_rate
start_time = start_frame/srcFps
stop_time = stop_frame/srcFps # todo something replicated here, already have start and stop times
logger.info('v2e DVS video will have constant-duration frames \n'
'at {}fps (accumulation time {}s), '
'\nDVS video will have {} frames with duration {}s '
'and playback duration {}s\n'
.format(eng(dvsFps), eng(1 / dvsFps),
dvsNumFrames, eng(dvsDuration),
eng(dvsPlaybackDuration)))
elif exposure_mode==ExposureMode.SOURCE:
logger.info(f'v2e DVS video will have constant-duration frames \n'
f'at the source video {eng(srcFps)} fps (accumulation time {eng(srcFrameIntervalS)}s)')
else:
logger.info(
'v2e DVS video will have constant-count '
'frames with {} events), '
.format(exposure_val))
# check one more time that we have an output width and height
if output_width is None or output_height is None:
logger.error("Either or both of output_width or output_height is None,\n"
"which means that they were not specified or could not be inferred from the input video. \n "
"Please see options for DVS camera sizes. \nYou can try the option --dvs346 for DAVIS346 camera as one well-supported option.")
v2e_quit(1)
num_pixels=output_width*output_height
hdr: bool = args.hdr
if hdr:
logger.info('Treating input as HDR logarithmic video')
scidvs:bool=args.scidvs
if scidvs:
logger.info('Simulating SCIDVS pixel')
emulator = EventEmulator(
pos_thres=pos_thres, neg_thres=neg_thres,
sigma_thres=sigma_thres, cutoff_hz=cutoff_hz,
leak_rate_hz=leak_rate_hz, shot_noise_rate_hz=shot_noise_rate_hz, photoreceptor_noise=args.photoreceptor_noise,
leak_jitter_fraction=args.leak_jitter_fraction,
noise_rate_cov_decades=args.noise_rate_cov_decades,
refractory_period_s=args.refractory_period,
seed=args.dvs_emulator_seed,
output_folder=output_folder, dvs_h5=dvs_h5, dvs_aedat2=dvs_aedat2, dvs_aedat4 = dvs_aedat4,
dvs_text=dvs_text, show_dvs_model_state=args.show_dvs_model_state,
save_dvs_model_state=args.save_dvs_model_state,
output_width=output_width, output_height=output_height,
device=torch_device,
cs_lambda_pixels=args.cs_lambda_pixels, cs_tau_p_ms=args.cs_tau_p_ms,
hdr=hdr,
scidvs=scidvs,
record_single_pixel_states=record_single_pixel_states,
label_signal_noise=label_signal_noise
)
if args.dvs_params is not None:
logger.warning(
f'--dvs_param={args.dvs_params} option overrides your '
f'selected options for threshold, threshold-mismatch, '
f'leak and shot noise rates')
emulator.set_dvs_params(args.dvs_params)
eventRenderer = EventRenderer(
output_path=output_folder,
dvs_vid=dvs_vid, preview=preview, full_scale_count=dvs_vid_full_scale,
exposure_mode=exposure_mode,
exposure_value=exposure_val,
area_dimension=area_dimension,
avi_frame_rate=args.avi_frame_rate)
if synthetic_input_next_frame_method is not None:
# array to batch events for rendering to DVS frames
events = np.zeros((0, 4), dtype=np.float32)
(fr, fr_time) = synthetic_input_instance.next_frame()
num_frames+=1
i = 0
with tqdm(total=synthetic_input_instance.total_frames(),
desc='dvs', unit='fr') as pbar:
with torch.no_grad():
while fr is not None:
newEvents = emulator.generate_events(fr, fr_time)
pbar.update(1)
i += 1
if newEvents is not None and newEvents.shape[0] > 0 \
and not args.skip_video_output:
events = np.append(events, newEvents, axis=0)
events = np.array(events)
if i % batch_size == 0:
eventRenderer.render_events_to_frames(
events, height=output_height,
width=output_width)
events = np.zeros((0, 4), dtype=np.float32)
(fr, fr_time) = synthetic_input_instance.next_frame()
num_frames+=1
# process leftover events
if len(events) > 0 and not args.skip_video_output:
eventRenderer.render_events_to_frames(
events, height=output_height, width=output_width)
else: # video file folder or (avi/mp4) file input
# timestamps of DVS start at zero and end with
# span of video we processed
srcVideoRealProcessedDuration = (stop_time-start_time) / \
input_slowmotion_factor
num_frames = srcNumFramesToBeProccessed
inputHeight = None
inputWidth = None
inputChannels = None
if start_frame > 0:
logger.info('skipping to frame {}'.format(start_frame))
for i in tqdm(range(start_frame), unit='fr', desc='src'):
if isinstance(cap,ImageFolderReader):
if i<start_frame-1:
ret,_=cap.read(skip=True)
else:
ret, _ = cap.read()
else:
ret, _ = cap.read()
if not ret:
raise ValueError(
'something wrong, got to end of file before '
'reaching start_frame')
logger.info(
'processing frames {} to {} from video input'.format(
start_frame, stop_frame))
c_l=0
c_r=None
c_t=0
c_b=None
if args.crop is not None:
c=args.crop
if len(c)!=4:
logger.error(f'--crop must have 4 elements (you specified --crop={args.crop}')
v2e_quit(1)
c_l=c[0] if c[0] > 0 else 0
c_r=-c[1] if c[1]>0 else None
c_t=c[2] if c[2]>0 else 0
c_b=-c[3] if c[3]>0 else None
logger.info(f'cropping video by (left,right,top,bottom)=({c_l},{c_r},{c_t},{c_b})')
with TemporaryDirectory() as source_frames_dir:
if os.path.isdir(input_file): # folder input
inputWidth = cap.frame_width
inputHeight = cap.frame_height
inputChannels = cap.frame_channels
else:
inputWidth = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
inputHeight = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
inputChannels = 1 if int(cap.get(cv2.CAP_PROP_MONOCHROME)) \
else 3
logger.info(
'Input video {} has W={} x H={} frames each with {} channels'
.format(input_file, inputWidth, inputHeight, inputChannels))
if (output_width is None) and (output_height is None):
output_width = inputWidth
output_height = inputHeight
logger.warning(
'output size ({}x{}) was set automatically to '
'input video size\n Are you sure you want this? '
'It might be slow.\n Consider using\n '
' --output_width=346 --output_height=260\n '
'to match Davis346.'
.format(output_width, output_height))
# set emulator output width and height for the last time
emulator.output_width = output_width
emulator.output_height = output_height
logger.info(
f'*** Stage 1/3: '
f'Resizing {srcNumFramesToBeProccessed} input frames '
f'to output size '
f'(with possible RGB to luma conversion)')
for inputFrameIndex in tqdm(
range(srcNumFramesToBeProccessed),
desc='rgb2luma', unit='fr'):
# read frame
ret, inputVideoFrame = cap.read()
num_frames+=1
if ret==False:
logger.warning(f'could not read frame {inputFrameIndex} from {cap}')
continue
if inputVideoFrame is None or np.shape(inputVideoFrame) == ():
logger.warning(f'empty video frame number {inputFrameIndex} in {cap}')
continue
if not ret or inputFrameIndex + start_frame > stop_frame:
break
if args.crop is not None:
# crop the frame, indices are y,x, UL is 0,0
if c_l+(c_r if c_r is not None else 0)>=inputWidth:
logger.error(f'left {c_l}+ right crop {c_r} is larger than image width {inputWidth}')
v2e_quit(1)
if c_t+(c_b if c_b is not None else 0)>=inputHeight:
logger.error(f'top {c_t}+ bottom crop {c_b} is larger than image height {inputHeight}')
v2e_quit(1)
inputVideoFrame= inputVideoFrame[c_t:c_b, c_l:c_r] # https://stackoverflow.com/questions/15589517/how-to-crop-an-image-in-opencv-using-python
if output_height and output_width and \
(inputHeight != output_height or
inputWidth != output_width):
dim = (output_width, output_height)
(fx, fy) = (float(output_width) / inputWidth,
float(output_height) / inputHeight)
inputVideoFrame = cv2.resize(
src=inputVideoFrame, dsize=dim, fx=fx, fy=fy,
interpolation=cv2.INTER_AREA)
if inputChannels == 3: # color
if inputFrameIndex == 0: # print info once
logger.info(
'\nConverting input frames from RGB color to luma')
# TODO would break resize if input is gray frames
# convert RGB frame into luminance.
inputVideoFrame = cv2.cvtColor(
inputVideoFrame, cv2.COLOR_BGR2GRAY) # much faster
# TODO add vid_orig output if not using slomo
# save frame into numpy records
save_path = os.path.join(
source_frames_dir, str(inputFrameIndex).zfill(8) + ".npy")
np.save(save_path, inputVideoFrame)
# print("Writing source frame {}".format(save_path), end="\r")
cap.release()
with TemporaryDirectory() as interpFramesFolder:
interpTimes = None
# make input to slomo
if slomo is not None and (auto_timestamp_resolution or slowdown_factor != NO_SLOWDOWN):
# interpolated frames are stored to tmpfolder as
# 1.png, 2.png, etc
logger.info(
f'*** Stage 2/3: SloMo upsampling from '
f'{source_frames_dir}')
interpTimes, avgUpsamplingFactor = slomo.interpolate(
source_frames_dir, interpFramesFolder,
(output_width, output_height))
avgTs = srcFrameIntervalS / avgUpsamplingFactor
logger.info(
'SloMo average upsampling factor={:5.2f}; '
'average DVS timestamp resolution={}s'
.format(avgUpsamplingFactor, eng(avgTs)))
# check for undersampling wrt the
# photoreceptor lowpass filtering
if cutoff_hz > 0:
logger.info('Using auto_timestamp_resolution. '
'checking if cutoff hz is ok given '
'sample rate {}'.format(1/avgTs))
check_lowpass(cutoff_hz, 1/avgTs, logger)
# read back to memory
interpFramesFilenames = all_images(interpFramesFolder)
# number of frames
n = len(interpFramesFilenames)
else:
logger.info(
f'*** Stage 2/3:turning npy frame files to png '
f'from {source_frames_dir}')
interpFramesFilenames = []
n = 0
src_files = sorted(
glob.glob("{}".format(source_frames_dir) + "/*.npy"))
for frame_idx, src_file_path in tqdm(
enumerate(src_files), desc='npy2png', unit='fr'):
src_frame = np.load(src_file_path)
tgt_file_path = os.path.join(
interpFramesFolder, str(frame_idx) + ".png")
interpFramesFilenames.append(tgt_file_path)
n += 1
cv2.imwrite(tgt_file_path, src_frame)
interpTimes = np.array(range(n))
# compute times of output integrated frames
nFrames = len(interpFramesFilenames)
# interpTimes is in units of 1 per input frame,
# normalize it to src video time range
f = srcVideoRealProcessedDuration/(
np.max(interpTimes)-np.min(interpTimes))
# compute actual times from video times
interpTimes = f*interpTimes
# debug
if slomo_stats_plot:
from matplotlib import pyplot as plt # TODO debug
dt = np.diff(interpTimes)
fig = plt.figure()
ax1 = fig.add_subplot(111)
ax1.set_title(
'Slo-Mo frame interval stats (close to continue)')
ax1.plot(interpTimes)
ax1.plot(interpTimes, 'x')
ax1.set_xlabel('frame')
ax1.set_ylabel('frame time (s)')
ax2 = ax1.twinx()
ax2.plot(dt*1e3)
ax2.set_ylabel('frame interval (ms)')
logger.info('close plot to continue')
fig.show()
# array to batch events for rendering to DVS frames
events = np.zeros((0, 4), dtype=np.float32)
logger.info(
f'*** Stage 3/3: emulating DVS events from '
f'{nFrames} frames')
# parepare extra steps for data storage
# right before event emulation
if args.ddd_output:
emulator.prepare_storage(nFrames, interpTimes)
# generate events from frames and accumulate events to DVS frames for output DVS video
with tqdm(total=nFrames, desc='dvs', unit='fr') as pbar:
with torch.no_grad():
for i in range(nFrames):
fr = read_image(interpFramesFilenames[i])
newEvents = emulator.generate_events(
fr, interpTimes[i])
pbar.update(1)
if newEvents is not None and \
newEvents.shape[0] > 0 \
and not args.skip_video_output:
events = np.append(events, newEvents, axis=0)
events = np.array(events)
if i % batch_size == 0:
eventRenderer.render_events_to_frames(
events, height=output_height,
width=output_width)
events = np.zeros((0, 4), dtype=np.float32)
# process leftover events
if len(events) > 0 and not args.skip_video_output:
eventRenderer.render_events_to_frames(
events, height=output_height, width=output_width)
# Clean up
eventRenderer.cleanup()
emulator.cleanup()
if slomo is not None:
slomo.cleanup()
if synthetic_input_instance is not None:
synthetic_input_instance.cleanup()
if num_frames == 0:
logger.error('no frames read from file')
totalTime = (time.time()-time_run_started)
framePerS = num_frames / totalTime
sPerFrame = totalTime / num_frames if num_frames>0 else None
throughputStr = (str(eng(framePerS)) + 'fr/s') \
if framePerS > 1 else (str(eng(sPerFrame)) + 's/fr')
timestr ='done processing {} frames in {}s ({})\n **************** see output folder {}'.format(num_frames,
eng(totalTime),
throughputStr,
output_folder)
logger.info('generated total {} events ({} on, {} off)'
.format(eng(emulator.num_events_total),
eng(emulator.num_events_on),
eng(emulator.num_events_off)))
total_time = emulator.t_previous
rate_total= emulator.num_events_total / total_time
rate_on_total= emulator.num_events_on / total_time
rate_off_total= emulator.num_events_off / total_time
rate_per_pixel=rate_total/num_pixels
rate_on_per_pixel=rate_on_total/num_pixels
rate_off_per_pixel=rate_off_total/num_pixels
logger.info(
f'Avg event rate for N={num_pixels} px and total time ={total_time:.3f} s'
f'\n\tTotal: {eng(rate_total)}Hz ({eng(rate_on_total)}Hz on, {eng(rate_off_total)}Hz off)'
f'\n\tPer pixel: {eng(rate_per_pixel)}Hz ({eng(rate_on_per_pixel)}Hz on, {eng(rate_off_per_pixel)}Hz off)')
if totalTime>60:
try:
from plyer import notification
logger.info(f'generating desktop notification')
notification.notify(title='v2e done', message=timestr,timeout=3)
except Exception as e:
logger.info(f'could not show notification: {e}')
# try to show desktop
# suppress folder opening if it's not necessary
if not output_folder is None:
try:
logger.info(f'showing {output_folder} in desktop')
desktop.open(os.path.abspath(output_folder))
except Exception as e:
logger.warning(
'{}: could not open {} in desktop'.format(e, output_folder))
logger.info(timestr)
sys.exit(0)
if __name__ == "__main__":
main()
v2e_quit()