-
Notifications
You must be signed in to change notification settings - Fork 5
/
ra-dns-check.py
executable file
·1365 lines (1288 loc) · 67 KB
/
ra-dns-check.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#!/usr/bin/env python3
#
# ra-dns-check.py, v2.2
#
# Parse, summarize, sort, and display RIPE Atlast measurement results for DNS queries
# Please see the file LICENSE for the license.
import argparse,argcomplete
# need ast to more safely parse config file
import ast
import configparser
import json
import logging
import mmap
import os
import re
import statistics
import sys
import time
from datetime import datetime
# to decompress RIPE Atlas probe data file
import bz2
import base64
# needed to fetch the probe properties file from RIPE
import urllib.request
# These RIPE python modules are usually installed with pip:
from ripe.atlas.cousteau import AtlasLatestRequest
from ripe.atlas.cousteau import AtlasResultsRequest
from ripe.atlas.cousteau import Probe
from ripe.atlas.sagan import DnsResult
from ripe.atlas.cousteau import Measurement
# for debugging
from pprint import pprint
#
# Valid log levels
valid_log_levels = ['DEBUG', 'INFO', 'WARN', 'ERROR', 'CRITICAL']
# Valid id server method
valid_id_server_method = ['quad9','google','cloudflare']
# Change "CRITICAL" to DEBUG if you want debugging-level logging *before* this
# script parses command line args, and subsequently sets the debug level:
logging.basicConfig(level=logging.CRITICAL)
###################
#
# Configurable settings
#
my_config_file = os.environ['HOME'] + '/.ra-dns-check.conf'
# Options can be set or changed on the command line of via config file.
#
# Help text is defined as strings here so it can be consistently presented
# in both the config file and --help (usage).
#
# These are dictionaries we'll iterate over to generate the config text,
# including a Big Blurb o' Help. There are three different dictionaries
# for the different data types (string, integer, boolean) because by
# default, ConfigParser reads everything in as a string, but there are
# ways to read specific sections or values as int or bool.
# A list of "legal" probe properties that can be reported.
reportable_probe_properties = ['probe_id', 'asn', 'country_code', 'ip_address', 'rt_a', 'rt_b', 'rt_diff', 'dns_response']
#
# Comment header (help) for the config file. If the config file does not
# exist, this script creates it from this string plus the preceding
# options_sample_dict_* dicts.
#
# Options specified in the config file can then also be overridden by what's specified on the command line.
#
sample_config_string_header = """;
; Config file for ra-dns-check.py
;
; This file is automatically created if it does not exist. After its
; initial creation, the script will add missing parameters to this config
; file, but otherwise should not overwrite any changes you make! (If you
; ever want to reset everything to the script defaults, you can rename or
; delete this file and the script will create a new one.)
;
; Some important notes on this file's syntax :
;
; 1) This file is read by the ConfigParser python module, and therefore
; (perhaps surprisingly) it expects Windoze INI syntax *NOT* python
; syntax. So:
; * do NOT enclose strings within quotes or double quotes
; (they will be passed along with the string and break things)
; * protect/escape any '%' character with another '%'
; * either ':' or '=' can be used to separate a config variable (key) and its value.
; * spaces are allowed *within* (as part of) a value or key!
; (So be careful of leading spaces before key names.
; E.g ' probe_properties_to_report = ...' will break!)
;
; 2) Do not remove or change the [DEFAULT] line, python's ConfigParser module depends upon it.
;
;;;;;;;;;;;;;;;;;;;;
[DEFAULT]
"""
# Define variable-type-specific sections of the config.
# The reasons we have these sections:
# 1) Python's ConfigParser module requires at least one section.
# 2) ConfigParser reads in all of the config values as strings. So by
# grouping them by type in the config, it's easier "up front" to loop
# through and cast them to the correct types, and then not worry about
# remembering to do the cases later as they get used.
options_sample_dict = {
'datetime1': {
'default': None,
'help': 'date-time to start 10-minute period for FIRST set of results (UTC).\n; Format: 1970-01-01_0000 OR the number of seconds since then (AKA "Unix time")',
'type': 'string'},
'datetime2': {
'default': None,
'help': 'date-time to start 10-minute period for SECOND set of results (UTC).\n; Format: 1970-01-01_0000 OR the number of seconds since then (AKA "Unix time")',
'type': 'string'},
'log_level': {
'default': 'WARN',
'help': 'The level of logging (debugging) messages to show. One of:' + str(valid_log_levels) + ' (default is WARN)',
'type': 'string'},
'oldest_atlas_result_datetime': {
'default': '2010 01 01 00:00:00',
'help': ' Wikipedia says 2010 was when RIPE Atlas was established, so we use that\n; as a starting point for when it might contain some data.',
'type': 'string'},
'probe_properties_to_report': {
'default': reportable_probe_properties,
'help': 'The list of probe properties to report. Must be a subset of:\n; ' + str(reportable_probe_properties),
'type': 'string'},
'ripe_atlas_probe_properties_raw_file': {
'default': os.environ['HOME'] + '/.RIPE_atlas_all_probe_properties.bz2',
'help': 'There are a couple of files used to locally cache probe data, the first comes directly from RIPE:',
'type': 'string'},
'ripe_atlas_probe_properties_json_cache_file': {
'default': os.environ['HOME'] + '/.RIPE_atlas_probe_properties_cache_file.json',
'help': 'The second cache file we generate, based upon probe info we request (one at a time) from the RIPE Atlas API.',
'type': 'string'},
'ripe_atlas_current_probe_properties_url': {
'default': 'https://ftp.ripe.net/ripe/atlas/probes/archive/meta-latest',
'help': 'Where to fetch the RA probe properties file from.',
'type': 'string'},
'split_char': {
'default': '.',
'help': 'character (delimiter) to split the string on (can occur in the string more than once.',
'type': 'string'},
'all_probes': {
'default': False,
'help': 'show information for probes present in *either* result sets, not just those present in *both* sets',
'type': 'boolean'},
'color': {
'default': True,
'help': 'colorize output',
'type': 'boolean'},
'no_color': {
'default': False,
'help': 'do NOT colorized output (AKA "colourised output")',
'type': 'boolean'},
'emphasis_chars': {
'default': False,
'help': 'add a trailing char (! or *) to aberrant sites and response times',
'type': 'boolean'},
'no_header': {
'default': False,
'help': 'Do NOT show the header above the probe list',
'type': 'boolean'},
'do_not_list_probes': {
'default': False,
'help': 'do NOT list the results for each probe',
'type': 'boolean'},
'list_slow_probes_only': {
'default': False,
'help': 'in per-probe list,show ONLY the probes reporting response times',
'type': 'boolean'},
'print_summary_stats': {
'default': False,
'help': 'show summary stats',
'type': 'boolean'},
'dns_response_item_occurence_to_return': {
'default': 1,
'help': 'Which item to return from the split-list. First element is 0. Default: 1',
'type': 'integer'},
'latency_diff_threshold': {
'default': 5,
'help': 'the amount of time difference (ms) that is significant when comparing latencies between tests. Default: 5',
'type': 'integer'},
'slow_threshold': {
'default': 50,
'help': 'Response times (ms) larger than this trigger color highlighting. Default: 50',
'type': 'integer'},
'raw_probe_properties_file_max_age': {
'default': 86400,
'help': 'The max age (seconds) of the RIPE Atlas probe info file (older than this and we download a new one). Default: 86400',
'type': 'integer'},
'scrape': {
'default': False,
'help': 'Scrape output for Prometheus',
'type': 'boolean'},
'include_probe_timestamp': {
'default': False,
'help': 'Default timestamp are not display in the Prometheus output, toggle this switch to display timestamp',
'type': 'boolean'},
'autocomplete': {
'default': False,
'help': 'Autocomplete for all options available',
'type': 'boolean'},
'probes': {
'default': [],
'help': 'Selected probes list eg. "919,166,1049"',
'type': 'list'},
'id_servermethod': {
'default': 'quad9',
'help': 'Select the method to handle id.server, option include [quad9,google,cloudflare]',
'type': 'string'},
'exclusion_list_file': {
'default': None,
'help': 'Filename for probe ID exclusion list',
'type': 'string'},
'scrape_staleness_seconds': {
'default': int(time.time()),
'help': 'Staleness of the data/records that exceed this value would be ignored. Staleness is determine between now() and record timestamp',
'type': 'integer'},
}
sample_config_string = sample_config_string_header
expected_config_items = options_sample_dict.keys()
# Iterate over the items in the options_sample_dict (defined above)
# and shove them into the big string "sample_config_dict" that will then be fed to ConfigParser.
for k in expected_config_items:
sample_config_string += (';\n')
sample_config_string += ('; ' + options_sample_dict[k]['help'] + '\n')
sample_config_string += (k + ' = ' + str(options_sample_dict[k]['default']) + '\n')
logging.debug(sample_config_string)
#
#
####################
#
# Argument processing and usage info (argparse lib automatically provides -h)
parser = argparse.ArgumentParser(description='Display statistics from RIPE Atlas DNS probes. Data can be from local files or this script will query RIPE Atlas for user-supplied Measurement IDs', formatter_class=argparse.RawDescriptionHelpFormatter, epilog='''Examples:
# Compare the results from two local files, "./foo" and "./bar" :
%(prog)s ./foo ./bar
# Compare the results from two items, "12345678" and "87654321"
# (if the items cannot be opened locally as files, but they are an 8-digit integer,
# the script will treat them as a RIPE Atlas measurement IDs, and query RIPE for them)
%(prog)s 123456789 987654321
# Same as example 2, but list the results for ALL probes, instead of only the probes with results in BOTH measurements:
%(prog)s -a 123456789 987654321
# Same as example 2, but do NOT colorize the output:
%(prog)s -C 123456789 987654321
# Same as example 2, but set the threshold for significant latency differences to 10 instead of the default:
%(prog)s -l 10 123456789 987654321
# Same as example 2, but list ONLY the probes that take longer to respond than the threshold (see -S)
%(prog)s -s 123456789 987654321
# Same as the previous example, but list ONLY the probes that take more than 100 ms to respond
%(prog)s -s -S 100 123456789 987654321
# Compare one measurement's (12016241) for two points in time: 20210101_0000 and 20210301_0000.
%(prog)s --datetime1 20210101_0000 --datetime2 20210301_0000 12016241
''')
parser.add_argument('--datetime1', '--dt1', help=options_sample_dict['datetime1']['help'], type=str, default=options_sample_dict['datetime1']['default'])
parser.add_argument('--datetime2', '--dt2', help=options_sample_dict['datetime2']['help'], type=str, default=options_sample_dict['datetime2']['default'])
parser.add_argument('-a', '--all_probes', help=options_sample_dict['all_probes']['help'], action='store_true', default=options_sample_dict['all_probes']['default'])
parser.add_argument('-c', '--color', '--colour', help=options_sample_dict['color']['help'], action="store_true", default=options_sample_dict['color']['default'])
parser.add_argument('-C', '--no_color', '--no_colour', help=options_sample_dict['no_color']['help'], action="store_true", default=options_sample_dict['no_color']['default'])
parser.add_argument('-e', '--emphasis_chars', help=options_sample_dict['emphasis_chars']['help'], action="store_true", default=options_sample_dict['emphasis_chars']['default'])
parser.add_argument('-E', '--exclusion_list_file', help=options_sample_dict['exclusion_list_file']['help'], type=str, default=options_sample_dict['exclusion_list_file']['default'])
parser.add_argument('-f', '--config_file', help='Read (and write) the config from specified file', type=str, default=my_config_file)
parser.add_argument('-H', '--no_header', help=options_sample_dict['no_header']['help'], action="store_true", default=options_sample_dict['no_header']['default'])
parser.add_argument('-i', '--dns_response_item_occurence_to_return', help=options_sample_dict['dns_response_item_occurence_to_return']['help'], type=int, default=options_sample_dict['dns_response_item_occurence_to_return']['default'])
parser.add_argument('-l', '--latency_diff_threshold', help=options_sample_dict['latency_diff_threshold']['help'], type=int, default=options_sample_dict['latency_diff_threshold']['default'])
parser.add_argument('--log_level', help=options_sample_dict['log_level']['help'], type=str, choices=valid_log_levels, default=options_sample_dict['log_level']['default'])
parser.add_argument('--oldest_atlas_result_datetime', help=options_sample_dict['oldest_atlas_result_datetime']['help'], type=str, default=options_sample_dict['oldest_atlas_result_datetime']['default'])
parser.add_argument('-P', '--do_not_list_probes', help=options_sample_dict['do_not_list_probes']['help'], action='store_true', default=options_sample_dict['do_not_list_probes']['default'])
parser.add_argument('--probe_properties_to_report', help=options_sample_dict['probe_properties_to_report']['help'], type=str, default=options_sample_dict['probe_properties_to_report']['default'])
parser.add_argument('-r', '--raw_probe_properties_file_max_age', help=options_sample_dict['raw_probe_properties_file_max_age']['help'], type=int, default=options_sample_dict['raw_probe_properties_file_max_age']['default'])
parser.add_argument('--ripe_atlas_current_probe_properties_url', help=options_sample_dict['ripe_atlas_current_probe_properties_url']['help'], type=str, default=options_sample_dict['ripe_atlas_current_probe_properties_url']['default'])
parser.add_argument('--ripe_atlas_probe_properties_json_cache_file', help=options_sample_dict['ripe_atlas_probe_properties_json_cache_file']['help'], type=str, default=options_sample_dict['ripe_atlas_probe_properties_json_cache_file']['default'])
parser.add_argument('--ripe_atlas_probe_properties_raw_file', help=options_sample_dict['ripe_atlas_probe_properties_raw_file']['help'], type=str, default=options_sample_dict['ripe_atlas_probe_properties_raw_file']['default'])
parser.add_argument('-s', '--list_slow_probes_only', help=options_sample_dict['list_slow_probes_only']['help'], action='store_true', default=options_sample_dict['list_slow_probes_only']['default'])
parser.add_argument('-S', '--slow_threshold', help=options_sample_dict['slow_threshold']['help'], type=int, default=options_sample_dict['slow_threshold']['default'])
parser.add_argument('-t', '--split_char', help=options_sample_dict['split_char']['help'], type=str, default=options_sample_dict['split_char']['default'])
parser.add_argument('-u', '--print_summary_stats', help=options_sample_dict['print_summary_stats']['help'], action='store_true', default=options_sample_dict['print_summary_stats']['default'])
parser.add_argument('--scrape', help=options_sample_dict['scrape']['help'], action='store_true', default=options_sample_dict['scrape']['default'])
parser.add_argument('--include_probe_timestamp', help=options_sample_dict['include_probe_timestamp']['help'], action='store_true', default=options_sample_dict['include_probe_timestamp']['default'])
parser.add_argument('--autocomplete', help=options_sample_dict['autocomplete']['help'], action='store_true', default=options_sample_dict['autocomplete']['default'])
parser.add_argument('--probes', help=options_sample_dict['probes']['help'], type=str, default=options_sample_dict['probes']['default'])
parser.add_argument('--id_servermethod', help=options_sample_dict['id_servermethod']['help'], type=str, choices=valid_id_server_method, default=options_sample_dict['id_servermethod']['default'])
parser.add_argument('--scrape_staleness_seconds', help=options_sample_dict['scrape_staleness_seconds']['help'], type=int, default=options_sample_dict['scrape_staleness_seconds']['default'])
parser.add_argument('filename_or_msmid', help='one or two local filenames or RIPE Atlas Measurement IDs', nargs='+')
parser.format_help()
argcomplete.autocomplete(parser)
args = parser.parse_known_args()
###pprint(args)
logger = logging.getLogger()
logger.setLevel(args[0].log_level)
####################
#
# Config file parse
#
my_config_file = args[0].config_file
raw_config = configparser.ConfigParser()
config_file_read = False
write_config_file = False
try:
if os.stat(my_config_file):
if os.access(my_config_file, os.R_OK):
logger.info('Found config file at %s; reading it now...\n' % my_config_file)
ro_cf = open(my_config_file, 'r')
config_file_string = ro_cf.read()
logger.debug(type(config_file_string))
logger.debug('config_file_string:')
logger.debug(config_file_string)
if re.search('STRING', config_file_string, re.MULTILINE):
old_style_cf = my_config_file + '.old-style'
logger.warning('Old-style config file found; it will be moved to %s' % old_style_cf)
logger.warning('A new-style config file with default values written at %s' % my_config_file)
os.rename(my_config_file,old_style_cf)
config_file_read = False
write_config_file = True
else:
raw_config.read_string(config_file_string)
config_file_read = True
else:
logger.critical('Config file exists at %s, but is not readable.\n' % my_config_file)
except FileNotFoundError:
logger.info('Config file does not exist at %s; will create a new one...\n' % my_config_file)
write_config_file = True
if not config_file_read:
raw_config.read_string(sample_config_string)
raw_config_options = set(raw_config['DEFAULT'].keys())
# This 'config' dict stores the merged config (from the config file and the sample above).
config = {}
#options_from_config_file = []
#logger.debug(raw_config.sections())
# If we read the a config file Loop through what's in the sample config
# and see if any items are missing from what was read from the config
# file. (Like if we've added some settings to the script, and it's
# reading in an older, pre-existing config file for the first time
# since the new option was added.)
if config_file_read:
logger.debug('options_sample_dict:')
logger.debug(set(options_sample_dict))
logger.debug('raw_config_options:')
logger.debug(set(raw_config_options))
for opt in (set(options_sample_dict) - set(raw_config_options)):
logger.info('%s missing from config file; setting it to default from script: %s' %
(opt, options_sample_dict[opt]['default']))
config[opt] = options_sample_dict[opt]['default']
write_config_file = True
#
# Loop through what's in the raw config and see if each variable is in
# the (following) list of expected config variables, so we can catch
# any unexpected ("illegal") parameters in the config file, rather
# than let a typo or some bit of random (non-comment) text in the
# config file go unnoticed.
for item in raw_config_options:
logger.debug('Checking %s to see if it is known...' % item)
if item in expected_config_items:
if getattr(args[0], item) != options_sample_dict[item]['default']:
config[item] = getattr(args[0], item)
else:
config[item] = raw_config['DEFAULT'].get(item)
else:
logger.critical('Unknown parameter in config file: %s\n' % item)
exit(1)
# Write out the config file
if write_config_file:
config_string_to_write = sample_config_string_header
# Iterate over the items in the three options_sample_dict_* (defined above)
# and shove them into the big string "sample_config_string" that will then be fed to ConfigParser.
for k in options_sample_dict.keys():
logger.debug('adding config item %s to config string' % k)
config_string_to_write += (';\n')
config_string_to_write += ('; ' + options_sample_dict[k]['help'] + '\n')
config_string_to_write += (k + ' = ' + str(config[k]) + '\n')
logger.debug('Config string to write:\n')
logger.debug(config_string_to_write)
logger.info('Writing config file at: %s\n' % my_config_file)
with open(my_config_file, 'w') as cf:
cf.write(config_string_to_write)
# What we get from configparser is a string. For
# probe_properties_to_report, we need convert this string to a list.
# (ast.literal_eval() is safer than plain eval())
probe_properties_to_report = ast.literal_eval(config['probe_properties_to_report'])
logger.debug('Config dict:')
for k, v in config.items():
logger.debug('%s : %s' % (k, v))
# Put the remaining command line arguments into a list to process as files
# or measurement IDs.
data_sources = args[0].filename_or_msmid
# We need an idea of current unix time to decide if user-supplied
# date-times are good.
# First, let's figure out what the current unix time is in UTC.
current_unixtime = int(time.time())
# unix-time representation of config['oldest_atlas_result_datetime'], which is hardcoded up above.
oldest_result_unixtime = int(time.mktime(time.strptime(str(config['oldest_atlas_result_datetime']), '%Y %m %d %H:%M:%S')))
##################################################
#
# Function definitions
#
##########
# Return true if a supplied number falls in between now hours and
# config['oldest_atlas_result_datetime']
def is_valid_unixtime(_possible_unixtime):
if isinstance(_possible_unixtime, int) and int(_possible_unixtime) < current_unixtime and int(_possible_unixtime) >= oldest_result_unixtime:
return True
else:
logger.debug(str(_possible_unixtime) + ' is not inbetween ' + str(oldest_result_unixtime) + ' and ' + str(current_unixtime) + '.\n')
return False
##########
# Try a few formats to convert the datetime string they've supplied into unixtime
def user_datetime_to_valid_unixtime(user_dt_string):
accepted_datetime_formats = [ '%Y%m%d', '%Y%m%d%H%M',
'%Y%m%d_%H%M', '%Y%m%d_%H:%M',
'%Y%m%d.%H%M', '%Y%m%d.%H:%M',
'%Y%m%d %H%M', '%Y%m%d %H:%M',
'%Y-%m-%d_%H%M', '%Y-%m-%d_%H:%M',
'%Y-%m-%d-%H%M', '%Y-%m-%d-%H:%M']
# First, check to see if what's supplied is a valid-looking integer
# representation of a reasonable unix time (seconds since the
# the 1 Jan 1970 Epoch)
if is_valid_unixtime(user_dt_string):
return int(user_dt_string)
# It's not unix time, so try to convert from some data time formats
for f in accepted_datetime_formats:
try:
# print (user_dt_string + ' / ' + f) and offset the time zone to UTC
_unixtime_candidate = int(time.mktime(time.strptime(user_dt_string, f))) - time.timezone
if is_valid_unixtime(_unixtime_candidate):
logger.debug('Accepted %i as valid unixtime.\n' % _unixtime_candidate)
return (_unixtime_candidate)
except ValueError:
...
# If fall out the bottom of the (above) for loop, then we do not have a valid time
logger.critical('Cannot validate "' + user_dt_string + '" as a date-time representation\n')
exit(2)
# A list that might contain the user-supplied time period durations
# durations = [args[0].duration1, args[0].duration2 ]
# A list that might contain the unixtime representation of the user-supplied start times
unixtimes = [0, 0]
#####
# ansi formatting chars for fancy printing
class fmt:
bold = '\033[1m'
clear = '\033[0m'
bright_green = '\033[92m'
bright_red = '\033[91m'
bright_yellow = '\033[93m'
bright_magenta = '\033[95m'
####
#####
# initialize data structs
probe_measurement_rt = {}
measurement_probe_response_times = {}
addr_string = {}
asn_string = {}
measurement_ids = []
header_format = []
header_words = []
#
probe_detail_line_format_string = ''
probe_detail_header_format_string = ''
#probe_detail_line_output_color = []
#probe_detail_line_output_no_color = []
# We're expecting to process one or two sets of results for comparison. Often,
# we're comparing two different measurement ids, but it's also possible to
# compare multiple sets of data called for the measurement id, so we
# create a results set id to organize and index the sets of results, instead of using
# the measurement id.
results_set_id = 0
# m_ are variables specific to measurement-sets
# p_ are variables specific to probes
# pm_ are variables specific to probe, per each measurement
#
# IP Address version -- some probe properties are either 4 or 6, like IP address or ASN
m_ip_version = {}
m_response_times = {}
m_timestamps = {}
m_total_response_time = {}
m_total_malformeds = {}
m_total_abuf_malformeds = {}
m_total_errors = {}
m_total_slow= {}
m_response_time_average = {}
m_response_time_std_dev = {}
m_total_responses = {}
#
# A dictionary for interesting properties associated with each probe, like
# their ASN, IP address, country, etc.
p_probe_properties = {}
#
pm_response_time = {}
pm_dns_server_substring = {}
#
m_seen_probe_ids = {}
m_seen_probe_ids_set = {}
# class probe_info:
# '''
# Class for Probe data this script might use
# '''
# def __init__(self, _id, address_v4, address_v6, asn_v4, asn_v6, prefix_v4, prefix_v6, country_code):
# self._id = _id
# self.address_v4 = address_v4
# self.address_v6 = address_v6
# self.asn_v4 = asn_v4
# self.asn_v6 = asn_v6
# self.prefix_v4 = prefix_v4
# self.prefix_v6 = prefix_v6
# self.country_code = country_code
# def id(self):
# if autocomplete option is set, register all the options for autocomplete then exit
if args[0].autocomplete:
prog = sys.argv[0].removeprefix("./")
print(f'source <(register-python-argcomplete {prog})')
os.system('source <(register-python-argcomplete {prog})')
exit()
# Validate the supplied date-times and stick them in a list
if args[0].datetime1:
logger.debug(args[0].datetime1)
unixtimes[0] = user_datetime_to_valid_unixtime(args[0].datetime1)
if args[0].datetime2:
logger.debug(args[0].datetime2)
unixtimes[1] = user_datetime_to_valid_unixtime(args[0].datetime2)
# Because this script is written to compare two measurement results, or
# just report one, this is kinda complicated:
#
# Set our last results set id to the length of the data_sources list. (It
# should be either 0 or 1, but maybe this script will be modified to
# compare more than two sets of data, so try not to block that...)
# The args parsing setup should prevent this from happening, but just in
# case, exit here if we have zero data sources.
if len(data_sources) == 0:
logger.critical('Please supply one or two local filenames or RIPE Atlas Measurement IDs.\n')
exit(3)
# They've supplied one msm or file...
elif len(data_sources) == 1:
# ...so see how many timedates they supplied...
if len(unixtimes) == 1:
# If we reach here, we have one data source and only one datetime,
# so only one set of results to show.
last_results_set_id = 0
# We have one data source and two times?
elif len(unixtimes) == 2:
# We set the second data source to be the same as the first,
# otherwise the main loop would need logic to handle it being unset.
data_sources.append(data_sources[0])
last_results_set_id = 1
# Somehow we have two many datetimes, so exit!
else:
logger.critical('Please supply no more than two date times instead of %d.\n' % len(unixtimes))
exit(3)
# They supplied two data sources:
elif len(data_sources) == 2:
last_results_set_id = 1
#
# They supplied something other than one or two data sources, which this script is not written to process.
else:
logger.critical('Please supply one or two local filenames or RIPE Atlas Measurement IDs.\n')
exit(3)
####################
#
# Process the data, either from a local file or by requesting it over the
# 'net from RIPE Atlas.
#
def process_request(_data_source, _results_set_id, _unixtime, probes = []):
logger.info('Trying to access data_source %s for unixtime %s\n' % (_data_source, _unixtime))
# First we try to open the _data_source as a local file. If it exists,
# read in the measurement results from a filename the user has
# supplied.
#
# This code currently reads everything in, but it should be
# modified to only load the data from the user-supplied time range,
# if the user supplied one.
try:
f = open(_data_source, "r")
results = json.load(f)
f.close()
if _unixtime != 0:
logger.critical('This script does not yet know how to read user-supplied time ranges out of local files.\n (But it can query the RIPE Atlas API for time ranges, so maybe you wanna do that instead?\n')
except:
logger.debug('cannot read from file: %s\n' % _data_source)
# If we are here, accessing _data_sources as a local file did not
# work. Next, we try to check to see if _data_source is an 8-digit
# number. If it is, then we assume it is an Atlas Measurement ID
# and query their API with it.
if re.match(r'^[0-9]{8}$', _data_source):
# use it to make the request, but the measurement ID in the
# returned data will be passed back to the code calling this
# function, potentially redefining the measurement ID from
# what the user supplied. (That really should not happen, but
# the world is a weird place.)
measurement_id = int(_data_source)
# If we have no unixtime to request results from, then we get the latest results
if _unixtime == 0:
kwargs = {
"msm_id": measurement_id,
"probe_ids": probes
}
logger.info('Fetching latest results for Measurement %i from RIPE Atlas API...\n' % measurement_id)
is_success, results = AtlasLatestRequest(**kwargs).create()
# We have a unixtime, so:
# * use it as a start time
# * add duration to it for the stoptime
# * request the results
else:
measurement = Measurement(id=measurement_id)
_stop_time = (_unixtime + measurement.interval - 300)
kwargs = {
"msm_id": measurement_id,
"start": _unixtime,
"stop": _stop_time,
"probe_ids": probes
}
logger.info('Fetching results for Measurement %i, start unixtime: %s stop unixtime: %s\n' % (measurement_id, _unixtime, _stop_time))
is_success, results = AtlasResultsRequest(**kwargs).create()
if not is_success:
logger.critical('Request of ' + _data_source + 'from RIPE Atlas failed.\n')
exit(11)
else:
logger.critical('Cannot read from ' + _data_source + ' and it does look like a RIPE Atlas Measurement ID\n')
sys.exit(12)
# Variables that start with a m_ are specific to measurements.
# All of the m_* dictionaries are initialized at the top of the script.
# Here, we are initializing the structure we will be writing into for this _results_set_id.
# (results set identifier)
m_ip_version[_results_set_id] = 0
m_total_responses[_results_set_id] = 0
m_total_response_time[_results_set_id] = 0
m_total_malformeds[_results_set_id] = 0
m_total_abuf_malformeds[_results_set_id] = 0
m_total_errors[_results_set_id] = 0
m_total_slow[_results_set_id] = 0
m_response_time_average[_results_set_id] = 0
m_response_time_std_dev[_results_set_id] = 0
#
m_response_times[_results_set_id] = []
m_timestamps[_results_set_id] = []
# The list of seen probe IDs for this measurement-result-set
m_seen_probe_ids[_results_set_id] = []
m_probe_ids_to_exclude = []
if args[0].exclusion_list_file:
try:
with open(args[0].exclusion_list_file, 'r') as f:
m_probe_ids_to_exclude = f.read().splitlines()
except IOError:
logger.critical ('Cannot read probe exclusion list from file: %s\n' % args[0].exclusion_list_file)
exit(13)
# Loop through each (probe) result that come back from the call to DnsResult.
for r in results:
# this next line parses the data in r:
dns_result = DnsResult(r, on_malformation=DnsResult.ACTION_IGNORE,
on_error=DnsResult.ACTION_IGNORE)
# TODO: It's important to note that
# 'on_error=DnsResult.ACTION_IGNORE' causes DnsResult to discard
# std.err -- this script should be updated to catch and report
# what's logged there.
# Probe exclusion list handling, doing it here as to do it as
# close to the source as possible. That is, as soon as we know
# the ID's of the probes, we exclude those we don't want.
if str(dns_result.probe_id) in m_probe_ids_to_exclude:
continue
#
# If the user supplied measurement ID(s) on the command line, we
# already have them, but if they supplied filenames to read from,
# we do not have one (or two). So here, we read and (re-)define
# the measurement_id, which is a tiny bit wasteful of CPU, but
# cheaper than deciding if we should read it our of the result and
# set measurement_id, or not.
measurement_id = int(dns_result.measurement_id)
# generate an response_set + probe_id to use as an index into
# various dicts with responses
results_and_probes_id = str(_results_set_id) + '-' + str(dns_result.probe_id)
# Add the probe_id to the seen list. We need to cast it to a
# string, because the corresponding probe IDs in probe_info data
# will be indexed by probe_id as a string. (Because python.)
m_seen_probe_ids[_results_set_id].append(str(dns_result.probe_id))
m_total_responses[_results_set_id] += 1
# Check for malformed responses or errors, and count them
if dns_result.is_malformed:
m_total_malformeds[_results_set_id] += 1
elif dns_result.is_error:
m_total_errors[_results_set_id] += 1
else:
# Even more (abuf) error checks...
#
# first check if there is even a dns_result.responses[0].abuf,
# because apparently sometimes there's not! :S (It seems like
# if there is not an abuf, we might want to skip some of the
# following code, but determining which lines can be skipped
# adds complexity to this bug fix, so johan is not going to do
# that right now.)
if dns_result.responses[0].abuf:
logger.debug('dns_result.responses[0].abuf: %s\n' % (dns_result.responses[0].abuf))
if dns_result.responses[0].abuf.is_malformed:
m_total_abuf_malformeds[_results_set_id] += 1
# try dns_result.responses[1].get:
if len(dns_result.responses) > 1: ### FIXME: Should this be 0 instead of 1?
if dns_result.responses[1].abuf:
if dns_result.responses[1].abuf.is_malformed:
m_total_abuf_malformeds[_results_set_id] += 1
# Appended results to the dicts...
m_response_times[_results_set_id].append(dns_result.responses[0].response_time)
m_total_response_time[_results_set_id] += (dns_result.responses[0].response_time)
if dns_result.responses[0].response_time > args[0].slow_threshold:
m_total_slow[_results_set_id] += 1
m_timestamps[_results_set_id].append(dns_result.created_timestamp)
#
pm_response_time[results_and_probes_id] = dns_result.responses[0].response_time
# Not all of the DNS responses Atlas receives contain answers,
# so we need to handle responses without them.
try:
dns_server_fqdn = dns_result.responses[0].abuf.answers[0].data[0]
#
# Split up the response text
if args[0].split_char == '!':
split_result = dns_server_fqdn
logger.debug('%s\n' % (dns_server_fqdn))
else:
split_result = dns_server_fqdn.split(args[0].split_char)
if len(split_result) > args[0].dns_response_item_occurence_to_return:
pm_dns_server_substring[results_and_probes_id] = split_result[args[0].dns_response_item_occurence_to_return]
else:
pm_dns_server_substring[results_and_probes_id] = dns_server_fqdn
except IndexError:
pm_dns_server_substring[results_and_probes_id] = 'no_reply'
except AttributeError:
pm_dns_server_substring[results_and_probes_id] = 'no_data'
measurement = Measurement(id=measurement_id)
logger.debug(dir(measurement))
m_ip_version[_results_set_id] = int(measurement.protocol)
logger.debug("Address family for measurement %i is %i\n" % (measurement_id, m_ip_version[_results_set_id]))
# Sort some of the lists of results
m_response_times[_results_set_id].sort()
m_timestamps[_results_set_id].sort()
m_seen_probe_ids[_results_set_id].sort()
logger.debug('m_seen_probe_ids[_results_set_id] is %d\n' % len(m_seen_probe_ids[_results_set_id]))
return measurement_id, results
# END def process_request
####################
#####################
#
# Functions related to RIPE Atlas probe data
#
#####
# A) Why use a local cache?
# AFAICT, one can only request info about one probe at a time from the
# RIPE Atlas API, and that can be a slow, latency-ful process.
#
# B) Why are there two cache files?
#
# RIPE publishes a (daily?) updated version of all the probe data in one
# bz2-compressed file via HTTPS or FTP, so we can download that
# periodically. The probe info is formatted as a 1-line JSON blob, that
# this script reads in and converts into a python dictionary.
#
# However, at the time of this writing (Apr. 2021) this file from RIPE
# seems to be missing information for many (like 35%+) of the probes. So
# this script was subsequently modified to add into the probe info cache
# (dictionary) the additional info that's pulled down by via the Atlas
# API.
#
# This script then caches that "combined" dictionary as an uncompressed JSON file.
#
# So the first thing this function does is read in the probe properties
# JSON cache file, if it exists. Then, if RIPE's raw file is newer, it
# loads into the prope properties dictionary the data from RIPE's
# (probably just downloaded) file on top of the cached file. Then it
# writes out that dictionary as a JSON file, for use next time.
#
def check_update_probe_properties_cache_file(pprf, ppcf, ppurl):
all_probes_dict = {}
# If the probe properties cache file exists, read it into the all probes dictionary.
try:
ppcf_statinfo = os.stat(ppcf)
ppcf_age = int(ppcf_statinfo.st_mtime)
logger.info('Reading in existing local JSON cache file %s...\n' % ppcf)
with open(ppcf, 'r') as f:
all_probes_dict = json.load(f)
except:
# The cache file does not seem to exist, so set the age to
# zero, to trigger rebuild.
logger.info('Local JSON cache file %s does not exist; generating it.\n' % ppcf)
ppcf_age = -1
try:
# Check to see if the raw file (that's downloaded from RIPE)
# exists, and if so get its last mtime.
pprf_statinfo = os.stat(pprf)
pprf_age = int(pprf_statinfo.st_mtime)
except:
# The raw file does not seem to exist, so set the age to zero,
# and assume the age evaluation will try trigger a download.
pprf_age = 0
# Check to see if the current time minus the raw file age is more than the expiry
if ((current_unixtime - pprf_age) > int(config['raw_probe_properties_file_max_age'])):
# Fetch a new raw file, and generate the JSON format cache file
try:
logger.info ('%s is out of date, so trying to fetch fresh probe data from RIPE...\n' % pprf)
urlretrievefilename, headers = urllib.request.urlretrieve(ppurl, filename=pprf)
html = open(pprf)
html.close()
except:
logger.critical('Cannot urlretrieve %s -- continuing without updating %s \n' %
(ppurl, pprf))
os.replace(pprf + '.old', pprf)
return(2)
# If the raw file is newer than the local JSON cache file, decompress
# and read it in on top of the probe properties cache dictionary.
if ppcf_age < pprf_age:
try:
all_probes_list = json.loads(bz2.BZ2File(pprf).read().decode()).get('objects')
except:
logger.critical ('Cannot read raw probe data from file: %s\n' % pprf)
return(1)
# What we end up with in all_probes_list is a python list, but a
# dictionary would be much more efficient keyed on the probe id would
# be much more efficient, so we're going to burn some electricity and
# convert the list into a dictionary.
logger.info ('Converting the RIPE Atlas probe data into a dictionary and indexing it...\n')
while len(all_probes_list) > 0:
probe_info = all_probes_list.pop()
probe_id = str(probe_info['id'])
all_probes_dict[probe_id] = probe_info
logger.debug('Seen probe IDs: ')
logger.debug(all_probes_dict.keys())
# now save that dictionary as a JSON file...
logger.info ('Saving the probe data dictionary as a JSON file at %s...\n' % ppcf)
with open(ppcf, 'w') as f:
json.dump(all_probes_dict, f)
logger.info('%s does not need to be updated.\n' % pprf)
return(0)
#
# END def check_update_probe_properties_cache_file
#
#####
#
# Return probe properties for one probe (Requires RIPE Atlas cousteau
# library) def report_probe_properties(prb_id): return
# (p_probe_properties[prb_id)) END def report_probe_properties
####################
#
# Load the probe properties, either from the cache or by requesting them from RIPE.
def load_probe_properties(probe_ids, ppcf):
probe_cache_hits = 0
probe_cache_misses = 0
matched_probe_info = {}
all_probes_dict = {}
#
logger.info ('Reading the probe data dictionary as a JSON file from %s...\n' % ppcf)
while True:
try:
with open(ppcf, 'r') as f:
all_probes_dict = json.load(f)
except:
logger.critical ('Cannot read probe data from file: %s\n' % ppcf)
logger.critical ('Regenerating probe data to file: %s\n' % ppcf)
_res = check_update_probe_properties_cache_file(config['ripe_atlas_probe_properties_raw_file'],
config['ripe_atlas_probe_properties_json_cache_file'],
config['ripe_atlas_current_probe_properties_url'])
if _res != 0:
logger.critical('Unexpected result when updating local cache files: %s' % _res)
continue
break
# Loop through the list of supplied (seen) probe ids and collect their
# info/meta data from either our local file or the RIPE Atlas API
logger.info ('Matching seen probes with probe data; will query RIPE Atlas API for probe info not in local cache...\n')
# Using python set() data strucure instead of checking ccahe hits or misses
# python set() element is unique and using set theory, we can identify the differences
# and discover new element not found in the caches. This reduce computational complexity
dns_probes = set(str(x) for x in probe_ids)
all_probes = set(all_probes_dict.keys())
new_probes = dns_probes.difference(all_probes)
for i in (dns_probes - new_probes):
matched_probe_info[i] = all_probes_dict[i]
for p in new_probes:
try:
ripe_result = Probe(id=p)
matched_probe_info[p] = {'asn_v4': ripe_result.asn_v4,
'asn_v6': ripe_result.asn_v6,
'country_code': ripe_result.country_code,
'lat': ripe_result.geometry['coordinates'][1],
'lon': ripe_result.geometry['coordinates'][0],
'address_v4': ripe_result.address_v4,
'address_v6': ripe_result.address_v6}
all_probes_dict[p] = matched_probe_info[p]
logger.debug('Probe %9s info fetched from RIPE' % p)
except:
# Otherwise, it's empty
# we did not find any information about the probe, so set values to '-'
matched_probe_info[p] = { 'asn_v4': '-',
'asn_v6': '-',
'country_code': '-',
'lat': '-',
'lon': '-',
'address_v4': '-',
'address_v6': '-' }
logger.debug('Failed to get info about probe ID %s in the local cache or from RIPE Atlas API.' % p)
logger.info('cache hits: %i cache misses: %i.\n' % (probe_cache_hits, probe_cache_misses))
# Write out the local JSON cache file
if len(new_probes) != 0:
with open(ppcf, mode='w') as f:
json.dump(all_probes_dict, f)
return(matched_probe_info)
####################
#
# Input a dictionary -> Output a single line of scrape formatted string
def dict_string(d):
dict_str = ""
for i,v in d.items():
# Ignore the string output if the value of a key is "None"
if v == "None":
dict_str += str(i) + "=\"" + "unknown" + "\","
else:
dict_str += str(i) + "=\"" + str(v) + "\","
return dict_str.rstrip(",")
#####################
#
# Input a base64 encoded bytes string -> output the last word of the string
def decode_base64(abuf):
locate_nsid = re.sub(r'(\\x..)|\'|(\\t)',' ',str(abuf)).split()
return locate_nsid[-1]
#####################
#
# Input any string -> output with santinization of special characters with exception of '-'
def sanitize_string(s):
# Regex to match special characters and remove it however need to preserve '-'
return re.sub(r'\W+','',s.replace('-','_'))
#####################
#
# Input any timestamp and staleness -> output None value if timestamp exceed staleness else output timestamp
def check_freshness(timestamp,staleness):
freshness = int(time.time()) - timestamp
if (freshness > staleness):
return None
else:
return timestamp
# END of all function defs
##################################################
######
#
# Data loading and summary stats reporting loop ...
try:
probes = args[0].probes.split(',')
except: