This repository has been archived by the owner on Oct 26, 2022. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 1
/
config.default.yml
93 lines (87 loc) · 4.3 KB
/
config.default.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
# ____ __ _____ _ __
# / __ \____ _____ / /_ ____ ____ _______ _/ ___/____ (_____/ ___ _____
# / / / / __ `/ __ \/ __ \/ __ \/ __ \/ ___/ / / \__ \/ __ \/ / __ / _ \/ ___/
# / /_/ / /_/ / / / / /_/ / /_/ / /_/ / / / /_/ ___/ / /_/ / / /_/ / __/ /
# /_____/\__,_/_/ /_/_.___/\____/\____/_/ \__,_/____/ .___/_/\__,_/\___/_/
# /_/
#
# A general purpose image spider based on Danbooru API.
general: # Global configuration related
log:
# Log level, There are the following types
# trace > debug > into > warning > error
level: info
# Log format, Please refer to
# https://loguru.readthedocs.io/en/stable/api/logger.html#record
format: >
<level>
<v>{level:^8}</v>
[{time:YYYY/MM/DD} {time:HH:mm:ss.SSS} <d>{module}:{name}</d>]</level>
{message}
version: 0.1.1 # Don't touch
# To facilitate the use of the established snippet
# it does not participate in reading
snippets:
proxy: &proxy ""
user-agents: &user-agents
- "Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.90 Safari/537.36"
- "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_4) AppleWebKit/605.1.15 (KHTML, like Gecko)"
- "Mozilla/5.0 (iPad; CPU OS 9_3_5 like Mac OS X) AppleWebKit/601.1.46 (KHTML, like Gecko) Mobile/13G36"
- "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36"
- "Mozilla/5.0 (iPhone; CPU iPhone OS 13_4_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.1 Mobile/15E148 Safari/604.1"
- "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36 Edge/15.15063"
- "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36"
- "Mozilla/5.0 (iPhone; CPU iPhone OS 12_4_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1.2 Mobile/15E148 Safari/604.1"
- "Mozilla/5.0 (iPhone; CPU iPhone OS 12_3_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1.1 Mobile/15E148 Safari/604.1"
- "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36"
- "Mozilla/5.0 (Windows NT 5.1; rv:36.0) Gecko/20100101 Firefox/36.0"
- "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/603.3.8 (KHTML, like Gecko)"
- "Mozilla/5.0 (Windows NT 6.3; WOW64; Trident/7.0; rv:11.0) like Gecko"
- "Mozilla/5.0 (Windows NT 5.1; rv:33.0) Gecko/20100101 Firefox/33.0"
- "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)"
- "Mozilla/5.0 (iPhone; CPU iPhone OS 11_4_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/11.0 Mobile/15E148 Safari/604.1"
- "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36"
- "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36 Edge/16.16299"
- "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.132 Safari/537.36"
- "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36"
spider:
images:
proxy: *proxy
user-agents: *user-agents
workers: 16 # Number of concurrent jobs
retries:
# Number of retries
times: 3
# If the delay is negative
# then any delay between 0 and 10 will be taken at random
delay: -1
lists:
proxy: *proxy
user-agents: *user-agents
size: 100 #Size of each page
queue-size: 5
max-page: 1000
workers: 4
retries:
times: 5
delay: 5
spiders:
- name: konachan
impl: danbooru-unified
config:
url: https://konachan.com/post.json
- name: yandere
impl: danbooru-unified
config:
url: https://yande.re/post.json
- name: danbooru
impl: danbooru-unified
config:
url: https://danbooru.donmai.us/posts.json
persistence:
database:
uri: sqlite:///data/database.sqlite3
connect-args:
check_same_thread: false
echo-sql-exec: false
path-depth: 3