-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy path__init__.py
42 lines (34 loc) · 1.36 KB
/
__init__.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
import contextlib
from pymongo import MongoClient
class Config(object):
''' config logic here '''
def __init__(self, settings):
self.settings = settings
# settings from scrapy
self.bot_name = self.settings.get("BOT_NAME")
self.host = self.settings.get('MONGO_PIPELINE_HOST', 'localhost')
self.keys = self.settings.get('MONGO_PIPELINE_KEYS', ['key', 'url'])
self.col_prefix = self.settings.getbool('MONGO_PIPELINE_COLNAME_BOTPREFIX', False)
self.db_suffix = self.settings.getbool('MONGO_PIPELINE_DBNAME_BOTSUFFIX', False)
def database_name(self):
dbname = self.settings.get('MONGO_PIPELINE_DBNAME')
if dbname is None: # not set fixed db name use bot name.
dbname = self.bot_name
elif self.db_suffix:
dbname = dbname +'.' + self.bot_name
return dbname
def collection_name(self, typename):
if self.col_prefix:
typename = self.bot_name + "." + typename
return typename
def __repr__(self):
return "\n".join(["%s:%s"%(n, getattr(self,n)) for n in ["bot_name", 'host', 'keys', 'col_prefix', 'db_suffix']])
@contextlib.contextmanager
def mongodb(settings):
config = Config(settings)
clt = MongoClient(config.host)
db = clt[config.database_name()]
try:
yield db
finally:
clt.close()