Merge pull request #81 from DankCity/55-switch-to-sqlite

55 switch to sqlite

.gitignore

@@ -64,3 +64,6 @@ target/
 # Virtual envs
 venv
 .venv*
+
+# Dankbot ignore
+dankbot.ini*

README.rst

@@ -33,30 +33,38 @@ Setup INI file
 ::
 
     cd /opt/dankbot
-    cp dankbot/dankbot.ini.sample dankbot/dankbot.ini
+    cp dankbot/dankbot.sample.ini dankbot/dankbot.ini
 
 Edit the INI file to fill in the missing token, username, and password fields:
 ::
 
-    (.venv35)➜  dankbot git:(master) ✗ cat dankbot/dankbot.ini.sample
+    (.venv35)➜  dankbot git:(master) ✗ cat dankbot/dankbot.sample.ini
+    [dankbot]
+    # Leave directory blank and dankbot will determine the best place to
+    # log to your platform
+    log_to_file: true
+    directory:
+    file_name: dankbot.log
+    backups: 5
+    max_bytes: 1000000
+
     [slack]
+    # Follow instructions at https://my.slack.com/services/new/bot
     token: <put here>
     channel: #random
 
     [reddit]
-    subreddits: dankmemes, fishpost, me_irl, 4chan
+    # r/dankmemes, r/funnygifs, etc
+    subreddits: dankmemes, funnygifs
 
     [imgur]
-    client_id: <client_id>
-    client_secret: <client Secret>
-
-    [mysql]
-    database: <db>
-    username: <username>
-    password: <password>
+    # Register at https://api.imgur.com/oauth2/addclient
+    # Select Anonymous usage
+    client_id: <your client ID>
+    client_secret: <your client secret>
 
     [misc]
-    include_nsfw: <boolean>
+    include_nsfw: false
     max_memes: 3
 
 Create and activate a virtual environment
@@ -75,13 +83,6 @@ Install the python package
     source env/bin/activate
     pip install -e .
 
-Create logging folder
----------------------
-::
-
-    sudo mkdir /var/log/dankbot
-    sudo chown <user> /var/log/dankbot
-
 Add an entry to your crontab
 -----------------------------
 Edit the crontab with your favorite editor

dankbot/dankbot.py

@@ -2,14 +2,13 @@
 
 import random
 from sys import platform
-from datetime import datetime as dt
 
 import praw
-import MySQLdb as mdb
 from retryz import retry
 from slacker import Slacker
 from praw.errors import HTTPException
 
+from dankbot.db import DB
 from dankbot import __version__ as dankbot_version
 from dankbot.memes import ImgurMeme, DankMeme, UndigestedError
 
@@ -24,10 +23,6 @@ def __init__(self, config, logger):
         self.slack_token = config['slack']['token']
         self.slack_channel = config['slack']['channel']
 
-        self.database = config['mysql']['database']
-        self.username = config['mysql']['username']
-        self.password = config['mysql']['password']
-
         self.include_nsfw = config.getboolean('misc', 'include_nsfw')
         self.max_memes = config.getint('misc', 'max_memes')
 
@@ -42,6 +37,9 @@ def __init__(self, config, logger):
         # Get logger
         self.logger = logger
 
+        # Create DB connection
+        self.db = DB(logger, create_db=True)  # pylint: disable=C0103
+
     def find_and_post_memes(self):
         """ Find memes from subreddits and post them to slack
         """
@@ -50,15 +48,22 @@ def find_and_post_memes(self):
         self.logger.info("Found {0} memes".format(len(memes)))
 
         # Filter out any known dank memes
-        filtered_memes = [m for m in memes if not self.in_collection(m)]
-        self.logger.info(
-            "Removed {0} known memes".format(len(memes) - len(filtered_memes)))
+        new_memes, old_memes = list(), list()
+        for meme in memes:
+            if self.db.in_collection(meme):
+                # Mark it as having been seen
+                self.db.update_collection(meme)
+                old_memes.append(meme)
+            else:
+                new_memes.append(meme)
+
+        self.logger.info("Removed {0} known memes".format(len(old_memes)))
 
         # Shuffle memes
-        random.shuffle(filtered_memes)
+        random.shuffle(new_memes)
 
         # Cut down to the max memes
-        pared_memes = filtered_memes[:self.max_memes]
+        pared_memes = new_memes[:self.max_memes]
         self.logger.info("Truncated to {0} memes".format(len(pared_memes)))
 
         # Bale here if nothing is left
@@ -109,9 +114,9 @@ def get_memes(self):
                     continue
 
                 if "imgur.com/" in meme.url:
-                    memes.append(ImgurMeme(meme.url, sub))
+                    memes.append(ImgurMeme(meme))
                 else:
-                    memes.append(DankMeme(meme.url, sub))
+                    memes.append(DankMeme(meme))
 
         return memes
 
@@ -120,56 +125,6 @@ def get_memes(self):
     def _get_memes_from_subreddit(client, subreddit):
         return client.get_subreddit(subreddit).get_hot()
 
-    def in_collection(self, meme):  # pragma no cover
-        '''
-        Checks to see if the supplied meme is already in the collection of known
-        memes
-        '''
-        query = "SELECT * FROM memes WHERE links = '%s'" % meme.link
-
-        con = mdb.connect(
-            'localhost',
-            self.username,
-            self.password,
-            self.database,
-            charset='utf8'
-        )
-
-        with con, con.cursor() as cur:
-            try:
-                resp = cur.execute(query)
-            except UnicodeEncodeError:
-                # Indicates a link with oddball characters, just ignore it
-                resp = True
-
-                log = "Bad character in meme: {0}"
-                self.logger.exception(log.format(meme))
-
-        return True if resp else False
-
-    def add_to_collection(self, meme):  # pragma no cover
-        '''
-        Adds a meme to the collection
-        '''
-        query = """INSERT INTO memes
-                   (links, sources, datecreated)
-                   VALUES
-                   ('%s', '%s', '%s')
-                """ % (meme.link, meme.source, str(dt.now()))
-
-        con = mdb.connect(
-            'localhost',
-            self.username,
-            self.password,
-            self.database,
-            charset='utf8'
-        )
-
-        with con, con.cursor() as cur:
-            cur.execute(query)
-
-        return
-
     def post_to_slack(self, memes):
         '''
         Post the memes to slack
@@ -191,7 +146,7 @@ def post_to_slack(self, memes):
             resp = slack.chat.post_message(self.slack_channel, message, as_user=True)
 
             if resp.successful:
-                self.add_to_collection(meme)
+                self.db.add_to_collection(meme)
                 ret_status = True
 
         return ret_status

dankbot/dankbot.sample.ini

@@ -22,12 +22,6 @@ subreddits: dankmemes, funnygifs
 client_id: <your client ID>
 client_secret: <your client secret>
 
-[mysql]
-# depricated, switching ti sqlite
-database: <db>
-username: <username>
-password: <password>
-
 [misc]
 include_nsfw: false
 max_memes: 3

dankbot/db.py

@@ -0,0 +1,90 @@
+import os
+from appdirs import user_data_dir
+import sqlite3 as lite
+from os.path import join
+from contextlib import closing
+
+from dankbot import APPNAME, APPAUTHOR
+
+DB_NAME = 'dankbot.db'
+
+
+class DB(object):
+    db_dir = None
+    db_path = None
+    db_con = None
+
+    def __init__(self, logger, db_dir=None, create_db=False):
+        self.logger = logger
+
+        self.db_dir = db_dir or user_data_dir(APPNAME, APPAUTHOR)
+        self.db_path = join(self.db_dir, DB_NAME)
+
+        if create_db is True:
+            self._create_db_and_table()
+
+    def _query(self, query, args=None):
+        if not args:
+            args = dict()
+
+        if self.db_con is None:
+            self.db_con = lite.connect(self.db_path)
+
+        with self.db_con, closing(self.db_con.cursor()) as cur:
+            cur.execute(query, args)
+            return cur.fetchall()
+
+    def _create_db_and_table(self):
+        """ Create the database and table if it doesn't exist
+        """
+        if os.path.exists(self.db_path):  # pragma: no cover
+            self.logger.info("Will not create database: already exists at:"
+                             " {0}".format(self.db_path))  # pylint: disable=W1202
+            return
+
+        # Create the data directory if it doesn't exist
+        if not os.path.exists(self.db_dir):  # pragma: no cover
+            os.makedirs(self.db_dir)
+
+        self.logger.info("Creating database at: {0}".format(self.db_path))  # pylint: disable=W1202
+
+        query = """CREATE TABLE IF NOT EXISTS memes (
+            reddit_id VARCHAR(255) PRIMARY KEY NOT NULL,
+            source VARCHAR(255) NOT NULL,
+            link VARCHAR(255) NOT NULL,
+            created DATETIME DEFAULT CURRENT_TIMESTAMP,
+            last_seen DATETIME DEFAULT CURRENT_TIMESTAMP
+            );"""
+
+        return self._query(query)
+
+    def add_to_collection(self, meme):
+        """ Add meme to database collection
+        """
+
+        query = """INSERT INTO memes (reddit_id, source, link)
+                   VALUES(?,?,?)"""
+
+        self._query(query, (meme.reddit_id, meme.source, meme.link))
+
+    def in_collection(self, meme):
+        query = "SELECT * FROM memes WHERE reddit_id=:reddit_id"
+
+        try:
+            resp = self._query(query, {'reddit_id': meme.reddit_id})
+        except UnicodeEncodeError:  # pragma: no cover
+            # Indicates a link with oddball characters, just ignore it
+            resp = True
+
+            log = "Bad character in meme: {0}"
+            self.logger.exception(log.format(meme))
+
+        return True if resp else False
+
+    def update_collection(self, meme):
+        """ Update the last_seen column with the current timestamp
+        """
+        query = """UPDATE memes SET last_seen=(CURRENT_TIMESTAMP)
+                   WHERE reddit_id=(:id)"""
+
+        self._query(query, {'id': meme.reddit_id})

dankbot/memes.py

@@ -9,9 +9,10 @@ class Meme(object):  # pylint: disable=R0903
     """
     Base class for meme objects
     """
-    def __init__(self, link, source):
-        self.link = link
-        self.source = source
+    def __init__(self, meme):
+        self.source = str(meme.subreddit)
+        self.reddit_id = meme.id
+        self.link = meme.url
 
     def __str__(self):
         return str(self.link)

setup.py

@@ -6,17 +6,16 @@
 
 import versioneer
 
-here = path.dirname(path.abspath(__file__))
+HERE = path.dirname(path.abspath(__file__))
 
 # Get the long description from the README file
-with open(path.join(here, 'README.rst')) as f:
-    long_description = f.read()
+with open(path.join(HERE, 'README.rst')) as f:
+    LONG_DESCRIPTION = f.read()
 
-install_requires = [
+INSTALL_REQUIRES = [
     'appdirs==1.4.0',
     'configparser==3.5.0',
     'imgurpython==1.1.7',
-    'mysqlclient==1.3.9',
     'praw==3.6.0',
     'retryz==0.1.8',
     'slacker==0.9.30',
@@ -33,10 +32,10 @@
     description="Slack bot for posting dank memes from Reddit",
     packages=find_packages(),
     license='MIT',
-    long_description=long_description,
+    long_description=LONG_DESCRIPTION,
     test_suite="tests",
     tests_require=['tox'],
-    install_requires=install_requires,
+    install_requires=INSTALL_REQUIRES,
     entry_points={
         "console_scripts": [
             "dankbot=dankbot.cli:main",