Merge branch 'dev' into master

Author: Official-Husko

.gitignore

@@ -10,4 +10,5 @@ old_config.json
 db/
 outdated
 runtime.log
-config.json.dev
+config.json.dev
+Build Release.bat

Build Release.bat

@@ -1 +1 @@
-".\.env\Scripts\activate" && pyinstaller --onefile --icon "icon.ico" --console --name "NN-Downloader" --upx-dir "Z:\Projects\Python\### UPX ###" --add-data="./.env/Lib/site-packages/grapheme/data/*;grapheme/data/" main.py && rmdir /s /q .\build && rmdir /s /q .\__pycache__ && del ".\NN-Downloader.spec"
+conda activate pyinstaller && pyinstaller --paths Z:\Projects\Python\NN-Downloader\.env\Lib\site-packages --hidden-import requests --hidden-import inquirer --hidden-import alive_progress --hidden-import termcolor --hidden-import xmltodict --add-data="Z:\Projects\Python\NN-Downloader\.env\Lib\site-packages\grapheme\data\*;grapheme/data/" --onefile --icon "icon.ico" --console --name "NN-Downloader" --upx-dir "Z:\Projects\Python\### UPX ###" main.py

main.py

@@ -6,7 +6,7 @@
 import sys
 import inquirer
 
-version = "1.5.0"
+version = "1.6.0"
 
 if os.name == 'nt':
     from ctypes import windll
@@ -15,7 +15,7 @@
 proxy_list = []
 header = {"User-Agent":f"nn-downloader/{version} (by Official Husko on GitHub)"}
 needed_folders = ["db", "media"]
-database_list = ["e621", "furbooru", "rule34", "e6ai"]
+database_list = ["e621", "furbooru", "rule34", "e6ai", "e926"]
 unsafe_chars = ["/", "\\", ":", "*", "?", "\"", "<", ">", "|", "\0", "$", "#", "@", "&", "%", "!", "`", "^", "(", ")", "{", "}", "[", "]", "=", "+", "~", ",", ";"]
 
 if sys.gettrace() is not None:
@@ -69,6 +69,7 @@ def clear_screen():
             oneTimeDownload = config["oneTimeDownload"]
             use_proxies = config["proxies"]
             checkForUpdates = config["checkForUpdates"]
+            ai_training = config["ai_training"]
         else:
             config = Config_Manager.creator()
             print(colored("New Config file generated. Please configure it for your use case and add API keys for needed services.", "green"))
@@ -106,10 +107,9 @@ def clear_screen():
 
         site = answers.get("selection").lower()
 
-        if site in ["multporn", "yiffer", "luscious"]:
-            pass
-        else: 
-            print(colored("Please enter the tags you want to use", "green"))
+        if site in ["e621", "e6ai", "e926"]:
+
+            print(colored("Please enter the tags you want to use.", "green"))
             user_tags = input(">> ").lower()
             while user_tags == "":    
                 print(colored("Please enter the tags you want.", "red"))
@@ -121,42 +121,17 @@ def clear_screen():
             max_sites = input(">> ").lower()
             print("")
 
-        if site == "e621":
-            apiUser = config["user_credentials"]["e621"]["apiUser"]
-            apiKey = config["user_credentials"]["e621"]["apiKey"]
-            if oneTimeDownload == True:
-                with open("db/e621.db", "r") as db_reader:
-                    database = db_reader.read().splitlines()
-            if apiKey == "" or apiUser == "":
-                print(colored("Please add your Api Key into the config.json", "red"))
-                sleep(5)
-            else:
-                output = E621.Fetcher(user_tags=user_tags, user_blacklist=config["blacklisted_tags"], proxy_list=proxy_list, max_sites=max_sites, user_proxies=config["proxies"], apiUser=apiUser, apiKey=apiKey, header=header, db=database)
-        
-        if site == "e6ai":
-            apiUser = config["user_credentials"]["e6ai"]["apiUser"]
-            apiKey = config["user_credentials"]["e6ai"]["apiKey"]
-            if oneTimeDownload == True:
-                with open("db/e6ai.db", "r") as db_reader:
-                    database = db_reader.read().splitlines()
-            if apiKey == "" or apiUser == "":
-                print(colored("Please add your Api Key into the config.json", "red"))
-                sleep(5)
-            else:
-                output = E6AI.Fetcher(user_tags=user_tags, user_blacklist=config["blacklisted_tags"], proxy_list=proxy_list, max_sites=max_sites, user_proxies=config["proxies"], apiUser=apiUser, apiKey=apiKey, header=header, db=database)
-     
-        elif site == "e926":
-            apiUser = config["user_credentials"]["e926"]["apiUser"]
-            apiKey = config["user_credentials"]["e926"]["apiKey"]
+            apiUser = config["user_credentials"][site]["apiUser"]
+            apiKey = config["user_credentials"][site]["apiKey"]
             if oneTimeDownload == True:
-                with open("db/e621.db", "r") as db_reader:
+                with open(f"db/{site}.db", "r") as db_reader:
                     database = db_reader.read().splitlines()
             if apiKey == "" or apiUser == "":
                 print(colored("Please add your Api Key into the config.json", "red"))
                 sleep(5)
             else:
-                output = E926.Fetcher(user_tags=user_tags, user_blacklist=config["blacklisted_tags"], proxy_list=proxy_list, max_sites=max_sites, user_proxies=config["proxies"], apiUser=apiUser, apiKey=apiKey, header=header, db=database)
-        
+                output = E6System.Fetcher(user_tags=user_tags, user_blacklist=config["blacklisted_tags"], proxy_list=proxy_list, max_sites=max_sites, user_proxies=config["proxies"], apiUser=apiUser, apiKey=apiKey, header=header, db=database, site=site, ai_training=ai_training)
+       
         elif site == "rule34":
             if oneTimeDownload == True:
                 with open("db/rule34.db", "r") as db_reader:

modules/__init__.py

@@ -6,9 +6,7 @@
 
 
 # Here are all modules for the sites that are supported
-from .e621 import E621
-from .e6ai import E6AI
-from .e926 import E926
+from .e6systems import E6System
 from .rule34 import RULE34
 from .furbooru import FURBOORU
 from .multporn import Multporn

modules/configManager.py

@@ -3,7 +3,7 @@
 from termcolor import colored
 import os
 
-def_config_version = 1.5
+def_config_version = 1.6
 
 class Config_Manager():
 
@@ -14,6 +14,7 @@ def creator():
             "checkForUpdates": True,
             "oneTimeDownload": True,
             "advancedMode": False,
+            "ai_training": False,
             "user_credentials": {
                 "e621": {
                     "apiUser": "",

modules/e621.py renamed to modules/deprecated/e621.py

@@ -0,0 +1,120 @@
+from requests.auth import HTTPBasicAuth
+import requests
+import random
+from termcolor import colored
+from alive_progress import alive_bar
+from time import sleep
+from datetime import datetime
+import os
+import json
+
+from main import unsafe_chars
+now = datetime.now()
+dt_now = now.strftime("%d-%m-%Y_%H-%M-%S")
+
+class E6System():
+    def Fetcher(user_tags, user_blacklist, proxy_list, max_sites, user_proxies, apiUser ,apiKey, header, db, site, ai_training):
+        try:
+            approved_list = []
+            page = 1
+            while True:
+                URL = f"https://{site}.net/posts.json?tags={user_tags}&limit=320&page={page}"
+                if user_proxies == True:
+                    proxy = random.choice(proxy_list)
+                    raw_req = requests.get(URL, headers=header, proxies=proxy, auth=HTTPBasicAuth(apiUser, apiKey))
+                else:
+                    raw_req = requests.get(URL, headers=header, auth=HTTPBasicAuth(apiUser, apiKey))
+
+                req = raw_req.json()
+
+                try:
+                    if req["message"] == "You cannot go beyond page 750. Please narrow your search terms.": 
+                        print(colored(req["message"] + " (API limit)", "red"))
+                        sleep(5)
+                        break
+                except:
+                    pass
+                
+                if req["posts"] == []:
+                    print(colored("No images found or all downloaded! Try different tags.", "yellow"))
+                    sleep(5)
+                    break
+
+                elif page == max_sites:
+                    print(colored(f"Finished Downloading {max_sites} of {max_sites} pages.", "yellow"))
+                    sleep(5)
+                    break
+                
+                else: 
+                    for item in req["posts"]:
+                        image_id = item["id"]
+                        image_address = item["file"]["url"]
+                        post_tags1 = item["tags"]["general"]
+                        post_tags2 = item["tags"]["species"]
+                        post_tags3 = item["tags"]["character"]
+                        if site == "e6ai":
+                            post_tags4 = item["tags"]["director"]
+                            post_tags5 = item["tags"]["meta"]
+                        else:
+                            post_tags4 = item["tags"]["copyright"]
+                            post_tags5 = item["tags"]["artist"]
+                        
+                        if ai_training == True:
+                            meta_tags = item["tags"]
+
+                        post_tags = post_tags1 + post_tags2 + post_tags3 + post_tags4 + post_tags5
+                        image_format = item["file"]["ext"]
+                        user_blacklist_lenght = len(user_blacklist)
+                        passed = 0
+
+                        for blacklisted_tag in user_blacklist:
+                            if blacklisted_tag in post_tags:
+                                break
+                            else:
+                                passed += 1
+                        if passed == user_blacklist_lenght and str(image_id) not in db and image_address != None:
+                            image_data = {"image_address": image_address, "image_format": image_format, "image_id": image_id, "meta_tags": meta_tags}
+                            approved_list.append(image_data)
+                        else:
+                            pass
+
+                # Download Each file
+                with alive_bar(len(approved_list), calibrate=1, dual_line=True, title='Downloading') as bar:
+                    for data in approved_list:
+                        image_address = data["image_address"]
+                        image_format = data["image_format"]
+                        image_id = data["image_id"]
+                        meta_tags = data["meta_tags"]
+                        bar.text = f'-> Downloading: {image_id}, please wait...'
+                        if user_proxies == True:
+                            proxy = random.choice(proxy_list)
+                            img_data = requests.get(image_address, proxies=proxy).content
+                        else:
+                            sleep(1)
+                            img_data = requests.get(image_address).content
+
+                        safe_user_tags = user_tags.replace(" ", "_")
+                        for char in unsafe_chars:
+                            safe_user_tags = safe_user_tags.replace(char, "")
+
+                        if not os.path.exists(f"media/{dt_now}_{safe_user_tags}"):
+                            os.mkdir(f"media/{dt_now}_{safe_user_tags}")
+                        if not os.path.exists(f"media/{dt_now}_{safe_user_tags}/meta") and ai_training == True:
+                            os.mkdir(f"media/{dt_now}_{safe_user_tags}/meta")
+                        with open(f"media/{dt_now}_{safe_user_tags}/{str(image_id)}.{image_format}", 'wb') as handler:
+                            handler.write(img_data)
+                        with open(f"media/{dt_now}_{safe_user_tags}/meta/{str(image_id)}.json", 'w') as handler:
+                            json.dump(meta_tags, handler, indent=6)
+                        with open(f"db/{site}.db", "a") as db_writer:
+                            db_writer.write(f"{str(image_id)}\n")
+                        bar()
+
+                print(colored(f"Page {page} Completed", "green"))
+                approved_list.clear()
+                page += 1
+                sleep(5)
+
+            return {"status": "ok"}
+        
+        except Exception as e:
+            return {"status": "error", "uinput": user_tags, "exception": str(e), "extra": raw_req.content}