WIP: movie_json_data analysis

.gitignore

@@ -0,0 +1 @@
+/img/

code/__main__.py

@@ -0,0 +1,7 @@
+import load_db as ldb
+import sys
+
+if __name__ == '__main__':
+    # execute only if run as the entry point into the program
+    ldb.main()
+    sys.exit(0)

code/config_handler.py

@@ -0,0 +1,49 @@
+from configparser import ConfigParser
+
+import pyodbc
+
+
+def load_config():
+    # Read configuration file
+    parser = ConfigParser()
+    parser.read('../dev.ini')
+    return parser
+
+
+def get_SqlConfig():
+    parser = load_config()
+    # Read corresponding file parameters
+    driver = parser.get("db", "driver")
+    database = parser.get("db", "database")
+    trusted_connection = parser.get("db", "trusted_connection")
+    server = parser.get("db", "server")
+    return driver, database, trusted_connection, server  # return required parameters
+
+
+def ms_sql_connection():
+    c = get_SqlConfig()
+    driver_ = c[0]
+    db_name = c[1]
+    server_ = c[3]
+    conn_info = ('DRIVER=' + driver_ + ';TrustServerCertificate=No;'
+                                       'DATABASE=' + db_name + ';SERVER=' + server_)
+    return pyodbc.connect(conn_info)
+
+
+# load json path from configuration file
+def json_path():
+    parser = load_config()
+    return parser.get("json", "movie_list")
+
+############create .ini file
+
+# config = ConfigParser()
+# config['db'] = {
+#     "driver": "{ODBC Driver 17 for SQL Server}",
+#     "database": "Movies_DB",
+#     "trusted_Connection": "yes",
+#     "server": "(localdb)\MSSQLLocalDB"
+#
+# }
+# with open('../dev.ini', 'w') as f:
+#     config.write(f)

code/extract_transform.py

@@ -0,0 +1,36 @@
+import json
+import pandas as pd
+import config_handler as ch
+
+
+def load():
+
+    with open(ch.json_path()) as f:
+        data = json.load(f)
+        # print(data)
+        df = pd.DataFrame(data)
+    return df
+
+
+# JSON to DataFrame
+def json_to_df():
+    dataframe = load()
+
+    # type conversion
+    # list to string for movie_genre_relationship table
+    dataframe['genres'] = dataframe['genres'].astype('str').apply(
+        lambda x: x.lower().strip().replace("[", "").replace("]", "")
+            .replace("\'", "").replace("\"", "").replace(", ", ","))
+
+    # extract relevant duration PT89M --> 89
+    dataframe['duration'] = dataframe['duration'].astype('str').apply(
+        lambda x: x.strip().replace("PT", "").replace("M", "")).astype(int)
+
+    # string to float conversion
+    dataframe['imdbRating'] = dataframe['imdbRating'].astype('float')
+
+    # handling names like Genelia D'Souza which was causing string handling issues
+    dataframe['actors'] = dataframe['actors'].astype('str').apply(
+        lambda x: x.lower().strip().replace("[", "").replace("]", "").
+            replace("\'", "").replace("\"", "").replace(", ", ","))
+    return dataframe

code/load_db.py

@@ -0,0 +1,69 @@
+import pyodbc
+import teardown as td
+import extract_transform as et
+import config_handler as db_
+
+
+# import pyodbc
+# from configparser import ConfigParser
+
+
+def main():
+    # For rerun purposes (incase of new daily feed)
+    td.db_cleanup()
+
+    # Extract Json and transform
+    dataframe = et.json_to_df()
+    # Queries
+    create_top_rated_movie = """CREATE TABLE [Top_rated_Movie] (
+                   Id int NOT NULL CONSTRAINT [Id] PRIMARY KEY,
+                   title char (100) NULL ,
+                   [year] [date] NULL ,
+                   [genres] [nvarchar] (50) NULL ,
+                   [duration] [int] NULL ,
+                   [releaseDate] [nvarchar] (50) NULL ,
+                   [actors] [nvarchar] (500) NULL ,
+                   [imdbRating] [varchar] (50) NULL ,
+                   ) ON [PRIMARY] """
+    create_movie_actor_relationship = """CREATE TABLE [Movie_Actor_Relationship] (
+                   	[movieID] [int] NOT NULL , 
+                   	[actor] [nvarchar] (500) NULL ,
+                   	[imdbRating] [float] (50) NULL ,) 
+                   """
+    create_movie_genre_relationship = """
+                    CREATE TABLE [Movie_Genre_Relationship] (
+                   	[movieID] [int] NOT NULL , 
+                   	[genre] [nvarchar] (50) NULL ,
+                   	[imdbRating] [float] (50) NULL ,
+                   )"""
+    # write to MS SQL db
+    create_query = '{0};{1};{2}'.format(create_top_rated_movie, create_movie_actor_relationship,
+                                        create_movie_genre_relationship)
+    conn = db_.ms_sql_connection()
+    cursor = conn.cursor()
+    # Create Tables
+    cursor.execute(create_query)
+    conn.commit()
+
+    # Inserting data in SQL Table:-
+    insert_top_rated_movie = """INSERT INTO dbo.Top_rated_Movie(Id,title,year,genres,actors,duration,
+                releaseDate,imdbRating) values (?,?,?,?,?,?,?,?)"""
+    insert_movie_actor_relationship = """INSERT INTO dbo.Movie_Actor_Relationship
+    (movieID,actor,imdbRating) values (?,?,?) """
+    insert_movie_genre_relationship = """INSERT INTO dbo.Movie_Genre_Relationship
+    (movieID,genre,imdbRating) values (?,?,?)"""
+    for index, row in dataframe.iterrows():
+
+        cursor.execute(insert_top_rated_movie,
+                       index, row.title, row.year, row['genres'], row['actors'], row.duration,
+                       row.releaseDate, row.imdbRating)
+        for actor in row['actors'].split(","):
+            cursor.execute(insert_movie_actor_relationship,
+                           index, actor, row.imdbRating)
+        for genre in row['genres'].split(","):
+            cursor.execute(insert_movie_genre_relationship,
+                           index, genre, row.imdbRating)
+
+    conn.commit()
+    cursor.close()
+    conn.close()

code/teardown.py

@@ -0,0 +1,12 @@
+import config_handler as db
+
+
+def db_cleanup():
+    conn = db.ms_sql_connection()
+    cursor = conn.cursor()
+    drop_tables = "DROP TABLE IF EXISTS [dbo].[Top_rated_Movie];" \
+                  + "DROP TABLE IF EXISTS  [dbo].[Movie_Actor_Relationship];" \
+                  + "DROP TABLE IF EXISTS  [dbo].[Movie_Genre_Relationship]"
+    cursor.execute(drop_tables)
+    cursor.commit()
+    conn.close()

dev.ini

@@ -0,0 +1,8 @@
+[db]
+driver = {ODBC Driver 17 for SQL Server}
+database = Movies_DB
+trusted_connection = yes
+server = (localdb)\MSSQLLocalDB
+
+[json]
+movie_list= ../json/top-rated-movies-02.json