bechdel.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 158,
   "id": "66a704df",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "here\n",
      "sldkfjdummykey\n",
      "> \u001b[31m\u001b[1mERROR   ❌ Failed to authenticate. Check the passed token is valid!\u001b[0m\n",
      "> \u001b[31m\u001b[1mERROR   ❌ Oops! Something failed in AutoNLP backend..\u001b[0m\n",
      "> \u001b[31m\u001b[1mERROR   Error code: 401; Details: '{\"error\":\"Unauthorized\"}'\u001b[0m\n",
      "Model trianing using autonlp (hugging face api)\n"
     ]
    }
   ],
   "source": [
    "# for interest, here is how I would make this a software project\n",
    "# training classifier using hugging face api (high, high level nlp)\n",
    "# then classifying script text content\n",
    "# dataset and csv are fomatted and ready to go for api (as in this sofware works)\n",
    "# will need a valid hugging face api to work- (will fail but illustrates working code) \n",
    "!python3 main.py --hugging_face --login --api_key sldkfjdummykey\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e21d0b71",
   "metadata": {},
   "source": [
    "####  Here is the software verison with and args parser that I wrote as an example of using hugging face api to train a binary text classifier and an illustration of how I would approach this problem as a piece of end to end software that runs in terminal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "id": "668c57e9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "usage: main.py [-h] [--input csv INPUT CSV] [--project PROJECT]\n",
      "               [--split SPLIT] [--col_mapping COL_MAPPING] [--files FILES]\n",
      "               [--api_key API_KEY] [--resize RESIZE] [--name NAME]\n",
      "               [--language LANGUAGE] [--task TASK] [--max_models MAX_MODELS]\n",
      "               [--create_project] [--hugging_face] [--send] [--login] [--make]\n",
      "               [--train]\n",
      "\n",
      "Training Calssifier\n",
      "\n",
      "optional arguments:\n",
      "  -h, --help            show this help message and exit\n",
      "  --input csv INPUT CSV\n",
      "                        relative loacation of input csv for training\n",
      "  --project PROJECT     poject name\n",
      "  --split SPLIT         dataset split\n",
      "  --col_mapping COL_MAPPING\n",
      "                        text:text, label:target\n",
      "  --files FILES         formated csv only 2 colls one for text one for target\n",
      "  --api_key API_KEY     api key from hugging_face account\n",
      "  --resize RESIZE       Resizes images by percentage as scalar\n",
      "  --name NAME           project name hugging face\n",
      "  --language LANGUAGE   lang in eg [en,sp,fr]\n",
      "  --task TASK           Resizes images by percentage as scalar\n",
      "  --max_models MAX_MODELS\n",
      "                        nuber of trainable models\n",
      "  --create_project      create_new hf project\n",
      "  --hugging_face        uses hugging face api to train model\n",
      "  --send                if entered will try to sen .csv\n",
      "  --login               if entered will try to sen .csv\n",
      "  --make                create_new hf project\n",
      "  --train               create_new hf project\n"
     ]
    }
   ],
   "source": [
    "# to show you what the args parser -help would return of this sofware \n",
    "!python3 main.py --help"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "id": "714f33ae",
   "metadata": {},
   "outputs": [],
   "source": [
    "#files\n",
    "import gdown \n",
    "import os\n",
    "import zipfile\n",
    "\n",
    "#data \n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# utils\n",
    "from pathlib import Path\n",
    "import time\n",
    "from tqdm import tqdm\n",
    "\n",
    "# web\n",
    "import urllib.request\n",
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "\n",
    "# evaluation metrics\n",
    "from scipy import stats\n",
    "from scipy.spatial import distance\n",
    "from sklearn.metrics import matthews_corrcoef\n",
    "from sklearn.metrics import confusion_matrix"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a914547c",
   "metadata": {},
   "source": [
    "### Start time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "id": "c933c023",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'20:46:46'"
      ]
     },
     "execution_count": 161,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#starting time\n",
    "time.strftime(\"%H:%M:%S\", time.localtime())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 195,
   "id": "d0e732c6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "README.md        main.py          \u001b[34mscripts_html\u001b[m\u001b[m\r\n",
      "bechdel.ipynb    \u001b[34mpre_scraped\u001b[m\u001b[m      scripts_html.zip\r\n",
      "classifier.py    requirments.txt  super_woman.png\r\n"
     ]
    }
   ],
   "source": [
    "!ls\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d5733549",
   "metadata": {},
   "source": [
    "\n",
    "\n",
    "<img src=\"super_woman.png\" width=50 height=50 />\n",
    "\n",
    "## **preamble**\n",
    "**bechdel criteria:**\n",
    "- 1. [x] (of film or literature) Has to have at least two women (discrete int) or $\\in \\mathbb{Z}$\n",
    "- 2. [x] Talk to each other in (continuous time) or $\\in \\mathbb{R}$\n",
    "- 3. [x] Talk about somthing other than a man (not.self) (binary) or $\\in \\mathbb{B}$ \n",
    " \n",
    " \n",
    "> here the problem is framed as one of comparing those that meet first condition with two different consecutive female first names in script segments and verifieable ground truth - that is the first condition a prediction of passing overall -- this is to gauge whether more detailed anaylis might be needed such as training an NLP classifier. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 200,
   "id": "8fa8ee8f",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Retrieving folder list\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing file 1sKXHb17Ah6Bl6nVMCUE--SnCGV-rvlzk scripts_html.zip\n",
      "Building directory structure completed\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Retrieving folder list completed\n",
      "Building directory structure\n",
      "Downloading...\n",
      "From: https://drive.google.com/uc?id=1sKXHb17Ah6Bl6nVMCUE--SnCGV-rvlzk\n",
      "To: /Users/fridades/Projects/bechdel/scripts_html.zip\n",
      "100%|██████████████████████████████████████| 16.1M/16.1M [00:01<00:00, 8.99MB/s]\n",
      "Download completed\n",
      "100%|█████████████████████████████████████████| 282/282 [00:16<00:00, 17.12it/s]\n"
     ]
    }
   ],
   "source": [
    "def get_scripts(did=None):\n",
    "    '''this funciton gets scripts using gdown from google drive zip\n",
    "    unzips and preservs file name'''\n",
    "    url = 'https://drive.google.com/drive/folders/1PWoip6Hkl-3WG9Syyd_IxOPeC-FsshDP?usp=sharing'\n",
    "    os.makedirs(did,exist_ok=True)\n",
    "    gdown.download_folder(url,output=None,quiet=False)\n",
    "    fid = Path('scripts_html.zip')\n",
    "    with zipfile.ZipFile(fid, 'r') as zip_fid:\n",
    "        zip_fid.extractall(fid.parent)\n",
    "# path to scripts \n",
    "did = Path('pre_scraped/')\n",
    "os.makedirs(did,exist_ok=True)\n",
    "get_scripts(did)\n",
    "did = Path('scripts_html/')\n",
    "# using path object as itterator to ge file in directory\n",
    "fids = [fid for fid in did.iterdir()]\n",
    "scripts = { }\n",
    "for fid in tqdm(fids):\n",
    "    with open(fid,'r') as handle:\n",
    "        scripts[fid.name.strip(fid.suffix)]=BeautifulSoup(handle, 'html.parser')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 194,
   "id": "87d6a342",
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean(script):\n",
    "    script = [element for element in script.find_all('b')]\n",
    "   \n",
    "    script = [element.get_text() for element in script]\n",
    " \n",
    "    script = [\" \".join(element.split()) for element in script\n",
    "              if \" \".join(element.split()) !='']\n",
    "    return [element.upper() for element in script]\n",
    "\n",
    "# make dict for ease of manipulation later and if need to save as .json\n",
    "names_dict = {script:clean(scripts[script]) for script in scripts}\n",
    "names_dict = {key.strip(', The').upper():names_dict[key] for key in names_dict}\n",
    "#clean names and make upper\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "id": "24109a92",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Downloading...\n",
      "From: https://www.cs.cmu.edu/Groups/AI/areas/nlp/corpora/names/female.txt\n",
      "To: /Users/fridades/Projects/bechdel/female.txt\n",
      "35.8kB [00:00, 58.1MB/s]                                                        \n",
      "Downloading...\n",
      "From: https://www.cs.cmu.edu/Groups/AI/areas/nlp/corpora/names/male.txt\n",
      "To: /Users/fridades/Projects/bechdel/male.txt\n",
      "20.5kB [00:00, 33.6MB/s]                                                        \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'male.txt'"
      ]
     },
     "execution_count": 167,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#get list of first names (man and woman from academic source)\n",
    "women_names ='https://www.cs.cmu.edu/Groups/AI/areas/nlp/corpora/names/female.txt'\n",
    "man_names = 'https://www.cs.cmu.edu/Groups/AI/areas/nlp/corpora/names/male.txt'\n",
    "gdown.download(women_names);gdown.download(man_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "id": "b0740285",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['# List of common female names.', '# Copyright (c) January 1991 by Mark Kantrowitz.', '# 4987 names', '# Thanks to Bill.Ross for about 1000 additional names.', '# Version 1.3 (29-MAR-94)'] \n",
      " ABAGAEL\n",
      "['# List of common male names.', '# Copyright (c) January 1991 by Mark Kantrowitz.', '# 2940 names', '# Thanks to Bill Ross for about 1000 additional names.', '# Version 1.3 (29-MAR-94)'] \n",
      " AAMIR\n"
     ]
    }
   ],
   "source": [
    "def read_text(fid):\n",
    "    '''gets names from txt removing preamble (prints)\n",
    "    prints first name converted to upper'''\n",
    "    with open(fid,'r') as hndl:\n",
    "        txt = hndl.read().splitlines()\n",
    "    preamble = txt[:5]\n",
    "    #convert all names to upper\n",
    "    txt = [name.upper() for name in txt[5:]]\n",
    "    print(preamble,'\\n', txt[1])\n",
    "    return txt\n",
    "\n",
    "# get lists\n",
    "fem_first, man_first = read_text('female.txt'), read_text('male.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "id": "0769948e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# might take a minute to parse and compute (but less time than training an nlp model :-))\n",
    "def bechdel_one(script_names,fems):\n",
    "    '''a generator function that yields whther or not condtion 1\n",
    "    is passed where 2 female names not the same in seqence are \n",
    "    a potential proxy for passing condition 1 of Bechdel test'''\n",
    "    for idx,name in enumerate(script_names[3:]):\n",
    "        if idx+1!=len(script_names):\n",
    "            dialoge = script_names[idx:idx+2]\n",
    "            #not the same person\n",
    "            not_solo = dialoge[0]!=dialoge[1]\n",
    "            # first person is in womens names\n",
    "            a_is_girl = dialoge[0] in fems\n",
    "            # secong perons is in womens names\n",
    "            b_is_girl = dialoge[1] in fems\n",
    "            yield all([not_solo,a_is_girl,b_is_girl])\n",
    "    \n",
    "passes_bechdel_one = {title:any(list(bechdel_one(names_dict[title],fem_first))) for title in names_dict}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "id": "72398578",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 170,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# passed condition one according to boolean process\n",
    "\n",
    "passes_bechdel_one['10 THINGS I HATE ABOUT YOU']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "id": "0f6eb192",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "189"
      ]
     },
     "execution_count": 171,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAQiklEQVR4nO3de5BedX3H8ffHRKkKCpqVQUAXMYK3Gu0OtlUQpFrUKtoqknpBtAY6Mpbx0uJlkNo6xQvasVZoKBmwIqJSRqp4oUhJdUDZQAwBQQHDkBjJKopYLDXw7R/P2eEx7JLNPs9mw8/3a+bM/p7vuX03OfvZs2fPeTZVhSSpLQ+a7wYkScNnuEtSgwx3SWqQ4S5JDTLcJalBC+e7AYBFixbV6OjofLchSQ8oq1at+klVjUw1b4cI99HRUcbHx+e7DUl6QEly83TzvCwjSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkN2iGeUJVaN3rCl+e7Be2g1p38kjnZrmfuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSg7Ya7klWJNmUZG1f7dwkq7tpXZLVXX00ya/65p02h71LkqYxkzcOOxP4BPCpyUJVvXpynOQU4Pa+5W+sqiVD6k+SNAtbDfeqWplkdKp5SQIcATx/yH1JkgYw6DX3A4Fbq+oHfbV9klyV5NIkB063YpJlScaTjE9MTAzYhiSp36DhvhQ4p+/1RuBxVfVM4G3AZ5I8YqoVq2p5VY1V1djIyMiAbUiS+s063JMsBP4UOHeyVlV3VdVPu/Eq4EbgSYM2KUnaNoOcuf8RcF1VrZ8sJBlJsqAbPwFYDNw0WIuSpG01k1shzwEuA/ZLsj7Jm7pZR/Kbl2QADgLWdLdGfgE4tqpuG2K/kqQZmMndMkunqb9hitp5wHmDtyVJGoRPqEpSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJatBM/obqiiSbkqztq52UZEOS1d304r5570pyQ5Lrk/zxXDUuSZreTM7czwQOm6L+sapa0k0XAiR5Cr0/nP3Ubp1PJlkwrGYlSTOz1XCvqpXAbTPc3uHAZ6vqrqr6IXADcMAA/UmSZmGQa+7HJVnTXbbZravtCdzSt8z6rnYfSZYlGU8yPjExMUAbkqQtzTbcTwX2BZYAG4FTtnUDVbW8qsaqamxkZGSWbUiSpjKrcK+qW6vq7qq6Bzidey+9bAD27lt0r64mSdqOZhXuSfboe/kKYPJOmguAI5PslGQfYDHwncFalCRtq4VbWyDJOcDBwKIk64H3AQcnWQIUsA44BqCqrknyOeBaYDPwlqq6e046lyRNa6vhXlVLpyifcT/LfwD4wCBNSZIG4xOqktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoO2Gu5JViTZlGRtX+3DSa5LsibJ+Ul27eqjSX6VZHU3nTaHvUuSpjGTM/czgcO2qF0EPK2qfhf4PvCuvnk3VtWSbjp2OG1KkrbFVsO9qlYCt21R+3pVbe5eXg7sNQe9SZJmaRjX3N8IfKXv9T5JrkpyaZIDp1spybIk40nGJyYmhtCGJGnSQOGe5D3AZuDsrrQReFxVPRN4G/CZJI+Yat2qWl5VY1U1NjIyMkgbkqQtzDrck7wB+BPgNVVVAFV1V1X9tBuvAm4EnjSEPiVJ22BW4Z7kMOCvgZdV1Z199ZEkC7rxE4DFwE3DaFSSNHMLt7ZAknOAg4FFSdYD76N3d8xOwEVJAC7v7ow5CHh/kl8D9wDHVtVtU25YkjRnthruVbV0ivIZ0yx7HnDeoE1JkgbjE6qS1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkho0o3BPsiLJpiRr+2qPSnJRkh90H3fr6kny8SQ3JFmT5Flz1bwkaWozPXM/Ezhsi9oJwMVVtRi4uHsN8CJgcTctA04dvE1J0raYUbhX1Urgti3KhwNndeOzgJf31T9VPZcDuybZYwi9SpJmaJBr7rtX1cZu/GNg9268J3BL33Lru9pvSLIsyXiS8YmJiQHakCRtaSi/UK2qAmob11leVWNVNTYyMjKMNiRJnUHC/dbJyy3dx01dfQOwd99ye3U1SdJ2Mki4XwAc1Y2PAr7YV399d9fM7wO3912+kSRtBwtnslCSc4CDgUVJ1gPvA04GPpfkTcDNwBHd4hcCLwZuAO4Ejh5yz5KkrZhRuFfV0mlmHTrFsgW8ZZCmJEmD8QlVSWqQ4S5JDTLcJalBM7rmvqMbPeHL892CdlDrTn7JfLcgzQvP3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSg2b9lr9J9gPO7Ss9ATgR2BV4MzDR1d9dVRfOdj+SpG0363CvquuBJQBJFgAbgPPp/UHsj1XVR4bRoCRp2w3rssyhwI1VdfOQtidJGsCwwv1I4Jy+18clWZNkRZLdhrQPSdIMDRzuSR4CvAz4fFc6FdiX3iWbjcAp06y3LMl4kvGJiYmpFpEkzdIwztxfBFxZVbcCVNWtVXV3Vd0DnA4cMNVKVbW8qsaqamxkZGQIbUiSJg0j3JfSd0kmyR59814BrB3CPiRJ22DWd8sAJHk48ALgmL7yh5IsAQpYt8U8SdJ2MFC4V9X/AI/eova6gTqSJA3MJ1QlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBg30N1QBkqwD7gDuBjZX1ViSRwHnAqP0/kj2EVX1s0H3JUmamWGduR9SVUuqaqx7fQJwcVUtBi7uXkuStpO5uixzOHBWNz4LePkc7UeSNIVhhHsBX0+yKsmyrrZ7VW3sxj8Gdt9ypSTLkownGZ+YmBhCG5KkSQNfcweeW1UbkjwGuCjJdf0zq6qS1JYrVdVyYDnA2NjYfeZLkmZv4DP3qtrQfdwEnA8cANyaZA+A7uOmQfcjSZq5gcI9ycOT7DI5Bl4IrAUuAI7qFjsK+OIg+5EkbZtBL8vsDpyfZHJbn6mqrya5AvhckjcBNwNHDLgfSdI2GCjcq+om4BlT1H8KHDrItiVJs+cTqpLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGjTrcE+yd5JLklyb5Jokf9XVT0qyIcnqbnrx8NqVJM3EIH9DdTPw9qq6MskuwKokF3XzPlZVHxm8PUnSbMw63KtqI7CxG9+R5HvAnsNqTJI0e0O55p5kFHgm8O2udFySNUlWJNltmnWWJRlPMj4xMTGMNiRJnYHDPcnOwHnA8VX1C+BUYF9gCb0z+1OmWq+qllfVWFWNjYyMDNqGJKnPQOGe5MH0gv3sqvp3gKq6tarurqp7gNOBAwZvU5K0LQa5WybAGcD3quqjffU9+hZ7BbB29u1JkmZjkLtlngO8Drg6yequ9m5gaZIlQAHrgGMG2IckaRYGuVvmm0CmmHXh7NuRJA2DT6hKUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDZqzcE9yWJLrk9yQ5IS52o8k6b7mJNyTLAD+GXgR8BRgaZKnzMW+JEn3NVdn7gcAN1TVTVX1f8BngcPnaF+SpC0snKPt7gnc0vd6PfDs/gWSLAOWdS9/meT6Oerlt80i4Cfz3cSOIh+c7w40BY/RPgMeo4+fbsZchftWVdVyYPl87b9VScaramy++5Cm4zG6fczVZZkNwN59r/fqapKk7WCuwv0KYHGSfZI8BDgSuGCO9iVJ2sKcXJapqs1JjgO+BiwAVlTVNXOxL92Hl7q0o/MY3Q5SVfPdgyRpyHxCVZIaZLhLUoMM9zmS5KQk75jvPiYlOT7Jw+a7D0nbh+H+2+N4wHCXfksY7kOU5D1Jvp/km8B+ffUlSS5PsibJ+Ul26+r7JvlqklVJ/jvJ/l39VUnWJvlukpVT7OfgJP+V5AtJrktydpJ08w5NclWSq5OsSLJTkrcCjwUuSXLJFNtbl+RD3TrfSfLErv7SJN/utvefSXbv6s9LsrqbrkqyS5I9kqzsamuTHDgH/8TaASQZ7Tvuvtcdhw9LcmKSK7r//+V9x+Rbk1zbHf+f7Wr3OYa6+ju7baxJ8rdd7eFJvtx9PaxN8ur5++wfQKrKaQgT8HvA1fTOjh8B3AC8o5u3BnheN34/8I/d+GJgcTd+NvCNbnw1sGc33nWKfR0M3E7v4bAHAZcBzwV+h97bPjypW+5TwPHdeB2waJre1wHv6cavB77UjXfj3juq/gI4pRv/B/CcbrwzvVtq3963jQXALvP9f+I0Z8f6KFB9x8AK4B3Ao/qW+Tfgpd34R8BO3XjX+zmGXkjvNsl0x/WXgIOAPwNO79v2I+f73+CBMHnmPjwHAudX1Z1V9Qu6h7aSPJLeAX1pt9xZwEFJdgb+EPh8ktXAvwB7dMt8CzgzyZvpBeVUvlNV66vqHmA1vS+4/YAfVtX3+/c1w/7P6fv4B914L+BrSa4G3gk8ta+/j3Y/EexaVZvpPbh2dJKTgKdX1R0z3K8emG6pqm9140/TO7k4pPtJ72rg+dx7vKwBzk7yWmBzV5vqGHphN10FXAnsDyymd7LzgiQfTHJgVd2+HT6/BzzDff48CPh5VS3pm54MUFXHAu+l9xYOq5I8eor17+ob383gD6TVFON/Aj5RVU8HjqH3kwFVdTK9M/mHAt9Ksn9VraT3jWQDvW9Mrx+wH+3YtnxApoBPAq/sjpfT6Y4X4CX03gL8WcAVSRZOdQzRO2P/h76vhydW1Rndycqz6IX83yc5cc4/uwYY7sOzEnh5kod21w9fCtCdZfys7xr064BLu7P7HyZ5FUB6ntGN962qb1fVicAEv/k+PffnemB08pr55L668R3ALvez7qv7Pl7WjR/Jve8JdNTkgl1/V1fVB+mdse+f5PHArVV1OvCv9L4Y1a7HJZn8Ce/PgW924590P5W+EiDJg4C9q+oS4G/oHVM7T3UM0Xui/Y3d+iTZM8ljkjwWuLOqPg18GI+tGZm3d4VsTVVdmeRc4LvAJnoH7KSjgNO6WxFvAo7u6q8BTk3yXuDB9N73/rvAh5Mspncmc3FXm0kP/5vkaHqXehZ2PZzWzV4OfDXJj6rqkClW3y3JGno/ESztaid12/oZ8A1gn65+fJJDgHuAa4Cv0Hv/oHcm+TXwS3rX7tWu64G3JFkBXAucSu93NGuBH3Pv8b8A+HR3eTLAx6vq50n+bstjqKruSvJk4LLud7G/BF4LPJHe18Q9wK+Bv9xen+QDmW8/IJKsA8aqyvfY1lYlGaX3S/enzXcvmp6XZSSpQZ65S1KDPHOXpAYZ7pLUIMNdkhpkuEtSgwx3SWrQ/wN02sN0f8NMQAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "passes_one_vector = np.array([passes_bechdel_one[key] for key in passes_bechdel_one])\n",
    "#convert bools to ints\n",
    "passes_one_vector = passes_one_vector.astype(int)\n",
    "len(passes_one_vector)\n",
    "plt.bar(['does not pass','passes'],np.bincount(passes_one_vector))\n",
    "np.count_nonzero(passes_one_vector)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "id": "4c458732",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'20:48:18'"
      ]
     },
     "execution_count": 172,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "time.strftime(\"%H:%M:%S\", time.localtime())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6fe9fd59",
   "metadata": {},
   "source": [
    "####  list of film titles that pass all three conditions from bechdeltest.com (here taken as **ground truth**)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "id": "673a5aa1",
   "metadata": {},
   "outputs": [],
   "source": [
    "bechdel_films = requests.get('https://bechdeltest.com/?list=all').text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "id": "aea8ee84",
   "metadata": {},
   "outputs": [],
   "source": [
    " bechdel_soup = BeautifulSoup(bechdel_films, 'html.parser')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "id": "e1b75c01",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "281"
      ]
     },
     "execution_count": 175,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "criteria = \"[There are two or more women in this movie and they talk to each other about something other than a man]\"\n",
    "films = bechdel_soup.find_all('a')\n",
    "titles_that_pass = [films[idx-2].text for idx,film in enumerate(films) if criteria in str(film.contents)]\n",
    "titles_that_pass = [title.upper().strip('THE ') for title in titles_that_pass if title !='']\n",
    "to_check = [title.strip(', The').upper() for title in list(names_dict.keys())]\n",
    "len(to_check)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "id": "a110e898",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 1*n d array (aka a vector) of those that pass all 1=pass 0=fail\n",
    "pass_all_vector = np.array([i in titles_that_pass for i in to_check]).astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "id": "571927a4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<BarContainer object of 2 artists>"
      ]
     },
     "execution_count": 177,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAARwklEQVR4nO3df5TldV3H8edLMCtRwZg4BNggrhr9Wm0OWYZhlKFmaD9QKiWyVjt4ipNapB2lXyfM0I79wJbYAyYiFnGkJIuIJD2izsK6LCC64HLYbd0df4QYRS68++N+5nBd7rAzc+/swMfn45x75nPf31/vge+89jvf+X7vN1WFJKkvj1rtBiRJk2e4S1KHDHdJ6pDhLkkdMtwlqUMHrnYDAIceemhNT0+vdhuS9IiycePGz1fV1KhpD4twn56eZnZ2drXbkKRHlCR3LDRtn6dlkhyV5JokNye5Kcmvt/oTk1yV5DPt6yGtniTvSLI1yeYkz5zctyJJWozFnHPfA7y2qo4FngWckeRY4Czg6qpaA1zd3gM8H1jTXuuA8ybetSTpIe0z3KtqZ1Vd38Z3A7cARwAnAxe12S4CXtzGJwPvqoHrgIOTHD7pxiVJC1vS1TJJpoFnAB8DDquqnW3S54DD2vgI4M6hxba32t7rWpdkNsns3NzcUvuWJD2ERYd7koOAy4Azq+rLw9Nq8AE1S/qQmqpaX1UzVTUzNTXyj72SpGVaVLgneTSDYL+4qv6+lXfNn25pX3e3+g7gqKHFj2w1SdJ+spirZQJcANxSVW8bmnQFcFobnwa8f6j+inbVzLOAu4ZO30iS9oPFXOf+bODlwI1JNrXaG4BzgPcleSVwB3BKm3Yl8AJgK3APcPokG5Yk7ds+w72qPgxkgcknjpi/gDPG7EuSNIaHxR2q45o+6wOr3YIeprad88LVbkFaFX5wmCR1yHCXpA4Z7pLUIcNdkjpkuEtShwx3SeqQ4S5JHTLcJalDhrskdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nqkOEuSR0y3CWpQ4a7JHVoMQ/I3pBkd5ItQ7VLk2xqr23zz1ZNMp3kf4amvXMFe5ckLWAxj9m7EPhz4F3zhap66fw4ybnAXUPz31ZVayfUnyRpGRbzgOxrk0yPmpYkwCnAj0y4L0nSGMY95348sKuqPjNUOzrJDUk+lOT4hRZMsi7JbJLZubm5MduQJA0bN9xPBS4Zer8TeFJVPQP4DeA9SR4/asGqWl9VM1U1MzU1NWYbkqRhyw73JAcCPwVcOl+rqnur6gttvBG4DXjquE1KkpZmnCP3HwU+VVXb5wtJppIc0MZPBtYAt4/XoiRpqRZzKeQlwEeBpyXZnuSVbdLL+NpTMgDPATa3SyP/Dnh1VX1xgv1KkhZhMVfLnLpA/RdH1C4DLhu/LUnSOLxDVZI6ZLhLUocMd0nqkOEuSR0y3CWpQ4a7JHXIcJekDhnuktQhw12SOmS4S1KHDHdJ6pDhLkkdMtwlqUOGuyR1yHCXpA4Z7pLUIcNdkjq0mMfsbUiyO8mWodrZSXYk2dReLxia9ttJtia5NcmPr1TjkqSFLebI/ULgpBH1t1fV2va6EiDJsQyerfqdbZm/nH9gtiRp/9lnuFfVtcBiH3J9MvDeqrq3qj4LbAWOG6M/SdIyjHPO/TVJNrfTNoe02hHAnUPzbG81SdJ+tNxwPw84BlgL7ATOXeoKkqxLMptkdm5ubpltSJJGWVa4V9Wuqrqvqu4HzueBUy87gKOGZj2y1UatY31VzVTVzNTU1HLakCQtYFnhnuTwobcvAeavpLkCeFmSxyQ5GlgDfHy8FiVJS3XgvmZIcglwAnBoku3Am4ETkqwFCtgGvAqgqm5K8j7gZmAPcEZV3bcinUuSFrTPcK+qU0eUL3iI+f8Q+MNxmpIkjcc7VCWpQ4a7JHXIcJekDhnuktQhw12SOmS4S1KHDHdJ6pDhLkkdMtwlqUOGuyR1yHCXpA4Z7pLUIcNdkjpkuEtShwx3SeqQ4S5JHTLcJalDhrskdWif4Z5kQ5LdSbYM1d6a5FNJNie5PMnBrT6d5H+SbGqvd65g75KkBSzmyP1C4KS9alcB31VV3wN8GvjtoWm3VdXa9nr1ZNqUJC3FPsO9qq4FvrhX7V+qak97ex1w5Ar0Jklapkmcc/8l4J+G3h+d5IYkH0py/EILJVmXZDbJ7Nzc3ATakCTNGyvck7wR2ANc3Eo7gSdV1TOA3wDek+Txo5atqvVVNVNVM1NTU+O0IUnay7LDPckvAj8B/HxVFUBV3VtVX2jjjcBtwFMn0KckaQmWFe5JTgJ+E/jJqrpnqD6V5IA2fjKwBrh9Eo1KkhbvwH3NkOQS4ATg0CTbgTczuDrmMcBVSQCua1fGPAf4vSRfBe4HXl1VXxy5YknSitlnuFfVqSPKFyww72XAZeM2JUkaj3eoSlKHDHdJ6pDhLkkdMtwlqUOGuyR1yHCXpA4Z7pLUIcNdkjpkuEtShwx3SeqQ4S5JHTLcJalDhrskdchwl6QOGe6S1CHDXZI6ZLhLUocWFe5JNiTZnWTLUO2JSa5K8pn29ZBWT5J3JNmaZHOSZ65U85Kk0RZ75H4hcNJetbOAq6tqDXB1ew/wfAYPxl4DrAPOG79NSdJSLCrcq+paYO8HXZ8MXNTGFwEvHqq/qwauAw5OcvgEepUkLdI459wPq6qdbfw54LA2PgK4c2i+7a0mSdpPJvIH1aoqoJayTJJ1SWaTzM7NzU2iDUlSM06475o/3dK+7m71HcBRQ/Md2Wpfo6rWV9VMVc1MTU2N0YYkaW/jhPsVwGltfBrw/qH6K9pVM88C7ho6fSNJ2g8OXMxMSS4BTgAOTbIdeDNwDvC+JK8E7gBOabNfCbwA2ArcA5w+4Z4lSfuwqHCvqlMXmHTiiHkLOGOcpiRJ4/EOVUnqkOEuSR0y3CWpQ4a7JHXIcJekDhnuktQhw12SOmS4S1KHDHdJ6tCi7lCVNJ7psz6w2i3oYWrbOS9ckfV65C5JHTLcJalDhrskdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nq0LJvYkryNODSodKTgTcBBwO/Asy1+huq6srlbkeStHTLDvequhVYC5DkAGAHcDmDB2K/var+ZBINSpKWblKnZU4EbquqOya0PknSGCYV7i8DLhl6/5okm5NsSHLIqAWSrEsym2R2bm5u1CySpGUaO9yTfAPwk8DfttJ5wDEMTtnsBM4dtVxVra+qmaqamZqaGrcNSdKQSRy5Px+4vqp2AVTVrqq6r6ruB84HjpvANiRJSzCJcD+VoVMySQ4fmvYSYMsEtiFJWoKxPs89yWOBHwNeNVT+4yRrgQK27TVNkrQfjBXuVfXfwLfsVXv5WB1JksbmHaqS1CHDXZI6ZLhLUocMd0nqkOEuSR0y3CWpQ4a7JHXIcJekDhnuktQhw12SOmS4S1KHDHdJ6pDhLkkdMtwlqUOGuyR1yHCXpA4Z7pLUIcNdkjo01mP2AJJsA+4G7gP2VNVMkicClwLTDJ6jekpVfWncbUmSFmdSR+7Praq1VTXT3p8FXF1Va4Cr23tJ0n6yUqdlTgYuauOLgBev0HYkSSNMItwL+JckG5Osa7XDqmpnG38OOGzvhZKsSzKbZHZubm4CbUiS5o19zh34oarakeRbgauSfGp4YlVVktp7oapaD6wHmJmZedB0SdLyjX3kXlU72tfdwOXAccCuJIcDtK+7x92OJGnxxgr3JI9N8rj5MfA8YAtwBXBam+004P3jbEeStDTjnpY5DLg8yfy63lNVH0zyCeB9SV4J3AGcMuZ2JElLMFa4V9XtwPeOqH8BOHGcdUuSls87VCWpQ4a7JHXIcJekDhnuktQhw12SOmS4S1KHDHdJ6pDhLkkdMtwlqUOGuyR1yHCXpA4Z7pLUIcNdkjpkuEtShwx3SeqQ4S5JHTLcJalDyw73JEcluSbJzUluSvLrrX52kh1JNrXXCybXriRpMcZ5zN4e4LVVdX17SPbGJFe1aW+vqj8Zvz1J0nIsO9yraiews43vTnILcMSkGpMkLd9EzrknmQaeAXyslV6TZHOSDUkOWWCZdUlmk8zOzc1Nog1JUjN2uCc5CLgMOLOqvgycBxwDrGVwZH/uqOWqan1VzVTVzNTU1LhtSJKGjBXuSR7NINgvrqq/B6iqXVV1X1XdD5wPHDd+m5KkpRjnapkAFwC3VNXbhuqHD832EmDL8tuTJC3HOFfLPBt4OXBjkk2t9gbg1CRrgQK2Aa8aYxuSpGUY52qZDwMZMenK5bcjSZoE71CVpA4Z7pLUIcNdkjpkuEtShwx3SeqQ4S5JHTLcJalDhrskdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nqkOEuSR0y3CWpQ4a7JHXIcJekDhnuktShFQv3JCcluTXJ1iRnrdR2JEkPtiLhnuQA4C+A5wPHMnho9rErsS1J0oOt1JH7ccDWqrq9qv4PeC9w8gptS5K0lwNXaL1HAHcOvd8OfP/wDEnWAeva268kuXWFevl6cyjw+dVu4uEib1ntDjSC++iQMffRb19owkqF+z5V1Xpg/Wptv1dJZqtqZrX7kBbiPrp/rNRpmR3AUUPvj2w1SdJ+sFLh/glgTZKjk3wD8DLgihXaliRpLytyWqaq9iR5DfDPwAHAhqq6aSW2pQfxVJce7txH94NU1Wr3IEmaMO9QlaQOGe6S1CHDfYUkOTvJ61a7j3lJzkzyzavdh6T9w3D/+nEmYLhLXycM9wlK8sYkn07yYeBpQ/W1Sa5LsjnJ5UkOafVjknwwycYk/5Hk6a3+s0m2JPlkkmtHbOeEJP+e5O+SfCrJxUnSpp2Y5IYkNybZkOQxSX4N+DbgmiTXjFjftiR/3Jb5eJKntPqLknysre9fkxzW6j+cZFN73ZDkcUkOT3Jtq21JcvwK/CfWw0CS6aH97pa2H35zkjcl+UT7/79+aJ/8tSQ3t/3/va32oH2o1V/f1rE5ye+22mOTfKD9PGxJ8tLV++4fQarK1wRewPcBNzI4On48sBV4XZu2GfjhNv494E/b+GpgTRt/P/BvbXwjcEQbHzxiWycAdzG4OexRwEeBHwK+kcHHPjy1zfcu4Mw23gYcukDv24A3tvErgH9s40N44IqqXwbObeN/AJ7dxgcxuKT2tUPrOAB43Gr/P/G1Yvv6NFBD+8AG4HXAE4fm+RvgRW38n8Bj2vjgh9iHnsfgMsm0/fofgecAPw2cP7TuJ6z2f4NHwssj98k5Hri8qu6pqi/TbtpK8gQGO/SH2nwXAc9JchDwg8DfJtkE/BVweJvnI8CFSX6FQVCO8vGq2l5V9wObGPzAPQ34bFV9enhbi+z/kqGvP9DGRwL/nORG4PXAdw7197b2G8HBVbWHwY1rpyc5G/juqrp7kdvVI9OdVfWRNn43g4OL57bf9G4EfoQH9pfNwMVJfgHY02qj9qHntdcNwPXA04E1DA52fizJW5IcX1V37Yfv7xHPcF89jwL+q6rWDr2+A6CqXg38DoOPcNiY5FtGLH/v0Pg+xr8hrUaM/wz486r6buBVDH4zoKrOYXAk/03AR5I8vaquZfAPyQ4G/zC9Ysx+9PC29w0yBfwl8DNtfzmftr8AL2TwEeDPBD6R5MBR+xCDI/Y/Gvp5eEpVXdAOVp7JIOT/IMmbVvy764DhPjnXAi9O8k3t/OGLANpRxpeGzkG/HPhQO7r/bJKfBcjA97bxMVX1sap6EzDH135Oz0O5FZieP2c+v602vht43EMs+9Khrx9t4yfwwGcCnTY/Y+vvxqp6C4Mj9qcn+XZgV1WdD/w1gx9G9etJSeZ/w/s54MNt/Pn2W+nPACR5FHBUVV0D/BaDfeqgUfsQgzvaf6ktT5Ijknxrkm8D7qmqdwNvxX1rUVbtUyF7U1XXJ7kU+CSwm8EOO+804J3tUsTbgdNb/eeB85L8DvBoBp97/0ngrUnWMDiSubrVFtPD/yY5ncGpngNbD+9sk9cDH0zyn1X13BGLH5JkM4PfCE5ttbPbur4E/BtwdKufmeS5wP3ATcA/Mfj8oNcn+SrwFQbn7tWvW4EzkmwAbgbOY/A3mi3A53hg/z8AeHc7PRngHVX1X0l+f+99qKruTfIdwEfb32K/AvwC8BQGPxP3A18FfnV/fZOPZH78gEiyDZipKj9jW/uUZJrBH92/a7V70cI8LSNJHfLIXZI65JG7JHXIcJekDhnuktQhw12SOmS4S1KH/h+myzgO8athRQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.bar(['does not pass','passes'],np.bincount(pass_all_vector))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "id": "29b45896",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "86"
      ]
     },
     "execution_count": 178,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.count_nonzero(pass_all_vector)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8619a886",
   "metadata": {},
   "source": [
    "#### PLCC vanilla correlation coeff (can be used for binary and give phi coefficinet) to compare method used to derive condition one with ground truth (human rated unambiguous passes of Bechdel test\n",
    "$$\\begin{equation}\n",
    "r = \\frac{{}\\sum_{i=1}^{n} (x_i - \\overline{x})(y_i -\\overline{y})}{\\sqrt{\\sum_{i=1}^{n} (x_i -\\overline{x})^2(y_i - \\overline{y})^2}}\n",
    "\\label{PLCC} \n",
    "\\end{equation}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "id": "f8624149",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'R value = -0.080 P value 0.183'"
      ]
     },
     "execution_count": 179,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pearson_r, p_val = stats.pearsonr(passes_one_vector,pass_all_vector)\n",
    "f'R value = {pearson_r:.3f} P value {p_val:.3f}'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3e6ef066",
   "metadata": {},
   "source": [
    "#### Shows that not very significant prediction and that either the method of using sequential female names is not very good or that it is a poor proxy for passing the Bechdel test"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0f64e8d3",
   "metadata": {},
   "source": [
    "#### hack proof using mathews (specifically binary) PLCC vanilla correlation coeff (can be used for binary as and give phi coefficinet)\n",
    "\n",
    "$$\\begin{equation}\n",
    "    MMC = \\frac{TP\\times TN - FP \\times FN}{\\sqrt{(TP+FP)(TP+FN)(TN+FP)(TN+FN)}}\n",
    "\\end{equation}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "id": "b14c5dcd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "' Mathews coef = -0.080'"
      ]
     },
     "execution_count": 180,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "f' Mathews coef = {matthews_corrcoef(pass_all_vector,passes_one_vector):.3f}'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bc2fa828",
   "metadata": {},
   "source": [
    "#### however perhaps more appropriate as shows 'set similarity' is Jacard's Distance (a bit like SSIM but for binary vectors)\n",
    "$$\\begin{equation}\n",
    "    d_j(A,B) = 1-J(A,B) = \\frac{| A \\cup B| - | A \\cap B|}{| A \\cup B|}\n",
    "\\end{equation}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "id": "ed725eb0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "' Jackards Dist = 0.761'"
      ]
     },
     "execution_count": 181,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "f' Jackards Dist = {distance.jaccard(pass_all_vector,passes_one_vector):.3f}'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "00b9917c",
   "metadata": {},
   "source": [
    "#### shows significant distance however about 1/3 of sample space is same \n",
    "> therefore while poor correlation (between those that pass condition 1 of bechdel and are in ground truth) ther is some promise that those that meet two fmale charachter names being used in seqece maight be good a prediction condition one but that the textuall content accounts for faliure of the Bechdel test after this (indicates that textuall analysis is needed)!!"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7a9d75c0",
   "metadata": {},
   "source": [
    "#### Show if condition of one by names method is a an accurate predictory of all three. \n",
    "\n",
    "$$\\begin{equation}\n",
    "    Accuracy = \\frac{TP+TN}{TP+TN+FP+FN}\n",
    "\\end{equation}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "id": "75a73404",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'accuracy = 0.399'"
      ]
     },
     "execution_count": 182,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tn, fp, fn, tp = confusion_matrix(pass_all_vector,passes_one_vector).ravel()\n",
    "\n",
    "f'accuracy = {sum([tp,tn])/sum([tn,fp,fn,tp]):.3f}'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "id": "62bc024a",
   "metadata": {},
   "outputs": [],
   "source": [
    "main_films = requests.get('https://imsdb.com/all-scripts.html').text\n",
    "main_films = BeautifulSoup(main_films, 'html.parser')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "id": "de266def",
   "metadata": {},
   "outputs": [],
   "source": [
    "script_urls = [ ]\n",
    "for i in main_films.find_all('a'):\n",
    "    if '/Movie Scripts/' in i['href']:\n",
    "        if i.text.strip(' ,The').upper() not in passes_bechdel_one:\n",
    "            url = 'https://imsdb.com'+str(i['href'])\n",
    "            script_urls.append(url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "id": "bec71d4f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'RESERVOIR DOGS SCRIP'"
      ]
     },
     "execution_count": 185,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "script_urls[0].split('/')[-1].strip('.html').strip(', The').upper()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "id": "816ae304",
   "metadata": {},
   "outputs": [],
   "source": [
    "'https://imsdb.com/Movie%20Scripts/Four%20Rooms%20Script.html'\n",
    "def get_script(parent_page):\n",
    "    '''gets script form parent html (script page \n",
    "    rather thatn script itself)'''\n",
    "    one_film = requests.get(parent_page).text\n",
    "    one_film = BeautifulSoup(one_film, 'html.parser')\n",
    "    for i in one_film.find_all('a'):\n",
    "        if 'Read' in i.text:\n",
    "            # get one script\n",
    "            one_script = requests.get(\n",
    "                'https://imsdb.com/Movie Scripts'+i['href']).text\n",
    "            return BeautifulSoup(one_script, 'html.parser')\n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "56ceb597",
   "metadata": {},
   "source": [
    "#### probably a bit exaggerated on the evasion of web crawl blocking but illustrates  some techniques for this"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "id": "2c94ab51",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|                                                     | 0/9 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GREMLINS 2 SCRIP not parsed\n",
      "FURY SCRIP not parsed\n",
      "CONTACT SCRIP not parsed\n",
      "DONNIE DARKO SCRIP not parsed\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 11%|█████                                        | 1/9 [00:48<06:29, 48.70s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WHAT ABOUT BOB? SCRIP not parsed\n",
      "O BROTHER WHERE ART THOU? SCRIP not parsed\n",
      "BATMAN FOREVER SCRIP not parsed\n",
      "BATMAN RETURNS SCRIP not parsed\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 22%|██████████                                   | 2/9 [01:36<05:38, 48.38s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BATMAN BEGINS SCRIP not parsed\n",
      "HARRY POTTER AND THE HALF-BLOOD PRINCE SCRIP not parsed\n",
      "OUTBREAK SCRIP not parsed\n",
      "EXECUTIVE DECISION SCRIP not parsed\n",
      "HARRY POTTER AND THE GOBLET OF FIRE SCRIP not parsed\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 33%|███████████████                              | 3/9 [02:25<04:51, 48.53s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "VALENTINE'S DAY SCRIP not parsed\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 44%|████████████████████                         | 4/9 [03:14<04:04, 48.85s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "UNFORGIVEN SCRIP not parsed\n",
      "MATCHSTICK MEN SCRIP not parsed\n",
      "HARRY POTTER AND THE CHAMBER OF SECRETS SCRIP not parsed\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 56%|█████████████████████████                    | 5/9 [04:04<03:16, 49.10s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GOODFELLAS SCRIP not parsed\n",
      "LETHAL WEAPON SCRIP not parsed\n",
      "CASABLANCA SCRIP not parsed\n",
      "BATMAN AND ROBIN SCRIP not parsed\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 67%|██████████████████████████████               | 6/9 [04:54<02:28, 49.43s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EYES WIDE SHUT SCRIP not parsed\n",
      "GINGER SNAPS SCRIP not parsed\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 78%|███████████████████████████████████          | 7/9 [05:46<01:39, 50.00s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WHO FRAMED ROGER RABBIT? SCRIP not parsed\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 89%|████████████████████████████████████████     | 8/9 [06:33<00:49, 49.38s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DARK KNIGHT, THE SCRIP not parsed\n",
      "HARRY POTTER AND THE DEATHLY HALLOWS PART 1 SCRIP not parsed\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████| 9/9 [07:25<00:00, 49.46s/it]\n"
     ]
    }
   ],
   "source": [
    "script_urls = np.array(script_urls)\n",
    "# random shuffle urls array so (evasion of blocking)\n",
    "np.random.shuffle(script_urls)\n",
    "constant = 100\n",
    "#batching scripts into 100 to take a \n",
    "#longer break of random time every 100 scrapes\n",
    "batches = [script_urls[i-constant:i] \n",
    "           for i in range(constant,len(script_urls),constant)]\n",
    "\n",
    "new_scripts =  { }\n",
    "for batch in tqdm(batches):\n",
    "    # random sleep of uniform probability (evasion of blocking)\n",
    "    for url in batch:\n",
    "        #sleep_for = sum(np.random.random_sample(1))*0.5\n",
    "        # micro sleep to emulate unpredictable behaviour\n",
    "        #time.sleep(sleep_for)\n",
    "        title = url.split('/')[-1].strip('.html').strip(', The').upper()\n",
    "        # try as some pages migh be irregular/not parsed etc\n",
    "        try:\n",
    "            script = clean(get_script(url))\n",
    "            new_scripts[title]=script\n",
    "        except:\n",
    "            print(title, 'not parsed')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "id": "e5a72057",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "866"
      ]
     },
     "execution_count": 146,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(new_scripts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "id": "90395cbe",
   "metadata": {},
   "outputs": [],
   "source": [
    "new_script = {key.replace('THE SCRIP','').replace('SCRIP',''):new_scripts[key] for key in new_scripts}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "id": "002dac64",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "there are now 1147 scripts scraped\n"
     ]
    }
   ],
   "source": [
    "all_scripts = {**names_dict, **new_script}\n",
    "print(f'there are now {len(all_scripts)} scripts scraped')\n",
    "passes_bechdel_one = {title:any(list(bechdel_one(all_scripts[title],fem_first))) for title in all_scripts}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "id": "7a4e06d6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "748"
      ]
     },
     "execution_count": 149,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAASBklEQVR4nO3df7DldV3H8edLVtQUWX5cd3B3bZnYJMsR8Y5h/nZHRzBdZlLSfrBDW1sNZUxpbuWYljNBTVn0g1qFWtRUohg2JYsWjHQEvQguCCJXgtndgL0arD8YNfLdH+ezw2G9u/fcvb/gw/Mxc+Z8vu/v53u+nwPf87rf+7nf79lUFZKkvjxuqQcgSZp/hrskdchwl6QOGe6S1CHDXZI6tGypBwBw7LHH1po1a5Z6GJL0qHL99dd/parGplv3iAj3NWvWMDExsdTDkKRHlSR3HWid0zKS1CHDXZI6ZLhLUocMd0nqkOEuSR0y3CWpQ4a7JHXIcJekDhnuktShR8QdqlLP1mz+2FIPQY9gd577mgV5Xc/cJalDhrskdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nqkOEuSR0y3CWpQzOGe5JnJrlx6PG1JOckOTrJlUlub89Htf5Jcn6SySQ7kpy88G9DkjRsxnCvqtuq6qSqOgl4HvAAcBmwGdheVWuB7W0Z4FRgbXtsAi5YgHFLkg5ittMy64AvV9VdwHpga6tvBU5v7fXAxTVwLbA8yXHzMVhJ0mhmG+5vBD7U2iuq6u7WvgdY0dorgZ1D2+xqtYdJsinJRJKJqampWQ5DknQwI4d7ksOB1wH/sP+6qiqgZrPjqtpSVeNVNT42NjabTSVJM5jNmfupwOeq6t62fO++6Zb2vKfVdwOrh7Zb1WqSpEUym3B/Ew9NyQBsAza09gbg8qH6me2qmVOAvUPTN5KkRTDSP9aR5MnAK4FfHCqfC1ySZCNwF3BGq18BnAZMMriy5qx5G60kaSQjhXtVfRM4Zr/aVxlcPbN/3wLOnpfRSZIOiXeoSlKHDHdJ6pDhLkkdMtwlqUOGuyR1yHCXpA4Z7pLUIcNdkjpkuEtShwx3SeqQ4S5JHTLcJalDhrskdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nqkOEuSR0aKdyTLE9yaZIvJrk1yQuSHJ3kyiS3t+ejWt8kOT/JZJIdSU5e2LcgSdrfqGfufwZ8vKpOBJ4D3ApsBrZX1Vpge1sGOBVY2x6bgAvmdcSSpBnNGO5JjgReAlwIUFXfqar7gfXA1tZtK3B6a68HLq6Ba4HlSY6b53FLkg5ilDP344Ep4G+T3JDkfUmeDKyoqrtbn3uAFa29Etg5tP2uVnuYJJuSTCSZmJqaOvR3IEn6HqOE+zLgZOCCqnou8E0emoIBoKoKqNnsuKq2VNV4VY2PjY3NZlNJ0gxGCfddwK6quq4tX8og7O/dN93Snve09buB1UPbr2o1SdIimTHcq+oeYGeSZ7bSOuAWYBuwodU2AJe39jbgzHbVzCnA3qHpG0nSIlg2Yr9fBT6Y5HDgDuAsBj8YLkmyEbgLOKP1vQI4DZgEHmh9JUmLaKRwr6obgfFpVq2bpm8BZ89tWJKkufAOVUnqkOEuSR0y3CWpQ4a7JHXIcJekDhnuktQhw12SOmS4S1KHDHdJ6pDhLkkdMtwlqUOGuyR1yHCXpA4Z7pLUIcNdkjpkuEtShwx3SeqQ4S5JHRop3JPcmeSmJDcmmWi1o5NcmeT29nxUqyfJ+Ukmk+xIcvJCvgFJ0veazZn7y6vqpKra92+pbga2V9VaYHtbBjgVWNsem4AL5muwkqTRzGVaZj2wtbW3AqcP1S+ugWuB5UmOm8N+JEmzNGq4F/BvSa5PsqnVVlTV3a19D7CitVcCO4e23dVqD5NkU5KJJBNTU1OHMHRJ0oEsG7Hfi6pqd5KnAVcm+eLwyqqqJDWbHVfVFmALwPj4+Ky2lSQd3Ehn7lW1uz3vAS4Dng/cu2+6pT3vad13A6uHNl/VapKkRTJjuCd5cpIj9rWBVwE3A9uADa3bBuDy1t4GnNmumjkF2Ds0fSNJWgSjTMusAC5Lsq//31fVx5N8FrgkyUbgLuCM1v8K4DRgEngAOGveRy1JOqgZw72q7gCeM039q8C6aeoFnD0vo5MkHRLvUJWkDhnuktQhw12SOmS4S1KHDHdJ6pDhLkkdMtwlqUOGuyR1yHCXpA4Z7pLUIcNdkjpkuEtShwx3SeqQ4S5JHTLcJalDhrskdchwl6QOGe6S1CHDXZI6NHK4JzksyQ1JPtqWj09yXZLJJB9JcnirP6EtT7b1axZo7JKkA5jNmfuvAbcOLZ8HvKeqTgDuAza2+kbgvlZ/T+snSVpEI4V7klXAa4D3teUArwAubV22Aqe39vq2TFu/rvWXJC2SZSP2+1PgN4Ej2vIxwP1V9WBb3gWsbO2VwE6Aqnowyd7W/yvDL5hkE7AJ4BnPeMYhDh/WbP7YIW+r/t157muWegjSkpjxzD3JjwN7qur6+dxxVW2pqvGqGh8bG5vPl5akx7xRztxfCLwuyWnAE4GnAn8GLE+yrJ29rwJ2t/67gdXAriTLgCOBr877yCVJBzTjmXtV/VZVraqqNcAbgauq6qeBq4HXt24bgMtbe1tbpq2/qqpqXkctSTqouVzn/jbg15NMMphTv7DVLwSOafVfBzbPbYiSpNka9Q+qAFTVJ4BPtPYdwPOn6fMt4A3zMDZJ0iHyDlVJ6pDhLkkdMtwlqUOGuyR1yHCXpA4Z7pLUIcNdkjpkuEtShwx3SeqQ4S5JHTLcJalDhrskdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nqkOEuSR0y3CWpQzOGe5InJvlMks8n+UKSd7X68UmuSzKZ5CNJDm/1J7TlybZ+zQK/B0nSfkY5c/828Iqqeg5wEvDqJKcA5wHvqaoTgPuAja3/RuC+Vn9P6ydJWkQzhnsNfKMtPr49CngFcGmrbwVOb+31bZm2fl2SzNeAJUkzG2nOPclhSW4E9gBXAl8G7q+qB1uXXcDK1l4J7ARo6/cCx0zzmpuSTCSZmJqamtObkCQ93EjhXlX/V1UnAauA5wMnznXHVbWlqsaranxsbGyuLydJGjKrq2Wq6n7gauAFwPIky9qqVcDu1t4NrAZo648Evjofg5UkjWaUq2XGkixv7ScBrwRuZRDyr2/dNgCXt/a2tkxbf1VV1TyOWZI0g2Uzd+E4YGuSwxj8MLikqj6a5Bbgw0neDdwAXNj6Xwi8P8kk8D/AGxdg3JKkg5gx3KtqB/Dcaep3MJh/37/+LeAN8zI6SdIh8Q5VSeqQ4S5JHTLcJalDhrskdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nqkOEuSR0y3CWpQ4a7JHXIcJekDhnuktQhw12SOmS4S1KHDHdJ6pDhLkkdGuUfyF6d5OoktyT5QpJfa/Wjk1yZ5Pb2fFSrJ8n5SSaT7Ehy8kK/CUnSw41y5v4g8BtV9SzgFODsJM8CNgPbq2otsL0tA5wKrG2PTcAF8z5qSdJBzRjuVXV3VX2utb8O3AqsBNYDW1u3rcDprb0euLgGrgWWJzluvgcuSTqwWc25J1kDPBe4DlhRVXe3VfcAK1p7JbBzaLNdrbb/a21KMpFkYmpqarbjliQdxMjhnuQpwD8C51TV14bXVVUBNZsdV9WWqhqvqvGxsbHZbCpJmsFI4Z7k8QyC/YNV9U+tfO++6Zb2vKfVdwOrhzZf1WqSpEUyytUyAS4Ebq2qPxlatQ3Y0NobgMuH6me2q2ZOAfYOTd9IkhbBshH6vBD4WeCmJDe22m8D5wKXJNkI3AWc0dZdAZwGTAIPAGfN54AlSTObMdyr6pNADrB63TT9Czh7juOSJM2Bd6hKUocMd0nqkOEuSR0y3CWpQ4a7JHXIcJekDhnuktQhw12SOmS4S1KHDHdJ6pDhLkkdMtwlqUOGuyR1yHCXpA4Z7pLUIcNdkjpkuEtShwx3SeqQ4S5JHZox3JNclGRPkpuHakcnuTLJ7e35qFZPkvOTTCbZkeTkhRy8JGl6o5y5/x3w6v1qm4HtVbUW2N6WAU4F1rbHJuCC+RmmJGk2Zgz3qroG+J/9yuuBra29FTh9qH5xDVwLLE9y3DyNVZI0okOdc19RVXe39j3AitZeCewc6rer1b5Hkk1JJpJMTE1NHeIwJEnTmfMfVKuqgDqE7bZU1XhVjY+Njc11GJKkIYca7vfum25pz3tafTeweqjfqlaTJC2iQw33bcCG1t4AXD5UP7NdNXMKsHdo+kaStEiWzdQhyYeAlwHHJtkF/C5wLnBJko3AXcAZrfsVwGnAJPAAcNYCjFmSNIMZw72q3nSAVeum6VvA2XMdlCRpbrxDVZI6ZLhLUocMd0nqkOEuSR0y3CWpQ4a7JHXIcJekDhnuktQhw12SOmS4S1KHDHdJ6pDhLkkdMtwlqUOGuyR1yHCXpA4Z7pLUIcNdkjpkuEtShwx3SerQgoR7klcnuS3JZJLNC7EPSdKBzXu4JzkM+EvgVOBZwJuSPGu+9yNJOrCFOHN/PjBZVXdU1XeADwPrF2A/kqQDWLYAr7kS2Dm0vAv40f07JdkEbGqL30hy2wKM5bHoWOArSz2IR4qct9Qj0DQ8RofM8Rj9/gOtWIhwH0lVbQG2LNX+e5VkoqrGl3oc0oF4jC6OhZiW2Q2sHlpe1WqSpEWyEOH+WWBtkuOTHA68Edi2APuRJB3AvE/LVNWDSX4F+FfgMOCiqvrCfO9HB+RUlx7pPEYXQapqqccgSZpn3qEqSR0y3CWpQ4b7AkjyziRvWepx7JPknCTft9TjkLR4DPfHhnMAw116DDHc50mS30nypSSfBJ45VD8pybVJdiS5LMlRrf4DST6e5Pok/5nkxFZ/Q5Kbk3w+yTXT7OdlST6R5NIkX0zywSRp69YluSHJTUkuSvKEJG8Gng5cneTqaV7vziR/2Lb5TJITWv21Sa5rr/fvSVa0+kuT3NgeNyQ5IslxSa5ptZuTvHgB/hPrESLJmqFj79Z2LH5fknck+Ww7BrYMHZdvTnJL+wx8uNW+5zhq9be219iR5F2t9uQkH2ufiZuT/OTSvftHkaryMccH8DzgJgZnx08FJoG3tHU7gJe29u8Bf9ra24G1rf2jwFWtfROwsrWXT7OvlwF7Gdwc9jjg08CLgCcy+NqHH2z9LgbOae07gWMPMPY7gd9p7TOBj7b2UTx0NdXPA3/c2v8MvLC1n8LgctrfGHqNw4Ajlvr/iY8FPd7XADV0HFwEvAU4eqjP+4HXtvZ/A09o7eUHOY5exeAyybRj+6PAS4CfAN479NpHLvV/g0fDwzP3+fFi4LKqeqCqvka7aSvJkQwO5v9o/bYCL0nyFODHgH9IciPwN8Bxrc+ngL9L8gsMgnI6n6mqXVX1XeBGBh+2ZwL/VVVfGt7XiOP/0NDzC1p7FfCvSW4C3gr88ND4/qT9RrC8qh5kcOPaWUneCTy7qr4+4n716LWzqj7V2h9gcILx8vbb3k3AK3jomNkBfDDJzwAPttp0x9Gr2uMG4HPAicBaBic8r0xyXpIXV9XeRXh/j3qG+9J4HHB/VZ009PghgKr6JeDtDL7C4fokx0yz/beH2v/H3G9Gq2nafw78RVU9G/hFBr8ZUFXnMjiTfxLwqSQnVtU1DH6Q7Gbwg+nMOY5Hj3z73yBTwF8Br2/HzHtpxwzwGgZfA34y8Nkky6Y7jhicsf/B0GfihKq6sJ2wnMwg5N+d5B0L/u46YLjPj2uA05M8qc0dvhagnWHcNzQH/bPAf7Sz+/9K8gaADDyntX+gqq6rqncAUzz8e3oO5jZgzb458337au2vA0ccZNufHHr+dGsfyUPfCbRhX8c2vpuq6jwGZ+wnJvl+4N6qei/wPgYfRPXtGUn2/Zb3U8AnW/sr7TfT1wMkeRywuqquBt7G4Lh6ynTHEYO72n+ubU+SlUmeluTpwANV9QHgj/D4GsmSfStkT6rqc0k+Anwe2MPgYN1nA/DX7VLEO4CzWv2ngQuSvB14PIPvvf888EdJ1jI4i9neaqOM4VtJzmIw1bOsjeGv2+otwMeT/HdVvXyazY9KsoPBbwRvarV3tte6D7gKOL7Vz0nycuC7wBeAf2Hw/UFvTfK/wDcYzN2rb7cBZye5CLgFuIDB32luBu7hoc/AYcAH2hRlgPOr6v4kv7//cVRV307yQ8Cn299ivwH8DHACg8/Fd4H/BX55sd7ko5lfP/AYl+ROYLyq/H5tjSTJGgZ/eP+RpR6LDsxpGUnqkGfuktQhz9wlqUOGuyR1yHCXpA4Z7pLUIcNdkjr0/w9sPHPTCKJEAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "passes_one_vector = np.array([passes_bechdel_one[key] for key in passes_bechdel_one])\n",
    "#convert bools to ints\n",
    "passes_one_vector = passes_one_vector.astype(int)\n",
    "len(passes_one_vector)\n",
    "plt.bar(['does not pass','passes'],np.bincount(passes_one_vector))\n",
    "np.count_nonzero(passes_one_vector)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "id": "d1322fe3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<BarContainer object of 2 artists>"
      ]
     },
     "execution_count": 150,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD4CAYAAAAAczaOAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAQFUlEQVR4nO3de7CdVX3G8e8DERRREuCUwST1MJKKVEelGcRSr2mpQG2YKXipSobSpu3QKkWtaXXEamcK2orSCzYKNVTGS6kOVK0ODSjVESRcTLiIZBBMIpejAmoZL5Rf/9grZRtPIDk72Sewvp+ZPXu9613v+66drP3sd6+9331SVUiS+rDbbHdAkjQ+hr4kdcTQl6SOGPqS1BFDX5I6Mme2O/Bw9t9//5qcnJztbkjSo8rVV1/9naqamG7dLh36k5OTrFmzZra7IUmPKklu39o6p3ckqSOGviR1xNCXpI4Y+pLUEUNfkjpi6EtSRwx9SeqIoS9JHTH0Jakju/QVuaOaXPGZ2e6CdlG3nXHsbHdBmhWe6UtSRwx9SeqIoS9JHTH0Jakjhr4kdcTQl6SOGPqS1BFDX5I68oihn+S8JHcnuX6obt8klyS5pd3Pa/VJcnaS9UnWJjlsaJtlrf0tSZbtnIcjSXo423Km/2HgZVvUrQBWV9UiYHVbBjgaWNRuy4FzYPAiAZwOPA84HDh98wuFJGl8HjH0q+py4HtbVC8FVrXyKuC4ofrza+AKYG6SA4HfBC6pqu9V1T3AJfz8C4kkaSeb6Zz+AVV1RyvfCRzQyvOBDUPtNra6rdX/nCTLk6xJsmZqamqG3ZMkTWfkD3KrqoDaAX3ZvL+VVbW4qhZPTEzsqN1Kkph56N/Vpm1o93e3+k3AwqF2C1rd1uolSWM009C/GNj8DZxlwEVD9Se2b/EcAdzXpoE+DxyVZF77APeoVidJGqNH/D39JB8FXgzsn2Qjg2/hnAF8IsnJwO3AK1rzzwLHAOuB+4GTAKrqe0neBVzV2r2zqrb8cFiStJM9YuhX1au3smrJNG0LOGUr+zkPOG+7eidJ2qG8IleSOmLoS1JHDH1J6oihL0kdMfQlqSOGviR1xNCXpI4Y+pLUEUNfkjpi6EtSRwx9SeqIoS9JHTH0Jakjhr4kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSOGviR1ZKTQT/JnSW5Icn2SjyZ5fJKDklyZZH2SjyfZo7Xdsy2vb+snd8gjkCRtsxmHfpL5wOuBxVX1TGB34FXAmcBZVXUwcA9wctvkZOCeVn9WaydJGqNRp3fmAE9IMgfYC7gDeClwYVu/CjiulZe2Zdr6JUky4vElSdthxqFfVZuAvwW+xSDs7wOuBu6tqgdas43A/FaeD2xo2z7Q2u+35X6TLE+yJsmaqampmXZPkjSNUaZ35jE4ez8IeArwROBlo3aoqlZW1eKqWjwxMTHq7iRJQ0aZ3vl14JtVNVVVPwU+CRwJzG3TPQALgE2tvAlYCNDW7wN8d4TjS5K20yih/y3giCR7tbn5JcCNwGXA8a3NMuCiVr64LdPWX1pVNcLxJUnbaZQ5/SsZfCB7DbCu7Wsl8BbgtCTrGczZn9s2ORfYr9WfBqwYod+SpBmY88hNtq6qTgdO36L6VuDwadr+CDhhlONJkkbjFbmS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSOGviR1xNCXpI4Y+pLUEUNfkjpi6EtSRwx9SeqIoS9JHTH0Jakjhr4kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpIyOFfpK5SS5M8vUkNyV5fpJ9k1yS5JZ2P6+1TZKzk6xPsjbJYTvmIUiSttWoZ/rvBz5XVYcAzwZuAlYAq6tqEbC6LQMcDSxqt+XAOSMeW5K0nWYc+kn2AV4InAtQVT+pqnuBpcCq1mwVcFwrLwXOr4ErgLlJDpzp8SVJ22+UM/2DgCngX5Jcm+RDSZ4IHFBVd7Q2dwIHtPJ8YMPQ9htb3c9IsjzJmiRrpqamRuieJGlLo4T+HOAw4Jyqei7wPzw0lQNAVRVQ27PTqlpZVYuravHExMQI3ZMkbWmU0N8IbKyqK9vyhQxeBO7aPG3T7u9u6zcBC4e2X9DqJEljMuPQr6o7gQ1Jnt6qlgA3AhcDy1rdMuCiVr4YOLF9i+cI4L6haSBJ0hjMGXH7PwUuSLIHcCtwEoMXkk8kORm4HXhFa/tZ4BhgPXB/aytJGqORQr+qrgMWT7NqyTRtCzhllONJkkbjFbmS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSOGviR1xNCXpI4Y+pLUEUNfkjpi6EtSRwx9SeqIoS9JHTH0Jakjhr4kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpIyOHfpLdk1yb5NNt+aAkVyZZn+TjSfZo9Xu25fVt/eSox5YkbZ8dcab/BuCmoeUzgbOq6mDgHuDkVn8ycE+rP6u1kySN0Uihn2QBcCzwobYc4KXAha3JKuC4Vl7almnrl7T2kqQxGfVM/33AnwMPtuX9gHur6oG2vBGY38rzgQ0Abf19rf3PSLI8yZoka6ampkbsniRp2IxDP8lvAXdX1dU7sD9U1cqqWlxViycmJnbkriWpe3NG2PZI4LeTHAM8Hngy8H5gbpI57Wx+AbCptd8ELAQ2JpkD7AN8d4TjS5K204zP9KvqL6pqQVVNAq8CLq2q1wCXAce3ZsuAi1r54rZMW39pVdVMjy9J2n4743v6bwFOS7KewZz9ua3+XGC/Vn8asGInHFuS9DBGmd75f1X1BeALrXwrcPg0bX4EnLAjjidJmhmvyJWkjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSOGviR1xNCXpI4Y+pLUEUNfkjpi6EtSRwx9SeqIoS9JHTH0Jakjhr4kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdmXHoJ1mY5LIkNya5IckbWv2+SS5Jcku7n9fqk+TsJOuTrE1y2I56EJKkbTPKmf4DwBur6lDgCOCUJIcCK4DVVbUIWN2WAY4GFrXbcuCcEY4tSZqBGYd+Vd1RVde08g+Am4D5wFJgVWu2CjiulZcC59fAFcDcJAfO9PiSpO23Q+b0k0wCzwWuBA6oqjvaqjuBA1p5PrBhaLONrW7LfS1PsibJmqmpqR3RPUlSM3LoJ9kb+Hfg1Kr6/vC6qiqgtmd/VbWyqhZX1eKJiYlRuydJGjJS6Cd5HIPAv6CqPtmq79o8bdPu7271m4CFQ5svaHWSpDEZ5ds7Ac4Fbqqq9w6tuhhY1srLgIuG6k9s3+I5ArhvaBpIkjQGc0bY9kjgdcC6JNe1ur8EzgA+keRk4HbgFW3dZ4FjgPXA/cBJIxxbkjQDMw79qvoSkK2sXjJN+wJOmenxJEmj84pcSeqIoS9JHTH0Jakjhr4kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6sgofy5R0ogmV3xmtrugXdRtZxy7U/brmb4kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOjL20E/ysiQ3J1mfZMW4jy9JPRtr6CfZHfhH4GjgUODVSQ4dZx8kqWfjPtM/HFhfVbdW1U+AjwFLx9wHSerWuH9Pfz6wYWh5I/C84QZJlgPL2+IPk9w8pr491u0PfGe2O7GryJmz3QNNwzE6ZMQx+tStrdjl/ohKVa0EVs52Px5rkqypqsWz3Q9paxyj4zHu6Z1NwMKh5QWtTpI0BuMO/auARUkOSrIH8Crg4jH3QZK6Ndbpnap6IMmfAJ8HdgfOq6obxtmHjjllpl2dY3QMUlWz3QdJ0ph4Ra4kdcTQl6SOGPpjluQdSd402/3YLMmpSfaa7X5IGg9DX6cChr7UCUN/DJK8Nck3knwJePpQ/XOSXJFkbZJPJZnX6p+W5HNJrk7y30kOafUnJLk+ydeSXD7NcV6c5AtJLkzy9SQXJElbtyTJtUnWJTkvyZ5JXg88BbgsyWXT7O+2JO9u23w1ycGt/uVJrmz7+68kB7T6FyW5rt2uTfKkJAcmubzVXZ/kBTvhn1i7gCSTQ+PupjYO90ry9iRXtf//lUNj8vVJbmzj/2Ot7ufGUKt/c9vH2iR/1eqemOQz7flwfZJXzt6jfxSpKm878Qb8CrCOwdn0k4H1wJvaurXAi1r5ncD7Wnk1sKiVnwdc2srrgPmtPHeaY70YuI/BRW+7AV8Bfg14PIOfv/il1u584NRWvg3Yfyt9vw14ayufCHy6lefx0De/fh/4u1b+D+DIVt6bwVeC3zi0j92BJ832/4m3nTbWJ4EaGgPnAW8C9h1q86/Ay1v528CerTz3YcbQUQy+zpk2rj8NvBD4HeCDQ/veZ7b/DR4NN8/0d74XAJ+qqvur6vu0i9GS7MNgoH+xtVsFvDDJ3sCvAv+W5Drgn4EDW5svAx9O8gcMAnQ6X62qjVX1IHAdgyfi04FvVtU3ho+1jf3/6ND981t5AfD5JOuANwO/PNS/97Z3EHOr6gEGF+SdlOQdwLOq6gfbeFw9Om2oqi+38kcYnHS8pL0zXAe8lIfGy1rggiSvBR5oddONoaPa7VrgGuAQYBGDk6DfSHJmkhdU1X1jeHyPeob+rmc34N6qes7Q7RkAVfVHwNsY/JTF1Un2m2b7Hw+V/5fRL8Cracp/D/xDVT0L+EMG7ySoqjMYnPk/AfhykkOq6nIGLzCbGLxgnThif7Rr2/LCnwL+CTi+jZcP0sYLcCyDn1o/DLgqyZzpxhCDM/y/GXo+HFxV57aTmMMYhP9fJ3n7Tn90jwGG/s53OXBckie0+cmXA7SzknuG5rhfB3yxvRv4ZpITADLw7FZ+WlVdWVVvB6b42d8xejg3A5Ob5+Q3H6uVfwA86WG2feXQ/VdaeR8e+s2kZZsbtv6tq6ozGZzhH5LkqcBdVfVB4EMMnqR67PrFJJvfEf4u8KVW/k57F3s8QJLdgIVVdRnwFgZjau/pxhCDK/h/r21PkvlJfiHJU4D7q+ojwHtwbG2TXe5XNh9rquqaJB8HvgbczWAgb7YM+ED7yuStwEmt/jXAOUneBjyOwd8d+BrwniSLGJz5rG5129KHHyU5icGU0ZzWhw+01SuBzyX5dlW9ZJrN5yVZy+AdxKtb3Tvavu4BLgUOavWnJnkJ8CBwA/CfDH5f6c1Jfgr8kMFnA3rsuhk4Jcl5wI3AOQw+A7oeuJOHxv/uwEfaNGeAs6vq3iTv2nIMVdWPkzwD+Er7DPiHwGuBgxk8Jx4Efgr88bge5KOZP8OgrUpyG7C4qvyNcz2iJJMMPux/5mz3RVvn9I4kdcQzfUnqiGf6ktQRQ1+SOmLoS1JHDH1J6oihL0kd+T8yhY5ytgZihQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "pass_all_vector = np.array([i in titles_that_pass for i in all_scripts]).astype(int)\n",
    "plt.bar(['does not pass','passes'],np.bincount(pass_all_vector))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "id": "4f4e9a57",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'R value = -0.022 P value 0.455'"
      ]
     },
     "execution_count": 151,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pearson_r, p_val = stats.pearsonr(passes_one_vector,pass_all_vector)\n",
    "f'R value = {pearson_r:.3f} P value {p_val:.3f}'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "id": "059a11d4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "' Mathews coef = -0.022'"
      ]
     },
     "execution_count": 152,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "f' Mathews coef = {matthews_corrcoef(pass_all_vector,passes_one_vector):.3f}'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "id": "393c2b88",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "' Jackards Dist = 0.935'"
      ]
     },
     "execution_count": 153,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "f' Jackards Dist = {distance.jaccard(pass_all_vector,passes_one_vector):.3f}'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "id": "27ca5f33",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'accuracy = 0.364'"
      ]
     },
     "execution_count": 154,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tn, fp, fn, tp = confusion_matrix(pass_all_vector,passes_one_vector).ravel()\n",
    "\n",
    "f'accuracy = {sum([tp,tn])/sum([tn,fp,fn,tp]):.3f}'"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}