grab_articles.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "from glob import glob\n",
    "\n",
    "from Bio import Entrez\n",
    "from Bio import Medline\n",
    "from datetime import datetime\n",
    "from dateutil import parser\n",
    "import pandas as pd\n",
    "import ast"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "searches = ['\"Laird AR\"[AUTH] AND (\"2012/01/01\"[PDAT] : \"3000/12/31\"[PDAT])',\n",
    "            '\"Sutherland MT\"[AUTH] AND (\"2012/01/01\"[PDAT] : \"3000/12/31\"[PDAT])']\n",
    "\n",
    "# Extract all publications matching term.\n",
    "Entrez.email = 'tsalo006@fiu.edu'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total number of publications containing \"Laird AR\"[AUTH] AND (\"2012/01/01\"[PDAT] : \"3000/12/31\"[PDAT]): 103\n",
      "Total number of publications containing \"Sutherland MT\"[AUTH] AND (\"2012/01/01\"[PDAT] : \"3000/12/31\"[PDAT]): 29\n"
     ]
    }
   ],
   "source": [
    "rows = []\n",
    "\n",
    "for TERM in searches:\n",
    "    h = Entrez.esearch(db='pubmed', retmax='2', term=TERM)\n",
    "    result = Entrez.read(h)\n",
    "    print('Total number of publications containing {0}: {1}'.format(TERM, result['Count']))\n",
    "    h_all = Entrez.esearch(db='pubmed', term=TERM, retmax=result['Count'])\n",
    "    result_all = Entrez.read(h_all)\n",
    "    ids_all = result_all['IdList']\n",
    "    h = Entrez.efetch(db='pubmed', id=ids_all, rettype='medline', retmode='text')\n",
    "    records = Medline.parse(h)\n",
    "\n",
    "    acceptable_formats = ['journal article', 'comparative study', 'editorial',\n",
    "                          'introductory journal article']\n",
    "    for record in records:\n",
    "        if any([type_.lower() in acceptable_formats for type_ in record.get('PT')]):\n",
    "            pmid = record.get('PMID')\n",
    "            pmcid = record.get('PMC', '')\n",
    "            \n",
    "            doi = [aid for aid in record.get('AID', []) if aid.endswith(' [doi]')]\n",
    "            if doi:\n",
    "                doi = doi[0].replace(' [doi]', '')\n",
    "            else:\n",
    "                doi = ''\n",
    "            \n",
    "            title = record.get('TI').rstrip('.')\n",
    "            authors = record.get('AU')\n",
    "\n",
    "            pub_date = parser.parse(record.get('DP'))\n",
    "            year = pub_date.year\n",
    "            month = pub_date.month\n",
    "            day = pub_date.day\n",
    "            \n",
    "            journal = record.get('TA')\n",
    "            volume = record.get('VI', '')\n",
    "            issue = record.get('IP', '')\n",
    "            pages = record.get('PG', '')\n",
    "            \n",
    "            abstract = record.get('AB', '')\n",
    "            \n",
    "            row = [pmid, pmcid, doi, title, authors, year, month,\n",
    "                   day, journal, volume, issue, pages, abstract]\n",
    "            rows += [row]\n",
    "\n",
    "df = pd.DataFrame(columns=['pmid', 'pmcid', 'doi', 'title', 'authors',\n",
    "                           'year', 'month', 'day',\n",
    "                           'journal', 'volume', 'issue', 'pages',\n",
    "                           'abstract'],\n",
    "                  data=rows)\n",
    "df = df.sort_values(by=['pmid'])\n",
    "df.to_csv('articles.csv', index=False)\n",
    "df = pd.read_csv('articles.csv')\n",
    "df = df.fillna('')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('papers/_posts/template_with_stuff.md', 'r') as fo:\n",
    "    template = fo.read()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "118\n",
      "114\n"
     ]
    }
   ],
   "source": [
    "old_papers = sorted(glob('papers/_posts/20*.md'))\n",
    "\n",
    "# One paper is by another MT Sutherland.\n",
    "# Something to do with mouse teeth.\n",
    "skip_pmids = ['28650075']\n",
    "\n",
    "# Add papers we already have pages for.\n",
    "old_pmids = skip_pmids\n",
    "for pap in old_papers:\n",
    "    with open(pap, 'r') as fo:\n",
    "        dat = fo.readlines()\n",
    "    line = [l for l in dat if l.startswith('pmid:')][0]\n",
    "    pmid = line.replace('pmid:', '').strip()\n",
    "    old_pmids.append(pmid)\n",
    "    old_pmids = [pmid for pmid in old_pmids if pmid]\n",
    "print len(old_papers)\n",
    "print len(old_pmids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['hum brain mapp' 'neuroinformatics' 'brain struct funct' 'neurol res int'\n",
      " 'neuroimage' 'cogn affect behav neurosci' 'plos one' 'cereb cortex'\n",
      " 'neurodegener dis' 'front neuroinform'\n",
      " 'j am acad child adolesc psychiatry' 'psychopharmacology (berl)'\n",
      " 'biol psychiatry' 'j pain' 'front hum neurosci' 'j neurosci'\n",
      " 'obesity (silver spring)' 'neuroimage clin' 'front aging neurosci'\n",
      " 'front neurosci' 'addict biol' 'brain lang' 'cortex' 'annu rev neurosci'\n",
      " 'biol psychol' 'j biomed semantics' 'jama psychiatry' 'j addict'\n",
      " 'neuropsychopharmacology' 'dev sci' 'neurosci biobehav rev'\n",
      " 'behav brain funct' 'front neuroendocrinol' 'j bone miner res'\n",
      " 'front behav neurosci' 'j psychopharmacol' 'trends mol med'\n",
      " 'dev cogn neurosci' 'mol psychiatry' 'netw neurosci' 'j sex med'\n",
      " 'sleep med rev' 'front ict' 'nat hum behav' 'exp clin psychopharmacol'\n",
      " 'sci adv' 'npj sci learn']\n"
     ]
    }
   ],
   "source": [
    "journals = df['journal'].str.lower().unique()\n",
    "print journals"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "for _, row in df.iterrows():\n",
    "    pmid = row['pmid']\n",
    "    if str(pmid) not in old_pmids:\n",
    "        authors = ast.literal_eval(row['authors'])\n",
    "        nick = [re.sub(r'\\W+', '', w) for w in row['title'].lower().split(' ')[:3]]\n",
    "        nickname = '{0}-{1}-{2}-{3}-{4}'.format(row['year'],\n",
    "                                                '{0:02d}'.format(int(row['month'])),\n",
    "                                                '{0:02d}'.format(int(row['day'])),\n",
    "                                                authors[0].split(' ')[0].lower(),\n",
    "                                                '-'.join(nick))\n",
    "        nickname = nickname.replace(':', '')\n",
    "        journal = row['journal']\n",
    "        image = '/assets/images/papers/{0}.png'.format('-'.join(journal.lower().split(' ')))\n",
    "        title = row['title'].replace('\"', \"'\")\n",
    "        completed = template.format(title=title, nickname=nickname,\n",
    "                                    authors=', '.join(authors), year=row['year'],\n",
    "                                    journal=journal, volume=row['volume'],\n",
    "                                    image=image,\n",
    "                                    issue=row['issue'], pages=row['pages'],\n",
    "                                    pmcid=row['pmcid'], doi=row['doi'], pmid=row['pmid'],\n",
    "                                    abstract=row['abstract'])\n",
    "        with open('papers/_posts/{0}.md'.format(nickname), 'w') as fo:\n",
    "            fo.write(completed)\n",
    "        \n",
    "        print('New file created for {0}'.format(pmid))"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda env:anaconda]",
   "language": "python",
   "name": "conda-env-anaconda-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}