Phamquangnghia

README.md

@@ -28,3 +28,4 @@ Technology is always changing. To embrace change in the world of technology, we
 
 
 
+

arstechnica_scraper/arstechnica_scraper.py

@@ -0,0 +1,38 @@
+from bs4 import BeautifulSoup
+import requests
+import pandas as pd
+
+ARTICLES_COUNT = 10
+titles_data = []
+links_data = []
+paragraphs = []
+
+def body(url) :   
+    article = requests.get(url)
+    doc = BeautifulSoup(article.text, "html.parser")
+
+    bodies = doc.find('article').find_all(lambda tag: tag.name == 'p' and not tag.attrs) 
+    bodies = [paragraph.text.strip() for paragraph in bodies]
+    body = ' '.join(bodies)
+    paragraphs.append(body)
+
+def article_links_and_title(url) :
+    page = requests.get(url)
+    doc = BeautifulSoup(page.text, "html.parser")
+
+    titles_and_links = [header.a for header in doc.find_all('h2')[:ARTICLES_COUNT]] 
+    for row in titles_and_links:
+        link = row.attrs['href']
+        title = row.text.strip()
+        titles_data.append(title)
+        links_data.append(link)  
+
+url = "https://arstechnica.com/gadgets/"
+article_links_and_title(url)
+for link in links_data:
+    body(link)
+
+df = pd.DataFrame({'Title': titles_data,
+                   'Link': links_data,
+                  'Body': paragraphs})
+df.to_csv("../data/arstechnica_data.csv", encoding='utf-8-sig', index=False)

bs4_scraper/bs4_scraper.py

@@ -0,0 +1,12 @@
+from bs4 import BeautifulSoup
+import requests
+url = "https://www.nature.com/articles/d41586-022-01320-y"
+page = requests.get(url)
+doc = BeautifulSoup(page.text, "html.parser")
+links = doc.find('article').find_all('a')
+f = open('./data/links.txt','a')
+f.write(str(doc.find('h1').string)+'\n')
+for link in links:
+    if ('href' in link.attrs) and ('http' in str(link.attrs['href'])):
+        f.write(str(link.attrs['href'])+'\n')
+f.close()

bs4_scraper/data/links.txt

@@ -0,0 +1,42 @@
+Black hole at the centre of our Galaxy imaged for the first time
+http://www.ncbi.nlm.nih.gov/entrez/query.fcgi?cmd=search&term=%22Davide%2BCastelvecchi%22
+https://scholar.google.co.uk/scholar?as_q=&btnG=Search+Scholar&as_sauthors=%22Davide%2BCastelvecchi%22
+https://twitter.com/intent/tweet?text=Black+hole+at+the+centre+of+our+Galaxy+imaged+for+the+first+time&url=https%3A%2F%2Fwww.nature.com%2Farticles%2Fd41586-022-01320-y
+http://www.facebook.com/sharer.php?u=https%3A%2F%2Fwww.nature.com%2Farticles%2Fd41586-022-01320-y
+mailto:?subject=Black hole at the centre of our Galaxy imaged for the first time&body=https%3A%2F%2Fwww.nature.com%2Farticles%2Fd41586-022-01320-y
+https://www.nature.com/articles/d41586-019-01155-0
+https://iopscience.iop.org/journal/2041-8205/page/Focus_on_First_Sgr_A_Results
+https://www.nature.com/articles/d41586-019-01155-0
+https://www.nature.com/articles/d41586-019-01155-0
+https://www.nature.com/articles/d41586-020-02717-3
+https://www.nature.com/articles/d41586-020-02717-3
+https://www.nature.com/articles/d41586-020-02764-w
+https://www.nature.com/articles/d41586-019-02726-x
+https://www.nature.com/articles/d41586-020-02764-w
+https://www.nature.com/articles/d41586-019-00458-6
+https://www.nature.com/news/how-to-hunt-for-a-black-hole-with-a-telescope-the-size-of-earth-1.21693
+https://www.nature.com/articles/d41586-019-00347-y
+https://doi.org/https%3A%2F%2Fdoi.org%2F10.3847%2F2041-8213%2Fac6674
+http://scholar.google.com/scholar_lookup?&title=&journal=Astrophys.%20J.%20Lett.&volume=930&publication_year=2022&author=Akiyama%2CK.
+https://citation-needed.springer.com/v2/references/10.1038/d41586-022-01320-y?format=refman&flavour=references
+https://www.nature.com/articles/d41586-019-01155-0
+https://www.nature.com/articles/d41586-019-01191-w
+https://www.nature.com/news/how-to-hunt-for-a-black-hole-with-a-telescope-the-size-of-earth-1.21693
+https://www.nature.com/articles/d41586-020-02764-w
+https://www.nature.com/articles/d41586-019-02726-x
+https://www.nature.com/articles/d41586-020-02717-3
+http://www.nature.com/articles/d41586-022-01276-z
+http://www.nature.com/articles/s41586-022-04635-y
+http://www.nature.com/articles/s41586-022-04608-1
+https://www.nature.com/naturecareers/
+https://www.nature.com/naturecareers/job/postdoctoral-fellowship-position-in-biomolecular-spectroscopy-the-university-of-british-columbia-ubc-758341
+https://www.nature.com/naturecareers/job/postdoctoral-fellow-for-translational-highfield-mri-german-cancer-research-center-in-the-helmholtz-association-dkfz-758334
+https://www.nature.com/naturecareers/job/phd-position-fmd-helmholtz-centre-for-environmental-research-ufz-758340
+https://www.nature.com/naturecareers/job/postdoc-in-proton-microscopy-all-genders-tenure-track-helmholtz-centre-for-heavy-ion-research-gmbh-gsi-758335
+https://www.nature.com/articles/d41586-019-01155-0
+https://www.nature.com/articles/d41586-019-01191-w
+https://www.nature.com/news/how-to-hunt-for-a-black-hole-with-a-telescope-the-size-of-earth-1.21693
+https://www.nature.com/articles/d41586-020-02764-w
+https://www.nature.com/articles/d41586-019-02726-x
+https://www.nature.com/articles/d41586-020-02717-3
+https://www.nature.com/info/privacy