General improvements and fixes

PyPaperBot/Crossref.py

@@ -42,33 +42,41 @@ def getPapersInfo(papers, scholar_search_link, restrict):
     papers_return = []
     num = 1
     for paper in papers:
-        title = paper[0].lower()
-        queries = {'query.bibliographic': title,'sort':'relevance',"select":"DOI,title,deposited,author,short-container-title"}
+        title = paper['title']
+        queries = {'query.bibliographic': title.lower(),'sort':'relevance',"select":"DOI,title,deposited,author,short-container-title"}
 
         print("Searching paper {} of {} on Crossref...".format(num,len(papers)))
         num += 1
 
         found_timestamp = 0
-        paper_found = Paper(title,paper[1],scholar_search_link, paper[2], paper[3])
-        for el in iterate_publications_as_json(max_results=30, queries=queries):
-
-            el_date = 0
-            if "deposited" in el and "timestamp" in el["deposited"]:
-                el_date = int(el["deposited"]["timestamp"])
-
-            if (paper_found.DOI==None or el_date>found_timestamp) and "title" in el and similarStrings(title ,el["title"][0].lower())>0.75:
-                found_timestamp = el_date
-
-                if "DOI" in el:
-                    paper_found.DOI = el["DOI"].strip().lower()
-                if "short-container-title" in el and len(el["short-container-title"])>0:
-                    paper_found.jurnal = el["short-container-title"][0]
+        paper_found = Paper(title,paper['link'],scholar_search_link, paper['cites'], paper['link_pdf'], paper['year'], paper['authors'])
+        while True:
+            try:
+                for el in iterate_publications_as_json(max_results=30, queries=queries):
 
-                if restrict==None or restrict!=1:    
-                    paper_found.setBibtex(getBibtex(paper_found.DOI))
+                    el_date = 0
+                    if "deposited" in el and "timestamp" in el["deposited"]:
+                        el_date = int(el["deposited"]["timestamp"])
+
+                    if (paper_found.DOI==None or el_date>found_timestamp) and "title" in el and similarStrings(title.lower() ,el["title"][0].lower())>0.75:
+                        found_timestamp = el_date
+
+                        if "DOI" in el:
+                            paper_found.DOI = el["DOI"].strip().lower()
+                        if "short-container-title" in el and len(el["short-container-title"])>0:
+                            paper_found.jurnal = el["short-container-title"][0]
+
+                        if restrict==None or restrict!=1:    
+                            paper_found.setBibtex(getBibtex(paper_found.DOI))
+
+                break
+            except ConnectionError as e:
+                print("Wait 10 seconds and try again...")
+                time.sleep(10)
+
 
         papers_return.append(paper_found)
 
         time.sleep(random.randint(1,10))
 
-    return papers_return
+    return papers_return

PyPaperBot/Downloader.py

@@ -64,8 +64,7 @@ def URLjoin(*args):
 
             faild = 0
             while p.downloaded==False and faild!=4:        
-                try:   
-
+                try:
                     dwn_source = 1 #1 scihub 2 scholar 
                     if faild==0 and p.DOI!=None:
                         url = URLjoin(NetInfo.SciHub_URL, p.DOI)
@@ -77,23 +76,22 @@ def URLjoin(*args):
                     if faild==3 and p.pdf_link!=None:
                         url = p.pdf_link
                         dwn_source = 2                        
-
+                    
                     if url!="":
                         r = requests.get(url, headers=NetInfo.HEADERS)
                         content_type = r.headers.get('content-type')
-
+                            
                         if dwn_source==1 and 'application/pdf' not in content_type:
                             time.sleep(random.randint(1,5))
-
+                                
                             pdf_link = getSchiHubPDF(r.text)
                             if(pdf_link != None):
                                 r = requests.get(pdf_link, headers=NetInfo.HEADERS)
                                 content_type = r.headers.get('content-type')
-
+        
                         if 'application/pdf' in content_type:
                             paper_files.append(saveFile(pdf_dir,r.content,p,dwn_source))
-
-                except:
+                except Exception:
                     pass
 
-                faild += 1
+                faild += 1

PyPaperBot/HTMLparsers.py

@@ -15,6 +15,8 @@ def schoolarParser(html):
             link = None
             link_pdf = None
             cites = None
+            year = None
+            authors = None
             for h3 in element.findAll("h3", class_="gs_rt"):
                 found = False
                 for a in h3.findAll("a"): 
@@ -27,8 +29,33 @@ def schoolarParser(html):
                      cites = int(a.text[8:])
                  if "[PDF]" in a.text:
                      link_pdf = a.get("href")
+            for div in element.findAll("div", class_="gs_a"):
+                try:
+                    authors, source_and_year, source = div.text.replace('\u00A0', ' ').split(" - ")
+                except ValueError:
+                    continue
+
+                if not authors.strip().endswith('\u2026'):
+                    # There is no ellipsis at the end so we know the full list of authors
+                    authors = authors.replace(', ', ';')
+                else:
+                    authors = None
+                try:
+                    year = int(source_and_year[-4:])
+                except ValueError:
+                    continue
+                if not (1000 <= year <= 3000):
+                    year = None
+                else:
+                    year = str(year)
             if title!=None:         
-                result.append((title, link, cites, link_pdf))
+                result.append({
+                    'title' : title,
+                    'link' : link,
+                    'cites' : cites,
+                    'link_pdf' : link_pdf,
+                    'year' : year,
+                    'authors' : authors})
     return result            
 
 

PyPaperBot/Paper.py

@@ -6,17 +6,20 @@
 """
 import bibtexparser
 import re
+import csv
+import os
 
 class Paper:
 
 
-    def __init__(self,title=None, scholar_link=None, scholar_page=None, cites=None, link_pdf=None):        
+    def __init__(self,title=None, scholar_link=None, scholar_page=None, cites=None, link_pdf=None, year=None, authors=None):        
         self.title = title
         self.scholar_page = scholar_page
         self.scholar_link = scholar_link 
         self.pdf_link = link_pdf
+        self.year = year
+        self.authors = authors
 
-        self.year = None
         self.jurnal = None
         self.cites_num = None
         self.bibtex = None
@@ -37,8 +40,11 @@ def setBibtex(self,bibtex):
 
         self.bibtex = bibtex
 
-        try: 
-            self.year=x[0]["year"] if "year" in x[0] else None
+        try:
+            if "year" in x[0]:
+                self.year=x[0]["year"]
+            if 'author' in x[0]:
+                self.authors = x[0]["author"]
             self.jurnal=x[0]["journal"].replace("\\","") if "journal" in x[0] else None
             if self.jurnal==None:
                  self.jurnal=x[0]["publisher"].replace("\\","") if "publisher" in x[0] else None
@@ -53,33 +59,36 @@ def canBeDownloaded(self):
         return False
 
 
-    def generateReport(papers, path):
-        def strFix(s):
-            if(len(str(s))==0):
-                return "None"
-            else:
-                return str(s).replace(",", "").rstrip('\n')
+    def generateReport(papers, path):            
+        with open(path, mode="w", encoding='utf-8', newline='', buffering=1) as w_file:
+            content = ["Name", "Scholar Link", "DOI", "Bibtex",
+                       "PDF Name", "Year", "Scholar page", "Journal",
+                       "Downloaded", "Downloaded from", "Authors"]
+            file_writer = csv.DictWriter(w_file, delimiter = ",", lineterminator=os.linesep, fieldnames=content)
+            file_writer.writeheader()
 
-
-        content = "Name,Scholar Link,DOI,Bibtex,PDF Name,Year,Scholar page,Journal,Downloaded,Downloaded from"
-        for p in papers:
-            pdf_name = p.getFileName() if p.downloaded==True else ""
-            bibtex_found = True if p.bibtex!=None else False
+            for p in papers:
+                pdf_name = p.getFileName() if p.downloaded==True else ""
+                bibtex_found = True if p.bibtex!=None else False
 
-            dwn_from = ""
-            if p.downloadedFrom == 1:
-                dwn_from = "SciHub"
-            if p.downloadedFrom == 2:
-                dwn_from = "Scholar"
-
-            content += ("\n"+strFix(p.title)+","+strFix(p.scholar_link)+","+
-                        strFix(p.DOI)+","+strFix(bibtex_found)+","+ strFix(pdf_name)+
-                        ","+strFix(p.year)+","+strFix(p.scholar_page)+","+
-                        strFix(p.jurnal)+","+strFix(p.downloaded)+","+strFix(dwn_from))
-
-        f = open(path, "w", encoding='utf-8-sig')
-        f.write(content)
-        f.close()
+                dwn_from = ""
+                if p.downloadedFrom == 1:
+                    dwn_from = "SciHub"
+                if p.downloadedFrom == 2:
+                    dwn_from = "Scholar"
+
+                file_writer.writerow({
+                        "Name" : p.title,
+                        "Scholar Link" : p.scholar_link,
+                        "DOI" : p.DOI,
+                        "Bibtex" : bibtex_found,
+                        "PDF Name" : pdf_name,
+                        "Year" : p.year,
+                        "Scholar page" : p.scholar_page,
+                        "Journal" : p.jurnal,
+                        "Downloaded" : p.downloaded,
+                        "Downloaded from" : dwn_from,
+                        "Authors" : p.authors})           
 
 
     def generateBibtex(papers, path):
@@ -100,4 +109,4 @@ def generateBibtex(papers, path):
 
 
 
-
+

PyPaperBot/Scholar.py

@@ -7,38 +7,34 @@
 
 
 def waithIPchange():
-    input("You have been blocked, change your IP or VPN than press Enter...")
-    print("Wait 30 seconds...")
-    time.sleep(30)
+    while True:
+        inp = input('You have been blocked, try changing your IP or using a VPN. '
+              'Press Enter to continue downloading, or type "exit" to stop and exit....')
+        if inp.strip().lower() == "exit":
+            return False
+        elif not inp.strip():
+            print("Wait 30 seconds...")
+            time.sleep(30)
+            return True
 
-
-def ScholarPapersInfo(query, scholar_pages, restrict, min_date=None):
-
+def scholar_requests(scholar_pages, url, restrict):
     javascript_error = "Sorry, we can't verify that you're not a robot when JavaScript is turned off"
-
-    url = "https://scholar.google.com/scholar?hl=en&q="+query+"&as_vis=1&as_sdt=1,5"
-    if min_date!=None:
-        url += "&as_ylo"+min_date
-
-    if len(query)>7 and (query[0:7]=="http://" or query[0:8]=="https://"):
-         url = query        
-
-
     to_download = []
-    last_blocked = False
-    i = 0
-    while i < scholar_pages:
-        html = requests.get(url, headers=NetInfo.HEADERS)
-        html = html.text
-
-        if javascript_error in html and last_blocked==False:
-            waithIPchange()
-            continue
-        else:
-            last_blocked=False
-
+    for i in scholar_pages:
+        while True:
+            res_url = url % (10 * (i - 1))
+            html = requests.get(res_url, headers=NetInfo.HEADERS)
+            html = html.text
+
+            if javascript_error in html:
+                is_continue = waithIPchange()
+                if not is_continue:
+                    return to_download
+            else:
+                break
+
         papers = schoolarParser(html)
-        print("\nGoogle Scholar page {} : {} papers found".format((i+1),len(papers)))
+        print("\nGoogle Scholar page {} : {} papers found".format(i,len(papers)))
 
         if(len(papers)>0):
             papersInfo = getPapersInfo(papers, url, restrict)
@@ -49,7 +45,19 @@ def ScholarPapersInfo(query, scholar_pages, restrict, min_date=None):
         else:
             print("Paper not found...")
 
-        i += 1
-        url += "&start=" + str(10*i)
+    return to_download
+
+
+
+def ScholarPapersInfo(query, scholar_pages, restrict, min_date=None):
+
+    url = r"https://scholar.google.com/scholar?hl=en&q="+query+"&as_vis=1&as_sdt=1,5&start=%d"
+    if min_date!=None:
+        url += "&as_ylo="+str(min_date)
+
+    if len(query)>7 and (query[0:7]=="http://" or query[0:8]=="https://"):
+         url = query        
+
+    to_download = scholar_requests(scholar_pages, url, restrict)
 
-    return [item for sublist in to_download for item in sublist]
+    return [item for sublist in to_download for item in sublist]