Ability to select any range of pages on scholar

PyPaperBot/Crossref.py

@@ -42,14 +42,14 @@ def getPapersInfo(papers, scholar_search_link, restrict):
     papers_return = []
     num = 1
     for paper in papers:
-        title = paper[0].lower()
+        title = paper['title'].lower()
         queries = {'query.bibliographic': title,'sort':'relevance',"select":"DOI,title,deposited,author,short-container-title"}
 
         print("Searching paper {} of {} on Crossref...".format(num,len(papers)))
         num += 1
 
         found_timestamp = 0
-        paper_found = Paper(title,paper[1],scholar_search_link, paper[2], paper[3])
+        paper_found = Paper(title,paper['link'],scholar_search_link, paper['cites'], paper['link_pdf'], paper['year'], paper['authors'])
         for el in iterate_publications_as_json(max_results=30, queries=queries):
 
             el_date = 0
@@ -71,4 +71,4 @@ def getPapersInfo(papers, scholar_search_link, restrict):
 
         time.sleep(random.randint(1,10))
 
-    return papers_return
+    return papers_return

PyPaperBot/Downloader.py

@@ -64,8 +64,7 @@ def URLjoin(*args):
 
             faild = 0
             while p.downloaded==False and faild!=4:        
-                try:   
-
+                try:
                     dwn_source = 1 #1 scihub 2 scholar 
                     if faild==0 and p.DOI!=None:
                         url = URLjoin(NetInfo.SciHub_URL, p.DOI)
@@ -77,23 +76,22 @@ def URLjoin(*args):
                     if faild==3 and p.pdf_link!=None:
                         url = p.pdf_link
                         dwn_source = 2                        
-
+                    
                     if url!="":
                         r = requests.get(url, headers=NetInfo.HEADERS)
                         content_type = r.headers.get('content-type')
-
+                            
                         if dwn_source==1 and 'application/pdf' not in content_type:
                             time.sleep(random.randint(1,5))
-
+                                
                             pdf_link = getSchiHubPDF(r.text)
                             if(pdf_link != None):
                                 r = requests.get(pdf_link, headers=NetInfo.HEADERS)
                                 content_type = r.headers.get('content-type')
-
+        
                         if 'application/pdf' in content_type:
                             paper_files.append(saveFile(pdf_dir,r.content,p,dwn_source))
-
-                except:
+                except Exception:
                     pass
 
-                faild += 1
+                faild += 1

PyPaperBot/HTMLparsers.py

@@ -15,6 +15,8 @@ def schoolarParser(html):
             link = None
             link_pdf = None
             cites = None
+            year = None
+            authors = None
             for h3 in element.findAll("h3", class_="gs_rt"):
                 found = False
                 for a in h3.findAll("a"): 
@@ -27,8 +29,28 @@ def schoolarParser(html):
                      cites = int(a.text[8:])
                  if "[PDF]" in a.text:
                      link_pdf = a.get("href")
+            for div in element.findAll("div", class_="gs_a"):
+                authors, source_and_year, source = div.text.replace('\u00A0', ' ').split(" - ")
+
+                if not authors.endswith('\u2026'):
+                    # There is no ellipsis at the end so we know the full list of authors
+                    authors = authors.replace(', ', ';')
+                try:
+                    year = int(source_and_year[-4:])
+                except ValueError:
+                    continue
+                if not (1000 <= year <= 3000):
+                    year = None
+                else:
+                    year = str(year)
             if title!=None:         
-                result.append((title, link, cites, link_pdf))
+                result.append({
+                    'title' : title,
+                    'link' : link,
+                    'cites' : cites,
+                    'link_pdf' : link_pdf,
+                    'year' : year,
+                    'authors' : authors})
     return result            
 
 

PyPaperBot/Paper.py

@@ -6,17 +6,20 @@
 """
 import bibtexparser
 import re
+import csv
+import os
 
 class Paper:
 
 
-    def __init__(self,title=None, scholar_link=None, scholar_page=None, cites=None, link_pdf=None):        
+    def __init__(self,title=None, scholar_link=None, scholar_page=None, cites=None, link_pdf=None, year=None, authors=None):        
         self.title = title
         self.scholar_page = scholar_page
         self.scholar_link = scholar_link 
         self.pdf_link = link_pdf
+        self.year = year
+        self.authors = authors
 
-        self.year = None
         self.jurnal = None
         self.cites_num = None
         self.bibtex = None
@@ -37,8 +40,11 @@ def setBibtex(self,bibtex):
 
         self.bibtex = bibtex
 
-        try: 
-            self.year=x[0]["year"] if "year" in x[0] else None
+        try:
+            if "year" in x[0]:
+                self.year=x[0]["year"]
+            if 'author' in x[0]:
+                self.authors = x[0]["author"]
             self.jurnal=x[0]["journal"].replace("\\","") if "journal" in x[0] else None
             if self.jurnal==None:
                  self.jurnal=x[0]["publisher"].replace("\\","") if "publisher" in x[0] else None
@@ -53,33 +59,36 @@ def canBeDownloaded(self):
         return False
 
 
-    def generateReport(papers, path):
-        def strFix(s):
-            if(len(str(s))==0):
-                return "None"
-            else:
-                return str(s).replace(",", "").rstrip('\n')
+    def generateReport(papers, path):            
+        with open(path, mode="w", encoding='utf-8', newline='', buffering=1) as w_file:
+            content = ["Name", "Scholar Link", "DOI", "Bibtex",
+                       "PDF Name", "Year", "Scholar page", "Journal",
+                       "Downloaded", "Downloaded from", "Authors"]
+            file_writer = csv.DictWriter(w_file, delimiter = ",", lineterminator=os.linesep, fieldnames=content)
+            file_writer.writeheader()
 
-
-        content = "Name,Scholar Link,DOI,Bibtex,PDF Name,Year,Scholar page,Journal,Downloaded,Downloaded from"
-        for p in papers:
-            pdf_name = p.getFileName() if p.downloaded==True else ""
-            bibtex_found = True if p.bibtex!=None else False
+            for p in papers:
+                pdf_name = p.getFileName() if p.downloaded==True else ""
+                bibtex_found = True if p.bibtex!=None else False
 
-            dwn_from = ""
-            if p.downloadedFrom == 1:
-                dwn_from = "SciHub"
-            if p.downloadedFrom == 2:
-                dwn_from = "Scholar"
-
-            content += ("\n"+strFix(p.title)+","+strFix(p.scholar_link)+","+
-                        strFix(p.DOI)+","+strFix(bibtex_found)+","+ strFix(pdf_name)+
-                        ","+strFix(p.year)+","+strFix(p.scholar_page)+","+
-                        strFix(p.jurnal)+","+strFix(p.downloaded)+","+strFix(dwn_from))
-
-        f = open(path, "w", encoding='utf-8-sig')
-        f.write(content)
-        f.close()
+                dwn_from = ""
+                if p.downloadedFrom == 1:
+                    dwn_from = "SciHub"
+                if p.downloadedFrom == 2:
+                    dwn_from = "Scholar"
+
+                file_writer.writerow({
+                        "Name" : p.title,
+                        "Scholar Link" : p.scholar_link,
+                        "DOI" : p.DOI,
+                        "Bibtex" : bibtex_found,
+                        "PDF Name" : pdf_name,
+                        "Year" : p.year,
+                        "Scholar page" : p.scholar_page,
+                        "Journal" : p.jurnal,
+                        "Downloaded" : p.downloaded,
+                        "Downloaded from" : dwn_from,
+                        "Authors" : p.authors})           
 
 
     def generateBibtex(papers, path):
@@ -100,4 +109,4 @@ def generateBibtex(papers, path):
 
 
 
-
+

PyPaperBot/Scholar.py

@@ -7,38 +7,34 @@
 
 
 def waithIPchange():
-    input("You have been blocked, change your IP or VPN than press Enter...")
-    print("Wait 30 seconds...")
-    time.sleep(30)
+    while True:
+        inp = input('You have been blocked, try changing your IP or using a VPN. '
+              'Press Enter to continue downloading, or type "exit" to stop and exit....')
+        if inp.strip().lower() == "exit":
+            return False
+        elif not inp.strip():
+            print("Wait 30 seconds...")
+            time.sleep(30)
+            return True
 
-
-def ScholarPapersInfo(query, scholar_pages, restrict, min_date=None):
-
+def scholar_requests(scholar_pages, url, restrict):
     javascript_error = "Sorry, we can't verify that you're not a robot when JavaScript is turned off"
-
-    url = "https://scholar.google.com/scholar?hl=en&q="+query+"&as_vis=1&as_sdt=1,5"
-    if min_date!=None:
-        url += "&as_ylo"+min_date
-
-    if len(query)>7 and (query[0:7]=="http://" or query[0:8]=="https://"):
-         url = query        
-
-
     to_download = []
-    last_blocked = False
-    i = 0
-    while i < scholar_pages:
-        html = requests.get(url, headers=NetInfo.HEADERS)
-        html = html.text
-
-        if javascript_error in html and last_blocked==False:
-            waithIPchange()
-            continue
-        else:
-            last_blocked=False
-
+    for i in scholar_pages:
+        while True:
+            res_url = url % (10 * (i - 1))
+            html = requests.get(res_url, headers=NetInfo.HEADERS)
+            html = html.text
+
+            if javascript_error in html:
+                is_continue = waithIPchange()
+                if not is_continue:
+                    return to_download
+            else:
+                break
+
         papers = schoolarParser(html)
-        print("\nGoogle Scholar page {} : {} papers found".format((i+1),len(papers)))
+        print("\nGoogle Scholar page {} : {} papers found".format(i,len(papers)))
 
         if(len(papers)>0):
             papersInfo = getPapersInfo(papers, url, restrict)
@@ -49,7 +45,19 @@ def ScholarPapersInfo(query, scholar_pages, restrict, min_date=None):
         else:
             print("Paper not found...")
 
-        i += 1
-        url += "&start=" + str(10*i)
+    return to_download
+
+
+
+def ScholarPapersInfo(query, scholar_pages, restrict, min_date=None):
+
+    url = r"https://scholar.google.com/scholar?hl=en&q="+query+"&as_vis=1&as_sdt=1,5&start=%d"
+    if min_date!=None:
+        url += "&as_ylo="+str(min_date)
+
+    if len(query)>7 and (query[0:7]=="http://" or query[0:8]=="https://"):
+         url = query        
+
+    to_download = scholar_requests(scholar_pages, url, restrict)
 
-    return [item for sublist in to_download for item in sublist]
+    return [item for sublist in to_download for item in sublist]

PyPaperBot/__main__.py

@@ -29,7 +29,7 @@ def start(query, scholar_pages, dwn_dir, min_date=None, num_limit=None, num_limi
             i +=  1
 
 
-    if restrict!=0:
+    if restrict!=0 and to_download:
         if filter_jurnal_file!=None:
            to_download = filterJurnals(to_download,filter_jurnal_file)
 
@@ -57,7 +57,9 @@ def main():
     parser.add_argument('--query', type=str, default=None, help='Query to make on Google Scholar or Google Scholar page link')
     parser.add_argument('--doi', type=str, default=None, help='DOI of the paper to download (this option uses only SciHub to download)')
     parser.add_argument('--doi-file', type=str, default=None, help='File .txt containing the list of paper\'s DOIs to download')
-    parser.add_argument('--scholar-pages', type=int, help='Number of Google Scholar pages to inspect. Each page has a maximum of 10 papers (required for --query)')
+    parser.add_argument('--scholar-pages', type=str, help='If given in %%d format, the number of pages to download from the beginning. '
+                                                          'If given in %%d-%%d format, the range of pages (starting from 1) to download (the end is included). '
+                                                          'Each page has a maximum of 10 papers (required for --query)')
     parser.add_argument('--dwn-dir', type=str, help='Directory path in which to save the results')
     parser.add_argument('--min-year', default=None, type=int, help='Minimal publication year of the paper to download')
     parser.add_argument('--max-dwn-year', default=None, type=int, help='Maximum number of papers to download sorted by year')
@@ -88,10 +90,25 @@ def main():
         print("Error: Only one option between '--max-dwn-year' and '--max-dwn-cites' can be used ")
         sys.exit()
 
-    if(args.query != None and args.scholar_pages==None):
-        print("Error: with --query provide also --scholar-pages")
-        sys.exit()
-
+    if(args.query != None):
+        if args.scholar_pages:
+            try:
+                split = args.scholar_pages.split('-')
+                if len(split) == 1:
+                    scholar_pages = range(1, int(split[0]) + 1)
+                elif len(split) == 2:
+                    start_page, end_page = [int(x) for x in split]
+                    scholar_pages = range(start_page, end_page + 1)
+                else:
+                    raise ValueError
+            except Exception:
+                print(r"Error: Invalid format for --scholar-pages option. Expected: %d or %d-%d, got: " + args.scholar_pages)
+                sys.exit()
+        else:
+            print("Error: with --query provide also --scholar-pages")
+            sys.exit()
+
+
     DOIs = None    
     if args.doi_file!=None:
         DOIs = [] 
@@ -116,7 +133,7 @@ def main():
         max_dwn_type = 1
 
 
-    start(args.query, args.scholar_pages, dwn_dir, args.min_year , max_dwn, max_dwn_type , args.journal_filter, args.restrict, DOIs, args.scihub_mirror)
+    start(args.query, scholar_pages, dwn_dir, args.min_year , max_dwn, max_dwn_type , args.journal_filter, args.restrict, DOIs, args.scihub_mirror)
 
 if __name__ == "__main__":
-    main()
+    main()