From 6b6449e034c7f97afef9e72a67bbd6bd7043d25b Mon Sep 17 00:00:00 2001 From: 23k_mal Date: Thu, 25 Mar 2021 04:24:09 +0300 Subject: [PATCH 1/8] Getting of year and authors from html --- PyPaperBot/Crossref.py | 6 ++--- PyPaperBot/Downloader.py | 51 ++++++++++++++++++--------------------- PyPaperBot/HTMLparsers.py | 26 +++++++++++++++++++- PyPaperBot/Paper.py | 19 +++++++++------ 4 files changed, 64 insertions(+), 38 deletions(-) diff --git a/PyPaperBot/Crossref.py b/PyPaperBot/Crossref.py index a2775f1..0e9502f 100644 --- a/PyPaperBot/Crossref.py +++ b/PyPaperBot/Crossref.py @@ -42,14 +42,14 @@ def getPapersInfo(papers, scholar_search_link, restrict): papers_return = [] num = 1 for paper in papers: - title = paper[0].lower() + title = paper['title'].lower() queries = {'query.bibliographic': title,'sort':'relevance',"select":"DOI,title,deposited,author,short-container-title"} print("Searching paper {} of {} on Crossref...".format(num,len(papers))) num += 1 found_timestamp = 0 - paper_found = Paper(title,paper[1],scholar_search_link, paper[2], paper[3]) + paper_found = Paper(title,paper['link'],scholar_search_link, paper['cites'], paper['link_pdf'], paper['year'], paper['authors']) for el in iterate_publications_as_json(max_results=30, queries=queries): el_date = 0 @@ -71,4 +71,4 @@ def getPapersInfo(papers, scholar_search_link, restrict): time.sleep(random.randint(1,10)) - return papers_return \ No newline at end of file + return papers_return diff --git a/PyPaperBot/Downloader.py b/PyPaperBot/Downloader.py index 7102427..7019763 100644 --- a/PyPaperBot/Downloader.py +++ b/PyPaperBot/Downloader.py @@ -64,36 +64,33 @@ def URLjoin(*args): faild = 0 while p.downloaded==False and faild!=4: - try: - dwn_source = 1 #1 scihub 2 scholar - if faild==0 and p.DOI!=None: - url = URLjoin(NetInfo.SciHub_URL, p.DOI) - if faild==1 and p.scholar_link!=None: - url = URLjoin(NetInfo.SciHub_URL, p.scholar_link) - if faild==2 and p.scholar_link!=None and p.scholar_link[-3:]=="pdf": - url = p.scholar_link - dwn_source = 2 - if faild==3 and p.pdf_link!=None: - url = p.pdf_link - dwn_source = 2 + dwn_source = 1 #1 scihub 2 scholar + if faild==0 and p.DOI!=None: + url = URLjoin(NetInfo.SciHub_URL, p.DOI) + if faild==1 and p.scholar_link!=None: + url = URLjoin(NetInfo.SciHub_URL, p.scholar_link) + if faild==2 and p.scholar_link!=None and p.scholar_link[-3:]=="pdf": + url = p.scholar_link + dwn_source = 2 + if faild==3 and p.pdf_link!=None: + url = p.pdf_link + dwn_source = 2 - if url!="": - r = requests.get(url, headers=NetInfo.HEADERS) - content_type = r.headers.get('content-type') + if url!="": + r = requests.get(url, headers=NetInfo.HEADERS) + content_type = r.headers.get('content-type') - if dwn_source==1 and 'application/pdf' not in content_type: - time.sleep(random.randint(1,5)) + if dwn_source==1 and 'application/pdf' not in content_type: + time.sleep(random.randint(1,5)) - pdf_link = getSchiHubPDF(r.text) - if(pdf_link != None): - r = requests.get(pdf_link, headers=NetInfo.HEADERS) - content_type = r.headers.get('content-type') + pdf_link = getSchiHubPDF(r.text) + if(pdf_link != None): + r = requests.get(pdf_link, headers=NetInfo.HEADERS) + content_type = r.headers.get('content-type') - if 'application/pdf' in content_type: - paper_files.append(saveFile(pdf_dir,r.content,p,dwn_source)) - - except: - pass + if 'application/pdf' in content_type: + paper_files.append(saveFile(pdf_dir,r.content,p,dwn_source)) + - faild += 1 \ No newline at end of file + faild += 1 diff --git a/PyPaperBot/HTMLparsers.py b/PyPaperBot/HTMLparsers.py index 71675e0..5207518 100644 --- a/PyPaperBot/HTMLparsers.py +++ b/PyPaperBot/HTMLparsers.py @@ -15,6 +15,8 @@ def schoolarParser(html): link = None link_pdf = None cites = None + year = None + authors = None for h3 in element.findAll("h3", class_="gs_rt"): found = False for a in h3.findAll("a"): @@ -27,8 +29,30 @@ def schoolarParser(html): cites = int(a.text[8:]) if "[PDF]" in a.text: link_pdf = a.get("href") + for div in element.findAll("div", class_="gs_a"): + authors, source_and_year, source = div.text.replace('\u00A0', ' ').split(" - ") + + if not authors.endswith('\u2026'): + # There is no ellipsis at the end so we know the full list of authors + authors = authors.replace(', ', ';') + else: + authors = None + try: + year = int(source_and_year[-4:]) + except ValueError: + pass + if not (1000 <= year <= 3000): + year = None + else: + year = str(year) if title!=None: - result.append((title, link, cites, link_pdf)) + result.append({ + 'title' : title, + 'link' : link, + 'cites' : cites, + 'link_pdf' : link_pdf, + 'year' : year, + 'authors' : authors}) return result diff --git a/PyPaperBot/Paper.py b/PyPaperBot/Paper.py index 04f68b8..0125416 100644 --- a/PyPaperBot/Paper.py +++ b/PyPaperBot/Paper.py @@ -10,13 +10,14 @@ class Paper: - def __init__(self,title=None, scholar_link=None, scholar_page=None, cites=None, link_pdf=None): + def __init__(self,title=None, scholar_link=None, scholar_page=None, cites=None, link_pdf=None, year=None, authors=None): self.title = title self.scholar_page = scholar_page self.scholar_link = scholar_link self.pdf_link = link_pdf + self.year = year + self.authors = authors - self.year = None self.jurnal = None self.cites_num = None self.bibtex = None @@ -37,8 +38,11 @@ def setBibtex(self,bibtex): self.bibtex = bibtex - try: - self.year=x[0]["year"] if "year" in x[0] else None + try: + if "year" in x[0]: + self.year=x[0]["year"] + if 'author' in x[0]: + self.authors = x[0]["author"] self.jurnal=x[0]["journal"].replace("\\","") if "journal" in x[0] else None if self.jurnal==None: self.jurnal=x[0]["publisher"].replace("\\","") if "publisher" in x[0] else None @@ -61,7 +65,7 @@ def strFix(s): return str(s).replace(",", "").rstrip('\n') - content = "Name,Scholar Link,DOI,Bibtex,PDF Name,Year,Scholar page,Journal,Downloaded,Downloaded from" + content = "Name,Scholar Link,DOI,Bibtex,PDF Name,Year,Scholar page,Journal,Downloaded,Downloaded from,Authors" for p in papers: pdf_name = p.getFileName() if p.downloaded==True else "" bibtex_found = True if p.bibtex!=None else False @@ -75,7 +79,8 @@ def strFix(s): content += ("\n"+strFix(p.title)+","+strFix(p.scholar_link)+","+ strFix(p.DOI)+","+strFix(bibtex_found)+","+ strFix(pdf_name)+ ","+strFix(p.year)+","+strFix(p.scholar_page)+","+ - strFix(p.jurnal)+","+strFix(p.downloaded)+","+strFix(dwn_from)) + strFix(p.jurnal)+","+strFix(p.downloaded)+","+strFix(dwn_from) + + ","+strFix(p.authors)) f = open(path, "w", encoding='utf-8-sig') f.write(content) @@ -100,4 +105,4 @@ def generateBibtex(papers, path): - \ No newline at end of file + From e65be7079dedbe2cbaa0349a068c857912db2672 Mon Sep 17 00:00:00 2001 From: 23k_mal Date: Thu, 25 Mar 2021 04:25:09 +0300 Subject: [PATCH 2/8] Fix url --- PyPaperBot/Scholar.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/PyPaperBot/Scholar.py b/PyPaperBot/Scholar.py index 64cea8d..e6bea16 100644 --- a/PyPaperBot/Scholar.py +++ b/PyPaperBot/Scholar.py @@ -18,7 +18,7 @@ def ScholarPapersInfo(query, scholar_pages, restrict, min_date=None): url = "https://scholar.google.com/scholar?hl=en&q="+query+"&as_vis=1&as_sdt=1,5" if min_date!=None: - url += "&as_ylo"+min_date + url += "&as_ylo="+str(min_date) if len(query)>7 and (query[0:7]=="http://" or query[0:8]=="https://"): url = query @@ -52,4 +52,4 @@ def ScholarPapersInfo(query, scholar_pages, restrict, min_date=None): i += 1 url += "&start=" + str(10*i) - return [item for sublist in to_download for item in sublist] \ No newline at end of file + return [item for sublist in to_download for item in sublist] From e27232ba1ffe82ab52cb8f81de5399c9a2a142a0 Mon Sep 17 00:00:00 2001 From: 23k_mal Date: Thu, 25 Mar 2021 04:48:52 +0300 Subject: [PATCH 3/8] Generating .csv using csv-module --- PyPaperBot/Paper.py | 55 ++++++++++++++++++++++++--------------------- 1 file changed, 29 insertions(+), 26 deletions(-) diff --git a/PyPaperBot/Paper.py b/PyPaperBot/Paper.py index 0125416..d78ee11 100644 --- a/PyPaperBot/Paper.py +++ b/PyPaperBot/Paper.py @@ -6,6 +6,7 @@ """ import bibtexparser import re +import csv class Paper: @@ -57,34 +58,36 @@ def canBeDownloaded(self): return False - def generateReport(papers, path): - def strFix(s): - if(len(str(s))==0): - return "None" - else: - return str(s).replace(",", "").rstrip('\n') + def generateReport(papers, path): + with open(path, mode="w", encoding='utf-8', newline='', buffering=1) as w_file: + content = ["Name", "Scholar Link", "DOI", "Bibtex", + "PDF Name", "Year", "Scholar page", "Journal", + "Downloaded", "Downloaded from", "Authors"] + file_writer = csv.DictWriter(w_file, delimiter = ",", fieldnames=content) + file_writer.writeheader() - - content = "Name,Scholar Link,DOI,Bibtex,PDF Name,Year,Scholar page,Journal,Downloaded,Downloaded from,Authors" - for p in papers: - pdf_name = p.getFileName() if p.downloaded==True else "" - bibtex_found = True if p.bibtex!=None else False + for p in papers: + pdf_name = p.getFileName() if p.downloaded==True else "" + bibtex_found = True if p.bibtex!=None else False - dwn_from = "" - if p.downloadedFrom == 1: - dwn_from = "SciHub" - if p.downloadedFrom == 2: - dwn_from = "Scholar" - - content += ("\n"+strFix(p.title)+","+strFix(p.scholar_link)+","+ - strFix(p.DOI)+","+strFix(bibtex_found)+","+ strFix(pdf_name)+ - ","+strFix(p.year)+","+strFix(p.scholar_page)+","+ - strFix(p.jurnal)+","+strFix(p.downloaded)+","+strFix(dwn_from) + - ","+strFix(p.authors)) - - f = open(path, "w", encoding='utf-8-sig') - f.write(content) - f.close() + dwn_from = "" + if p.downloadedFrom == 1: + dwn_from = "SciHub" + if p.downloadedFrom == 2: + dwn_from = "Scholar" + + file_writer.writerow({ + "Name" : p.title, + "Scholar Link" : p.scholar_link, + "DOI" : p.DOI, + "Bibtex" : bibtex_found, + "PDF Name" : pdf_name, + "Year" : p.year, + "Scholar page" : p.scholar_page, + "Journal" : p.jurnal, + "Downloaded" : p.downloaded, + "Downloaded from" : dwn_from, + "Authors" : p.authors}) def generateBibtex(papers, path): From 0f33c7d1edf590885675664215a9f71b1879a40f Mon Sep 17 00:00:00 2001 From: 23k_mal Date: Thu, 25 Mar 2021 04:54:31 +0300 Subject: [PATCH 4/8] try-except --- PyPaperBot/Downloader.py | 53 ++++++++++++++++++++-------------------- 1 file changed, 27 insertions(+), 26 deletions(-) diff --git a/PyPaperBot/Downloader.py b/PyPaperBot/Downloader.py index 7019763..c7e255e 100644 --- a/PyPaperBot/Downloader.py +++ b/PyPaperBot/Downloader.py @@ -64,33 +64,34 @@ def URLjoin(*args): faild = 0 while p.downloaded==False and faild!=4: + try: + dwn_source = 1 #1 scihub 2 scholar + if faild==0 and p.DOI!=None: + url = URLjoin(NetInfo.SciHub_URL, p.DOI) + if faild==1 and p.scholar_link!=None: + url = URLjoin(NetInfo.SciHub_URL, p.scholar_link) + if faild==2 and p.scholar_link!=None and p.scholar_link[-3:]=="pdf": + url = p.scholar_link + dwn_source = 2 + if faild==3 and p.pdf_link!=None: + url = p.pdf_link + dwn_source = 2 - dwn_source = 1 #1 scihub 2 scholar - if faild==0 and p.DOI!=None: - url = URLjoin(NetInfo.SciHub_URL, p.DOI) - if faild==1 and p.scholar_link!=None: - url = URLjoin(NetInfo.SciHub_URL, p.scholar_link) - if faild==2 and p.scholar_link!=None and p.scholar_link[-3:]=="pdf": - url = p.scholar_link - dwn_source = 2 - if faild==3 and p.pdf_link!=None: - url = p.pdf_link - dwn_source = 2 - - if url!="": - r = requests.get(url, headers=NetInfo.HEADERS) - content_type = r.headers.get('content-type') - - if dwn_source==1 and 'application/pdf' not in content_type: - time.sleep(random.randint(1,5)) + if url!="": + r = requests.get(url, headers=NetInfo.HEADERS) + content_type = r.headers.get('content-type') - pdf_link = getSchiHubPDF(r.text) - if(pdf_link != None): - r = requests.get(pdf_link, headers=NetInfo.HEADERS) - content_type = r.headers.get('content-type') - - if 'application/pdf' in content_type: - paper_files.append(saveFile(pdf_dir,r.content,p,dwn_source)) - + if dwn_source==1 and 'application/pdf' not in content_type: + time.sleep(random.randint(1,5)) + + pdf_link = getSchiHubPDF(r.text) + if(pdf_link != None): + r = requests.get(pdf_link, headers=NetInfo.HEADERS) + content_type = r.headers.get('content-type') + + if 'application/pdf' in content_type: + paper_files.append(saveFile(pdf_dir,r.content,p,dwn_source)) + except Exception: + pass faild += 1 From 8cbee2ac32967311fdd2e2001fff74c5d675febe Mon Sep 17 00:00:00 2001 From: 23k_mal Date: Thu, 25 Mar 2021 20:06:56 +0300 Subject: [PATCH 5/8] fix a little bug --- PyPaperBot/HTMLparsers.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/PyPaperBot/HTMLparsers.py b/PyPaperBot/HTMLparsers.py index 5207518..06677f9 100644 --- a/PyPaperBot/HTMLparsers.py +++ b/PyPaperBot/HTMLparsers.py @@ -35,12 +35,10 @@ def schoolarParser(html): if not authors.endswith('\u2026'): # There is no ellipsis at the end so we know the full list of authors authors = authors.replace(', ', ';') - else: - authors = None try: year = int(source_and_year[-4:]) except ValueError: - pass + continue if not (1000 <= year <= 3000): year = None else: From 6b0a1a063ccd8fb7c309bdb9c291442770211623 Mon Sep 17 00:00:00 2001 From: 23k_mal Date: Tue, 30 Mar 2021 20:40:19 +0300 Subject: [PATCH 6/8] fix csv writer --- PyPaperBot/Paper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PyPaperBot/Paper.py b/PyPaperBot/Paper.py index d78ee11..a0208a4 100644 --- a/PyPaperBot/Paper.py +++ b/PyPaperBot/Paper.py @@ -63,7 +63,7 @@ def generateReport(papers, path): content = ["Name", "Scholar Link", "DOI", "Bibtex", "PDF Name", "Year", "Scholar page", "Journal", "Downloaded", "Downloaded from", "Authors"] - file_writer = csv.DictWriter(w_file, delimiter = ",", fieldnames=content) + file_writer = csv.DictWriter(w_file, delimiter = ",", lineterminator=os.linesep, fieldnames=content) file_writer.writeheader() for p in papers: From e808a9be40d6d3377ad7d25d6abd203d0a62adb0 Mon Sep 17 00:00:00 2001 From: 23k_mal Date: Thu, 1 Apr 2021 00:53:15 +0300 Subject: [PATCH 7/8] Add missing import --- PyPaperBot/Paper.py | 1 + 1 file changed, 1 insertion(+) diff --git a/PyPaperBot/Paper.py b/PyPaperBot/Paper.py index a0208a4..54ea3dc 100644 --- a/PyPaperBot/Paper.py +++ b/PyPaperBot/Paper.py @@ -7,6 +7,7 @@ import bibtexparser import re import csv +import os class Paper: From ec10aca4947c0d737dd81ecaabb89124dd24e4d9 Mon Sep 17 00:00:00 2001 From: 23k_mal Date: Fri, 2 Apr 2021 03:15:24 +0300 Subject: [PATCH 8/8] Ability to select any range of pages on scholar --- PyPaperBot/Scholar.py | 68 +++++++++++++++++++++++------------------- PyPaperBot/__main__.py | 33 +++++++++++++++----- README.md | 54 ++++++++++++++++++++++----------- 3 files changed, 100 insertions(+), 55 deletions(-) diff --git a/PyPaperBot/Scholar.py b/PyPaperBot/Scholar.py index e6bea16..5d21371 100644 --- a/PyPaperBot/Scholar.py +++ b/PyPaperBot/Scholar.py @@ -7,38 +7,34 @@ def waithIPchange(): - input("You have been blocked, change your IP or VPN than press Enter...") - print("Wait 30 seconds...") - time.sleep(30) + while True: + inp = input('You have been blocked, try changing your IP or using a VPN. ' + 'Press Enter to continue downloading, or type "exit" to stop and exit....') + if inp.strip().lower() == "exit": + return False + elif not inp.strip(): + print("Wait 30 seconds...") + time.sleep(30) + return True - -def ScholarPapersInfo(query, scholar_pages, restrict, min_date=None): - +def scholar_requests(scholar_pages, url, restrict): javascript_error = "Sorry, we can't verify that you're not a robot when JavaScript is turned off" - - url = "https://scholar.google.com/scholar?hl=en&q="+query+"&as_vis=1&as_sdt=1,5" - if min_date!=None: - url += "&as_ylo="+str(min_date) - - if len(query)>7 and (query[0:7]=="http://" or query[0:8]=="https://"): - url = query - - to_download = [] - last_blocked = False - i = 0 - while i < scholar_pages: - html = requests.get(url, headers=NetInfo.HEADERS) - html = html.text - - if javascript_error in html and last_blocked==False: - waithIPchange() - continue - else: - last_blocked=False - + for i in scholar_pages: + while True: + res_url = url % (10 * (i - 1)) + html = requests.get(res_url, headers=NetInfo.HEADERS) + html = html.text + + if javascript_error in html: + is_continue = waithIPchange() + if not is_continue: + return to_download + else: + break + papers = schoolarParser(html) - print("\nGoogle Scholar page {} : {} papers found".format((i+1),len(papers))) + print("\nGoogle Scholar page {} : {} papers found".format(i,len(papers))) if(len(papers)>0): papersInfo = getPapersInfo(papers, url, restrict) @@ -49,7 +45,19 @@ def ScholarPapersInfo(query, scholar_pages, restrict, min_date=None): else: print("Paper not found...") - i += 1 - url += "&start=" + str(10*i) + return to_download + + + +def ScholarPapersInfo(query, scholar_pages, restrict, min_date=None): + + url = r"https://scholar.google.com/scholar?hl=en&q="+query+"&as_vis=1&as_sdt=1,5&start=%d" + if min_date!=None: + url += "&as_ylo="+str(min_date) + + if len(query)>7 and (query[0:7]=="http://" or query[0:8]=="https://"): + url = query + + to_download = scholar_requests(scholar_pages, url, restrict) return [item for sublist in to_download for item in sublist] diff --git a/PyPaperBot/__main__.py b/PyPaperBot/__main__.py index c005431..3609c4c 100644 --- a/PyPaperBot/__main__.py +++ b/PyPaperBot/__main__.py @@ -29,7 +29,7 @@ def start(query, scholar_pages, dwn_dir, min_date=None, num_limit=None, num_limi i += 1 - if restrict!=0: + if restrict!=0 and to_download: if filter_jurnal_file!=None: to_download = filterJurnals(to_download,filter_jurnal_file) @@ -57,7 +57,9 @@ def main(): parser.add_argument('--query', type=str, default=None, help='Query to make on Google Scholar or Google Scholar page link') parser.add_argument('--doi', type=str, default=None, help='DOI of the paper to download (this option uses only SciHub to download)') parser.add_argument('--doi-file', type=str, default=None, help='File .txt containing the list of paper\'s DOIs to download') - parser.add_argument('--scholar-pages', type=int, help='Number of Google Scholar pages to inspect. Each page has a maximum of 10 papers (required for --query)') + parser.add_argument('--scholar-pages', type=str, help='If given in %%d format, the number of pages to download from the beginning. ' + 'If given in %%d-%%d format, the range of pages (starting from 1) to download (the end is included). ' + 'Each page has a maximum of 10 papers (required for --query)') parser.add_argument('--dwn-dir', type=str, help='Directory path in which to save the results') parser.add_argument('--min-year', default=None, type=int, help='Minimal publication year of the paper to download') parser.add_argument('--max-dwn-year', default=None, type=int, help='Maximum number of papers to download sorted by year') @@ -88,10 +90,25 @@ def main(): print("Error: Only one option between '--max-dwn-year' and '--max-dwn-cites' can be used ") sys.exit() - if(args.query != None and args.scholar_pages==None): - print("Error: with --query provide also --scholar-pages") - sys.exit() - + if(args.query != None): + if args.scholar_pages: + try: + split = args.scholar_pages.split('-') + if len(split) == 1: + scholar_pages = range(1, int(split[0]) + 1) + elif len(split) == 2: + start_page, end_page = [int(x) for x in split] + scholar_pages = range(start_page, end_page + 1) + else: + raise ValueError + except Exception: + print(r"Error: Invalid format for --scholar-pages option. Expected: %d or %d-%d, got: " + args.scholar_pages) + sys.exit() + else: + print("Error: with --query provide also --scholar-pages") + sys.exit() + + DOIs = None if args.doi_file!=None: DOIs = [] @@ -116,7 +133,7 @@ def main(): max_dwn_type = 1 - start(args.query, args.scholar_pages, dwn_dir, args.min_year , max_dwn, max_dwn_type , args.journal_filter, args.restrict, DOIs, args.scihub_mirror) + start(args.query, scholar_pages, dwn_dir, args.min_year , max_dwn, max_dwn_type , args.journal_filter, args.restrict, DOIs, args.scihub_mirror) if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/README.md b/README.md index 0676e3f..dadb71e 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,13 @@ [![Donate](https://img.shields.io/badge/Donate-PayPal-green.svg)](https://www.paypal.me/ferru97) # PyPaperBot + PyPaperBot is a Python tool for **downloading scientific papers** using Google Scholar, Crossref, and SciHub. The tool tries to download papers from different sources such as PDF provided by Scholar, Scholar related links, and Scihub. PyPaerbot is also able to download the **bibtex** of each paper. ## Features + - Download papers given a query - Download papers given paper's DOIs - Download papers given a Google Scholar link @@ -13,6 +15,7 @@ PyPaerbot is also able to download the **bibtex** of each paper. - Filter downloaded paper by year, journal and citations number ## Installation + Use `pip` to install from pypi: ```bash @@ -20,25 +23,28 @@ pip install PyPaperBot ``` ## How to use + PyPaperBot arguments: -| Arguments | Description | Type -| ------------- | ------------- |------------- | -| \-\-query | Query to make on Google Scholar or Google Scholar page link |string| -| \-\-doi |DOI of the paper to download (this option uses only SciHub to download) |string| -| \-\-doi-file |File .txt containing the list of paper's DOIs to download |string| -| \-\-scholar-pages | Number of Google Scholar pages to inspect. Each page has a maximum of 10 papers |int| -| \-\-dwn-dir | Directory path in which to save the result |string| -| \-\-min-year | Minimal publication year of the paper to download |int| -| \-\-max-dwn-year | Maximum number of papers to download sorted by year |int| -| \-\-max-dwn-cites | Maximum number of papers to download sorted by number of citations |int| -| \-\-journal-filter | CSV file path of the journal filter (More info on github) |string| -| \-\-restrict | 0:Download only Bibtex - 1:Down load only papers PDF |int| -| \-\-scihub-mirror | Mirror for downloading papers from sci-hub. If not set, it is selected automatically |string| -| \-h | Shows the help |--| +| Arguments | Description | Type | +| ------------------ | ---------------------------------------------------------------------------------------- | ------ | +| \-\-query | Query to make on Google Scholar or Google Scholar page link | string | +| \-\-doi | DOI of the paper to download (this option uses only SciHub to download) | string | +| \-\-doi-file | File .txt containing the list of paper's DOIs to download | string | +| \-\-scholar-pages | Number or range of Google Scholar pages to inspect. Each page has a maximum of 10 papers | string | +| \-\-dwn-dir | Directory path in which to save the result | string | +| \-\-min-year | Minimal publication year of the paper to download | int | +| \-\-max-dwn-year | Maximum number of papers to download sorted by year | int | +| \-\-max-dwn-cites | Maximum number of papers to download sorted by number of citations | int | +| \-\-journal-filter | CSV file path of the journal filter (More info on github) | string | +| \-\-restrict | 0:Download only Bibtex - 1:Down load only papers PDF | int | +| \-\-scihub-mirror | Mirror for downloading papers from sci-hub. If not set, it is selected automatically | string | +| \-h | Shows the help | -- | ### Note + You can use only one of the arguments in the following groups + - *\-\-query*, *\-\-doi-file*, and *\-\-doi* - *\-\-max-dwn-year* and *and max-dwn-cites* @@ -51,43 +57,57 @@ The argument *\-\-journal-filter* require the path of a CSV containing a list o The argument *\-\-doi-file* require the path of a txt file containing the list of paper's DOIs to download organized with one DOI per line [Example](https://github.com/ferru97/PyPaperBot/blob/master/file_examples/papers.txt) ## SchiHub access + If access to SciHub is blocked in your country, consider using a free VPN service like [ProtonVPN](https://protonvpn.com/) ## Example -Download a maximum of 30 papers given a query and starting from 2018 using the mirror https://sci-hub.do: + +Download a maximum of 30 papers from the first 3 pages given a query and starting from 2018 using the mirror https://sci-hub.do: + ```bash python -m PyPaperBot --query="Machine learning" --scholar-pages=3 --min-year=2018 --dwn-dir="C:\User\example\papers" --scihub-mirror="https://sci-hub.do" ``` +Download papers from pages 4 to 7 (7th included) given a query: + +```bash +python -m PyPaperBot --query="Machine learning" --scholar-pages=4-7 --dwn-dir="C:\User\example\papers" +``` + Download a paper given the DOI: + ```bash python -m PyPaperBot --doi="10.0086/s41037-711-0132-1" --dwn-dir="C:\User\example\papers"` ``` Download papers given a file containing the DOIs: + ```bash python -m PyPaperBot --doi-file="C:\User\example\papers\file.txt" --dwn-dir="C:\User\example\papers"` ``` If it doesn't work, try to use *py* instead of *python* i.e. + ```bash py -m PyPaperBot --doi="10.0086/s41037-711-0132-1" --dwn-dir="C:\User\example\papers"` ``` ## Contributions + Feel free to contribute to this project by proposing any change, fix, and enhancement on the **dev** branch ### To do + - Tests - Code documentation - General improvements ## Disclaimer + This application is for educational purposes only. I do not take responsibility for what you choose to do with this application. ## Donation + If you like this project, you can give me a cup of coffee :) [![paypal](https://www.paypalobjects.com/en_US/i/btn/btn_donateCC_LG.gif)](https://www.paypal.me/ferru97) - -