diff --git a/PyPaperBot/Crossref.py b/PyPaperBot/Crossref.py index a2775f1..613974f 100644 --- a/PyPaperBot/Crossref.py +++ b/PyPaperBot/Crossref.py @@ -42,33 +42,41 @@ def getPapersInfo(papers, scholar_search_link, restrict): papers_return = [] num = 1 for paper in papers: - title = paper[0].lower() - queries = {'query.bibliographic': title,'sort':'relevance',"select":"DOI,title,deposited,author,short-container-title"} + title = paper['title'] + queries = {'query.bibliographic': title.lower(),'sort':'relevance',"select":"DOI,title,deposited,author,short-container-title"} print("Searching paper {} of {} on Crossref...".format(num,len(papers))) num += 1 found_timestamp = 0 - paper_found = Paper(title,paper[1],scholar_search_link, paper[2], paper[3]) - for el in iterate_publications_as_json(max_results=30, queries=queries): - - el_date = 0 - if "deposited" in el and "timestamp" in el["deposited"]: - el_date = int(el["deposited"]["timestamp"]) - - if (paper_found.DOI==None or el_date>found_timestamp) and "title" in el and similarStrings(title ,el["title"][0].lower())>0.75: - found_timestamp = el_date - - if "DOI" in el: - paper_found.DOI = el["DOI"].strip().lower() - if "short-container-title" in el and len(el["short-container-title"])>0: - paper_found.jurnal = el["short-container-title"][0] + paper_found = Paper(title,paper['link'],scholar_search_link, paper['cites'], paper['link_pdf'], paper['year'], paper['authors']) + while True: + try: + for el in iterate_publications_as_json(max_results=30, queries=queries): - if restrict==None or restrict!=1: - paper_found.setBibtex(getBibtex(paper_found.DOI)) + el_date = 0 + if "deposited" in el and "timestamp" in el["deposited"]: + el_date = int(el["deposited"]["timestamp"]) + + if (paper_found.DOI==None or el_date>found_timestamp) and "title" in el and similarStrings(title.lower() ,el["title"][0].lower())>0.75: + found_timestamp = el_date + + if "DOI" in el: + paper_found.DOI = el["DOI"].strip().lower() + if "short-container-title" in el and len(el["short-container-title"])>0: + paper_found.jurnal = el["short-container-title"][0] + + if restrict==None or restrict!=1: + paper_found.setBibtex(getBibtex(paper_found.DOI)) + + break + except ConnectionError as e: + print("Wait 10 seconds and try again...") + time.sleep(10) + papers_return.append(paper_found) time.sleep(random.randint(1,10)) - return papers_return \ No newline at end of file + return papers_return diff --git a/PyPaperBot/Downloader.py b/PyPaperBot/Downloader.py index 7102427..c7e255e 100644 --- a/PyPaperBot/Downloader.py +++ b/PyPaperBot/Downloader.py @@ -64,8 +64,7 @@ def URLjoin(*args): faild = 0 while p.downloaded==False and faild!=4: - try: - + try: dwn_source = 1 #1 scihub 2 scholar if faild==0 and p.DOI!=None: url = URLjoin(NetInfo.SciHub_URL, p.DOI) @@ -77,23 +76,22 @@ def URLjoin(*args): if faild==3 and p.pdf_link!=None: url = p.pdf_link dwn_source = 2 - + if url!="": r = requests.get(url, headers=NetInfo.HEADERS) content_type = r.headers.get('content-type') - + if dwn_source==1 and 'application/pdf' not in content_type: time.sleep(random.randint(1,5)) - + pdf_link = getSchiHubPDF(r.text) if(pdf_link != None): r = requests.get(pdf_link, headers=NetInfo.HEADERS) content_type = r.headers.get('content-type') - + if 'application/pdf' in content_type: paper_files.append(saveFile(pdf_dir,r.content,p,dwn_source)) - - except: + except Exception: pass - faild += 1 \ No newline at end of file + faild += 1 diff --git a/PyPaperBot/HTMLparsers.py b/PyPaperBot/HTMLparsers.py index 71675e0..8d6784c 100644 --- a/PyPaperBot/HTMLparsers.py +++ b/PyPaperBot/HTMLparsers.py @@ -15,6 +15,8 @@ def schoolarParser(html): link = None link_pdf = None cites = None + year = None + authors = None for h3 in element.findAll("h3", class_="gs_rt"): found = False for a in h3.findAll("a"): @@ -27,8 +29,33 @@ def schoolarParser(html): cites = int(a.text[8:]) if "[PDF]" in a.text: link_pdf = a.get("href") + for div in element.findAll("div", class_="gs_a"): + try: + authors, source_and_year, source = div.text.replace('\u00A0', ' ').split(" - ") + except ValueError: + continue + + if not authors.strip().endswith('\u2026'): + # There is no ellipsis at the end so we know the full list of authors + authors = authors.replace(', ', ';') + else: + authors = None + try: + year = int(source_and_year[-4:]) + except ValueError: + continue + if not (1000 <= year <= 3000): + year = None + else: + year = str(year) if title!=None: - result.append((title, link, cites, link_pdf)) + result.append({ + 'title' : title, + 'link' : link, + 'cites' : cites, + 'link_pdf' : link_pdf, + 'year' : year, + 'authors' : authors}) return result diff --git a/PyPaperBot/Paper.py b/PyPaperBot/Paper.py index 04f68b8..54ea3dc 100644 --- a/PyPaperBot/Paper.py +++ b/PyPaperBot/Paper.py @@ -6,17 +6,20 @@ """ import bibtexparser import re +import csv +import os class Paper: - def __init__(self,title=None, scholar_link=None, scholar_page=None, cites=None, link_pdf=None): + def __init__(self,title=None, scholar_link=None, scholar_page=None, cites=None, link_pdf=None, year=None, authors=None): self.title = title self.scholar_page = scholar_page self.scholar_link = scholar_link self.pdf_link = link_pdf + self.year = year + self.authors = authors - self.year = None self.jurnal = None self.cites_num = None self.bibtex = None @@ -37,8 +40,11 @@ def setBibtex(self,bibtex): self.bibtex = bibtex - try: - self.year=x[0]["year"] if "year" in x[0] else None + try: + if "year" in x[0]: + self.year=x[0]["year"] + if 'author' in x[0]: + self.authors = x[0]["author"] self.jurnal=x[0]["journal"].replace("\\","") if "journal" in x[0] else None if self.jurnal==None: self.jurnal=x[0]["publisher"].replace("\\","") if "publisher" in x[0] else None @@ -53,33 +59,36 @@ def canBeDownloaded(self): return False - def generateReport(papers, path): - def strFix(s): - if(len(str(s))==0): - return "None" - else: - return str(s).replace(",", "").rstrip('\n') + def generateReport(papers, path): + with open(path, mode="w", encoding='utf-8', newline='', buffering=1) as w_file: + content = ["Name", "Scholar Link", "DOI", "Bibtex", + "PDF Name", "Year", "Scholar page", "Journal", + "Downloaded", "Downloaded from", "Authors"] + file_writer = csv.DictWriter(w_file, delimiter = ",", lineterminator=os.linesep, fieldnames=content) + file_writer.writeheader() - - content = "Name,Scholar Link,DOI,Bibtex,PDF Name,Year,Scholar page,Journal,Downloaded,Downloaded from" - for p in papers: - pdf_name = p.getFileName() if p.downloaded==True else "" - bibtex_found = True if p.bibtex!=None else False + for p in papers: + pdf_name = p.getFileName() if p.downloaded==True else "" + bibtex_found = True if p.bibtex!=None else False - dwn_from = "" - if p.downloadedFrom == 1: - dwn_from = "SciHub" - if p.downloadedFrom == 2: - dwn_from = "Scholar" - - content += ("\n"+strFix(p.title)+","+strFix(p.scholar_link)+","+ - strFix(p.DOI)+","+strFix(bibtex_found)+","+ strFix(pdf_name)+ - ","+strFix(p.year)+","+strFix(p.scholar_page)+","+ - strFix(p.jurnal)+","+strFix(p.downloaded)+","+strFix(dwn_from)) - - f = open(path, "w", encoding='utf-8-sig') - f.write(content) - f.close() + dwn_from = "" + if p.downloadedFrom == 1: + dwn_from = "SciHub" + if p.downloadedFrom == 2: + dwn_from = "Scholar" + + file_writer.writerow({ + "Name" : p.title, + "Scholar Link" : p.scholar_link, + "DOI" : p.DOI, + "Bibtex" : bibtex_found, + "PDF Name" : pdf_name, + "Year" : p.year, + "Scholar page" : p.scholar_page, + "Journal" : p.jurnal, + "Downloaded" : p.downloaded, + "Downloaded from" : dwn_from, + "Authors" : p.authors}) def generateBibtex(papers, path): @@ -100,4 +109,4 @@ def generateBibtex(papers, path): - \ No newline at end of file + diff --git a/PyPaperBot/Scholar.py b/PyPaperBot/Scholar.py index 64cea8d..5d21371 100644 --- a/PyPaperBot/Scholar.py +++ b/PyPaperBot/Scholar.py @@ -7,38 +7,34 @@ def waithIPchange(): - input("You have been blocked, change your IP or VPN than press Enter...") - print("Wait 30 seconds...") - time.sleep(30) + while True: + inp = input('You have been blocked, try changing your IP or using a VPN. ' + 'Press Enter to continue downloading, or type "exit" to stop and exit....') + if inp.strip().lower() == "exit": + return False + elif not inp.strip(): + print("Wait 30 seconds...") + time.sleep(30) + return True - -def ScholarPapersInfo(query, scholar_pages, restrict, min_date=None): - +def scholar_requests(scholar_pages, url, restrict): javascript_error = "Sorry, we can't verify that you're not a robot when JavaScript is turned off" - - url = "https://scholar.google.com/scholar?hl=en&q="+query+"&as_vis=1&as_sdt=1,5" - if min_date!=None: - url += "&as_ylo"+min_date - - if len(query)>7 and (query[0:7]=="http://" or query[0:8]=="https://"): - url = query - - to_download = [] - last_blocked = False - i = 0 - while i < scholar_pages: - html = requests.get(url, headers=NetInfo.HEADERS) - html = html.text - - if javascript_error in html and last_blocked==False: - waithIPchange() - continue - else: - last_blocked=False - + for i in scholar_pages: + while True: + res_url = url % (10 * (i - 1)) + html = requests.get(res_url, headers=NetInfo.HEADERS) + html = html.text + + if javascript_error in html: + is_continue = waithIPchange() + if not is_continue: + return to_download + else: + break + papers = schoolarParser(html) - print("\nGoogle Scholar page {} : {} papers found".format((i+1),len(papers))) + print("\nGoogle Scholar page {} : {} papers found".format(i,len(papers))) if(len(papers)>0): papersInfo = getPapersInfo(papers, url, restrict) @@ -49,7 +45,19 @@ def ScholarPapersInfo(query, scholar_pages, restrict, min_date=None): else: print("Paper not found...") - i += 1 - url += "&start=" + str(10*i) + return to_download + + + +def ScholarPapersInfo(query, scholar_pages, restrict, min_date=None): + + url = r"https://scholar.google.com/scholar?hl=en&q="+query+"&as_vis=1&as_sdt=1,5&start=%d" + if min_date!=None: + url += "&as_ylo="+str(min_date) + + if len(query)>7 and (query[0:7]=="http://" or query[0:8]=="https://"): + url = query + + to_download = scholar_requests(scholar_pages, url, restrict) - return [item for sublist in to_download for item in sublist] \ No newline at end of file + return [item for sublist in to_download for item in sublist] diff --git a/PyPaperBot/__main__.py b/PyPaperBot/__main__.py index c005431..3609c4c 100644 --- a/PyPaperBot/__main__.py +++ b/PyPaperBot/__main__.py @@ -29,7 +29,7 @@ def start(query, scholar_pages, dwn_dir, min_date=None, num_limit=None, num_limi i += 1 - if restrict!=0: + if restrict!=0 and to_download: if filter_jurnal_file!=None: to_download = filterJurnals(to_download,filter_jurnal_file) @@ -57,7 +57,9 @@ def main(): parser.add_argument('--query', type=str, default=None, help='Query to make on Google Scholar or Google Scholar page link') parser.add_argument('--doi', type=str, default=None, help='DOI of the paper to download (this option uses only SciHub to download)') parser.add_argument('--doi-file', type=str, default=None, help='File .txt containing the list of paper\'s DOIs to download') - parser.add_argument('--scholar-pages', type=int, help='Number of Google Scholar pages to inspect. Each page has a maximum of 10 papers (required for --query)') + parser.add_argument('--scholar-pages', type=str, help='If given in %%d format, the number of pages to download from the beginning. ' + 'If given in %%d-%%d format, the range of pages (starting from 1) to download (the end is included). ' + 'Each page has a maximum of 10 papers (required for --query)') parser.add_argument('--dwn-dir', type=str, help='Directory path in which to save the results') parser.add_argument('--min-year', default=None, type=int, help='Minimal publication year of the paper to download') parser.add_argument('--max-dwn-year', default=None, type=int, help='Maximum number of papers to download sorted by year') @@ -88,10 +90,25 @@ def main(): print("Error: Only one option between '--max-dwn-year' and '--max-dwn-cites' can be used ") sys.exit() - if(args.query != None and args.scholar_pages==None): - print("Error: with --query provide also --scholar-pages") - sys.exit() - + if(args.query != None): + if args.scholar_pages: + try: + split = args.scholar_pages.split('-') + if len(split) == 1: + scholar_pages = range(1, int(split[0]) + 1) + elif len(split) == 2: + start_page, end_page = [int(x) for x in split] + scholar_pages = range(start_page, end_page + 1) + else: + raise ValueError + except Exception: + print(r"Error: Invalid format for --scholar-pages option. Expected: %d or %d-%d, got: " + args.scholar_pages) + sys.exit() + else: + print("Error: with --query provide also --scholar-pages") + sys.exit() + + DOIs = None if args.doi_file!=None: DOIs = [] @@ -116,7 +133,7 @@ def main(): max_dwn_type = 1 - start(args.query, args.scholar_pages, dwn_dir, args.min_year , max_dwn, max_dwn_type , args.journal_filter, args.restrict, DOIs, args.scihub_mirror) + start(args.query, scholar_pages, dwn_dir, args.min_year , max_dwn, max_dwn_type , args.journal_filter, args.restrict, DOIs, args.scihub_mirror) if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/README.md b/README.md index 0676e3f..dadb71e 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,13 @@ [![Donate](https://img.shields.io/badge/Donate-PayPal-green.svg)](https://www.paypal.me/ferru97) # PyPaperBot + PyPaperBot is a Python tool for **downloading scientific papers** using Google Scholar, Crossref, and SciHub. The tool tries to download papers from different sources such as PDF provided by Scholar, Scholar related links, and Scihub. PyPaerbot is also able to download the **bibtex** of each paper. ## Features + - Download papers given a query - Download papers given paper's DOIs - Download papers given a Google Scholar link @@ -13,6 +15,7 @@ PyPaerbot is also able to download the **bibtex** of each paper. - Filter downloaded paper by year, journal and citations number ## Installation + Use `pip` to install from pypi: ```bash @@ -20,25 +23,28 @@ pip install PyPaperBot ``` ## How to use + PyPaperBot arguments: -| Arguments | Description | Type -| ------------- | ------------- |------------- | -| \-\-query | Query to make on Google Scholar or Google Scholar page link |string| -| \-\-doi |DOI of the paper to download (this option uses only SciHub to download) |string| -| \-\-doi-file |File .txt containing the list of paper's DOIs to download |string| -| \-\-scholar-pages | Number of Google Scholar pages to inspect. Each page has a maximum of 10 papers |int| -| \-\-dwn-dir | Directory path in which to save the result |string| -| \-\-min-year | Minimal publication year of the paper to download |int| -| \-\-max-dwn-year | Maximum number of papers to download sorted by year |int| -| \-\-max-dwn-cites | Maximum number of papers to download sorted by number of citations |int| -| \-\-journal-filter | CSV file path of the journal filter (More info on github) |string| -| \-\-restrict | 0:Download only Bibtex - 1:Down load only papers PDF |int| -| \-\-scihub-mirror | Mirror for downloading papers from sci-hub. If not set, it is selected automatically |string| -| \-h | Shows the help |--| +| Arguments | Description | Type | +| ------------------ | ---------------------------------------------------------------------------------------- | ------ | +| \-\-query | Query to make on Google Scholar or Google Scholar page link | string | +| \-\-doi | DOI of the paper to download (this option uses only SciHub to download) | string | +| \-\-doi-file | File .txt containing the list of paper's DOIs to download | string | +| \-\-scholar-pages | Number or range of Google Scholar pages to inspect. Each page has a maximum of 10 papers | string | +| \-\-dwn-dir | Directory path in which to save the result | string | +| \-\-min-year | Minimal publication year of the paper to download | int | +| \-\-max-dwn-year | Maximum number of papers to download sorted by year | int | +| \-\-max-dwn-cites | Maximum number of papers to download sorted by number of citations | int | +| \-\-journal-filter | CSV file path of the journal filter (More info on github) | string | +| \-\-restrict | 0:Download only Bibtex - 1:Down load only papers PDF | int | +| \-\-scihub-mirror | Mirror for downloading papers from sci-hub. If not set, it is selected automatically | string | +| \-h | Shows the help | -- | ### Note + You can use only one of the arguments in the following groups + - *\-\-query*, *\-\-doi-file*, and *\-\-doi* - *\-\-max-dwn-year* and *and max-dwn-cites* @@ -51,43 +57,57 @@ The argument *\-\-journal-filter* require the path of a CSV containing a list o The argument *\-\-doi-file* require the path of a txt file containing the list of paper's DOIs to download organized with one DOI per line [Example](https://github.com/ferru97/PyPaperBot/blob/master/file_examples/papers.txt) ## SchiHub access + If access to SciHub is blocked in your country, consider using a free VPN service like [ProtonVPN](https://protonvpn.com/) ## Example -Download a maximum of 30 papers given a query and starting from 2018 using the mirror https://sci-hub.do: + +Download a maximum of 30 papers from the first 3 pages given a query and starting from 2018 using the mirror https://sci-hub.do: + ```bash python -m PyPaperBot --query="Machine learning" --scholar-pages=3 --min-year=2018 --dwn-dir="C:\User\example\papers" --scihub-mirror="https://sci-hub.do" ``` +Download papers from pages 4 to 7 (7th included) given a query: + +```bash +python -m PyPaperBot --query="Machine learning" --scholar-pages=4-7 --dwn-dir="C:\User\example\papers" +``` + Download a paper given the DOI: + ```bash python -m PyPaperBot --doi="10.0086/s41037-711-0132-1" --dwn-dir="C:\User\example\papers"` ``` Download papers given a file containing the DOIs: + ```bash python -m PyPaperBot --doi-file="C:\User\example\papers\file.txt" --dwn-dir="C:\User\example\papers"` ``` If it doesn't work, try to use *py* instead of *python* i.e. + ```bash py -m PyPaperBot --doi="10.0086/s41037-711-0132-1" --dwn-dir="C:\User\example\papers"` ``` ## Contributions + Feel free to contribute to this project by proposing any change, fix, and enhancement on the **dev** branch ### To do + - Tests - Code documentation - General improvements ## Disclaimer + This application is for educational purposes only. I do not take responsibility for what you choose to do with this application. ## Donation + If you like this project, you can give me a cup of coffee :) [![paypal](https://www.paypalobjects.com/en_US/i/btn/btn_donateCC_LG.gif)](https://www.paypal.me/ferru97) - -