diff --git a/PyPaperBot/Downloader.py b/PyPaperBot/Downloader.py index fe82adf..3807c80 100644 --- a/PyPaperBot/Downloader.py +++ b/PyPaperBot/Downloader.py @@ -44,9 +44,12 @@ def saveFile(file_name,content, paper,dwn_source): paper.downloadedFrom = dwn_source -def downloadPapers(papers, dwnl_dir, num_limit): +def downloadPapers(papers, dwnl_dir, num_limit, SciHub_URL=None): def URLjoin(*args): return "/".join(map(lambda x: str(x).rstrip('/'), args)) + + if SciHub_URL: + NetInfo.SciHub_URL = SciHub_URL if NetInfo.SciHub_URL==None: setSciHubUrl() diff --git a/PyPaperBot/Scholar.py b/PyPaperBot/Scholar.py index 9e40aa4..64cea8d 100644 --- a/PyPaperBot/Scholar.py +++ b/PyPaperBot/Scholar.py @@ -12,11 +12,14 @@ def waithIPchange(): time.sleep(30) -def ScholarPapersInfo(query, scholar_pages, restrict): +def ScholarPapersInfo(query, scholar_pages, restrict, min_date=None): javascript_error = "Sorry, we can't verify that you're not a robot when JavaScript is turned off" url = "https://scholar.google.com/scholar?hl=en&q="+query+"&as_vis=1&as_sdt=1,5" + if min_date!=None: + url += "&as_ylo"+min_date + if len(query)>7 and (query[0:7]=="http://" or query[0:8]=="https://"): url = query @@ -30,7 +33,6 @@ def ScholarPapersInfo(query, scholar_pages, restrict): if javascript_error in html and last_blocked==False: waithIPchange() - i -= 1 continue else: last_blocked=False diff --git a/PyPaperBot/__init__.py b/PyPaperBot/__init__.py index a975903..62737bb 100644 --- a/PyPaperBot/__init__.py +++ b/PyPaperBot/__init__.py @@ -1 +1 @@ -__version__= "1.0.1" \ No newline at end of file +__version__= "1.0.2" diff --git a/PyPaperBot/__main__.py b/PyPaperBot/__main__.py index 350c510..c005431 100644 --- a/PyPaperBot/__main__.py +++ b/PyPaperBot/__main__.py @@ -9,12 +9,12 @@ from .Crossref import getPapersInfoFromDOIs -def start(query, scholar_pages, dwn_dir, min_date=None, num_limit=None, num_limit_type=None, filter_jurnal_file=None, restrict=None, DOIs=None): +def start(query, scholar_pages, dwn_dir, min_date=None, num_limit=None, num_limit_type=None, filter_jurnal_file=None, restrict=None, DOIs=None, SciHub_URL=None): to_download = [] if DOIs==None: print("Query: {}".format(query)) - to_download = ScholarPapersInfo(query, scholar_pages, restrict) + to_download = ScholarPapersInfo(query, scholar_pages, restrict, min_date) else: print("Downloading papers from DOIs\n") num = 1 @@ -42,7 +42,7 @@ def start(query, scholar_pages, dwn_dir, min_date=None, num_limit=None, num_limi if num_limit_type!=None and num_limit_type==1: to_download.sort(key=lambda x: int(x.sc_cites) if x.sc_cites!=None else 0, reverse=True) - downloadPapers(to_download, dwn_dir, num_limit) + downloadPapers(to_download, dwn_dir, num_limit, SciHub_URL) Paper.generateReport(to_download,dwn_dir+"result.csv") Paper.generateBibtex(to_download,dwn_dir+"bibtex.bib") @@ -64,6 +64,7 @@ def main(): parser.add_argument('--max-dwn-cites', default=None, type=int, help='Maximum number of papers to download sorted by number of citations') parser.add_argument('--journal-filter', default=None, type=str ,help='CSV file path of the journal filter (More info on github)') parser.add_argument('--restrict', default=None, type=int ,choices=[0,1], help='0:Download only Bibtex - 1:Down load only papers PDF') + parser.add_argument('--scihub-mirror', default=None, type=str, help='Mirror for downloading papers from sci-hub. If not set, it is selected automatically') args = parser.parse_args() @@ -115,8 +116,7 @@ def main(): max_dwn_type = 1 - start(args.query, args.scholar_pages, dwn_dir, args.min_year , max_dwn, max_dwn_type , args.journal_filter, args.restrict, DOIs) - + start(args.query, args.scholar_pages, dwn_dir, args.min_year , max_dwn, max_dwn_type , args.journal_filter, args.restrict, DOIs, args.scihub_mirror) if __name__ == "__main__": main() \ No newline at end of file diff --git a/README.md b/README.md index 39a7e6d..0676e3f 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,7 @@ PyPaperBot arguments: | \-\-max-dwn-cites | Maximum number of papers to download sorted by number of citations |int| | \-\-journal-filter | CSV file path of the journal filter (More info on github) |string| | \-\-restrict | 0:Download only Bibtex - 1:Down load only papers PDF |int| +| \-\-scihub-mirror | Mirror for downloading papers from sci-hub. If not set, it is selected automatically |string| | \-h | Shows the help |--| ### Note @@ -53,9 +54,9 @@ The argument *\-\-doi-file* require the path of a txt file containing the list If access to SciHub is blocked in your country, consider using a free VPN service like [ProtonVPN](https://protonvpn.com/) ## Example -Download a maximum of 30 papers given a query and starting from 2018: +Download a maximum of 30 papers given a query and starting from 2018 using the mirror https://sci-hub.do: ```bash -python -m PyPaperBot --query="Machine learning" --scholar-pages=3 --min-year=2018 --dwn-dir="C:\User\example\papers" +python -m PyPaperBot --query="Machine learning" --scholar-pages=3 --min-year=2018 --dwn-dir="C:\User\example\papers" --scihub-mirror="https://sci-hub.do" ``` Download a paper given the DOI: diff --git a/requirements.txt b/requirements.txt index be605b1..cbbab16 100644 Binary files a/requirements.txt and b/requirements.txt differ diff --git a/setup.py b/setup.py index 1f238bf..a6a779a 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ setuptools.setup( name = 'PyPaperBot', packages = setuptools.find_packages(), - version = '1.0.1', + version = '1.0.2', license='MIT', description = 'PyPaperBot is a Python tool for downloading scientific papers using Google Scholar, Crossref, and SciHub.', long_description=long_description, @@ -30,8 +30,8 @@ 'isort==5.4.2', 'lazy-object-proxy==1.4.3', 'mccabe==0.6.1', - 'numpy==1.19.1', - 'pandas==1.1.1', + 'numpy==1.20.1', + 'pandas==1.2.2', 'pylint==2.6.0', 'pyparsing==2.4.7', 'python-dateutil==2.8.1', @@ -55,8 +55,9 @@ 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', ], entry_points={ 'console_scripts': ["PyPaperBot=PyPaperBot.__main__:main"], }, -) \ No newline at end of file +)