Version 1.0.2

PyPaperBot/Downloader.py

@@ -44,9 +44,12 @@ def saveFile(file_name,content, paper,dwn_source):
     paper.downloadedFrom = dwn_source
 
 
-def downloadPapers(papers, dwnl_dir, num_limit):
+def downloadPapers(papers, dwnl_dir, num_limit, SciHub_URL=None):
     def URLjoin(*args):
         return "/".join(map(lambda x: str(x).rstrip('/'), args))
+
+    if SciHub_URL:
+        NetInfo.SciHub_URL = SciHub_URL
 
     if NetInfo.SciHub_URL==None:
         setSciHubUrl()

PyPaperBot/Scholar.py

@@ -12,11 +12,14 @@ def waithIPchange():
     time.sleep(30)
 
 
-def ScholarPapersInfo(query, scholar_pages, restrict):
+def ScholarPapersInfo(query, scholar_pages, restrict, min_date=None):
 
     javascript_error = "Sorry, we can't verify that you're not a robot when JavaScript is turned off"
 
     url = "https://scholar.google.com/scholar?hl=en&q="+query+"&as_vis=1&as_sdt=1,5"
+    if min_date!=None:
+        url += "&as_ylo"+min_date
+
     if len(query)>7 and (query[0:7]=="http://" or query[0:8]=="https://"):
          url = query        
 
@@ -30,7 +33,6 @@ def ScholarPapersInfo(query, scholar_pages, restrict):
 
         if javascript_error in html and last_blocked==False:
             waithIPchange()
-            i -= 1
             continue
         else:
             last_blocked=False

PyPaperBot/__init__.py

@@ -1 +1 @@
-__version__= "1.0.1"
+__version__= "1.0.2"

PyPaperBot/__main__.py

@@ -9,12 +9,12 @@
 from .Crossref import getPapersInfoFromDOIs
 
 
-def start(query, scholar_pages, dwn_dir, min_date=None, num_limit=None, num_limit_type=None, filter_jurnal_file=None, restrict=None, DOIs=None):
+def start(query, scholar_pages, dwn_dir, min_date=None, num_limit=None, num_limit_type=None, filter_jurnal_file=None, restrict=None, DOIs=None, SciHub_URL=None):
 
     to_download = []
     if DOIs==None:
         print("Query: {}".format(query)) 
-        to_download = ScholarPapersInfo(query, scholar_pages, restrict)
+        to_download = ScholarPapersInfo(query, scholar_pages, restrict, min_date)
     else:
         print("Downloading papers from DOIs\n")
         num = 1
@@ -42,7 +42,7 @@ def start(query, scholar_pages, dwn_dir, min_date=None, num_limit=None, num_limi
         if num_limit_type!=None and num_limit_type==1:       
             to_download.sort(key=lambda x: int(x.sc_cites) if x.sc_cites!=None else 0, reverse=True)
 
-        downloadPapers(to_download, dwn_dir, num_limit)
+        downloadPapers(to_download, dwn_dir, num_limit, SciHub_URL)
 
     Paper.generateReport(to_download,dwn_dir+"result.csv")
     Paper.generateBibtex(to_download,dwn_dir+"bibtex.bib")
@@ -64,6 +64,7 @@ def main():
     parser.add_argument('--max-dwn-cites', default=None, type=int, help='Maximum number of papers to download sorted by number of citations')
     parser.add_argument('--journal-filter', default=None, type=str ,help='CSV file path of the journal filter (More info on github)')
     parser.add_argument('--restrict', default=None, type=int ,choices=[0,1], help='0:Download only Bibtex - 1:Down load only papers PDF')
+    parser.add_argument('--scihub-mirror', default=None, type=str, help='Mirror for downloading papers from sci-hub. If not set, it is selected automatically')
 
     args = parser.parse_args()
 
@@ -115,8 +116,7 @@ def main():
         max_dwn_type = 1
 
 
-    start(args.query, args.scholar_pages, dwn_dir, args.min_year , max_dwn, max_dwn_type , args.journal_filter, args.restrict, DOIs)
-
+    start(args.query, args.scholar_pages, dwn_dir, args.min_year , max_dwn, max_dwn_type , args.journal_filter, args.restrict, DOIs, args.scihub_mirror)
 
 if __name__ == "__main__":
     main()

README.md

@@ -34,6 +34,7 @@ PyPaperBot arguments:
 | \-\-max-dwn-cites  | Maximum number of papers to download sorted by number of citations  |int|
 | \-\-journal-filter  | CSV file path of the journal filter (More info on github)  |string|
 | \-\-restrict  | 0:Download only Bibtex - 1:Down load only papers PDF  |int|
+| \-\-scihub-mirror  | Mirror for downloading papers from sci-hub. If not set, it is selected automatically  |string|
 | \-h  | Shows the help  |--|
 
 ### Note
@@ -53,9 +54,9 @@ The argument *\-\-doi-file*  require the path of a txt file containing the list
 If access to SciHub is blocked in your country, consider using a free VPN service like [ProtonVPN](https://protonvpn.com/)
 
 ## Example
-Download a maximum of 30 papers given a query and starting from 2018:
+Download a maximum of 30 papers given a query and starting from 2018 using the mirror https://sci-hub.do:
 ```bash
-python -m PyPaperBot --query="Machine learning" --scholar-pages=3  --min-year=2018 --dwn-dir="C:\User\example\papers"
+python -m PyPaperBot --query="Machine learning" --scholar-pages=3  --min-year=2018 --dwn-dir="C:\User\example\papers" --scihub-mirror="https://sci-hub.do"
 ```
 
 Download a paper given the DOI:

setup.py

@@ -6,7 +6,7 @@
 setuptools.setup(
   name = 'PyPaperBot',        
   packages = setuptools.find_packages(),
-  version = '1.0.1',     
+  version = '1.0.2',     
   license='MIT', 
   description = 'PyPaperBot is a Python tool for downloading scientific papers using Google Scholar, Crossref, and SciHub.',
   long_description=long_description,
@@ -30,8 +30,8 @@
         'isort==5.4.2',
         'lazy-object-proxy==1.4.3',
         'mccabe==0.6.1',
-        'numpy==1.19.1',
-        'pandas==1.1.1',
+        'numpy==1.20.1',
+        'pandas==1.2.2',
         'pylint==2.6.0',
         'pyparsing==2.4.7',
         'python-dateutil==2.8.1',
@@ -55,8 +55,9 @@
     'Programming Language :: Python :: 3.6',
     'Programming Language :: Python :: 3.7',
     'Programming Language :: Python :: 3.8',
+    'Programming Language :: Python :: 3.9',
   ],
   entry_points={
     'console_scripts': ["PyPaperBot=PyPaperBot.__main__:main"],
   },
-)
+)