diff --git a/requirements.txt b/requirements.txt index 5e5b67547..21c8f7983 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,3 @@ -elasticsearch=8.10.1 \ No newline at end of file +elasticsearch==8.10.1 +selenium==4.14.0 +webdriver-manager==4.0.1 diff --git a/src/search/scrape_documents.py b/src/search/scrape_documents.py index e69de29bb..5675dcbde 100644 --- a/src/search/scrape_documents.py +++ b/src/search/scrape_documents.py @@ -0,0 +1,221 @@ +import time + +from selenium import webdriver +from selenium.webdriver.chrome.options import Options +from selenium.webdriver.chrome.service import Service as ChromeService +from selenium.webdriver.common.by import By +from selenium.webdriver.support import expected_conditions as EC +from selenium.webdriver.support.ui import Select +from selenium.webdriver.support.wait import WebDriverWait +from webdriver_manager.chrome import ChromeDriverManager + +numOfRetries = 3 + +def initBrowser(): + WINDOW_SIZE = "1920,1080" + chrome_options = Options() + chrome_options.add_argument("--headless") + chrome_options.add_argument("--window-size=%s" % WINDOW_SIZE) + chrome_options.add_argument("no-sandbox") + chrome_options.add_argument("--disable-extensions") + driver = webdriver.Chrome(service=ChromeService(ChromeDriverManager().install()), options=chrome_options) + return driver + + +# Given a keyword, get the first n number of documents from yahoo finance +# Documents are returned as a list of url string +# [ +# 'https://finance.yahoo.com/news/retail-investors-can-now-bet-on-music-royalties-we-want-people-to-have-access-150221575.html', +# 'https://finance.yahoo.com/news/foxconn-faces-tax-audit-land-024521324.html', +# 'https://finance.yahoo.com/news/iphone-assembler-hon-hai-dives-010601477.html', +# 'https://finance.yahoo.com/m/05189731-ff16-33e1-b077-dfead8ad1cb9/paypal-s-new-boss.html', +# ] +def getDocumentsUrls(keyword, n): + """ + :param string keyword: keyword to search + :param int n: number of docs to return + + """ + + url = "https://finance.yahoo.com/" + cur = 0 + + while cur < numOfRetries: + try: + driver = initBrowser() + except: + print("There was an issue initializing browser window") + continue + driver.get(url) + try: + WebDriverWait(driver, 5).until( + EC.presence_of_element_located((By.ID, "header-search-form")) + ) + break + except: + cur += 1 + print("Could not load " + url + ", retrying") + driver.close() + + if cur == numOfRetries: + print("Could not load " + url + ", exiting") + driver.quit() + return [] + + try: + WebDriverWait(driver, 10).until( + EC.presence_of_element_located((By.ID, "header-search-form")) + ) + except: + print("Could not load yahoo finace") + driver.quit() + return [] + + searchBar = driver.find_element(By.XPATH, "//form[contains(@action, '/quote')]") + searchBarInput = searchBar.find_element(By.TAG_NAME, "input") + searchBarInput.send_keys(keyword) + searchBarButton = searchBar.find_element(By.ID, "header-desktop-search-button") + searchBarButton.click() + + try: + WebDriverWait(driver, 10).until( + EC.presence_of_element_located((By.XPATH, "//div[contains(@id, 'mrt-node-quoteNewsStream')]")) + ) + except: + print("Could not load results") + driver.quit() + return [] + + resultsDiv = driver.find_element(By.XPATH, "//div[contains(@id, 'mrt-node-quoteNewsStream')]") + results = resultsDiv.find_elements(By.TAG_NAME, "li") + filteredResults = [] + for result in results: + try: + result.find_element(By.XPATH, ".//a[text()='Ad']") + except: + filteredResults.append(result) + + prev = 0 + while len(filteredResults) < n: + print("{} urls found".format(len(filteredResults))) + if (prev == len(filteredResults)): + print("Maximum number of documents found related to " + keyword) + break + prev = len(filteredResults) + driver.execute_script("arguments[0].scrollIntoView();",results[-1]) + time.sleep(1) + + results = resultsDiv.find_elements(By.TAG_NAME, "li") + filteredResults = [] + for result in results: + try: + result.find_element(By.XPATH, ".//a[text()='Ad']") + except: + filteredResults.append(result) + + res = [] + for result in filteredResults[:n]: + a = result.find_element(By.TAG_NAME, "a") + res.append(a.get_attribute('href')) + + driver.quit() + return res + +def processDocumentUrl(driver, url): + print("Processing " + url) + cur = 0 + while cur < numOfRetries: + driver.get(url) + try: + WebDriverWait(driver, 5).until( + EC.presence_of_element_located((By.XPATH, "//h1[@data-test-locator='headline']")) + ) + break + except: + cur += 1 + print("Could not load " + url + ", retrying") + + if cur == numOfRetries: + print("Could not load " + url + ", skiping") + return + + title = driver.find_element(By.XPATH, "//h1[@data-test-locator='headline']").text + + if driver.find_elements(By.XPATH, "//div[contains(@class, 'caas-readmore')]//*[text()='Continue reading']"): + body = driver.find_element(By.CLASS_NAME, "caas-body") + text = body.find_elements(By.TAG_NAME, 'p') + a = body.find_elements(By.TAG_NAME, "a") + print("External link") + return { + 'base_url': url, + 'title': title, + 'body': "\n".join([p.text for p in text]), + 'urls': [_.get_attribute('href') for _ in a] + } + + readMoreButtons = driver.find_elements(By.XPATH, "//div[contains(@class, 'caas-readmore')]//button") + if readMoreButtons: + expandButton = readMoreButtons[0] + expandButton.click() + time.sleep(1) + + body = driver.find_element(By.CLASS_NAME, "caas-body") + text = body.find_elements(By.TAG_NAME, 'p') + a = body.find_elements(By.TAG_NAME, "a") + + return { + 'base_url': url, + 'title': title, + 'body': "\n".join([p.text for p in text]), + 'urls': [_.get_attribute('href') for _ in a] + } + + +# Given a list of urls, process the urls and return a list of documents +# [ +# { +# 'base_url': 'https://finance.yahoo.com/m/05189731-ff16-33e1-b077-dfead8ad1cb9/paypal-s-new-boss.html', +# 'title': "PayPal's New Boss", +# 'body': "When PayPal Holdings reports third quarter earnings on Nov. 1, it'll give new Chief Executive Alex Chriss his first chance to lay out his turnaround strategy." +# 'urls': [] +# }, +# { +# ... +# }, +# ... +# ] +# +# +# Note that this method cannot process the urls recursively +def processUrls(urls): + try: + driver = initBrowser() + except: + print("There was an issue initializing browser window") + return [] + + res = [] + for url in urls: + doc = processDocumentUrl(driver, url) + if doc: + res.append(doc) + + return res + + + +# urls = getDocumentsUrls("Apple", 10) +# f = open("urls.txt", "w") +# for url in urls: +# f.write(url + "\n") + +# f.close() + + +f = open("urls.txt", "r") +urls = f.read().rstrip().split("\n") +docs = processUrls(urls) + +print(docs[0]['body']) + +