Python 点击";“显示更多”;按钮,直到显示表中的所有数据,然后从表中获取所有数据
我需要从这个页面的表中获取所有数据 但我需要单击“显示更多”按钮,直到显示所有数据 但是,无论我单击“显示更多”按钮多少次,该表始终有30行Python 点击";“显示更多”;按钮,直到显示表中的所有数据,然后从表中获取所有数据,python,selenium,selenium-webdriver,Python,Selenium,Selenium Webdriver,我需要从这个页面的表中获取所有数据 但我需要单击“显示更多”按钮,直到显示所有数据 但是,无论我单击“显示更多”按钮多少次,该表始终有30行 import sys import time from pyvirtualdisplay import Display from selenium import webdriver from selenium.webdriver.common.by import By from selenium.webdriver.support.ui import W
import sys
import time
from pyvirtualdisplay import Display
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import TimeoutException
import json
class Scrape:
display = None
driver = None
def __init__(self):
# Start display
self.display = Display(visible=0, size=(800, 600))
self.display.start()
# Init driver
self.driver = webdriver.Firefox()
self.driver.wait = WebDriverWait(self.driver, 5)
self.load_page()
time.sleep(5)
self.close()
def load_page(self):
data = []
url = 'https://www.nets.eu/dk-da/l%C3%B8sninger/Registreringsnumre'
xpath = '//table[@class="itera-DataTable"]/tbody/tr'
self.driver.get(url)
try:
table = self.driver.wait.until(EC.presence_of_element_located(
(By.CLASS_NAME, 'itera-DataTable')))
print 'Table found!'
i = 1
while True:
button = self.driver.wait.until(EC.presence_of_element_located(
(By.CLASS_NAME, 'itera-nextbatchbox')))
print 'Button %d found!' % (i)
row_count = len(self.driver.find_elements_by_xpath(xpath))
print row_count
button.click()
i += 1
if i > 5:
break
i = 1
for tr in self.driver.find_elements_by_xpath(xpath):
print 'TR %d' % (i)
tr_data = []
tds = tr.find_elements_by_tag_name('td')
if tds:
tr_data.append([td.text for td in tds])
data.append(tr_data)
i += 1
#print json.dumps(data)
except TimeoutException:
self.error('Table not found')
def error(self, str):
self.close()
print>>sys.stderr, str
sys.exit(1)
def close(self):
if self.driver is not None:
self.driver.quit()
self.display.stop()
if __name__ == '__main__':
Scrape()
看起来根本原因是点击按钮不起作用。使用JavaScriptExecutor单击按钮将解决此问题。请参阅下面的代码
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import TimeoutException
import time
driver = webdriver.Chrome()
data = []
url = 'https://www.nets.eu/dk-da/l%C3%B8sninger/Registreringsnumre'
xpath = '//table[@class="itera-DataTable"]/tbody/tr'
driver.get(url)
try:
table = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.CLASS_NAME, 'itera-DataTable')))
print('Table found!')
except TimeoutException:
print('Table not found')
row_count = len(driver.find_elements_by_xpath(xpath))
print(row_count)
while True:
try:
button = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.CSS_SELECTOR, '.itera-nextbatchbox')))
print('Button found!')
except TimeoutException:
break
driver.execute_script("arguments[0].click();", button)
time.sleep(1)
try:
WebDriverWait(driver, 10).until(lambda driver: len(driver.find_elements_by_xpath( xpath)) > row_count)
time.sleep(1)
row_count = len(driver.find_elements_by_xpath( xpath))
print(row_count)
except TimeoutException:
print('No more rows. Rows count: ' + str(len(driver.find_elements_by_xpath( xpath))))
那么数据库中有多少数据?您需要多少页的数据?您确实可以访问数据库数据,因为您有权限刮取此网站,对吗?Itera.RegNumbers.js中定义的see ore link的点击事件功能,我认为您出现问题的可能原因来自两个方面。1.在单击“查看更多”链接之前,您没有等待页面加载,浏览器正忙于呈现页面,无法响应单击事件。2.单击“查看更多”链接时,Itera.RegNumbers.js尚未完成加载。一种快速检测的方法是在self.driver.get(url)之后添加sleep 15秒;