解析导航栏的url

 1 #coding=utf-8 2 ?3 """ 4 是为了获取XXX系统菜单的url 5 使用的是selenium登录，获取网页的内容，然后用beautifulsoup来解析 6 """ 7 import unittest 8 import time 9 from selenium import webdriver10 from bs4 import BeautifulSoup11 12 # 登录url13 url = ‘http://XXXX.XXXX.com/‘ # 系统的url14 username = ‘XXXX‘15 password = ‘XXXXX‘16 17 class GetUrl(unittest.TestCase):18 ????def setUp(self):19 ????????self.dr = webdriver.Chrome()20 ????????self.dr.get(url)21 22 ????def tearDown(self):23 ????????self.dr.quit()24 25 ????def _login(self):26 ????????self.dr.find_element_by_id(‘username‘).send_keys(username) # 输入用户名27 ????????self.dr.find_element_by_id(‘password‘).send_keys(password) # 输入密码28 ????????# self.dr.find_element_by_id(‘verifycode‘).send_keys(‘XXXXX‘) 这里原来是需要验证码的，后来取消掉了29 ????????self.dr.find_element_by_id(‘weblogin‘).click() # 点击登录按钮30 ????????time.sleep(3)31 32 ????def _gethtmlcontent(self):33 ????????"""获取当前页面的html的所有内容"""34 ????????content = self.dr.page_source # 将该页面的内容 返回给content保存起来方便后面解析35 ????????return content36 37 ????def _geturl(self,pagesource):38 ????????"""39 ????????找出所有的a标签，然后筛选掉非导航连接的a标签。返回的是一个dict40 ????????"""41 ????????result = dict()42 ????????soup = BeautifulSoup(pagesource, "lxml")43 ????????eles = soup.find_all("a")44 ????????flag = 045 ????????for ele in eles:46 ????????????if ‘#‘ in ele[‘href‘]:47 ????????????????continue48 ????????????tmp = ele.string49 ????????????if tmp is not None and ‘@‘ not in tmp:50 ????????????????flag += 151 ????????????????ele_url = ele[‘href‘].split(‘?‘)[0]52 ????????????????# print(‘{0} ==> {1}‘.format(tmp,ele_url))53 ????????????????result[tmp] = ele_url54 55 ????????# print(‘Find out {0} datas.‘.format(len(result)))56 ????????return result57 58 ????def _writetotxt(self,contents):59 ????????"""60 ????????将结果写入文件中61 ????????"""62 ????????print(‘写入开始‘)63 ????????with open(‘urlcontent.txt‘,‘w‘) as f:64 ???????????for title,value in contents.items():65 ????????????????f.write(‘{0} ==> {1}\n‘.format(title,value))66 ????????print(‘写入完毕‘)67 68 ????def test_run(self):69 ????????self._login()70 ????????pagesources = self._gethtmlcontent()71 ????????result = self._geturl(pagesources)72 ????????self._writetotxt(result)73 74 75 if __name__ == ‘__main__‘:76 ????unittest.main()

 1 #coding=utf-8 2 ?3 """ 4 是为了获取XXX系统菜单的url 5 使用的是selenium登录，查找元素，获取元素的属性 6 """ 7 from selenium import webdriver 8 import unittest 9 import time10 11 # 登录url12 url = ‘http://XXX.XXX.com/‘13 username = ‘XXX‘14 password = ‘XXX‘15 16 class GetUrl(unittest.TestCase):17 ????def setUp(self):18 ????????self.dr = webdriver.Chrome()19 ????????self.dr.get(url)20 21 ????def tearDown(self):22 ????????self.dr.quit()23 24 ????def _login(self):25 ????????# time.sleep(2)26 ????????self.dr.find_element_by_id(‘username‘).send_keys(username)27 ????????self.dr.find_element_by_id(‘password‘).send_keys(password)28 ????????# self.dr.find_element_by_id(‘verifycode‘).send_keys(‘XXXXX‘)29 ????????self.dr.find_element_by_id(‘weblogin‘).click()30 ????????time.sleep(3)31 32 ????def _geturl(self):
　　　　　　# 这里返回的是一个list，然后里面是一个个字典33 ????????result = list()34 ????????eles = self.dr.find_elements_by_css_selector(‘menu.u-menu a‘)35 ????????for ele in eles:36 ????????????tmp = dict()37 ????????????href = ele.get_attribute(‘href‘).split(‘?‘)[0]38 ????????????# 获取菜单 的名称39 ????????????name = ele.get_attribute(‘innerHTML‘)40 ????????????if "<i>" not in name:41 ????????????????tmp[‘name‘] = name.strip()42 ????????????????tmp[‘href‘] = href43 ????????????????result.append(tmp)44 ????????????????# print(‘name: {0},href: {1}‘.format(name,href))45 ????????return result46 47 ????def _writetotxt(self,contents):48 ????????print("一共{0}条数据".format(len(contents)))49 ????????print(‘写入开始‘)50 ????????with open(‘urlcontent.txt‘,‘w‘) as f:51 ????????????for content in contents:52 ????????????????f.write(‘{0} ==> {1}\n‘.format(content[‘name‘],content[‘href‘]))53 ????????print(‘写入完毕‘)54 55 ????def test_run(self):56 ????????self._login()57 ????????self._writetotxt(self._geturl())58 ????????59 60 if __name__ == ‘__main__‘:61 ????unittest.main()