[爬虫]爬取炎黄云后台到期的库存信息(含图片验证码识别)

要求：

爬取所有机房的到期的服务器

代码：

#单   位:常州旺龙
#作   者:OLDNI
#开发日期:2023/11/3

'''
爬取炎黄云到期的服务器数据
'''


import os
import time

from bs4 import BeautifulSoup
from ddddocr import DdddOcr
from openpyxl import Workbook
from openpyxl.styles import Font, PatternFill, Alignment
from selenium import webdriver
from selenium.webdriver.common.by import By

#登录网站，失败会重新登录
def browser_login():
    #创建浏览器
    options=webdriver.ChromeOptions()
    # options.add_argument('--headless')
    options.add_experimental_option('detach',True)
    options.add_argument('--disable-blink-features=AutomationControlled')
    # 隐藏"正在受到自动软件的控制"这几个字
    options.add_experimental_option("excludeSwitches", ["enable-automation"])
    options.add_experimental_option('useAutomationExtension', False)

    #需要设置为全局变量，不然后面的就无法使用browser了
    global browser
    browser=webdriver.Chrome(options=options)
    browser.implicitly_wait(10)
    browser.get('http://w.yhyun.com/index.php/meihao2021/Login/index.html')

    browser.find_element(By.CSS_SELECTOR,'.login_input.login_user.defaultinput').send_keys('13685234796')
    browser.find_element(By.CSS_SELECTOR,'.login_input.login_password').send_keys('www.123.nyc')
    #验证码识别
    ocr=DdddOcr()
    img_tag=browser.find_element(By.CSS_SELECTOR,'.show-captcha')
    img_tag.screenshot('code.png')
    with open('code.png','rb') as f:
        img_content=f.read()
        result=ocr.classification(img_content)
    browser.find_element(By.CSS_SELECTOR,'.login_input.login_verify.defaultinput').send_keys(result)
    browser.get_screenshot_as_file('firstpage.png')

    browser.find_element(By.CSS_SELECTOR,'.login_submit').click()
    #登录前的地址http://w.yhyun.com/index.php/meihao2021/Login/index.html
    print(browser.current_url)
    time.sleep(5)
    #登录后的地址http://w.yhyun.com/index.php/meihao2021/Index/index.html
    print(browser.current_url)

    #判断是否登录成功，不成功重新登录
    if browser.current_url != 'http://w.yhyun.com/index.php/meihao2021/Index/index.html':
        browser.quit()
        print('登录失败,正重新运行程序！')
        browser_login()

#调用函数
browser_login()

#打开库存100条
#爬取到期服务器数据
url_list=[
    'http://w.yhyun.com/index.php/meihao2021/baremetal/ovedueList.html?_field=&_order=&page=1&pagesize=100&keywordType=ip&keyword=',
]

#最后汇总
statistics=[]

# 创建excel表
wb = Workbook()
sheet = wb.active
header = ['线路', '机型', 'IP', 'IPMI', 'IP数量', '用户ID', '购买时间','到期时间' ]
sheet.append(header)

for url in url_list:
    new_tab=f'window.open("{url}")'
    browser.execute_script(new_tab)
    browser.switch_to.window(browser.window_handles[-1])

    #判断库存有多少页
    html=browser.page_source
    soup=BeautifulSoup(html,'html.parser')
    try:
        pages=int(soup.select_one('.num').text)
    except:
        pages=1
    print(f'当前线路共有{pages}页')

    # 计算库存总条数
    total = 0

    for p in range(1,(pages+1)):
        #找到当页所有tr标签元素，一条tr对应一条库存
        tr_tag_list=soup.select('#server_list_table>tbody>tr')
        print(f'库存条数:{len(tr_tag_list)}')
        #计算库存总条数
        total+=len(tr_tag_list)
        for tr_tag in tr_tag_list:
            line=tr_tag.select_one('td:nth-child(2)').text.strip()
            type=tr_tag.select_one('td:nth-child(3)').text.strip()
            ip_addr=tr_tag.select_one('td:nth-child(4)').text.strip()
            ipmi=tr_tag.select_one('td:nth-child(5)').text.strip()
            ip_num=tr_tag.select_one('td:nth-child(6)').text.strip()
            user_id=tr_tag.select_one('td:nth-child(9)').text.strip()
            start_time=tr_tag.select_one('td:nth-child(12)').text.strip()
            end_time=tr_tag.select_one('td:nth-child(13)').text.strip()
            data=[line,type,ip_addr,ipmi,ip_num,user_id,start_time,end_time]
            sheet.append(data)
        print(f'已处理第{p}页数据')
        #如果只有一页就不需要点击下一页，如果最后一页也不点击下一页
        if pages!=1 or p!=pages:
            browser.find_element(By.CSS_SELECTOR,'tbody > tr > td.right > div > table > tbody td:nth-child(4)').click()
        html = browser.page_source
        soup = BeautifulSoup(html, 'html.parser')
    statistics.append(f'到期服务器数:{total}')
    browser.close()
    browser.switch_to.window(browser.window_handles[0])
browser.close()
sheet.append(statistics)
"""
设置表格样式
"""
#设置列宽
column_number_and_dimensions={'A':40,'B':25,'C':20,'D':20,'E':10,'F':15,'G':30,'H':30}
for n,d in column_number_and_dimensions.items():
    sheet.column_dimensions[n].width = d
#冻结窗格
sheet.freeze_panes='B2'
#设置表头样式
cells=sheet[1]
font=Font(bold=True)
patternfill=PatternFill(fill_type='solid',fgColor='FFFF00')
alignment=Alignment(horizontal='center')
for cell in cells:
    cell.font=font
    cell.fill=patternfill
    cell.alignment=alignment
#D列内容凡是"禁用"就填充红色(解决条件格式问题)
patternfill=PatternFill(fill_type='solid',fgColor='FF0000')
cells=sheet['D']
for cell in cells:
    if cell.value=='禁用':
        cell.fill=patternfill
cells=sheet['F']
for cell in cells:
    if cell.value=='异常':
        cell.fill=patternfill

wb.save('鸟云平台到期服务器数据.xlsx')
os.startfile('鸟云平台到期服务器数据.xlsx')