8684公交面向过程爬取

import json
import re
import urllib.request
import urllib.parse
from bs4 import BeautifulSoup

def main():
    url = 'http://js.8684.cn/citys/city_boxInf.min.js'
    #按照拼接获取城市
    request = get_city_request(url)
    #获取响应的对象
    response = get_city_response(request)


#获得第一层数据城市
def get_city_request(url):
    headers = {
        'User - Agent':'Mozilla / 5.0(Windows NT 6.1;WOW64) AppleWebKit / 537.36(KHTML, likeGecko) Chrome / 67.0.3396.99Safari / 537.36'
    }
    request = urllib.request.Request(url=url,headers=headers)
    return request
#获得响应的对象
def get_city_response(request):

    response = urllib.request.urlopen(request)
    data = response.read().decode("utf8").split("=")
    res = []
    #获取汉字版的数字
    for i in range(len(data)):
        res.append(data[2])
    html = res[0].split(",")
    for i in html:
        # print(i)
        #通过正则匹配拼接
        a = re.findall(r'(\w+):',i)

        print('正在爬取%s' %a)
        #创建每个省的文件夹
        fp = open(a[0]+'.txt','w',encoding="utf8")
        #拼接url
        url = 'http://'+a[0]+'.8684.cn'
        #调用请求对象函数
        request = get_city_request(url)
        #调用第二层请求对象函数
        response = get_response(request)

        data = get_data(response,url)
        print(msg_list)
        for i in msg_list:
            print(type(i))
            fp.write(str(i))
        print("%s爬取结束" %a)
#调用第二层响应的对象
def get_response(request):
    try:
        response = urllib.request.urlopen(request)

        return response
    except:
        response=''
        return response
#获取二层数据
def get_data(response,url):
    try:
        #实例化一个soup
        data = response.read().decode("utf8")
        soup = BeautifulSoup(data,'lxml')
        #获取以数字开头的公交车信息
        html1 = soup.select('.bus_kt_r1 > a')
        html2 = soup.select('.bus_kt_r2 > a')
        #将两个获取的列表相加
        html = html1+html2
        #遍历循环HTML文件里面的数据
        for i in html:
            #拼接路径
            print(url)
            url1 = url + i['href']
            print('url',url1)
            #调用获取请求参数的函数
            request  = get_city_request(url1)
            #调用第二层获取响应数据的函数
            response = get_response(request)
            #调用第三层的数据
            data = get_data_san(response,url)
    except:
        print("过去")
        data=""
#获取三层数据

def get_data_san(response,url):
    global  msg_list
    msg_list = []
    data = response.read().decode("utf8")
    #实例化一个soup
    soup = BeautifulSoup(data,'lxml')
    html = soup.select("#con_site_1 > a")
    for  i in html:
        #先截取url，在拼接路径
        url = url.split("cn")[0] + 'cn'  + i['href']
        # print(url)
        #调用获取请求对象
        request = get_city_request(url)
        #调用第二层的响应对象
        response = get_response(request)
        #调用第四层数据
        text = get_data_si(response)
        msg_list.append(text)
    print('msg',msg_list)
#获取第四层数据
def get_data_si(res):
    data = res.read().decode("utf8")
    #实例化一个soup
    soup = BeautifulSoup(data,'lxml')
    #获取具体的信息
    #获得公交名字
    bus_name = soup.select(".bus_i_t1 > h1")[0].text
    #获得公交运行时间
    bus_run_time = soup.select('.bus_i_t4')[0].text
    #获得公交的票价
    bus_fares = soup.select('.bus_i_t4')[1].text.split(" ")[-1]
    #获得公交的运行公司
    bus_company = soup.select('.bus_i_t4')[2].text
    #获得更新时间
    bus_msg_update_time  = soup.select('.bus_i_t4')[-1].text.split("：")[-1]
    #获取正向行驶的路线
    up_run = soup.select(".bus_line_site")[0]
    up = up_run.select('div > div > a')
    up_temp = []
    for  i in up:
        up_temp.append(i.text)
    # print(up_temp)
    try:
        #获取反向行驶路线
        down_run = soup.select(".bus_line_site")[1]
        down = down_run.select('div > div > a')
        down_temp = []
        for i in down:
            down_temp.append(i.text)
        # print(down_temp)
    except:
        down_temp = []
    items = {
        '公交名字':bus_name,
        '运行时间':bus_run_time,
        '票价':bus_fares,
        '公交运行公司':bus_company,
        '路线更新时间':bus_msg_update_time,
        '正向行驶路线':up_temp,
        '反向行驶路线':down_temp
    }
    # print(items)
    return items


if __name__ == '__main__':
    main()