1
import requests
2
from bs4 import BeautifulSoup
3
import bs4
4

5
def getHTMLText(url):
6
    try:
7
        r = requests.get(url, timeout=30)
8
        r.raise_for_status()
9
        r.encoding = r.apparent_encoding
10
        return r.text
11
    except:
12
        return ""
13

14
def fillUnivList(ulist, html):
15
    soup = BeautifulSoup(html, "html.parser")
16
    for tr in soup.find('tbody').children:
17
        if isinstance(tr, bs4.element.Tag):
18
            tds = tr('td')
19
            ulist.append([tds[0].string, tds[1].string, tds[3].string])
20

21
def printUnivList(ulist, num):
22
    tplt = "{0:^10}\t{1:{3}^10}\t{2:^10}"
23
    print(tplt.format("排名","学校名称","总分",chr(12288)))
24
    for i in range(num):
25
        u=ulist[i]
26
        print(tplt.format(u[0],u[1],u[2],chr(12288)))
27

28
def main():
29
    uinfo = []
30
    url = 'https://www.zuihaodaxue.cn/zuihaodaxuepaiming2016.html'
31
    html = getHTMLText(url)
32
    fillUnivList(uinfo, html)
33
    printUnivList(uinfo, 20) # 20 univs
34
main()

1
import requests
2
import re
3

4
def getHTMLText(url):
5
    try:
6
        kv = \
7
            {
8
                'user-agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.99 Safari/537.36 Edg/97.0.1072.76',
9
                'Cookie':'【去审查元素自己看看】'
10
            }
11
        r = requests.get(url, timeout=30, headers=kv)
12
        r.raise_for_status()
13
        r.encoding = r.apparent_encoding
14
        return r.text
15
    except:
16
        print("getHTMLText")
17
        return ""
18

19
def parsePage(ilt, html):
20
    try:
21
        plt = re.findall(r'\"view_price\"\:\"[\d\.]*\"',html)
22
        tlt = re.findall(r'\"raw_title\"\:\".*?\"',html)
23
        for i in range(len(plt)):
24
            price = eval(plt[i].split(':')[1])
25
            title = eval(tlt[i].split(':')[1])
26
            ilt.append([price , title])
27
    except:
28
        print("parsePage")
29

30
def printGoodsList(ilt):
31
    tplt = "{:4}\t{:8}\t{:16}"
32
    print(tplt.format("序号", "价格", "商品名称"))
33
    count = 0
34
    for g in ilt:
35
        count = count + 1
36
        print(tplt.format(count, g[0], g[1]))
37

38
goods = '书包'
39
depth = 3
40
start_url = 'https://s.taobao.com/search?q=' + goods
41
infoList = []
42
for i in range(depth):
43
    try:
44
        url = start_url + '&s=' + str(44*i)
45
        html = getHTMLText(url)
46
        parsePage(infoList, html)
47
    except:
48
        continue
49
printGoodsList(infoList)

1
import requests
2
from bs4 import BeautifulSoup
3
import traceback
4
import re
5

6
def getHTMLText(url, code="utf-8"):
7
    try:
8
        r = requests.get(url)
9
        r.raise_for_status()
10
        r.encoding = code
11
        return r.text
12
    except:
13
        return ""
14

15
def getStockList(lst, stockURL):
16
    html = getHTMLText(stockURL, "GB2312")
17
    soup = BeautifulSoup(html, 'html.parser')
18
    a = soup.find_all('a')
19
    for i in a:
20
        try:
21
            href = i.attrs['href']
22
            lst.append(re.findall(r"[s][hz]\d{6}", href)[0])
23
        except:
24
            continue
25

26
def getStockInfo(lst, stockURL, fpath):
27
    count = 0
28
    for stock in lst:
29
        url = stockURL + stock + ".html"
30
        html = getHTMLText(url)
31
        try:
32
            if html=="":
33
                continue
34
            infoDict = {}
35
            soup = BeautifulSoup(html, 'html.parser')
36
            stockInfo = soup.find('div',attrs={'class':'stock-bets'})
37

38
            name = stockInfo.find_all(attrs={'class':'bets-name'})[0]
39
            infoDict.update({'股票名称': name.text.split()[0]})
40

41
            keyList = stockInfo.find_all('dt')
42
            valueList = stockInfo.find_all('dd')
43
            for i in range(len(keyList)):
44
                key = keyList[i].text
45
                val = valueList[i].text
46
                infoDict[key] = val
47

48
            with open(fpath, 'a', encoding='utf-8') as f:
49
                f.write( str(infoDict) + '\n' )
50
                count = count + 1
51
                print("\r当前进度: {:.2f}%".format(count*100/len(lst)),end="")
52
        except:
53
            count = count + 1
54
            print("\r当前进度: {:.2f}%".format(count*100/len(lst)),end="")
55
            continue
56

57
def main():
58
    stock_list_url = 'https://quote.eastmoney.com/stocklist.html'
59
    stock_info_url = 'https://gupiao.baidu.com/stock/'
60
    output_file = 'D:/BaiduStockInfo.txt'
61
    slist=[]
62
    getStockList(slist, stock_list_url)
63
    getStockInfo(slist, stock_info_url, output_file)
64

65
main()

1
import scrapy
2
from scrapy_dangdang_095.items import ScrapyDangdang095Item
3

4
class DangSpider(scrapy.Spider):
5
    name = 'dang'
6
    # 如果是多页下载的话 那么必须要调整的是allowed_domains的范围 一般情况下只写域名
7
    allowed_domains = ['category.dangdang.com']
8
    start_urls = ['http://category.dangdang.com/cp01.01.02.00.00.00.html']
9
    base_url = 'http://category.dangdang.com/pg'
10
    page = 1
11

12
    def parse(self, response):
13
#       pipelines 下载数据
14
#       items     定义数据结构的
15
#         src = //ul[@id="component_59"]/li//img/@src
16
#         alt = //ul[@id="component_59"]/li//img/@alt
17
#         price = //ul[@id="component_59"]/li//p[@class="price"]/span[1]/text()
18
#         所有的seletor的对象 都可以再次调用xpath方法
19
        li_list = response.xpath('//ul[@id="component_59"]/li')
20

21
        for li in li_list:
22
            src = li.xpath('.//img/@data-original').extract_first()
23
            # 第一张图片和其他的图片的标签的属性是不一样的
24
            # 第一张图片的src是可以使用的  其他的图片的地址是data-original
25
            if src:
26
                src = src
27
            else:
28
                src = li.xpath('.//img/@src').extract_first()
29

30
            name = li.xpath('.//img/@alt').extract_first()
31
            price = li.xpath('.//p[@class="price"]/span[1]/text()').extract_first()
32

33
            book = ScrapyDangdang095Item(src=src,name=name,price=price)
34

35
            # 获取一个book就将book交给pipelines
36
            yield book
37

38

39
#       每一页的爬取的业务逻辑全都是一样的，所以我们只需要将执行的那个页的请求再次调用parse方法就可以了
40
        if self.page < 100:
41
            self.page = self.page + 1
42
            url = self.base_url + str(self.page) + '-cp01.01.02.00.00.00.html'
43

44
#             怎么去调用parse方法
45
#             scrapy.Request就是scrpay的get请求
46
#             url就是请求地址
47
#             callback是你要执行的那个函数  注意不需要加（）
48
            yield scrapy.Request(url=url,callback=self.parse)

1
from itemadapter import ItemAdapter
2

3
# 如果想使用管道的话 那么就必须在settings中开启管道
4
class ScrapyDangdang095Pipeline:
5
    def open_spider(self,spider):
6
        self.fp = open('book.json','w',encoding='utf-8')
7

8
    def process_item(self, item, spider):
9
        self.fp.write(str(item))
10
        return item
11

12
    def close_spider(self,spider):
13
        self.fp.close()
14

15
import urllib.request
16

17
# 多条管道开启
18
# 定义管道类 + 在settings中开启管道
19
#'scrapy_dangdang_095.pipelines.DangDangDownloadPipeline':301
20
class DangDangDownloadPipeline:
21
    def process_item(self, item, spider):
22
        url = 'http:' + item.get('src')
23
        filename = './books/' + item.get('name') + '.jpg'
24
        urllib.request.urlretrieve(url = url, filename= filename)
25
        return item