Python抓取站酷作品图片并保存至本地

2018年9月6日15:08:20 发表评论 3,842 views

本文最后更新于2018年9月7日，已超过 1 年没有更新，如果文章内容或图片资源失效，请留言反馈，我们会及时处理，谢谢！

思路

1、通过 requests 模块请求网页，获取到网页的源码，其中包含了作品集的 URL

2、通过 BeautifulSoup 提取到作品集的 URL ，返回为列表，使用 for 循环遍历作品集

3、遍历访问作品集时，通过 BeautifulSoup 提取到图片的 URL，进行名称格式化后，返回

4、保存图片的 URL 至本地磁盘

5、遍历多页作品集，使用多线程

嗯。。。思路大概是这个样子吧，新手入门- -

模块编写

1、抓取主页（start_app）

从网页源码中可以看出，其主要内容包含在class为 work-list-box 的这个div内，并且每个作品又是单独包含在 class 为 card-box 这个div内的，所以首先，通过 BeautifulSoup 生成 soup 对象后，使用 soup.find class=work-list-box 找到位于该 class 内的元素，再通过 find_all class= card-box 生成作品集的一个列表，代码为：

def start_app(self):
    print('[Message]正在抓取 - %s' % self.url)
    try:
        response = requests.get(self.url, headers=self.headers)
        if response.status_code == 200:
            soup = BeautifulSoup(response.text, 'lxml')
            page_content = soup.find('div', class_="work-list-box")
            card_box_list = page_content.find_all('div', class_="card-box")
        else:
            print('[Error]文档获取失败，状态为 - %s' % (self.url, response.status_code))
    except Exception as e:
        print(e)

def start_app(self):

print('[Message]正在抓取 - %s' % self.url)

try:

response = requests.get(self.url, headers=self.headers)

if response.status_code == 200:

soup = BeautifulSoup(response.text, 'lxml')

page_content = soup.find('div', class_="work-list-box")

card_box_list = page_content.find_all('div', class_="card-box")

else:

print('[Error]文档获取失败，状态为 - %s' % (self.url, response.status_code))

except Exception as e:

print(e)

2、抓取内容

1 中获取到的作品集list，在该模块进行遍历，通过 BeautifulSoup 方法获取到作品集的 URL，标题，作者，其中，标题 + 作者作为保存图片文件的路径，作品集的URL通过 requests 模块请求，再次抓取作品集内部的图片

def get_content(self, item):
    title = item.find('a', class_="title-content")
    avatar = item.find('div', class_="card-item")
    if title is not None and avatar is not None:
        titlename = title.text
        authorname = avatar.find('a')['title']
        path_str = "[%s]-[%s]" % (authorname, titlename)
        path_str_mk = self.save_path('D:/img/' + self.file_name(path_str))
        href = title['href']
        try:
            html = requests.get(href, headers=self.headers)
            if html.status_code == 200:
                img_url_list = self.get_img_link(html.text)
                if path_str_mk is None:
                    return
                else:
                    for img_url in img_url_list:
                        self.save_img(img_url, path_str_mk)
            print("[Error]文档 %s 获取失败，状态为 - %s，" % (href, html.status_code))
        except Exception as e:
            print(e)

def get_content(self, item):

title = item.find('a', class_="title-content")

avatar = item.find('div', class_="card-item")

if title is not None and avatar is not None:

titlename = title.text

authorname = avatar.find('a')['title']

path_str = "[%s]-[%s]" % (authorname, titlename)

path_str_mk = self.save_path('D:/img/' + self.file_name(path_str))

href = title['href']

try:

html = requests.get(href, headers=self.headers)

if html.status_code == 200:

img_url_list = self.get_img_link(html.text)

if path_str_mk is None:

return

else:

for img_url in img_url_list:

self.save_img(img_url, path_str_mk)

print("[Error]文档 %s 获取失败，状态为 - %s，" % (href, html.status_code))

except Exception as e:

print(e)

def get_img_link(self, html):
    soup = BeautifulSoup(html, 'lxml')
    work_show_box = soup.find('div', class_="work-show-box")
    res = work_show_box.find_all('div', class_="reveal-work-wrap")
    img_list = []
    for item in res:
        img = item.find("img")
        if img is not None:
            img_url = str(img['src']).split('@')[0]
            img_list.append(img_url)
        else:
            print('[Warning]没有图片')
            continue
    return img_list

def get_img_link(self, html):

soup = BeautifulSoup(html, 'lxml')

work_show_box = soup.find('div', class_="work-show-box")

res = work_show_box.find_all('div', class_="reveal-work-wrap")

img_list = []

for item in res:

img = item.find("img")

if img is not None:

img_url = str(img['src']).split('@')[0]

img_list.append(img_url)

else:

print('[Warning]没有图片')

continue

return img_list

3、处理保存路径和保存文件名，保存文件

def save_path(self, file_path):
    file_name_s = file_path.split("/")
    file_name = file_name_s[len(file_name_s) - 1]
    file_name_s[len(file_name_s) - 1] = file_name
    path = "/".join(file_name_s)
    if not os.path.exists(path):
        os.mkdir(path)
    return path

def file_name(self, file_name):
    file_stop_str = ['\\', '/', '*', '?', ':', '"', '<', '>', '|']
    for item2 in file_stop_str:
        file_name = file_name.replace(item2, '-')
    return file_name

def save_img(self, url, path):
    download_name = str(path) + '/' + str(url).split('/')[-1]
    res = requests.get(url, headers=self.headers)
    if res.status_code == 200:
        f = open(download_name, 'wb')
        f.write(res.content)
        f.close()
        print("[Message]下载成功 - %s" % download_name)
    else:
        print('[Error]下载失败 - %s' % download_name)

def save_path(self, file_path):

file_name_s = file_path.split("/")

file_name = file_name_s[len(file_name_s) - 1]

file_name_s[len(file_name_s) - 1] = file_name

path = "/".join(file_name_s)

if not os.path.exists(path):

os.mkdir(path)

return path

def file_name(self, file_name):

file_stop_str = ['\\', '/', '*', '?', ':', '"', '<', '>', '|']

for item2 in file_stop_str:

file_name = file_name.replace(item2, '-')

return file_name

def save_img(self, url, path):

download_name = str(path) + '/' + str(url).split('/')[-1]

res = requests.get(url, headers=self.headers)

if res.status_code == 200:

f = open(download_name, 'wb')

f.write(res.content)

f.close()

print("[Message]下载成功 - %s" % download_name)

else:

print('[Error]下载失败 - %s' % download_name)

完整代码

# -*- coding:utf-8 -*-
__author__ = 'wx'
import requests,os
from bs4 import BeautifulSoup
import threading
class zcool_img(object):
    def __init__(self,url):
        self.url = url
        self.headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.87 Safari/537.36'
        }
    def start_app(self):
        print('[Message]正在抓取 - %s' % self.url)
        try:
            # 请求主页
            response = requests.get(self.url,headers=self.headers)
            if response.status_code == 200:
                # 返回BeautifulSoup对象
                soup = BeautifulSoup(response.text,'lxml')
                # 寻找作品集div
                page_content = soup.find('div',class_="work-list-box")
                # 在作品集div中，寻找class为card-box，返回即为该页面所有作品集的一个列表
                card_box_list = page_content.find_all('div',class_="card-box")
                # 遍历该列表
                for item in card_box_list:
                    # 执行get_content函数，获取内容
                    self.get_content(item)
            else:
                print('[Error]文档获取失败，状态为 - %s' % (self.url, response.status_code))
        except Exception as e:
            print(e)
    def get_content(self,item):
        # 此处传入的 item 为一个作品集 的源码，通过BeautifulSoup 标签寻找，获取 title avator信息
        title = item.find('a',class_="title-content")
        avatar = item.find('div', class_="card-item")
        if title is not None and avatar is not None:
            titlename = title.text
            authorname = avatar.find('a')['title']
            # 处理保存路径，格式化为 [作品名]-[作者名]
            path_str = "[%s]-[%s]" % (authorname, titlename)
            # 调用路径处理函数，返回最终的路径
            path_str_mk = self.save_path('D:/img/' + self.file_name(path_str))
            # 获取作品集的 URL
            href = title['href']
            try:
                # 请求作品集URL
                html = requests.get(href,headers=self.headers)
                if html.status_code == 200:
                    # 通过 get_img_link函数，得到该作品集下的所有图片URL
                    img_url_list = self.get_img_link(html.text)
                    if path_str_mk is None:
                        return
                    else:
                        # 调用save_img函数，传入图片URL，保存至本地
                        for img_url in img_url_list:
                            self.save_img(img_url,path_str_mk)
                print("[Error]文档 %s 获取失败，状态为 - %s，" % (href, html.status_code))

            except Exception as e:
                print(e)
    def save_img(self,url,path):
        download_name = str(path) + '/' + str(url).split('/')[-1]
        res = requests.get(url,headers=self.headers)
        if res.status_code == 200:
            f = open(download_name,'wb')
            f.write(res.content)
            f.close()
            print("[Message]下载成功 - %s" % download_name)
        else:
            print('[Error]下载失败 - %s' % download_name)

    def get_img_link(self,html):
        # BeautifulSoup处理 作品集源码
        soup = BeautifulSoup(html,'lxml')
        # 获取图片
        work_show_box = soup.find('div',class_="work-show-box")
        res = work_show_box.find_all('div',class_="reveal-work-wrap")
        img_list = []
        for item in res:
            img = item.find("img")
            if img is not None:
                img_url = str(img['src']).split('@')[0]
                img_list.append(img_url)
            else:
                print('[Warning]没有图片')
                continue
        return img_list
    def save_path(self,file_path):
        file_name_s = file_path.split("/")
        file_name = file_name_s[len(file_name_s) - 1]
        file_name_s[len(file_name_s) - 1] = file_name
        path = "/".join(file_name_s)
        if not os.path.exists(path):
            os.mkdir(path)
        return path

    def file_name(self,file_name):
        file_stop_str = ['\\', '/', '*', '?', ':', '"', '<', '>', '|']
        for item2 in file_stop_str:
            file_name = file_name.replace(item2, '-')
        return file_name
def main():
    threads = []
    for i in range(1,22):
        url = 'https://www.zcool.com.cn/?p='+ str(i) + '#tab_anchor'
        zcoolOper = zcool_img(url)
        threads.append(threading.Thread(target=zcoolOper.start_app))
    for item in threads:
        item.start()

if __name__ == '__main__':
    main()

100

101

102

103

104

105

106

107

108

109

110

111

112

# -*- coding:utf-8 -*-

__author__ = 'wx'

import requests,os

from bs4 import BeautifulSoup

import threading

class zcool_img(object):

def __init__(self,url):

self.url = url

self.headers = {

'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.87 Safari/537.36'

}

def start_app(self):

print('[Message]正在抓取 - %s' % self.url)

try:

# 请求主页

response = requests.get(self.url,headers=self.headers)

if response.status_code == 200:

# 返回BeautifulSoup对象

soup = BeautifulSoup(response.text,'lxml')

# 寻找作品集div

page_content = soup.find('div',class_="work-list-box")

# 在作品集div中，寻找class为card-box，返回即为该页面所有作品集的一个列表

card_box_list = page_content.find_all('div',class_="card-box")

# 遍历该列表

for item in card_box_list:

# 执行get_content函数，获取内容

self.get_content(item)

else:

print('[Error]文档获取失败，状态为 - %s' % (self.url, response.status_code))

except Exception as e:

print(e)

def get_content(self,item):

# 此处传入的 item 为一个作品集的源码，通过BeautifulSoup 标签寻找，获取 title avator信息

title = item.find('a',class_="title-content")

avatar = item.find('div', class_="card-item")

if title is not None and avatar is not None:

titlename = title.text

authorname = avatar.find('a')['title']

# 处理保存路径，格式化为 [作品名]-[作者名]

path_str = "[%s]-[%s]" % (authorname, titlename)

# 调用路径处理函数，返回最终的路径

path_str_mk = self.save_path('D:/img/' + self.file_name(path_str))

# 获取作品集的 URL

href = title['href']

try:

# 请求作品集URL

html = requests.get(href,headers=self.headers)

if html.status_code == 200:

# 通过 get_img_link函数，得到该作品集下的所有图片URL

img_url_list = self.get_img_link(html.text)

if path_str_mk is None:

return

else:

# 调用save_img函数，传入图片URL，保存至本地

for img_url in img_url_list:

self.save_img(img_url,path_str_mk)

print("[Error]文档 %s 获取失败，状态为 - %s，" % (href, html.status_code))

except Exception as e:

print(e)

def save_img(self,url,path):

download_name = str(path) + '/' + str(url).split('/')[-1]

res = requests.get(url,headers=self.headers)

if res.status_code == 200:

f = open(download_name,'wb')

f.write(res.content)

f.close()

print("[Message]下载成功 - %s" % download_name)

else:

print('[Error]下载失败 - %s' % download_name)

def get_img_link(self,html):

# BeautifulSoup处理作品集源码

soup = BeautifulSoup(html,'lxml')

# 获取图片

work_show_box = soup.find('div',class_="work-show-box")

res = work_show_box.find_all('div',class_="reveal-work-wrap")

img_list = []

for item in res:

img = item.find("img")

if img is not None:

img_url = str(img['src']).split('@')[0]

img_list.append(img_url)

else:

print('[Warning]没有图片')

continue

return img_list

def save_path(self,file_path):

file_name_s = file_path.split("/")

file_name = file_name_s[len(file_name_s) - 1]

file_name_s[len(file_name_s) - 1] = file_name

path = "/".join(file_name_s)

if not os.path.exists(path):

os.mkdir(path)

return path

def file_name(self,file_name):

file_stop_str = ['\\', '/', '*', '?', ':', '"', '<', '>', '|']

for item2 in file_stop_str:

file_name = file_name.replace(item2, '-')

return file_name

def main():

threads = []

for i in range(1,22):

url = 'https://www.zcool.com.cn/?p='+ str(i) + '#tab_anchor'

zcoolOper = zcool_img(url)

threads.append(threading.Thread(target=zcoolOper.start_app))

for item in threads:

item.start()

if __name__ == '__main__':

main()

运行结果

微信小程序
关注微信小程序

微信公众号
关注微信公众号

思路

模块编写

1、抓取主页（start_app）

2、抓取内容

3、处理保存路径和保存文件名，保存文件

完整代码

运行结果

发表评论取消回复