初步爬取 deviantart gallery 的图片

由于ajax中iid参数(每次网站服务器动态生成)无法解密，所以只能爬取首页的20张。本周计划使用selenium爬取所有。
ajax问题示例: https://www.deviantart.com/dapi/v1/gallery/23512439?iid=594m47c9b8a1ce472e884d249d0d1d401cf0-jo77dlpy-1.0&mp=2
其中iid=594m47c9b8a1ce472e884d249d0d1d401cf0可以在首页源码中得到，但是其后的 jo77dlpy 这八位的参数无法获取到(每次动态生成，刷新页面都会导致其变化。似乎与cookies有关，但通过复制浏览器生成的cookies，再使用requests访问仍然会被服务器拒绝访问)。

import re
import requests
from requests.exceptions import RequestException
from bs4 import BeautifulSoup
import time
import os


def get_one_page(url: str) -> str:
    '''requests获取网页源码'''
    # 设置浏览器标识user-agent
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 ' +
            '(KHTML, like Gecko) Chrome/69.0.3497.81 Safari/537.36'
    }
    try:
        response = requests.get(url, headers=headers)
        if response.status_code == 200:
            return response.text
        return None
    except RequestException as e:
        print(e)
        return None


def get_urls(text: str) -> list:
    '''解析网页获取url并提取到列表里'''
    soup = BeautifulSoup(text, 'html5lib')
    items = soup.select('#gmi- span[class~=thumb]')
    for item in items:
        yield item.attrs['href']


def parse_one_page(text: str) -> dict:
    '''解析图片地址并提取到列表里'''
    soup = BeautifulSoup(text, 'html5lib')
    item = soup.select('img.dev-content-full')[0]
    return item.attrs['src']


def download_img(img_url: str):
    img_name = re.findall('.*/(.*?)_by', img_url, re.S)[0]
    r = requests.get(img_url, stream=True)  # stream loading
    file_name = './temp/wlop/{}.jpg'.format(img_name)
    with open(file_name, 'wb') as f:
        for chunk in r.iter_content(chunk_size=32):
            f.write(chunk)
    print('%s download successful!' % (img_name))


def main():
    '''调用上面函数爬取网页'''
    base_url = 'https://www.deviantart.com/wlop/gallery/'
    base_html = get_one_page(base_url)

    os.makedirs('./temp/wlop/', exist_ok=True)

    for url in get_urls(base_html):
        time.sleep(1)
        html = get_one_page(url)
        img_url = parse_one_page(html)
        download_img(img_url)


if __name__ == '__main__':
    main()
    print('finished')