Python crawler tutorial - Crawler 5K resolution ultra clear beautiful wallpaper source code

Posted by delmardata on Fri, 07 Feb 2020 15:40:21 +0100

brief introduction

In fact, the choice of wallpaper can largely see the inner world of the computer owner. Some people like the scenery, some people like the starry sky, some people like beautiful women, some people like animals. However, one day you will have aesthetic fatigue, but when you decide to change the wallpaper, you will find that the online wallpaper is either low resolution or watermarked.

Project source code

# -*- coding:utf-8 -*-

from requests import get
from filetype import guess
from os import rename
from os import makedirs
from os.path import exists
from json import loads
from contextlib import closing


# File downloader
def Down_load(file_url, file_full_name, now_photo_count, all_photo_count):
    headers = {"User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36"}

    # Start downloading pictures
    with closing(get(file_url, headers=headers, stream=True)) as response:
        chunk_size = 1024  # Single request maximum
        content_size = int(response.headers['content-length'])  # Total file size
        data_count = 0 # Current transferred size
        with open(file_full_name, "wb") as file:
            for data in response.iter_content(chunk_size=chunk_size):
                file.write(data)
                done_block = int((data_count / content_size) * 50)
                data_count = data_count + len(data)
                now_jd = (data_count / content_size) * 100
                print("\r %s: [%s%s] %d%% %d/%d" % (file_full_name, done_block * '█', ' ' * (50 - 1 - done_block), now_jd, now_photo_count, all_photo_count), end=" ")

    # After downloading a picture, get the picture extension and add it
    file_type = guess(file_full_name)
    rename(file_full_name, file_full_name + '.' + file_type.extension)



# Crawling different types of pictures
def crawler_photo(type_id, photo_count):

    # Latest 1, hottest 2, girl 3, star 4
    if(type_id == 1):
        url = 'https://service.paper.meiyuan.in/api/v2/columns/flow/5c68ffb9463b7fbfe72b0db0?page=1&per_page=' + str(photo_count)
    elif(type_id == 2):
        url = 'https://service.paper.meiyuan.in/api/v2/columns/flow/5c69251c9b1c011c41bb97be?page=1&per_page=' + str(photo_count)
    elif(type_id == 3):
        url = 'https://service.paper.meiyuan.in/api/v2/columns/flow/5c81087e6aee28c541eefc26?page=1&per_page=' + str(photo_count)
    elif(type_id == 4):
        url = 'https://service.paper.meiyuan.in/api/v2/columns/flow/5c81f64c96fad8fe211f5367?page=1&per_page=' + str(photo_count)

    # Get picture list data
    headers = {"User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36"}
    respond = get(url, headers=headers)
    photo_data = loads(respond.content)

    # Number of downloaded pictures
    now_photo_count = 1

    # Number of all pictures
    all_photo_count = len(photo_data)

    # Start downloading and saving 5K resolution wallpaper
    for photo in photo_data:

        # Create a folder to store the pictures we downloaded
        if not exists('./' + str(type_id)):
            makedirs('./' + str(type_id))

        # Picture link ready to download
        file_url = photo['urls']['raw']

        # The name of the picture to be downloaded, excluding the extension
        file_name_only = file_url.split('/')
        file_name_only = file_name_only[len(file_name_only) -1]

        # Prepare to save to local full path
        file_full_name = './' + str(type_id) + '/' + file_name_only

        # Start downloading pictures
        Down_load(file_url, file_full_name, now_photo_count, all_photo_count)
        now_photo_count = now_photo_count + 1



if __name__ == '__main__':

    # Latest 1, hottest 2, girl 3, star 4
    # Take pictures (girls) of type 3 and prepare to take 20000 in total
    wall_paper_id = 1
    wall_paper_count = 10
    while(True):

        # Newline character
        print('\n\n')

        # Choose wallpaper type
        wall_paper_id = input("Wallpaper type: latest wallpaper 1, Hottest wallpaper 2, Girls wallpaper 3, Star Wallpaper 4\n Please enter the number to select 5 K Ultra clear wallpaper type:")
        # Judge whether the input is correct
        while(wall_paper_id  != str(1) and wall_paper_id  != str(2) and wall_paper_id  != str(3) and wall_paper_id  != str(4)):
            wall_paper_id = input("Wallpaper type: latest wallpaper 1, Hottest wallpaper 2, Girls wallpaper 3, Star Wallpaper 4\n Please enter the number to select 5 K Ultra clear wallpaper type:")


        # Select the number of wallpapers to download
        wall_paper_count = input("Please enter 5 to download K Number of ultra clean Wallpapers:")
        # Judge whether the input is correct
        while(int(wall_paper_count) <= 0):
            wall_paper_count = input("Please enter 5 to download K Number of ultra clean Wallpapers:")


        # Start to crawl 5K high-definition wallpaper
        print("Downloading 5 K Super clean wallpaper, please wait")
        crawler_photo(int(wall_paper_id), int(wall_paper_count))
        print('\n Download 5 K HD Wallpaper success!')

For beginners, the concept of Python is unclear. What can Python do? What route should we follow when we learn it? What direction should we go after it? If you want to know more about it, you can copy Youdao cloud notes and link to the browser to open it. http://note.youdao.com/noteshare?id=e4fa02e7b56d7909a27674cdb3da08aa

Learning video materials, development tools, etc. are shared free of charge, and professional teachers answer questions

103 original articles published, 66 praised, 10000 visitors+
Private letter follow

Topics: Python Windows JSON