BeautifulSoup find_all ("img")이 모든 사이트에서 작동하지 않습니다.

Question

어떤 웹 사이트에서나 이미지를 다운로드 할 수있는 Python 스크립트를 작성하려고합니다. 그것은 효과가 있지만 일치하지 않습니다. 특히 find_all ("img")은 두 번째 url에 대해 그렇게하지 않습니다. 스크립트는 다음과 같습니다.

# works for http://proof.nationalgeographic.com/2016/02/02/photo-of-the-day-best-of-january-3/ 
# but not http://www.nationalgeographic.com/photography/proof/2017/05/lake-chad-desertification/ 
import requests 
from PIL import Image 
from io import BytesIO 
from bs4 import BeautifulSoup 

def url_to_image(url, filename): 
    # get HTTP response, open as bytes, save the image 
    # http://docs.python-requests.org/en/master/user/quickstart/#binary-response-content 
    req = requests.get(url) 
    i = Image.open(BytesIO(req.content)) 
    i.save(filename) 

# open page, get HTML request and parse with BeautifulSoup 
html = requests.get("http://proof.nationalgeographic.com/2016/02/02/photo-of-the-day-best-of-january-3/") 
soup = BeautifulSoup(html.text, "html.parser") 

# find all JPEGS in our soup and write their "src" attribute to array 
urls = [] 
for img in soup.find_all("img"): 
    if img["src"].endswith("jpg"): 
     print("endswith jpg") 
     urls.append(str(img["src"])) 
    print(str(img)) 

jpeg_no = 00 
for url in urls: 
    url_to_image(url, filename="NatGeoPix/" + str(jpeg_no) + ".jpg") 
    jpeg_no += 1 

Answer 1

이미지는 실패한 페이지에서 JavaScript로 렌더링됩니다.

예를 (당신이 Web-scraping JavaScript page with Python을 볼 dryscrape 사용하지 않을 경우) 먼저 dryscrape

로 페이지를 렌더링

import requests 
from PIL import Image 
from io import BytesIO 
from bs4 import BeautifulSoup 
import dryscrape 

def url_to_image(url, filename): 
    # get HTTP response, open as bytes, save the image 
    # http://docs.python-requests.org/en/master/user/quickstart/#binary-response-content 
    req = requests.get(url) 
    i = Image.open(BytesIO(req.content)) 
    i.save(filename) 

# open page, get HTML request and parse with BeautifulSoup 

session = dryscrape.Session() 
session.visit("http://www.nationalgeographic.com/photography/proof/2017/05/lake-chad-desertification/") 
response = session.body() 
soup = BeautifulSoup(response, "html.parser") 

# find all JPEGS in our soup and write their "src" attribute to array 
urls = [] 
for img in soup.find_all("img"): 
    if img["src"].endswith("jpg"): 
     print("endswith jpg") 
     urls.append(str(img["src"])) 
     print(str(img)) 

jpeg_no = 00 
for url in urls: 
    url_to_image(url, filename="NatGeoPix/" + str(jpeg_no) + ".jpg") 
    jpeg_no += 1 

그러나 나는 또한 당신이 절대 URL이 아닌 상대 일이 있는지 확인합니다 :

이

import requests 
from PIL import Image 
from io import BytesIO 
from bs4 import BeautifulSoup 
import dryscrape 
from urllib.parse import urljoin 


def url_to_image(url, filename): 
    # get HTTP response, open as bytes, save the image 
    # http://docs.python-requests.org/en/master/user/quickstart/#binary-response-content 
    req = requests.get(url) 
    i = Image.open(BytesIO(req.content)) 
    i.save(filename) 

# open page, get HTML request and parse with BeautifulSoup 
base = "http://www.nationalgeographic.com/photography/proof/2017/05/lake-chad-desertification/" 
session = dryscrape.Session() 
session.visit(base) 
response = session.body() 
soup = BeautifulSoup(response, "html.parser") 

# find all JPEGS in our soup and write their "src" attribute to array 
urls = [] 
for img in soup.find_all("img"): 
    if img["src"].endswith("jpg"): 
     print("endswith jpg") 
     urls.append(str(img["src"])) 
     print(str(img)) 

jpeg_no = 00 
for url in urls: 
    if url.startswith('http'): 
     absoute = url 
    else: 
     absoute = urljoin(base, url) 
    print (absoute) 
    url_to_image(absoute, filename="NatGeoPix/" + str(jpeg_no) + ".jpg") 
    jpeg_no += 1