BeautifulSoup find_all ("img") funktioniert nicht für alle Websites

Question

Ich versuche ein Python-Skript zu schreiben, um Bilder von einer beliebigen Website herunterzuladen. Es funktioniert, aber inkonsistent. Insbesondere tut find_all ("img") dies nicht für die zweite URL. Das Skript lautet:

# works for http://proof.nationalgeographic.com/2016/02/02/photo-of-the-day-best-of-january-3/ 
# but not http://www.nationalgeographic.com/photography/proof/2017/05/lake-chad-desertification/ 
import requests 
from PIL import Image 
from io import BytesIO 
from bs4 import BeautifulSoup 

def url_to_image(url, filename): 
    # get HTTP response, open as bytes, save the image 
    # http://docs.python-requests.org/en/master/user/quickstart/#binary-response-content 
    req = requests.get(url) 
    i = Image.open(BytesIO(req.content)) 
    i.save(filename) 

# open page, get HTML request and parse with BeautifulSoup 
html = requests.get("http://proof.nationalgeographic.com/2016/02/02/photo-of-the-day-best-of-january-3/") 
soup = BeautifulSoup(html.text, "html.parser") 

# find all JPEGS in our soup and write their "src" attribute to array 
urls = [] 
for img in soup.find_all("img"): 
    if img["src"].endswith("jpg"): 
     print("endswith jpg") 
     urls.append(str(img["src"])) 
    print(str(img)) 

jpeg_no = 00 
for url in urls: 
    url_to_image(url, filename="NatGeoPix/" + str(jpeg_no) + ".jpg") 
    jpeg_no += 1 

Answer 1

Die Bilder werden mit JavaScript auf der Seite gerendert, die fehlschlägt. zuerst die Seite mit dryscrape

machen (Wenn Sie nicht wollen, verwenden dryscrape siehe Web-scraping JavaScript page with Python) zum Beispiel

import requests 
from PIL import Image 
from io import BytesIO 
from bs4 import BeautifulSoup 
import dryscrape 

def url_to_image(url, filename): 
    # get HTTP response, open as bytes, save the image 
    # http://docs.python-requests.org/en/master/user/quickstart/#binary-response-content 
    req = requests.get(url) 
    i = Image.open(BytesIO(req.content)) 
    i.save(filename) 

# open page, get HTML request and parse with BeautifulSoup 

session = dryscrape.Session() 
session.visit("http://www.nationalgeographic.com/photography/proof/2017/05/lake-chad-desertification/") 
response = session.body() 
soup = BeautifulSoup(response, "html.parser") 

# find all JPEGS in our soup and write their "src" attribute to array 
urls = [] 
for img in soup.find_all("img"): 
    if img["src"].endswith("jpg"): 
     print("endswith jpg") 
     urls.append(str(img["src"])) 
     print(str(img)) 

jpeg_no = 00 
for url in urls: 
    url_to_image(url, filename="NatGeoPix/" + str(jpeg_no) + ".jpg") 
    jpeg_no += 1 

Aber ich würde auch prüfen, ob Sie eine absolute URL keinen relativen haben:

import requests 
from PIL import Image 
from io import BytesIO 
from bs4 import BeautifulSoup 
import dryscrape 
from urllib.parse import urljoin 


def url_to_image(url, filename): 
    # get HTTP response, open as bytes, save the image 
    # http://docs.python-requests.org/en/master/user/quickstart/#binary-response-content 
    req = requests.get(url) 
    i = Image.open(BytesIO(req.content)) 
    i.save(filename) 

# open page, get HTML request and parse with BeautifulSoup 
base = "http://www.nationalgeographic.com/photography/proof/2017/05/lake-chad-desertification/" 
session = dryscrape.Session() 
session.visit(base) 
response = session.body() 
soup = BeautifulSoup(response, "html.parser") 

# find all JPEGS in our soup and write their "src" attribute to array 
urls = [] 
for img in soup.find_all("img"): 
    if img["src"].endswith("jpg"): 
     print("endswith jpg") 
     urls.append(str(img["src"])) 
     print(str(img)) 

jpeg_no = 00 
for url in urls: 
    if url.startswith('http'): 
     absoute = url 
    else: 
     absoute = urljoin(base, url) 
    print (absoute) 
    url_to_image(absoute, filename="NatGeoPix/" + str(jpeg_no) + ".jpg") 
    jpeg_no += 1