Gibt es einen besseren Ansatz, BeautifulSoup in meinen Python-Web-Crawler-Codes zu verwenden?

Question

Ich versuche, Informationen von URLs auf einer Seite zu crawlen und sie in einer Textdatei zu speichern.

Ich habe große Hilfe bei der Frage erhielt How to get the right source code with Python from the URLs using my web crawler? und ich versuche zu verwenden, was ich in BeautifulSoup gelernt haben meine Codes basieren zu beenden auf diese Frage.

Aber wenn ich meine Codes anschaue, obwohl sie mein Bedürfnis befriedigt haben, aber sie sehen ziemlich durcheinander aus. Kann mir jemand helfen, sie ein wenig zu optimieren, besonders auf dem BeautifulSoup Teil? Wie der Teil der InfoListen und der Teil saveInfo. Danke!

Hier meine Codes sind:

import requests 
from bs4 import BeautifulSoup 
from urlparse import urljoin 

url = 'http://bbs.skykiwi.com/forum.php?mod=forumdisplay&fid=55&typeid=470&sortid=231&filter=typeid&pageNum=1&page=1' 

#To get the source code from the url 
def getsourse(url): 
    header = {'User-Agent': 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 10.0; WOW64; Trident/8.0; Touch)'} 
    html = requests.get(url, headers=header) 
    return html.content 

#To get all the links in current page 
def getallLinksinPage(sourceCode): 
    soup = BeautifulSoup(sourceCode) 
    return [a["href"] for a in soup.select("#threadlist a.xst")] 

#To save the info in the info.txt file 
def saveinfo(infoLists): 
    f = open('info.txt', 'a') 
    for each in infoLists: 
     f.writelines('Job Title: ' + str(each['title'].encode('utf-8')) + '\n') 
     f.writelines('Company Name: ' + str(each['companyName'].encode('utf-8')) + '\n') 
     f.writelines('Company Address: ' + str(each['address'].encode('utf-8')) + '\n') 
     f.writelines('Job Position: ' + str(each['position'].encode('utf-8')) + '\n') 
     f.writelines('Salary: ' + str(each['salary'].encode('utf-8')) + '\n') 
     f.writelines('Full/Part time: ' + str(each['jobType'].encode('utf-8')) + '\n') 
     f.writelines('Company Tel: ' + str(each['tel'].encode('utf-8')) + '\n') 
     f.writelines('Company Email: ' + str(each['email'].encode('utf-8')) + '\n') 
     f.writelines('WorkTime: ' + str(each['workTime'].encode('utf-8')) + '\n\n') 
    f.close() 

sourceCode = getsourse(url) # source code of the url page 
allLinksinPage = getallLinksinPage(sourceCode) #a List of the urls in current page 
linkNum=1 
infoLists=[] 
for eachLink in allLinksinPage: 
    print('Now downloading link '+str(linkNum)) 
    url = 'http://bbs.skykiwi.com/' 
    realUrl=urljoin(url, eachLink) 
    html = getsourse(realUrl) 
    soup= BeautifulSoup(html) 
    infoList={} #To save the following info,such as title companyName etc 
    infoList['title']=soup.find(attrs={'id':'thread_subject'}).string 
    infoList2=[] #To temporarily save info except 'title' 
    #FROM HERE IT GETS MESSY... 
    for line in soup.find_all(attrs={'class':'typeoption'}): # first locate the bigClass 
     for td in line.find_all('td'): # then locate all the 'td's 
      infoList2.append(td.string) 
     try: 
      for eachInfo in infoList2: 
       infoList['companyName'] = infoList2[0] 
       infoList['address'] = infoList2[1] 
       infoList['position'] = infoList2[2] 
       infoList['salary'] = infoList2[3] 
       infoList['jobType'] = infoList2[4] 
       infoList['tel'] = infoList2[5] 
       infoList['email'] = infoList2[6] 
       infoList['workTime'] = infoList2[7] 
     finally: 
      linkNum += 1 # To print link number 
    infoLists.append(infoList) 

saveinfo(infoLists) 

Answer 1

zip() Verwendung und eine list comprehension würde Lesbarkeit dramatisch verbessern:

headers = ['companyName', 'address', 'position', 'salary', 'jobType', 'tel', 'email', 'workTime'] 

infoLists = [dict(zip(headers, [item.string for item in line.find_all('td')[:8]])) 
      for line in soup.select(".typeoption")]