7
Mein Golang-Programm (URL-Monitor) hat ein Speicherleck, es wird schließlich vom Kernel (Oom) getötet. die env:Golang Programmspeicherleck?
$ go version
go version go1.0.3
$ go env
GOARCH="amd64"
GOBIN=""
GOCHAR="6"
GOEXE=""
GOGCCFLAGS="-g -O2 -fPIC -m64 -pthread"
GOHOSTARCH="amd64"
GOHOSTOS="linux"
GOOS="linux"
GOPATH="/data/apps/go"
GOROOT="/usr/local/go"
GOTOOLDIR="/usr/local/go/pkg/tool/linux_amd64"
CGO_ENABLED="1"
Code:
package main
import (
"bytes"
"database/sql"
"flag"
"fmt"
_ "github.com/Go-SQL-Driver/MySQL"
"ijinshan.com/cfg"
"log"
"net"
"net/http"
"net/smtp"
"os"
"strconv"
"strings"
"sync"
"time"
)
var (
Log *log.Logger
Conf cfg.KVConfig
Debug bool
CpuCore int
HttpTransport = &http.Transport{
Dial: func(netw, addr string) (net.Conn, error) {
deadline := time.Now().Add(30 * time.Second)
c, err := net.DialTimeout(netw, addr, 20*time.Second)
if err != nil {
return nil, err
}
c.SetDeadline(deadline)
return c, nil
},
DisableKeepAlives: true,
}
HttpClient = &http.Client{
Transport: HttpTransport,
}
WG sync.WaitGroup
)
const (
LogFileFlag = os.O_WRONLY | os.O_CREATE | os.O_APPEND
LogFileMode = 0644
LogFlag = log.LstdFlags | log.Lshortfile
GET_VIDEO_SQL = `SELECT B.Name, A.TSID, A.Chapter, A.ChapterNum,
IFNULL(A.Website, ''), IFNULL(A.Descr, ''),
IFNULL(A.VideoId, ''), IFNULL(AndroidWebURL, ''), IFNULL(IOSWebURL, ''),
IFNULL(AndroidURL, ''), IFNULL(AndroidURL2, ''), IFNULL(IOSURL, '')
FROM Video A INNER JOIN TVS B ON A.TSID = B.ID LIMIT 200`
HtmlHead = `<table border=1 width=100% height=100%><tr><td>节目名
</td><td>tsid</td><td>章节</td><td>章节号</td><td>描述
</td><td>videoid</td><td>网站</td><td>地址</td></tr>`
HtmlTail = "</table>"
)
type videoInfo struct {
name string
tsid uint
chapter string
chapterNum uint
descr string
videoId string
website string
androidWebUrl string
iosWebUrl string
androidUrl string
androidUrl2 string
iosUrl string
}
func init() {
var (
confFile string
err error
)
// parse command argument:w
flag.StringVar(&confFile, "c", "./vsmonitor.conf", " set config file path")
flag.Parse()
// read config
if Conf, err = cfg.Read(confFile); err != nil {
panic(fmt.Sprintf("Read config file \"%s\" failed (%s)",
confFile, err.Error()))
}
// open log file
file, err := os.OpenFile(Conf["log.file"], LogFileFlag, LogFileMode)
if err != nil {
panic(fmt.Sprintf("OpenFile \"%s\" failed (%s)", Conf["log.file"],
err.Error()))
}
// init LOG
Log = log.New(file, "", LogFlag)
Debug = false
i, err := strconv.ParseInt(Conf["cpucore.num"], 10, 32)
if err != nil {
panic(fmt.Sprintf("ParseInt \"%s\" failed (%s)", Conf["cpucore.num"],
err.Error()))
}
CpuCore = int(i)
}
func getHttpStatusCode(url string) int {
if url == "" {
return 200
}
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return 0
}
req.Header.Add("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_3) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/24.0.1312.57 Safari/537.17")
req.Header.Add("Connection", "close")
resp, err := HttpClient.Do(req)
if err != nil {
return 0
}
defer resp.Body.Close()
return resp.StatusCode
}
func sendMail(host, user, pwd, from, to, subject, body, mailType string) error {
auth := smtp.PlainAuth("", user, pwd, strings.Split(host, ":")[0])
cntType := fmt.Sprintf("Content-Type: text/%s;charset=UTF-8", mailType)
msg := fmt.Sprintf("To: %s\r\nFrom: %s<%s>\r\nSubject: %s\r\n%s\r\n\r\n%s",
to, from, user, subject, cntType, body)
return smtp.SendMail(host, auth, user, strings.Split(to, ","), []byte(msg))
}
func getVideos(videoChan chan *videoInfo, htmlBuf *bytes.Buffer) error {
defer HttpTransport.CloseIdleConnections()
db, err := sql.Open("mysql", Conf["weikan.mysql"])
if err != nil {
return err
}
rows, err := db.Query(GET_VIDEO_SQL)
if err != nil {
db.Close()
return err
}
for rows.Next() {
video := &videoInfo{}
err = rows.Scan(&video.name, &video.tsid, &video.chapter,
&video.chapterNum,
&video.website, &video.descr, &video.videoId, &video.androidWebUrl,
&video.iosWebUrl, &video.androidUrl, &video.androidUrl2,
&video.iosUrl)
if err != nil {
db.Close()
return err
}
videoChan <- video
WG.Add(1)
}
db.Close()
// wait check url finish
WG.Wait()
// send mail
for {
if htmlBuf.Len() == 0 {
Log.Print("no error found!!!!!!!!")
break
}
Log.Print("found error !!!!!!!!")
/*
err := sendMail("smtp.gmail.com:587", "xxxx",
"xxx", "xxx <xxx>",
Conf["mail.to"], "xxxxx",
HtmlHead+htmlBuf.String()+HtmlTail, "html")
if err != nil {
Log.Printf("sendMail failed (%s)", err.Error())
time.Sleep(10 * time.Second)
continue
}
*/
Log.Print("send mail")
break
}
Log.Print("reset buf")
htmlBuf.Reset()
return nil
}
func checkUrl(videoChan chan *videoInfo, errChan chan string) {
defer func() {
if err := recover(); err != nil {
Log.Print("rouintes failed : ", err)
}
}()
for {
video := <-videoChan
ok := true
errUrl := ""
if code := getHttpStatusCode(video.androidWebUrl); code >= 400 {
errUrl += fmt.Sprintf("%s (%d)<br />",
video.androidWebUrl, code)
ok = false
}
if code := getHttpStatusCode(video.iosWebUrl); code >= 400 {
errUrl += fmt.Sprintf("%s (%d)<br />",
video.iosWebUrl, code)
ok = false
}
if code := getHttpStatusCode(video.androidUrl); code >= 400 {
errUrl += fmt.Sprintf("%s (%d)<br />",
video.androidUrl, code)
ok = false
}
if code := getHttpStatusCode(video.androidUrl2); code >= 400 {
errUrl += fmt.Sprintf("%s (%d)<br />",
video.androidUrl2, code)
ok = false
}
if code := getHttpStatusCode(video.iosUrl); code >= 400 {
errUrl += fmt.Sprintf("%s (%d)<br />",
video.iosUrl, code)
ok = false
}
if !ok {
errChan <- fmt.Sprintf(`<tr><td>%s</td><td>%d</td><td>%s</td>
<td>%d</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td></tr>`,
video.name, video.tsid, video.chapter, video.chapterNum,
video.descr, video.videoId,
video.website, errUrl)
Log.Printf("\"%s\" (%s) —— \"%s\" checkurl err", video.name,
video.chapter, video.descr)
} else {
Log.Printf("\"%s\" (%s) —— \"%s\" checkurl ok", video.name,
video.chapter, video.descr)
WG.Done()
}
}
}
func mergeErr(errChan chan string, htmlBuf *bytes.Buffer) {
defer func() {
if err := recover(); err != nil {
Log.Print("rouintes failed : ", err)
}
}()
for {
html := <-errChan
_, err := htmlBuf.WriteString(html)
if err != nil {
Log.Printf("htmlBuf WriteString \"%s\" failed (%s)", html,
err.Error())
panic(err)
}
WG.Done()
}
}
func main() {
videoChan := make(chan *videoInfo, 100000)
errChan := make(chan string, 100000)
htmlBuf := &bytes.Buffer{}
defer func() {
if err := recover(); err != nil {
Log.Print("rouintes failed : ", err)
}
}()
// check url
for i := 0; i < CpuCore; i++ {
go checkUrl(videoChan, errChan)
}
// merge error string then send mail
go mergeErr(errChan, htmlBuf)
for {
// get Video and LiveSrc video source
if err := getVideos(videoChan, htmlBuf); err != nil {
Log.Printf("getVideos failed (%s)", err.Error())
time.Sleep(10 * time.Second)
continue
}
// time.Sleep(1 * time.Hour)
}
Log.Print("exit...")
}
der Code hat vier funcs:
getHttpStatusCode
freien Ressourcennutzung resp.Body.Close()
sendmail
Ich brauche nicht die Ressource manuell
mergeErr
concat der err Zeichenfolge frei
durch die Verwendung eines htmlBuf (* bytes.Buffer)getVideos
Zuerst erhält es die Video-URLs und sendet sie dann an videoChan, dann wartet es darauf, dass alle Routinen ihre Prüfjobs beenden. dann sendmail und htmlBuf zurücksetzen.
Ich finde keine Ressource, die frei, aber benötigt.
$ top
zeigt:
PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
6451 root 20 0 3946m 115m 2808 S 0.7 0.2 6:11.20 vsmonitor
der VIRT und RES wachsen ...
Speicherprofilierungs:
(pprof) top
Total: 10.8 MB
2.3 21.2% 21.2% 2.3 21.2% main.main
2.0 18.5% 39.8% 2.0 18.5% bufio.NewWriterSize
1.5 13.9% 53.7% 1.5 13.9% bufio.NewReaderSize
1.5 13.9% 67.6% 1.5 13.9% compress/flate.NewReader
0.5 4.6% 72.2% 0.5 4.6% net.newFD
0.5 4.6% 76.8% 0.5 4.6% net.sockaddrToTCP
0.5 4.6% 81.5% 4.5 41.7% net/http.(*Transport).getConn
0.5 4.6% 86.1% 2.5 23.2% net/http.(*persistConn).readLoop
0.5 4.6% 90.7% 0.5 4.6% net/textproto.(*Reader).ReadMIMEHeader
0.5 4.6% 95.4% 0.5 4.6% net/url.(*URL).ResolveReference
Rekursionen kann schließlich eine Menge Haufen Platz in Anspruch nehmen, wie gehen geteilten Stapel verwendet, wo wachsen statt ein fester Speicherbereich zu sein. Suchen Sie nach endlosen (ausreichend großen) Rekursionen. – nemo
aber mein Code enthält nur ** loop ** – Terry
Lassen Sie das Profiling etwas länger laufen, bis man das tatsächliche Wachstum sehen kann? – nemo