func (e NotFoundError) Error() string {
return e.Message
}
type RemoteError struct {
Host string
Err error
}
func (e *RemoteError) Error() string {
return e.Err.Error()
}
var UserAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1541.0 Safari/537.36"
// HttpGet gets the specified resource. ErrNotFound is returned if the
// server responds with status 404.
func HttpGet(client *http.Client, url string, header http.Header) (io.ReadCloser, error) {
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return nil, err
}
req.Header.Set("User-Agent", UserAgent)
for k, vs := range header {
req.Header[k] = vs
}
resp, err := client.Do(req)
if err != nil {
return nil, &RemoteError{req.URL.Host, err}
}
if resp.StatusCode == 200 {
return resp.Body, nil
}
resp.Body.Close()
if resp.StatusCode == 404 { // 403 can be rate limit error. || resp.StatusCode == 403 {
err = NotFoundError{"Resource not found: " + url}
} else {
err = &RemoteError{req.URL.Host, fmt.Errorf("get %s -> %d", url, resp.StatusCode)}
}
return nil, err
}
// HttpGetBytes gets the specified resource. ErrNotFound is returned if the server
// responds with status 404.
func HttpGetBytes(client *http.Client, url string, header http.Header) ([]byte, error) {
rc, err := HttpGet(client, url, header)
if err != nil {
return nil, err
}
defer rc.Close()
return ioutil.ReadAll(rc)
}
// HttpGetToFile gets the specified resource and writes to file.
// ErrNotFound is returned if the server responds with status 404.
func HttpGetToFile(client *http.Client, url string, header http.Header, fileName string) error {
rc, err := HttpGet(client, url, header)
if err != nil {
return err
}
defer rc.Close()
var img = regexp.MustCompile(`href=\"javascript:goView\((\d+)`)
var imgPattern = regexp.MustCompile(`id="mainImage" src=\"../upload(.*?).jpg`)
var totalTask int
