CHttp

#ifndef _HTTP_H_031105_
#define _HTTP_H_031105_

#include <map>

using namespace std;

class CHttp
{
private:
    string m_strUrl;    // url
    int *m_sock;        // socket

public:
    CHttp();
    virtual ~CHttp();


    //strUrl:  待抓取的网页对应的URL
    //fileBuf: 网页体信息
    //fileHead:网页头信息
    //location:网页如果重定向对应的URL
    //sock:套接子文件描述符
    int Fetch(string strUrl, char **fileBuf, 
        char **fileHead, char **location, int* sock);

private:
    //下面4个私有的成员函数--被 Fetch()函数调用
    //通过IO复用的方法读取网页头信息
    int read_header(int sock, char *headerPtr);
    
    //创建套接字文件描述符
    int CreateSocket(const char *host, int port);

    //被CreateSocket()调用,通过IO复用的方法连接目标服务器
    int nonb_connect(int, struct sockaddr*, int);

    //检测*buf所指的内存空间剩余值是否大于more,不够再加more+1单位的内存空间
    int checkBufSize(char **buf, int *bufsize, int more);

};

extern pthread_mutex_t mutexMemory;

#endif /* _HTTP_H_031105_ */

#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <strings.h>
#include <errno.h>
#include <netdb.h>
#include <unistd.h>
#include <netinet/in.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <sys/time.h>
#include <fcntl.h>
#include <iostream>
#include "Http.h"

//#include "Tse.h"
#include "CommonDef.h"
#include "Url.h"
//#include "Page.h"
#include "StrFun.h"

char *userAgent = NULL;
int timeout = DEFAULT_TIMEOUT;//设置最长的等待时间30秒
int hideUserAgent = 0;

CHttp::CHttp()
{
}

CHttp::~CHttp()
{
}


    /*
         * Actually downloads the page, registering a hit (donation)
         *      If the fileBuf passed in is NULL, the url is downloaded and then
         *      freed; otherwise the necessary space is allocated for fileBuf.
         *      Returns size of download on success, 
            -1 on error is set,
             -2 out of ip block,
             -3 invalid host,
            -4 MIME is imag/xxx
             -300 on 301.
         */



/*

function:

success: return bytesRead[网页体信息的真实的字节数]

fail:    return -1  各种其他的错误

         return -2  在IP阻塞范围内

         return -3  无效的主机号

         return -4  image/text类型

         return -300 网页重定向

strUrl:  待抓取的网页对应的URL

fileBuf: 网页体信息

fileHead:网页头信息

location:网页如果重定向对应的URL

sock:套接子文件描述符

*/
int CHttp::Fetch(string strUrl, char **fileBuf, char **fileHeadBuf, char **location, int* nPSock )
{
    char *tmp, *url, *requestBuf, *pageBuf;
    const char *host, *path;
    int sock, bytesRead = 0, bufsize = REQUEST_BUF_SIZE;
    int ret = -1, tempSize, selectRet;
    int port = 80;


    if( strUrl.empty() )//空的URL肯定不能抓取到网页
    {
        cout << "strUrl is NULL" << endl;
        return -1;
    }

    /* Copy the url passed in into a buffer we can work with, change, etc. */
/*
    url = (char*)malloc(strUrl.length()+1);
    if( url == NULL ){
        cout << "can not allocate enought memory for url" << endl;
        return -1;
    } else {
        memset(url, 0,strUrl.length()+1);
        memcpy(url, strUrl.c_str(), strUrl.length() );
    }
*/
    //pthread_mutex_lock(&mutexMemory);
    url = strdup(strUrl.c_str());//复制url
    //pthread_mutex_unlock(&mutexMemory);
    if( url == NULL )//分配失败
    {
        cout << "!error: stdup() in Fetch()" << endl;
        return -1;
    }

    // parse the url
    CUrl u;
    if( u.ParseUrlEx(url) == false )
    {
        //如果没有"http://"协议号，肯定会解析错误
        cout << "ParseUrlEx error in Fetch(): " << strUrl << endl;
        return -1;
    }

    host = u.m_sHost.c_str();
    path = u.m_sPath.c_str();
    if( u.m_nPort > 0 ) port = u.m_nPort;

    /* Compose a request string */
    //pthread_mutex_lock(&mutexMemory);

    /*构造HTTP请求报文:  假设strUrl="http://www.baidu.com/ecjtu/nihao.html"*/
    // GET /ecjtu/nihao.html HTTP/1.0\r\n
    requestBuf = (char*)malloc(bufsize);
    //pthread_mutex_unlock(&mutexMemory);
    if(requestBuf == NULL)
    {
        if (url)
        {
            //pthread_mutex_lock(&mutexMemory);
            free(url);
            url=NULL;
            //pthread_mutex_unlock(&mutexMemory);
        }
        cout << "can not allocate enought memory for requestBuf" << endl;
        return -1;
    }
    requestBuf[0] = 0;

    if( strlen(path) < 1 )//说明请求的是根目录下的网页
    {
        // GET / HTTP/1.0\r\n
        /* The url has no '/' in it, assume the user is making a root-level
                 *      request */
        tempSize = strlen("GET /") + strlen(HTTP_VERSION) +2;
/*
        if( tempSize > bufsize ){
            free(url);
            free(requestBuf);
            cout << "tempSize larger than bufsize" << endl;
            return -1;
        }
*/

        if(checkBufSize(&requestBuf, &bufsize, tempSize) ||    snprintf(requestBuf, bufsize, "GET / %s\r\n", HTTP_VERSION) < 0 ){
            /*int snprintf(char *restrict buf, size_t n, const char * restrict  format, ...);
             函数说明:最多从源串中拷贝n－1个字符到目标串中，然后再在后面加一个0。所以如果目标串的大小为n
             的话，将不会溢出。*/

            //pthread_mutex_lock(&mutexMemory);
            if (url)
            {
                 free(url); 
                 url=NULL;
            }
            if (requestBuf)
            {
                 free(requestBuf); 
                 requestBuf=NULL;
            }
            //pthread_mutex_unlock(&mutexMemory);
            cout << "1.checkBuffSize(&requestBuf..) error" << endl;
            return -1;
        }

    }
    else//说明请求的是非根目录下的网页
    {
        tempSize = strlen("GET ") + strlen(path) + strlen(HTTP_VERSION) + 4;

        if(checkBufSize(&requestBuf, &bufsize, tempSize) ||    snprintf(requestBuf, bufsize, "GET %s %s\r\n", path, HTTP_VERSION) < 0)
        {

            //pthread_mutex_lock(&mutexMemory);
            if (url)
            {
                 free(url); 
                 url=NULL;
            }
            if (requestBuf)
            {
                 free(requestBuf); 
                 requestBuf=NULL;
            }
            //pthread_mutex_unlock(&mutexMemory);
            cout << "2._checkBuffSize(&requestBuf..) error" << endl;
            return -1;
        }

    }


    /* Use Host: even though 1.0 doesn't specify it.  Some servers
         *      won't play nice if we don't send Host, and it shouldn't hurt anything */
    tempSize = (int)strlen("Host: ") + (int)strlen(host) + 3;/* +3 for "\r\n\0" */

    if(checkBufSize(&requestBuf, &bufsize, tempSize + 128)){
        //pthread_mutex_lock(&mutexMemory);
        if (url)
        {
             free(url); url=NULL;
        }
        if (requestBuf)
        {
             free(requestBuf); requestBuf=NULL;
        }
        //pthread_mutex_unlock(&mutexMemory);
        cout << "3._checkBuffSize(&requestBuf..) error" << endl;
        return -1;
    }

    strcat(requestBuf, "Host: ");
    strcat(requestBuf, host);
    strcat(requestBuf, "\r\n");

    if(!hideUserAgent && userAgent == NULL) {

        tempSize = (int)strlen("User-Agent: ") +
            (int)strlen(DEFAULT_USER_AGENT) + (int)strlen(VERSION) + 4;
        if(checkBufSize(&requestBuf, &bufsize, tempSize)) {
            //pthread_mutex_lock(&mutexMemory);
            if (url)
            {
                 free(url); url=NULL;
            }
            if (requestBuf)
            {
                 free(requestBuf); requestBuf=NULL;
            }
            //pthread_mutex_unlock(&mutexMemory);
            cout << "4._checkBuffSize(&requestBuf..) error" << endl;
            return -1;
        }
        strcat(requestBuf, "User-Agent: ");
        strcat(requestBuf, DEFAULT_USER_AGENT);
        strcat(requestBuf, "/");
        strcat(requestBuf, VERSION);
        strcat(requestBuf, "\r\n");

    } else if(!hideUserAgent) {

        tempSize = (int)strlen("User-Agent: ") + (int)strlen(userAgent) + 3;
        if(checkBufSize(&requestBuf, &bufsize, tempSize)) {

            //pthread_mutex_lock(&mutexMemory);
            if (url)
            {
                 free(url); url=NULL;
            }
            if (requestBuf)
            {
                 free(requestBuf); requestBuf=NULL;
            }
            //pthread_mutex_unlock(&mutexMemory);
            cout << "5._checkBuffSize(&requestBuf..) error" << endl;
            return -1;
        }
        strcat(requestBuf, "User-Agent: ");
        strcat(requestBuf, userAgent);
        strcat(requestBuf, "\r\n");
    }

    //tempSize = (int)strlen("Connection: Close\n\n");
    tempSize = (int)strlen("Connection: Keep-Alive\r\n\r\n");
    if(checkBufSize(&requestBuf, &bufsize, tempSize)) {
        //pthread_mutex_lock(&mutexMemory);
        if (url)
        {
             free(url); url=NULL;
        }
        if (requestBuf)
        {
             free(requestBuf); requestBuf=NULL;
        }
        //pthread_mutex_unlock(&mutexMemory);
        cout << "6._checkBuffSize(&requestBuf..) error" << endl;
        return -1;
    }


    //strcat(requestBuf, "Connection: Close\n\n");
    strcat(requestBuf, "Connection: Keep-Alive\r\n\r\n");


    /* Now free any excess memory allocated to the buffer */
    //pthread_mutex_lock(&mutexMemory);
    //重新调整requestBuf的内存空间，释放多余的内存空间
    tmp = (char *)realloc(requestBuf, strlen(requestBuf) + 1);
    //pthread_mutex_unlock(&mutexMemory);
    if(tmp == NULL){
        //pthread_mutex_lock(&mutexMemory);
        if (url)
        {
             free(url); url=NULL;
        }
        if (requestBuf)
        {
             free(requestBuf); requestBuf=NULL;
        }
        //pthread_mutex_unlock(&mutexMemory);
        cout << "realloc for tmp error" << endl;
        return -1;
    }
    requestBuf = tmp;

    if( *nPSock != -1 ){
        sock = *nPSock;
        cout << "using privous socket " << *nPSock << endl;
    }else{

        // cout << "1.get a new one" << endl;
        sock = CreateSocket( host, port );
        if(sock == -1) { // invalid host
            //pthread_mutex_lock(&mutexMemory);
            if (url)
            {
                 free(url); url=NULL;
            }
            if (requestBuf)
            {
                 free(requestBuf); requestBuf=NULL;
            }
            //pthread_mutex_unlock(&mutexMemory);
            return -3;
        }
        if(sock == -2) { // out of ip block
            //pthread_mutex_lock(&mutexMemory);
            if (url)
            {
                 free(url); url=NULL;
            }
            if (requestBuf)
            {
                 free(requestBuf); requestBuf=NULL;
            }
            //pthread_mutex_unlock(&mutexMemory);
            //cout << "2.not able to MakeSocket" << endl;
            return -2;
        }
    }
    


    ret = write(sock, requestBuf, strlen(requestBuf));
    if( ret == 0 ){
        cout << "requestBuf is " << requestBuf << endl;
        cout << "write nothing" << endl;
        //pthread_mutex_lock(&mutexMemory);
        if (url)
        {
            free(url); url=NULL;
        }
        if (requestBuf)
        {
            free(requestBuf); requestBuf=NULL;
        }
        //pthread_mutex_unlock(&mutexMemory);
        close(sock);
        *nPSock = -1;
        return -1;
        
    }
    if( ret == -1){
        //cout << "write error" << endl;
        // sock is invalid,we should make a new one
        close(sock);
        *nPSock  = -1;

        cout << "2.close previous socket " << *nPSock << " and get a new one" << endl;
        //maybe sock is dead,try again
        sock = CreateSocket( host, port );
        if(sock == -1) { 
            //pthread_mutex_lock(&mutexMemory);
            if (url)
            {
                free(url); url=NULL;
            }
            if (requestBuf)
            {
                free(requestBuf); requestBuf=NULL;
            }
            //pthread_mutex_unlock(&mutexMemory);
            cout << "3.not able to MakeSocket" << endl;
            return -1;
        }
        if(sock == -2) { 
            //pthread_mutex_lock(&mutexMemory);
            if (url)
            {
                free(url); url=NULL;
            }
            if (requestBuf)
            {
                free(requestBuf); requestBuf=NULL;
            }
            //pthread_mutex_unlock(&mutexMemory);
            cout << "4.not able to MakeSocket" << endl;
            return -1;
        }
        if(write(sock, requestBuf, strlen(requestBuf)) == -1){
            //pthread_mutex_lock(&mutexMemory);
            if (url)
            {
                free(url); url=NULL;
            }
            if (requestBuf)
            {
                free(requestBuf); requestBuf=NULL;
            }
            //pthread_mutex_unlock(&mutexMemory);
            close(sock);
            *nPSock = -1;
            cout << "write error" << endl;
            return -1;
        }
    }

    //pthread_mutex_lock(&mutexMemory);
    if (url)
    {
        free(url); url=NULL;
    }
    if (requestBuf)
    {
        free(requestBuf); requestBuf=NULL;
    }
    //pthread_mutex_unlock(&mutexMemory);


    char headerBuf[HEADER_BUF_SIZE];
    /* Grab enough of the response to get the metadata */
    memset( headerBuf,0,HEADER_BUF_SIZE );
    //cout << "old sock is " << sock << endl;
    ret = read_header(sock, headerBuf);
    //cout << "ret = " << ret << endl;
    if(ret < 0) { 
        close(sock); 
        *nPSock = -1;
        return -1;
    }

    //cout << headerBuf << endl;
    if( strlen(headerBuf) == 0 ){
        cout << "strlen(headerBuf) = 0" << headerBuf << endl;
        cout << "strUrl: " << strUrl << endl << endl;;
        close(sock);
                *nPSock = -1;
        return -1;
    }



     //解析网页头信息
    CPage iPage;
    iPage.ParseHeaderInfo(headerBuf);
    if (iPage.m_nStatusCode == -1)
    {
        close(sock);
        *nPSock = -1;
        cout << "headerBuf: " << headerBuf << endl;
        cout << "!header error: not find HTTP" << endl;
        return -1;
    }



    // deal with http://net.cs.pku.edu.cn/~cnds
    if (iPage.m_nStatusCode == 301 || iPage.m_nStatusCode == 302)
    {
        if (iPage.m_sLocation.empty() || iPage.m_sLocation.size()>URL_LEN)
        {    
            close(sock);
            *nPSock = -1;
            cout << headerBuf << endl;
            cout << "!error: Location" << endl;
            return -1;
        }
        else
        {
            //pthread_mutex_lock(&mutexMemory);
            char *loc=strdup(iPage.m_sLocation.c_str());
            //pthread_mutex_unlock(&mutexMemory);
            *location = loc;
            close(sock);
            *nPSock = -1;
            return -300;//重定向了
        }
    }

    if(iPage.m_nStatusCode<200 || iPage.m_nStatusCode>299 ){
        close(sock);
        *nPSock = -1;
        cout << "!header code = " << iPage.m_nStatusCode << endl;
        return -1;
    }

    // when crawling images for ImgSE, remember to comment the paragraph
    // when crawling plain text for SE, remember to open the paragraph
    // paragraph begin
    if( iPage.m_sContentType.find("image") != string::npos )
    { // 
        close(sock);
        *nPSock = -1;
        return -4;
    }
    // paragraph end

    if (iPage.m_nContentLength == -1)
    {
        close(sock);
        *nPSock = -1;
        cout << headerBuf << endl;
        cout << "!error: Content-length" << endl;
        return -1;
    }

    if (iPage.m_nContentLength==0 || iPage.m_nContentLength<20)
    { // Allocate enough memory to hold the page 
        iPage.m_nContentLength = DEFAULT_PAGE_BUF_SIZE;
    }


    if (iPage.m_nContentLength > MAX_PAGE_BUF_SIZE)
    {
         cout<<"这个网页的长度大于5M，我过滤掉它！"<<endl;
        cout << "the page discarded due to its size " 
            << iPage.m_nContentLength 
            << " is larger than " << MAX_PAGE_BUF_SIZE << endl;
        close(sock);
        *nPSock = -1;
        return -1;
    }

    //pthread_mutex_lock(&mutexMemory);
    pageBuf = (char *)malloc(iPage.m_nContentLength);
    //pthread_mutex_unlock(&mutexMemory);
    if(pageBuf == NULL){
        close(sock);
        *nPSock = -1;
        cout << "malloc for pageBuf" << endl;
        return -1;
    }
    
    /* Begin reading the body of the file */
    //开始读取网页体信息
    fd_set rfds;
    struct timeval tv;
    int flags;
    //将sock套接子文件描述符设置为非阻塞的方式
    flags=fcntl(sock,F_GETFL,0);
    if(flags<0)
    {
        close(sock);
        *nPSock = -1;
        if (pageBuf)
        {
            //pthread_mutex_lock(&mutexMemory);
            free(pageBuf);
            pageBuf=NULL;
            //pthread_mutex_unlock(&mutexMemory);
        }
        cout << "1.fcntl() error " << endl;
        return -1;
    }
    
    
    flags|=O_NONBLOCK;
    if(fcntl(sock,F_SETFL,flags)<0){
        close(sock);
        *nPSock = -1;
        if (pageBuf)
        {
            free(pageBuf); pageBuf=NULL;
        }
        cout << "2.fcntl() error " << endl;
        return -1;
    }


    //挂一个while()循环读取网页体信息
    int pre_ret=0;
    while(ret > 0)
    {
        FD_ZERO(&rfds);//清理rfds读文件描述符集合
        FD_SET(sock, &rfds);//将sock加到rfds读文件描述符集合中
        if( bytesRead == iPage.m_nContentLength )
        {
            tv.tv_sec = 1;
        }
        else
        {
            tv.tv_sec = timeout;
        }
        tv.tv_usec = 0;

        if(DEFAULT_TIMEOUT >= 0)
            selectRet = select(sock+1, &rfds, NULL, NULL, &tv);//IO复用
        else            /* No timeout, can block indefinately */
            selectRet = select(sock+1, &rfds, NULL, NULL, NULL);

        if(selectRet == 0 && timeout < 0)//超时
        {
            close(sock);
            *nPSock = -1;
            if (pageBuf)
            {
                //pthread_mutex_lock(&mutexMemory);
                free(pageBuf);
                pageBuf=NULL;
                //pthread_mutex_unlock(&mutexMemory);
            }
            cout << "selectRet == 0 && timeout < 0" << endl;
            return -1;
        } 
        else if(selectRet == -1)//select()函数出错
        {
            close(sock);
            *nPSock = -1;
            if (pageBuf)
            {
                free(pageBuf);
                pageBuf=NULL;
            }
            cout << "selectRet == -1" << endl;
            return -1;
        }

        //每次最多接收iPage.m_nContentLength字节--缓冲区的大小为iPage.m_nContentLength
        ret = read(sock, pageBuf + bytesRead, iPage.m_nContentLength);
        //ret = read(sock, (char*)pageBuf.c_str() + bytesRead, iPage.m_nContentLength);

        if(ret == 0) break;
        if(ret == -1 && pre_ret==0)//read()函数出错
        {
            close(sock);
            *nPSock = -1;
            if (pageBuf)
            {
                //pthread_mutex_lock(&mutexMemory);
                free(pageBuf); pageBuf=NULL;
                //pthread_mutex_unlock(&mutexMemory);
            }
            cout << "read()'s retval=-1" << endl;
            return -1;
        }
        else if( ret == -1 && pre_ret )
        {
            //cout << "2. pre_ret = " << pre_ret << endl;
/*
            if( bytesRead < iPage.m_nContentLength){    // meaning we lost the connection too soon
                cout << "lost the connection too soon" << endl;
                freeOpageBuf);
                return -1;
            }
*/
            break;
        }

        pre_ret = ret;
        //cout << "1.pre_ret = " << pre_ret << endl;

        bytesRead += ret;


            /* To be tolerant of inaccurate Content-Length fields, we'll
             *      allocate another read-sized chunk to make sure we have
             *      enough room.
             */
        if(ret > 0) {
            //pthread_mutex_lock(&mutexMemory);
            pageBuf = (char *)realloc(pageBuf, bytesRead + iPage.m_nContentLength);
            //pthread_mutex_unlock(&mutexMemory);
            if(pageBuf == NULL) {
                close(sock);
                *nPSock = -1;
                if (pageBuf)
                {
                    //pthread_mutex_lock(&mutexMemory);
                    free(pageBuf); pageBuf=NULL;
                    //pthread_mutex_unlock(&mutexMemory);
                }
                cout << "realloc()" << endl;
                return -1;
            }
        }

    }

    /*
     * The download buffer is too large.  Trim off the safety padding.
    */

    //pthread_mutex_lock(&mutexMemory);
    pageBuf = (char *)realloc(pageBuf, bytesRead+1);
    //pthread_mutex_unlock(&mutexMemory);
    if(pageBuf == NULL){
        close(sock);
        *nPSock = -1;
        if (pageBuf)
        {
            //pthread_mutex_lock(&mutexMemory);
            free(pageBuf); pageBuf=NULL;
            //pthread_mutex_unlock(&mutexMemory);
        }
        cout << "2.realloc()" << endl;
        return -1;
    }


    pageBuf[bytesRead] = '\0';


    if(fileBuf == NULL){    /* They just wanted us to "hit" the url */
        if (pageBuf)
        {
            //pthread_mutex_lock(&mutexMemory);
            free(pageBuf); pageBuf=NULL;
            //pthread_mutex_unlock(&mutexMemory);
        }
    }else{



        char *tmp;
        //tmp = (char *)malloc(HEADER_BUF_SIZE);
        //pthread_mutex_lock(&mutexMemory);
        tmp = (char *)malloc(strlen(headerBuf)+1);
        //pthread_mutex_unlock(&mutexMemory);
            if(tmp == NULL){
                    close(sock);
            *nPSock = -1;
            if (pageBuf)
            {
                //pthread_mutex_lock(&mutexMemory);
                free(pageBuf); pageBuf=NULL;
                //pthread_mutex_unlock(&mutexMemory);
            }
            cout << "malloc() for headerBuf" << endl;
                    return -1;
            }
        //memcpy( tmp, headerBuf, HEADER_BUF_SIZE-1 );
        strncpy( tmp, headerBuf, strlen(headerBuf)+1 );
        *fileHeadBuf = tmp;

        *fileBuf = pageBuf;
    }
        
    //close(sock);
    *nPSock = sock;
    return bytesRead;
}
    




/*

function: 创建套接字文件描述符，并且调用nonb_connect()同目标服务器进行连接

success:  return sock[成功创建的套接子文件描述符]

fail:     return -1  其他错误

          return -2  在IP阻塞范围内

*/
int CHttp::CreateSocket(const char *host, int port)
{
    int sock;        // Socket descriptor
    struct sockaddr_in sa;    // Socket address


    unsigned long   inaddr;
    int ret;

    CUrl url;
    char *ip = url.GetIpByHost(host);//通过主机号得到IP地址

    if( ip == NULL )//获得失败
    { // gethostbyname() error in GetIpByHost()
        //cout << "invalid host: " << host << endl;
        return -1;

    } 
    else 
    {
        // filter ip (decide whether it is inside the ip block)
        if( url.IsValidIp(ip) )//在IP阻塞范围内
        {
            // inside
            inaddr = (unsigned long)inet_addr(ip);//将字符串IP转化为32位的网络字节序

            if( inaddr == INADDR_NONE )
            {
                // release the buffer, be careful
                //pthread_mutex_lock(&mutexMemory);
                delete [] ip; ip = NULL;
                //pthread_mutex_unlock(&mutexMemory);
                cout << "invalid ip " << ip << endl;
                return -1;
            }

            memcpy((char *)&sa.sin_addr, (char *)&inaddr, sizeof(inaddr));

            // release the buffer, be carful
            //pthread_mutex_lock(&mutexMemory);
            delete [] ip; ip = NULL;
            //pthread_mutex_unlock(&mutexMemory);

        } 
        else//在IP阻塞范围外
        { // out of ip block
            // release the buffer, be carful
            //pthread_mutex_lock(&mutexMemory);
            delete [] ip; ip = NULL;
            //pthread_mutex_unlock(&mutexMemory);
            //cout << "out of ip block: " << host << endl;
            return -2;
        }
    }


    /* Copy host address from hostent to (server) socket address */
    sa.sin_family = AF_INET;        
    sa.sin_port = htons(port);    /* Put portnum into sockaddr */

    sock = -1;
    sock = socket(AF_INET, SOCK_STREAM, 0);//创建套接字文件描述符
    if(sock < 0 ) //创建失败
    { 
        cout << "socket() in CreateSocket" << endl;
        return -1;
    }

    int optval = 1;
    if (setsockopt (sock, SOL_SOCKET, SO_REUSEADDR,(char *)&optval, sizeof (optval)) < 0)
        //SOL_SOCKET 通用套接字选项
        //SO_REUSEADDR 表示允许本地地址重用
    {

        cout << "setsockopt() in CreateSocket" << endl;
        close(sock);
        return -1;
    }

        //ret = connect(sock, (struct sockaddr *)&sa, sizeof(sa));
        ret = nonb_connect(sock, (struct sockaddr *)&sa, DEFAULT_TIMEOUT);
        if(ret == -1) { 
        cout << "nonb_connect() in CreateSocket" << endl;
        close(sock);
        return -1; 
    }

        return sock;//返回套接字文件描述符
}


/* function:通过IO复用的方法读取网页头信息
 success: return  bytesRead[网页头信息的真实长度]
 fail:    return  -1
 */
int CHttp::read_header(int sock, char *headerPtr)
{
    fd_set rfds;//读文件描述符集合
    struct timeval tv;
    int bytesRead = 0, newlines = 0, ret, selectRet;

    int flags;

    flags=fcntl(sock,F_GETFL,0);//将sock套接子文件描述符设置为非阻塞方式
    if(flags<0)
    {
        cout << "1.fcntl() in read_header()< 0" << endl;
        return -1;
    }
    
    flags|=O_NONBLOCK;
    if(fcntl(sock,F_SETFL,flags)<0)
    {
        cout << "2.fcntl() < 0 in read_header()" << endl;
        return -1;
    }

    //挂一个while()循环来读取网页头信息
    while(newlines != 2 && bytesRead != HEADER_BUF_SIZE-1)
    {
        FD_ZERO(&rfds);//清理读文件描述符集合
        FD_SET(sock, &rfds);//将套接字文件描述符加到读文件描述符集合中
        tv.tv_sec = timeout;//设置最长的等待时间
        tv.tv_usec = 0;

        if(timeout >= 0)
            selectRet = select(sock+1, &rfds, NULL, NULL, &tv);
        else   //最一个参数设置为NULL，表示阻塞操作会一直等待，直到莫个监视的文件集合中的某个文件描述符符合返回条件
            selectRet = select(sock+1, &rfds, NULL, NULL, NULL);

        if(selectRet == 0 && timeout < 0) 
        {
            cout << "selectRet == 0 && timeout < 0" << endl;
            return -1;
        }
        else if(selectRet == -1) //select()出错
        {
            cout << "selectRet == 0 && timeout < 0 else" << endl;
            return -1;
        }

        ret = read(sock, headerPtr, 1);
        if(ret == -1)
        {
            cout << "!error: read() in read_header()" << endl;
            return -1;
        }

        bytesRead++;
                
        if(*headerPtr == '\r')
        {                 /* Ignore CR */
            /* Basically do nothing special, just don't set newlines
             *      to 0 */
            headerPtr++;
            continue;
        }
        else if(*headerPtr == '\n')             /* LF is the separator */
            newlines++;
        else    
            newlines = 0;
                
        headerPtr++;

    }
        
    //headerPtr -= 3;         /* Snip the trailing LF's */
                  /* to be compatible with Tianwang format, we have to retain them*/
    headerPtr -= 2;
    *headerPtr = '\0';
    //cout << "in it " << headerPtr << endl;
    return bytesRead;
}



/*
 function:被CreateSocket()调用,通过IO复用的方法连接目标服务器
 success: return 0;
 fail:    return -1;
 sockfd:  套接子文件描述符
 sa:      服务器套接子地址结构
 sec:     最长的等待时间
 */
int CHttp::nonb_connect(int sockfd,struct sockaddr* sa,int sec)
{
    int flags;
    int status;
    fd_set mask;//写文件描述符集合
    struct timeval timeout;

    //set the socket as nonblocking
    flags=fcntl(sockfd,F_GETFL,0);//将套接子文件描述符设置为非阻塞方式

    if(flags<0) return -1;
    flags|=O_NONBLOCK;//设置非阻塞方式
    if(fcntl(sockfd,F_SETFL,flags) < 0)
    {
        cout << "1.fcntl() in nonb_connect" << endl;
        return -1;
    }

    if( connect(sockfd,sa,sizeof(struct sockaddr)) == 0)//立刻连接上了
    {
        flags&=~O_NONBLOCK;//因为上面已经设置了非阻塞方式，所以我们这里有必要重新设置阻塞方式--相当于复位
        fcntl(sockfd,F_SETFL,flags);
        return sockfd;//connected immediately
        }

    FD_ZERO(&mask);//清理写文件描述符集合mask
    FD_SET(sockfd,&mask);//将sockfd套接字文件描述符加到文件描述符集合mask中
    timeout.tv_sec=sec;//设置最长的等待时间
    timeout.tv_usec=0;
    status=select(sockfd+1,NULL,&mask,NULL,&timeout);//IO复用

    switch(status){
        case -1:        // Select error, set the socket as default blocking  //select()出错
            flags&=~O_NONBLOCK;
            fcntl(sockfd,F_SETFL,flags);
            cout << "2.fcntl() in nonb_connect" << endl;
            return -1;
        case 0:         //Connection timed out.//连接超时
            flags&=~O_NONBLOCK;
            fcntl(sockfd,F_SETFL,flags);
            cout << "3.fcntl() in nonb_connect" << endl;
            return -1;
        default:         // Connected successfully.//连接成功
            FD_CLR(sockfd,&mask);
            flags&=~O_NONBLOCK;
            fcntl(sockfd,F_SETFL,flags);
            return 0;
    }
}

/*
function: 检测*buf所指的内存空间剩余值是否大于more,不过再加more+1单位的内存空间
success:  return 0;
fail:     return -1;
*/
int CHttp::checkBufSize(char **buf, int *bufsize, int more)
{
    char *tmp;
    int roomLeft = *bufsize - (strlen(*buf) + 1);//*buf内存空间的剩余值

    if(roomLeft > more) return 0;//剩余值大于more返回0

    //pthread_mutex_lock(&mutexMemory);
    tmp = (char *)realloc(*buf, *bufsize + more + 1);//剩余值不够more,这个时候我们要调整内存空间的长度，长度加more+1
    //pthread_mutex_unlock(&mutexMemory);
    if(tmp == NULL) return -1;//没有调整成功返回-1

    *buf = tmp;
    *bufsize += more + 1;
    return 0;//调整成功
}