OSDN Git Service

epoll実装見直し版(ざっくり動作確認済み)
author3644889 <3644889@1ed66053-1c2d-0410-8867-f7571e6e31d3>
Tue, 9 Mar 2010 09:06:44 +0000 (09:06 +0000)
committer3644889 <3644889@1ed66053-1c2d-0410-8867-f7571e6e31d3>
Tue, 9 Mar 2010 09:06:44 +0000 (09:06 +0000)
git-svn-id: http://10.144.169.20/repos/um/branches/l7vsd-3.x-ramiel-epoll@10015 1ed66053-1c2d-0410-8867-f7571e6e31d3

l7vsd/include/tcp_session.h
l7vsd/src/tcp_session.cpp

index 14b7c0d..ff052aa 100644 (file)
@@ -45,6 +45,8 @@
 
 #define    PARAM_UP_BUFFER_SIZE    "upstream_buffer_size"
 #define    PARAM_DOWN_BUFFER_SIZE    "downstream_buffer_size"
+#define    PARAM_EPOLL_TRIGGER    "epoll_trigger"
+#define    PARAM_EPOLL_TIMEOUT    "epoll_timeout"
 
 namespace l7vs{
 
@@ -317,15 +319,26 @@ namespace l7vs{
 
             // epoll using member
             #define EVENT_NUM       2
-            #define EPOLL_TIMEOUT   50      //[microsecond]
             struct epoll_event  up_client_events[EVENT_NUM];
             struct epoll_event  up_realserver_events[EVENT_NUM];
+            struct epoll_event  up_sorryserver_events[EVENT_NUM];
             struct epoll_event  down_client_events[EVENT_NUM];
             struct epoll_event  down_realserver_events[EVENT_NUM];
+            struct epoll_event  down_sorryserver_events[EVENT_NUM];
             int    up_client_epollfd;
             int    up_realserver_epollfd;
+            int    up_sorryserver_epollfd;
             int    down_client_epollfd;
             int    down_realserver_epollfd;
+            int    down_sorryserver_epollfd;
+            bool   up_client_epollfd_registered;
+            bool   up_realserver_epollfd_registered;
+            bool   up_sorryserver_epollfd_registered;
+            bool   down_client_epollfd_registered;
+            bool   down_realserver_epollfd_registered;
+            bool   down_sorryserver_epollfd_registered;
+            bool   is_epoll_edge_trigger;
+            int    epoll_timeout;
 
             //! handshake timer handler
             //! @param[in]        error is error code object
index 5092aa5..88aba88 100644 (file)
@@ -283,8 +283,19 @@ namespace l7vs{
         // epoll impliment
         up_client_epollfd = epoll_create( EVENT_NUM );
         up_realserver_epollfd = epoll_create( EVENT_NUM );
+        up_sorryserver_epollfd = epoll_create( EVENT_NUM );
         down_client_epollfd = epoll_create( EVENT_NUM );
         down_realserver_epollfd = epoll_create( EVENT_NUM );
+        down_sorryserver_epollfd = epoll_create( EVENT_NUM );
+        up_client_epollfd_registered = false;
+        up_realserver_epollfd_registered = false;
+        up_sorryserver_epollfd_registered = false;
+        down_client_epollfd_registered = false;
+        down_realserver_epollfd_registered = false;
+        down_sorryserver_epollfd_registered = false;
+        is_epoll_edge_trigger = true;
+        epoll_timeout = 50;
+
     }
     //! destructor
     tcp_session::~tcp_session(){
@@ -366,6 +377,20 @@ namespace l7vs{
             downstream_buffer_size    = int_val;
         }
 
+        int_val    = param.get_int( PARAM_COMP_SESSION, PARAM_EPOLL_TRIGGER, vs_err );
+        if((likely( !vs_err )) && (0 < int_val)){
+            if (int_val == 1) {
+                is_epoll_edge_trigger = true;
+            } else if (int_val == 2) {
+                is_epoll_edge_trigger = false;
+            }
+        }
+
+        int_val    = param.get_int( PARAM_COMP_SESSION, PARAM_EPOLL_TIMEOUT, vs_err );
+        if((likely( !vs_err )) && (0 < int_val)){
+            epoll_timeout = int_val;
+        }
+
         protocol_module = parent_service.get_protocol_module();
 
         if(unlikely( protocol_module == NULL )){
@@ -435,6 +460,13 @@ namespace l7vs{
             }
         }
 
+        up_client_epollfd_registered = false;
+        up_realserver_epollfd_registered = false;
+        up_sorryserver_epollfd_registered = false;
+        down_client_epollfd_registered = false;
+        down_realserver_epollfd_registered = false;
+        down_sorryserver_epollfd_registered = false;
+
         return msg;
     }
 
@@ -1488,11 +1520,28 @@ namespace l7vs{
         UP_THREAD_FUNC_TYPE_TAG func_tag;
 
         struct epoll_event event;
-        event.events = EPOLLET | EPOLLIN | EPOLLHUP;
         if ( !ssl_flag ) {
-            epoll_ctl( up_client_epollfd, EPOLL_CTL_ADD, client_socket.get_socket().native(), &event );
+            event.data.fd = client_socket.get_socket().native();
         } else {
-            epoll_ctl( up_client_epollfd, EPOLL_CTL_ADD, client_ssl_socket.get_socket().lowest_layer().native(), &event );
+            event.data.fd = client_ssl_socket.get_socket().lowest_layer().native();
+        }
+        if (is_epoll_edge_trigger) {
+            event.events = EPOLLET | EPOLLIN | EPOLLHUP;
+        } else {
+            event.events = EPOLLIN | EPOLLHUP;
+        }
+        bool add_flag = false;
+        if (!up_client_epollfd_registered) {
+            if (epoll_ctl( up_client_epollfd, EPOLL_CTL_ADD, event.data.fd, &event ) < 0) {
+                std::stringstream buf;
+                buf << "up_thread_client_receive : epoll_ctl EPOLL_CTL_ADD error : ";
+                buf << strerror(errno);
+                Logger::putLogWarn( LOG_CAT_L7VSD_SESSION, 999, buf.str(), __FILE__, __LINE__ );
+                boost::this_thread::yield();
+                return;
+            }
+            up_client_epollfd_registered = true;
+            add_flag = true;
         }
 
         while( true ){
@@ -1504,6 +1553,56 @@ namespace l7vs{
                 }
             }
 
+            if (is_epoll_edge_trigger && (!add_flag)) {
+                if (epoll_ctl( up_client_epollfd, EPOLL_CTL_MOD, event.data.fd, &event ) < 0) {
+                    std::stringstream buf;
+                    buf << "up_thread_client_receive : epoll_ctl EPOLL_CTL_MOD error : ";
+                    buf << strerror(errno);
+                    Logger::putLogWarn( LOG_CAT_L7VSD_SESSION, 999, buf.str(), __FILE__, __LINE__ );
+                    boost::this_thread::yield();
+                    return;
+                }
+            }
+
+            int ret_fds = epoll_wait( up_client_epollfd, up_client_events, EVENT_NUM, epoll_timeout );
+            if (ret_fds <= 0) {
+                if (ret_fds == 0) {
+                    std::stringstream buf;
+                    buf << "up_thread_client_receive : epoll_wait timeout ";
+                    buf << epoll_timeout;
+                    buf << "mS";
+                    Logger::putLogInfo( LOG_CAT_L7VSD_SESSION, 999, buf.str(), __FILE__, __LINE__ );
+                } else {
+                    std::stringstream buf;
+                    buf << "up_thread_client_receive : epoll_wait error : ";
+                    buf << strerror(errno);
+                    Logger::putLogWarn( LOG_CAT_L7VSD_SESSION, 999, buf.str(), __FILE__, __LINE__ );
+                }
+                boost::this_thread::yield();
+                return;
+            }
+
+            bool event_check = false;
+            for (int i = 0; i < ret_fds; ++i) {
+                if (up_client_events[i].data.fd == event.data.fd) {
+                    if (up_client_events[i].events & EPOLLIN) {
+                        event_check = true;
+                        break;
+                    } else if (up_client_events[i].events & EPOLLHUP) {
+                        std::stringstream buf;
+                        buf << "up_thread_client_receive : epoll_wait EPOLLHUP";
+                        Logger::putLogError( LOG_CAT_L7VSD_SESSION, 999, buf.str(), __FILE__, __LINE__ );
+                        down_thread_exit(process_type);
+                        return;
+                    }
+                }
+            }
+            if (!event_check) {
+                Logger::putLogWarn( LOG_CAT_L7VSD_SESSION, 999, "up_thread_client_receive : epoll_wait unknown fd", __FILE__, __LINE__ );
+                boost::this_thread::yield();
+                return;
+            }
+
             if (!ssl_flag) {
                 recv_size = client_socket.read_some(boost::asio::buffer(data_buff,MAX_BUFFER_SIZE), ec);
             } else {
@@ -1562,22 +1661,6 @@ namespace l7vs{
                     break;
                 }
             }
-
-            event.events = EPOLLET | EPOLLIN | EPOLLHUP;
-            if ( !ssl_flag ) {
-                epoll_ctl( up_client_epollfd, EPOLL_CTL_MOD, client_socket.get_socket().native(), &event );
-            } else {
-                epoll_ctl( up_client_epollfd, EPOLL_CTL_MOD, client_ssl_socket.get_socket().lowest_layer().native(), &event );
-            }
-
-            epoll_wait( up_client_epollfd, up_client_events, EVENT_NUM, EPOLL_TIMEOUT );
-        }
-
-        event.events = EPOLLET | EPOLLIN | EPOLLHUP;
-        if ( !ssl_flag ) {
-            epoll_ctl( up_client_epollfd, EPOLL_CTL_DEL, client_socket.get_socket().native(), &event );
-        } else {
-            epoll_ctl( up_client_epollfd, EPOLL_CTL_DEL, client_ssl_socket.get_socket().lowest_layer().native(), &event );
         }
 
         up_thread_function_pair    func    = up_thread_function_array[func_tag];
@@ -1748,8 +1831,25 @@ namespace l7vs{
         UP_THREAD_FUNC_TYPE_TAG func_tag;
 
         struct epoll_event event;
-        event.events = EPOLLET | EPOLLOUT | EPOLLHUP;
-        epoll_ctl( up_realserver_epollfd, EPOLL_CTL_ADD, send_socket->second->get_socket().native(), &event );
+        event.data.fd = send_socket->second->get_socket().native();
+        if (is_epoll_edge_trigger) {
+            event.events = EPOLLET | EPOLLOUT | EPOLLHUP;
+        } else {
+            event.events = EPOLLOUT | EPOLLHUP;
+        }
+        bool add_flag = false;
+        if (!up_realserver_epollfd_registered) {
+            if (epoll_ctl( up_realserver_epollfd, EPOLL_CTL_ADD, event.data.fd, &event ) < 0) {
+                std::stringstream buf;
+                buf << "up_thread_realserver_send : epoll_ctl EPOLL_CTL_ADD error : ";
+                buf << strerror(errno);
+                Logger::putLogWarn( LOG_CAT_L7VSD_SESSION, 999, buf.str(), __FILE__, __LINE__ );
+                boost::this_thread::yield();
+                return;
+            }
+            up_realserver_epollfd_registered = true;
+            add_flag = true;
+        }
 
         while( true ){
             {
@@ -1760,6 +1860,56 @@ namespace l7vs{
                 }
             }
 
+            if (is_epoll_edge_trigger && (!add_flag)) {
+                if (epoll_ctl( up_realserver_epollfd, EPOLL_CTL_MOD, event.data.fd, &event ) < 0) {
+                    std::stringstream buf;
+                    buf << "up_thread_realserver_send : epoll_ctl EPOLL_CTL_MOD error : ";
+                    buf << strerror(errno);
+                    Logger::putLogWarn( LOG_CAT_L7VSD_SESSION, 999, buf.str(), __FILE__, __LINE__ );
+                    boost::this_thread::yield();
+                    return;
+                }
+            }
+
+            int ret_fds = epoll_wait( up_realserver_epollfd, up_realserver_events, EVENT_NUM, epoll_timeout );
+            if (ret_fds <= 0) {
+                if (ret_fds == 0) {
+                    std::stringstream buf;
+                    buf << "up_thread_realserver_send : epoll_wait timeout ";
+                    buf << epoll_timeout;
+                    buf << "mS";
+                    Logger::putLogInfo( LOG_CAT_L7VSD_SESSION, 999, buf.str(), __FILE__, __LINE__ );
+                } else {
+                    std::stringstream buf;
+                    buf << "up_thread_realserver_send : epoll_wait error : ";
+                    buf << strerror(errno);
+                    Logger::putLogWarn( LOG_CAT_L7VSD_SESSION, 999, buf.str(), __FILE__, __LINE__ );
+                }
+                boost::this_thread::yield();
+                return;
+            }
+
+            bool event_check = false;
+            for (int i = 0; i < ret_fds; ++i) {
+                if (up_realserver_events[i].data.fd == event.data.fd) {
+                    if (up_realserver_events[i].events & EPOLLOUT) {
+                        event_check = true;
+                        break;
+                    } else if (up_realserver_events[i].events & EPOLLHUP) {
+                        std::stringstream buf;
+                        buf << "up_thread_realserver_send : epoll_wait EPOLLHUP";
+                        Logger::putLogError( LOG_CAT_L7VSD_SESSION, 999, buf.str(), __FILE__, __LINE__ );
+                        down_thread_exit(process_type);
+                        return;
+                    }
+                }
+            }
+            if (!event_check) {
+                Logger::putLogWarn( LOG_CAT_L7VSD_SESSION, 999, "up_thread_realserver_send : epoll_wait unknown fd", __FILE__, __LINE__ );
+                boost::this_thread::yield();
+                return;
+            }
+
             send_size = send_socket->second->write_some(boost::asio::buffer(data_buff.data()+send_data_size,data_size-send_data_size),ec);
 
             if( !ec ){
@@ -1808,16 +1958,8 @@ namespace l7vs{
                     break;
                 }
             }
-
-            event.events = EPOLLET | EPOLLOUT | EPOLLHUP;
-            epoll_ctl( up_realserver_epollfd, EPOLL_CTL_MOD, send_socket->second->get_socket().native(), &event );
-
-            epoll_wait( up_realserver_epollfd, up_realserver_events, EVENT_NUM, EPOLL_TIMEOUT );
         }
 
-        event.events = EPOLLET | EPOLLOUT | EPOLLHUP;
-        epoll_ctl( up_realserver_epollfd, EPOLL_CTL_DEL, send_socket->second->get_socket().native(), &event );
-
         up_thread_function_pair    func    = up_thread_function_array[func_tag];
         if( unlikely( !func.second ) ){
             //Error not find function map
@@ -2216,8 +2358,25 @@ namespace l7vs{
         UP_THREAD_FUNC_TYPE_TAG func_tag;
 
         struct epoll_event event;
-        event.events = EPOLLET | EPOLLOUT | EPOLLHUP;
-        epoll_ctl( up_realserver_epollfd, EPOLL_CTL_ADD, sorryserver_socket.second->get_socket().native(), &event );
+        event.data.fd = sorryserver_socket.second->get_socket().native();
+        if (is_epoll_edge_trigger) {
+            event.events = EPOLLET | EPOLLOUT | EPOLLHUP;
+        } else {
+            event.events = EPOLLOUT | EPOLLHUP;
+        }
+        bool add_flag = false;
+        if (!up_sorryserver_epollfd_registered) {
+            if (epoll_ctl( up_sorryserver_epollfd, EPOLL_CTL_ADD, event.data.fd, &event ) < 0) {
+                std::stringstream buf;
+                buf << "up_thread_sorryserver_send : epoll_ctl EPOLL_CTL_ADD error : ";
+                buf << strerror(errno);
+                Logger::putLogWarn( LOG_CAT_L7VSD_SESSION, 999, buf.str(), __FILE__, __LINE__ );
+                boost::this_thread::yield();
+                return;
+            }
+            up_sorryserver_epollfd_registered = true;
+            add_flag = true;
+        }
 
         while( true ){
             {
@@ -2228,6 +2387,56 @@ namespace l7vs{
                 }
             }
 
+            if (is_epoll_edge_trigger && (!add_flag)) {
+                if (epoll_ctl( up_sorryserver_epollfd, EPOLL_CTL_MOD, event.data.fd, &event ) < 0) {
+                    std::stringstream buf;
+                    buf << "up_thread_sorryserver_send : epoll_ctl EPOLL_CTL_MOD error : ";
+                    buf << strerror(errno);
+                    Logger::putLogWarn( LOG_CAT_L7VSD_SESSION, 999, buf.str(), __FILE__, __LINE__ );
+                    boost::this_thread::yield();
+                    return;
+                }
+            }
+
+            int ret_fds = epoll_wait( up_sorryserver_epollfd, up_sorryserver_events, EVENT_NUM, epoll_timeout );
+            if (ret_fds <= 0) {
+                if (ret_fds == 0) {
+                    std::stringstream buf;
+                    buf << "up_thread_sorryserver_send : epoll_wait timeout ";
+                    buf << epoll_timeout;
+                    buf << "mS";
+                    Logger::putLogInfo( LOG_CAT_L7VSD_SESSION, 999, buf.str(), __FILE__, __LINE__ );
+                } else {
+                    std::stringstream buf;
+                    buf << "up_thread_sorryserver_send : epoll_wait error : ";
+                    buf << strerror(errno);
+                    Logger::putLogWarn( LOG_CAT_L7VSD_SESSION, 999, buf.str(), __FILE__, __LINE__ );
+                }
+                boost::this_thread::yield();
+                return;
+            }
+
+            bool event_check = false;
+            for (int i = 0; i < ret_fds; ++i) {
+                if (up_sorryserver_events[i].data.fd == event.data.fd) {
+                    if (up_sorryserver_events[i].events & EPOLLOUT) {
+                        event_check = true;
+                        break;
+                    } else if (up_sorryserver_events[i].events & EPOLLHUP) {
+                        std::stringstream buf;
+                        buf << "up_thread_sorryserver_send : epoll_wait EPOLLHUP";
+                        Logger::putLogError( LOG_CAT_L7VSD_SESSION, 999, buf.str(), __FILE__, __LINE__ );
+                        down_thread_exit(process_type);
+                        return;
+                    }
+                }
+            }
+            if (!event_check) {
+                Logger::putLogWarn( LOG_CAT_L7VSD_SESSION, 999, "up_thread_sorryserver_send : epoll_wait unknown fd", __FILE__, __LINE__ );
+                boost::this_thread::yield();
+                return;
+            }
+
             send_size = sorryserver_socket.second->write_some(boost::asio::buffer(data_buff.data()+send_data_size,data_size-send_data_size),ec);
 
             if(!ec){
@@ -2275,16 +2484,8 @@ namespace l7vs{
                     break;
                 }
             }
-
-            event.events = EPOLLET | EPOLLOUT | EPOLLHUP;
-            epoll_ctl( up_realserver_epollfd, EPOLL_CTL_MOD, sorryserver_socket.second->get_socket().native(), &event );
-
-            epoll_wait( up_realserver_epollfd, up_realserver_events, EVENT_NUM, EPOLL_TIMEOUT );
         }
 
-        event.events = EPOLLET | EPOLLOUT | EPOLLHUP;
-        epoll_ctl( up_realserver_epollfd, EPOLL_CTL_DEL, sorryserver_socket.second->get_socket().native(), &event );
-
         up_thread_function_pair    func    = up_thread_function_array[func_tag];
         if(unlikely( !func.second )){
             //Error not find function map
@@ -2701,8 +2902,25 @@ namespace l7vs{
         DOWN_THREAD_FUNC_TYPE_TAG func_tag;
 
         struct epoll_event event;
-        event.events = EPOLLET | EPOLLIN | EPOLLHUP;
-        epoll_ctl( down_realserver_epollfd, EPOLL_CTL_ADD, down_thread_current_receive_realserver_socket->second->get_socket().native(), &event );
+        event.data.fd = down_thread_current_receive_realserver_socket->second->get_socket().native();
+        if (is_epoll_edge_trigger) {
+            event.events = EPOLLET | EPOLLIN | EPOLLHUP;
+        } else {
+            event.events = EPOLLIN | EPOLLHUP;
+        }
+        bool add_flag = false;
+        if (!down_realserver_epollfd_registered) {
+            if (epoll_ctl( down_realserver_epollfd, EPOLL_CTL_ADD, event.data.fd, &event ) < 0) {
+                std::stringstream buf;
+                buf << "doun_thread_realserver_receive : epoll_ctl EPOLL_CTL_ADD error : ";
+                buf << strerror(errno);
+                Logger::putLogWarn( LOG_CAT_L7VSD_SESSION, 999, buf.str(), __FILE__, __LINE__ );
+                boost::this_thread::yield();
+                return;
+            }
+            down_realserver_epollfd_registered = true;
+            add_flag = true;
+        }
 
         while( true ){
             {
@@ -2713,6 +2931,56 @@ namespace l7vs{
                 }
             }
 
+            if (is_epoll_edge_trigger && (!add_flag)) {
+                if (epoll_ctl( down_realserver_epollfd, EPOLL_CTL_MOD, event.data.fd, &event ) < 0) {
+                    std::stringstream buf;
+                    buf << "down_thread_realserver_receive : epoll_ctl EPOLL_CTL_MOD error : ";
+                    buf << strerror(errno);
+                    Logger::putLogWarn( LOG_CAT_L7VSD_SESSION, 999, buf.str(), __FILE__, __LINE__ );
+                    boost::this_thread::yield();
+                    return;
+                }
+            }
+
+            int ret_fds = epoll_wait( down_realserver_epollfd, down_realserver_events, EVENT_NUM, epoll_timeout );
+            if (ret_fds <= 0) {
+                if (ret_fds == 0) {
+                    std::stringstream buf;
+                    buf << "down_thread_realserver_receive : epoll_wait timeout ";
+                    buf << epoll_timeout;
+                    buf << "mS";
+                    Logger::putLogInfo( LOG_CAT_L7VSD_SESSION, 999, buf.str(), __FILE__, __LINE__ );
+                } else {
+                    std::stringstream buf;
+                    buf << "down_thread_realserver_receive : epoll_wait error : ";
+                    buf << strerror(errno);
+                    Logger::putLogWarn( LOG_CAT_L7VSD_SESSION, 999, buf.str(), __FILE__, __LINE__ );
+                }
+                boost::this_thread::yield();
+                return;
+            }
+
+            bool event_check = false;
+            for (int i = 0; i < ret_fds; ++i) {
+                if (down_realserver_events[i].data.fd == event.data.fd) {
+                    if (down_realserver_events[i].events & EPOLLIN) {
+                        event_check = true;
+                        break;
+                    } else if (down_realserver_events[i].events & EPOLLHUP) {
+                        std::stringstream buf;
+                        buf << "down_thread_realserver_receive : epoll_wait EPOLLHUP";
+                        Logger::putLogError( LOG_CAT_L7VSD_SESSION, 999, buf.str(), __FILE__, __LINE__ );
+                        down_thread_exit(process_type);
+                        return;
+                    }
+                }
+            }
+            if (!event_check) {
+                Logger::putLogWarn( LOG_CAT_L7VSD_SESSION, 999, "down_thread_realserver_receive : epoll_wait unknown fd", __FILE__, __LINE__ );
+                boost::this_thread::yield();
+                return;
+            }
+
             recv_size = down_thread_current_receive_realserver_socket->second->read_some(boost::asio::buffer(data_buff,MAX_BUFFER_SIZE),ec);
 
             boost::asio::ip::tcp::endpoint server_endpoint = down_thread_current_receive_realserver_socket->first;
@@ -2763,16 +3031,8 @@ namespace l7vs{
                     break;
                 }
             }
-
-            event.events = EPOLLET | EPOLLIN | EPOLLHUP;
-            epoll_ctl( down_realserver_epollfd, EPOLL_CTL_MOD, down_thread_current_receive_realserver_socket->second->get_socket().native(), &event );
-
-            epoll_wait( down_realserver_epollfd, down_realserver_events, EVENT_NUM, EPOLL_TIMEOUT );
         }
 
-        event.events = EPOLLET | EPOLLIN | EPOLLHUP;
-        epoll_ctl( down_realserver_epollfd, EPOLL_CTL_DEL, down_thread_current_receive_realserver_socket->second->get_socket().native(), &event );
-
         down_thread_function_pair    func    = down_thread_function_array[func_tag];
         if(unlikely( !func.second ) ){
             //Error not find function map
@@ -3032,11 +3292,28 @@ namespace l7vs{
         DOWN_THREAD_FUNC_TYPE_TAG func_tag;
 
         struct epoll_event event;
-        event.events = EPOLLET | EPOLLOUT | EPOLLHUP;
         if ( !ssl_flag ) {
-            epoll_ctl( down_client_epollfd, EPOLL_CTL_ADD, client_socket.get_socket().native(), &event );
+            event.data.fd = client_socket.get_socket().native();
+        } else {
+            event.data.fd = client_ssl_socket.get_socket().lowest_layer().native();
+        }
+        if (is_epoll_edge_trigger) {
+            event.events = EPOLLET | EPOLLOUT | EPOLLHUP;
         } else {
-            epoll_ctl( down_client_epollfd, EPOLL_CTL_ADD, client_ssl_socket.get_socket().lowest_layer().native(), &event );
+            event.events = EPOLLOUT | EPOLLHUP;
+        }
+        bool add_flag = false;
+        if (!down_client_epollfd_registered) {
+            if (epoll_ctl( down_client_epollfd, EPOLL_CTL_ADD, event.data.fd, &event ) < 0) {
+                std::stringstream buf;
+                buf << "down_thread_client_send : epoll_ctl EPOLL_CTL_ADD error : ";
+                buf << strerror(errno);
+                Logger::putLogWarn( LOG_CAT_L7VSD_SESSION, 999, buf.str(), __FILE__, __LINE__ );
+                boost::this_thread::yield();
+                return;
+            }
+            down_client_epollfd_registered = true;
+            add_flag = true;
         }
 
         while( true ){
@@ -3048,6 +3325,56 @@ namespace l7vs{
                 }
             }
 
+            if (is_epoll_edge_trigger && (!add_flag)) {
+                if (epoll_ctl( down_client_epollfd, EPOLL_CTL_MOD, event.data.fd, &event ) < 0) {
+                    std::stringstream buf;
+                    buf << "down_thread_client_send : epoll_ctl EPOLL_CTL_MOD error : ";
+                    buf << strerror(errno);
+                    Logger::putLogWarn( LOG_CAT_L7VSD_SESSION, 999, buf.str(), __FILE__, __LINE__ );
+                    boost::this_thread::yield();
+                    return;
+                }
+            }
+
+            int ret_fds = epoll_wait( down_client_epollfd, down_client_events, EVENT_NUM, epoll_timeout );
+            if (ret_fds <= 0) {
+                if (ret_fds == 0) {
+                    std::stringstream buf;
+                    buf << "down_thread_client_send : epoll_wait timeout ";
+                    buf << epoll_timeout;
+                    buf << "mS";
+                    Logger::putLogInfo( LOG_CAT_L7VSD_SESSION, 999, buf.str(), __FILE__, __LINE__ );
+                } else {
+                    std::stringstream buf;
+                    buf << "down_thread_client_send : epoll_wait error : ";
+                    buf << strerror(errno);
+                    Logger::putLogWarn( LOG_CAT_L7VSD_SESSION, 999, buf.str(), __FILE__, __LINE__ );
+                }
+                boost::this_thread::yield();
+                return;
+            }
+
+            bool event_check = false;
+            for (int i = 0; i < ret_fds; ++i) {
+                if (down_client_events[i].data.fd == event.data.fd) {
+                    if (down_client_events[i].events & EPOLLOUT) {
+                        event_check = true;
+                        break;
+                    } else if (down_client_events[i].events & EPOLLHUP) {
+                        std::stringstream buf;
+                        buf << "down_thread_client_send : epoll_wait EPOLLHUP";
+                        Logger::putLogError( LOG_CAT_L7VSD_SESSION, 999, buf.str(), __FILE__, __LINE__ );
+                        down_thread_exit(process_type);
+                        return;
+                    }
+                }
+            }
+            if (!event_check) {
+                Logger::putLogWarn( LOG_CAT_L7VSD_SESSION, 999, "down_thread_client_send : epoll_wait unknown fd", __FILE__, __LINE__ );
+                boost::this_thread::yield();
+                return;
+            }
+
             if (!ssl_flag) {
                 send_size = client_socket.write_some(boost::asio::buffer(data_buff.data()+send_data_size,data_size-send_data_size),ec);
             } else {
@@ -3107,22 +3434,6 @@ namespace l7vs{
                     break;
                 }
             }
-
-            event.events = EPOLLET | EPOLLOUT | EPOLLHUP;
-            if ( !ssl_flag ) {
-                epoll_ctl( down_client_epollfd, EPOLL_CTL_MOD, client_socket.get_socket().native(), &event );
-            } else {
-                epoll_ctl( down_client_epollfd, EPOLL_CTL_MOD, client_ssl_socket.get_socket().lowest_layer().native(), &event );
-            }
-
-            epoll_wait( down_client_epollfd, up_client_events, EVENT_NUM, EPOLL_TIMEOUT );
-        }
-
-        event.events = EPOLLET | EPOLLOUT | EPOLLHUP;
-        if ( !ssl_flag ) {
-            epoll_ctl( down_client_epollfd, EPOLL_CTL_DEL, client_socket.get_socket().native(), &event );
-        } else {
-            epoll_ctl( down_client_epollfd, EPOLL_CTL_DEL, client_ssl_socket.get_socket().lowest_layer().native(), &event );
         }
 
         down_thread_function_pair    func    = down_thread_function_array[func_tag];
@@ -3221,53 +3532,138 @@ namespace l7vs{
         down_thread_data_dest_side.initialize();
         boost::array<char,MAX_BUFFER_SIZE>& data_buff = down_thread_data_dest_side.get_data();
         boost::system::error_code ec;
-        size_t recv_size = sorryserver_socket.second->read_some(boost::asio::buffer(data_buff,MAX_BUFFER_SIZE),ec);
-        boost::asio::ip::tcp::endpoint sorry_endpoint = sorryserver_socket.first;
-        down_thread_data_dest_side.set_endpoint(sorry_endpoint);
+        size_t recv_size;
         DOWN_THREAD_FUNC_TYPE_TAG func_tag;
-        if(!ec){
-            if(recv_size > 0){
-                //----Debug log----------------------------------------------------------------------
-                if (unlikely(LOG_LV_DEBUG == Logger::getLogLevel(LOG_CAT_L7VSD_SESSION))){
-                    std::stringstream buf;
-                    buf << "Thread ID[";
-                    buf << boost::this_thread::get_id();
-                    buf << "] down_thread_sorryserver_receive";
-                    buf << " receive data size[";
-                    buf << recv_size;
-                    buf << "] from [";
-                    buf << sorry_endpoint;
-                    buf << "]";
-                    Logger::putLogDebug( LOG_CAT_L7VSD_SESSION, 46, buf.str(), __FILE__, __LINE__ );
+
+        struct epoll_event event;
+        bool add_flag = false;
+        if (!down_sorryserver_epollfd_registered) {
+            event.data.fd = sorryserver_socket.second->get_socket().native();
+            if (is_epoll_edge_trigger) {
+                event.events = EPOLLET | EPOLLIN | EPOLLHUP;
+            } else {
+                event.events = EPOLLIN | EPOLLHUP;
+            }
+            if (epoll_ctl( down_sorryserver_epollfd, EPOLL_CTL_ADD, event.data.fd, &event ) < 0) {
+                std::stringstream buf;
+                buf << "doun_thread_sorryserver_receive : epoll_ctl EPOLL_CTL_ADD error : ";
+                buf << strerror(errno);
+                Logger::putLogWarn( LOG_CAT_L7VSD_SESSION, 999, buf.str(), __FILE__, __LINE__ );
+                boost::this_thread::yield();
+                return;
+            }
+            down_sorryserver_epollfd_registered = true;
+            add_flag = true;
+        }
+
+        while( true ){
+            {
+                rd_scoped_lock scoped_lock(exit_flag_update_mutex);
+                if(unlikely(exit_flag)){
+                    func_tag = DOWN_FUNC_EXIT;
+                    break;
                 }
-                //----Debug log----------------------------------------------------------------------
-                down_thread_data_dest_side.set_size(recv_size);
-                protocol_module_base::EVENT_TAG module_event = protocol_module->handle_sorryserver_recv(down_thread_id,sorry_endpoint,data_buff,recv_size);
-                std::map< protocol_module_base::EVENT_TAG ,DOWN_THREAD_FUNC_TYPE_TAG >::iterator func_type = down_thread_module_event_map.find(module_event);
-                if(unlikely( func_type == down_thread_module_event_map.end() )){
-                    //Error unknown protocol_module_base::EVENT_TAG return
+            }
+
+            if (is_epoll_edge_trigger && (!add_flag)) {
+                if (epoll_ctl( down_sorryserver_epollfd, EPOLL_CTL_MOD, event.data.fd, &event ) < 0) {
                     std::stringstream buf;
-                    buf << "Thread ID[";
-                    buf << boost::this_thread::get_id();
-                    buf << "] protocol_module returnd illegal EVENT_TAG : ";
-                    buf << module_event;    
-                    Logger::putLogError( LOG_CAT_L7VSD_SESSION, 88, buf.str(), __FILE__, __LINE__ );
-                    down_thread_exit(process_type);
+                    buf << "down_thread_sorryserver_receive : epoll_ctl EPOLL_CTL_MOD error : ";
+                    buf << strerror(errno);
+                    Logger::putLogWarn( LOG_CAT_L7VSD_SESSION, 999, buf.str(), __FILE__, __LINE__ );
+                    boost::this_thread::yield();
                     return;
                 }
-                func_tag = func_type->second;
-            }else{
-                func_tag = DOWN_FUNC_SORRYSERVER_RECEIVE;
+            }
+
+            int ret_fds = epoll_wait( down_sorryserver_epollfd, down_sorryserver_events, EVENT_NUM, epoll_timeout );
+            if (ret_fds <= 0) {
+                if (ret_fds == 0) {
+                    std::stringstream buf;
+                    buf << "down_thread_sorryserver_receive : epoll_wait timeout ";
+                    buf << epoll_timeout;
+                    buf << "mS";
+                    Logger::putLogInfo( LOG_CAT_L7VSD_SESSION, 999, buf.str(), __FILE__, __LINE__ );
+                } else {
+                    std::stringstream buf;
+                    buf << "down_thread_sorryserver_receive : epoll_wait error : ";
+                    buf << strerror(errno);
+                    Logger::putLogWarn( LOG_CAT_L7VSD_SESSION, 999, buf.str(), __FILE__, __LINE__ );
+                }
                 boost::this_thread::yield();
+                return;
             }
-        }else{
-            if(ec == boost::asio::error::try_again){
-                func_tag = DOWN_FUNC_SORRYSERVER_RECEIVE;
+
+            bool event_check = false;
+            for (int i = 0; i < ret_fds; ++i) {
+                if (down_sorryserver_events[i].data.fd == event.data.fd) {
+                    if (down_sorryserver_events[i].events & EPOLLIN) {
+                        event_check = true;
+                        break;
+                    } else if (down_sorryserver_events[i].events & EPOLLHUP) {
+                        std::stringstream buf;
+                        buf << "down_thread_sorryserver_receive : epoll_wait EPOLLHUP";
+                        Logger::putLogError( LOG_CAT_L7VSD_SESSION, 999, buf.str(), __FILE__, __LINE__ );
+                        down_thread_exit(process_type);
+                        return;
+                    }
+                }
+            }
+            if (!event_check) {
+                Logger::putLogWarn( LOG_CAT_L7VSD_SESSION, 999, "down_thread_sorryserver_receive : epoll_wait unknown fd", __FILE__, __LINE__ );
                 boost::this_thread::yield();
+                return;
+            }
+
+            recv_size = sorryserver_socket.second->read_some(boost::asio::buffer(data_buff,MAX_BUFFER_SIZE),ec);
+
+            boost::asio::ip::tcp::endpoint sorry_endpoint = sorryserver_socket.first;
+            down_thread_data_dest_side.set_endpoint(sorry_endpoint);
+            if(!ec){
+                if(recv_size > 0){
+                    //----Debug log----------------------------------------------------------------------
+                    if (unlikely(LOG_LV_DEBUG == Logger::getLogLevel(LOG_CAT_L7VSD_SESSION))){
+                        std::stringstream buf;
+                        buf << "Thread ID[";
+                        buf << boost::this_thread::get_id();
+                        buf << "] down_thread_sorryserver_receive";
+                        buf << " receive data size[";
+                        buf << recv_size;
+                        buf << "] from [";
+                        buf << sorry_endpoint;
+                        buf << "]";
+                        Logger::putLogDebug( LOG_CAT_L7VSD_SESSION, 46, buf.str(), __FILE__, __LINE__ );
+                    }
+                    //----Debug log----------------------------------------------------------------------
+                    down_thread_data_dest_side.set_size(recv_size);
+                    protocol_module_base::EVENT_TAG module_event = protocol_module->handle_sorryserver_recv(down_thread_id,sorry_endpoint,data_buff,recv_size);
+                    std::map< protocol_module_base::EVENT_TAG ,DOWN_THREAD_FUNC_TYPE_TAG >::iterator func_type = down_thread_module_event_map.find(module_event);
+                    if(unlikely( func_type == down_thread_module_event_map.end() )){
+                        //Error unknown protocol_module_base::EVENT_TAG return
+                        std::stringstream buf;
+                        buf << "Thread ID[";
+                        buf << boost::this_thread::get_id();
+                        buf << "] protocol_module returnd illegal EVENT_TAG : ";
+                        buf << module_event;    
+                        Logger::putLogError( LOG_CAT_L7VSD_SESSION, 88, buf.str(), __FILE__, __LINE__ );
+                        down_thread_exit(process_type);
+                        return;
+                    }
+                    func_tag = func_type->second;
+                }else{
+                    //func_tag = DOWN_FUNC_SORRYSERVER_RECEIVE;
+                    //boost::this_thread::yield();
+                }
             }else{
-                func_tag = DOWN_FUNC_SORRYSERVER_DISCONNECT;
+                if(ec == boost::asio::error::try_again){
+                    //func_tag = DOWN_FUNC_SORRYSERVER_RECEIVE;
+                    //boost::this_thread::yield();
+                }else{
+                    func_tag = DOWN_FUNC_SORRYSERVER_DISCONNECT;
+                }
             }
         }
+
         down_thread_function_pair    func    = down_thread_function_array[func_tag];
         if(unlikely( !func.second )){
             //Error not find function map