l7vsd/include/virtualservice.h

   1 /*!
   2  *      @file   virtualservice.h
   3  *      @brief  VirtualService classes
   4  *
   5  * L7VSD: Linux Virtual Server for Layer7 Load Balancing
   6  * Copyright (C) 2009  NTT COMWARE Corporation.
   7  *
   8  * This program is free software; you can redistribute it and/or
   9  * modify it under the terms of the GNU Lesser General Public
  10  * License as published by the Free Software Foundation; either
  11  * version 2.1 of the License, or (at your option) any later version.
  12  *
  13  * This program is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  16  * Lesser General Public License for more details.
  17  *
  18  * You should have received a copy of the GNU Lesser General Public
  19  * License along with this library; if not, write to the Free Software
  20  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
  21  * 02110-1301 USA
  22  *
  23  **********************************************************************/
  24
  25 #ifndef VIRTUALSERVICE_H
  26 #define VIRTUALSERVICE_H
  27
  28 #include <sched.h>
  29 #include <net/if.h>
  30 #include <unistd.h>
  31 #include <sched.h>
  32 #include <error.h>
  33 #include <string>
  34 #include <map>
  35 #include <vector>
  36 #include <boost/noncopyable.hpp>
  37 #include <boost/shared_ptr.hpp>
  38 #include <boost/asio.hpp>
  39 #include <boost/thread.hpp>
  40
  41 #include "error_code.h"
  42 #include "l7vsd.h"
  43 #include "realserver.h"
  44 #include "virtualservice_element.h"
  45 #include "tcp_session.h"
  46 #include "udp_session.h"
  47 #include "session_thread_control.h"
  48 #include "replication.h"
  49
  50 #include "protocol_module_base.h"
  51 #include "schedule_module_base.h"
  52
  53 #include "atomic.h"
  54 #include "lockfree_queue.h"
  55 #include "lockfree_hashmap.h"
  56
  57 #define PARAM_RS_SIDE_NIC_NAME          "nic_realserver_side"
  58 #define PARAM_POOLSIZE_KEY_NAME         "session_thread_pool_size"
  59 #define PARAM_BPS_CALC_INTERVAL         "throughput_calc_interval"
  60 #define PARAM_REP_INTERVAL              "interval"
  61
  62 #define PROTOMOD_NOTLOAD_ERROR_MSG      "Protocol Module not loaded"
  63 #define SCHEDMOD_NOTLOAD_ERROR_MSG      "Schedule Module not loaded"
  64 #define PROTOMOD_LOAD_ERROR_MSG         "Protocol Module load error"
  65 #define SCHEDMOD_LOAD_ERROR_MSG         "Schedule Module load error"
  66
  67 #define REP_BLOCK_SIZE_ERR_MSG          "Replication area block size error"
  68 #define REP_AREA_SIZE_ERR_MSG           "Replication area size error"
  69
  70 #define REP_AREA_NAME                   "virtualservice"
  71
  72 //! SSL method default
  73 #define DEFAULT_SSL_METHOD              boost::asio::ssl::context::sslv23       //! SSLv23_method
  74 //! SSL context default
  75 #define DEFAULT_CA_DIR                  "/etc/l7vs/sslproxy/"
  76 #define DEFAULT_CERT_CHAIN_DIR          "/etc/l7vs/sslproxy/"
  77 #define DEFAULT_PRIVATE_KEY_DIR         "/etc/l7vs/sslproxy/"
  78 #define DEFAULT_PRIVATE_KEY_FILETYPE    boost::asio::ssl::context::pem          //! SSL_FILETYPE_PEM
  79 #define DEFAULT_PRIVATE_KEY_PASSWD_DIR  "/etc/l7vs/sslproxy/"
  80 #define DEFAULT_VERIFY_OPTIONS          (SSL_VERIFY_PEER | SSL_VERIFY_FAIL_IF_NO_PEER_CERT)
  81 #define DEFAULT_VERIFY_CERT_DEPTH       9
  82 #define DEFAULT_SSL_OPTIONS             (SSL_OP_ALL | SSL_OP_NO_SSLv2 | SSL_OP_SINGLE_DH_USE)
  83 #define DEFAULT_TMP_DH_DIR              "/etc/l7vs/sslproxy/"
  84 #define DEFAULT_CIPHER_LIST             "ALL:!ADH:!LOW:!EXP:!MD5:@STRENGTH"
  85 #define MAX_PASSWD_SIZE                 256
  86 //! SSL session cache default
  87 #define DEFAULT_SESSION_CACHE_MODE      (SSL_SESS_CACHE_SERVER | SSL_SESS_CACHE_NO_AUTO_CLEAR)  //! "on"
  88 #define DEFAULT_SESSION_CACHE_SIZE      SSL_SESSION_CACHE_MAX_SIZE_DEFAULT      //! 20480
  89 #define DEFAULT_SESSION_CACHE_TIMEOUT   300
  90 //! SSL handshake timeout default
  91 #define DEFAULT_HANDSHAKE_TIMEOUT       30
  92
  93 namespace l7vs{
  94
  95 class   l7vsd;
  96 class   replication;
  97 class   session_thread_control;
  98
  99 //!     @class  virtualservice_base
 100 //!     @brief  virtualservice base class is base class of tcp virtual service and udp virtual service.
 101 class   virtualservice_base : boost::noncopyable{
 102 public:
 103         //! shared_ptr session_thread_control typedef
 104         typedef boost::shared_ptr<boost::asio::deadline_timer>  deadline_timer_ptr_type;
 105         //! tcp endpoint typedef
 106         typedef boost::asio::ip::tcp::endpoint                                  tcp_endpoint_type;
 107         //! udp endpoint typedef
 108         typedef boost::asio::ip::udp::endpoint                                  udp_endpoint_type;
 109
 110         typedef std::list<l7vs::realserver>
 111                                                                         rslist_type;
 112         typedef boost::function< rslist_type::iterator (void)>
 113                                                                         rslist_iterator_begin_func_type;
 114         typedef boost::function< rslist_type::iterator (void)>
 115                                                                         rslist_iterator_end_func_type;
 116         typedef boost::function< rslist_type::iterator (rslist_type::iterator)>
 117                                                                         rslist_iterator_next_func_type;
 118         typedef boost::function< void (         const boost::thread::id,
 119                                                                                 rslist_iterator_begin_func_type,
 120                                                                                 rslist_iterator_end_func_type,
 121                                                                                 rslist_iterator_next_func_type,
 122                                                                                 boost::asio::ip::tcp::endpoint& ) >
 123                                                                 tcp_schedule_func_type;
 124
 125         typedef l7vs::atomic<unsigned long long>        AUUL;
 126
 127         //!     @struct replication_header replication header structure
 128         struct  replication_header{
 129                 unsigned int    data_num;
 130                 replication_header() : data_num( 0 ) {}
 131         };
 132         //!     @struct replication_data replication data structure
 133         struct  replication_data{
 134                 bool                            udpmode;
 135                 char                            tcp_endpoint[48];
 136                 char                            udp_endpoint[48];
 137                 long long                       sorry_maxconnection;
 138                 char                            sorry_endpoint[48];
 139                 bool                            sorry_flag;
 140                 unsigned long long      qos_up;
 141                 unsigned long long      qos_down;
 142         };
 143
 144         const static int        SESSION_POOL_NUM_DEFAULT        = 256;          //! Default count for number of session-pool size
 145         const static long       BPS_INTERVAL_DEFAULT            = 500;          //! bps calcurate interval(500ms)
 146         const static long       REP_INTERVAL_DEFAULT            = 500;          //! replication-data create interval(500ms)
 147         const static int        MAX_REPLICATION_DATA_NUM        = 64;           //! Maximum count value of replication data array
 148         const static int        OPERATION_TIMEOUT                       = 1;            //! Operation timed out value
 149         const static int        REFCOUNT_WAIT_INTERVAL          = 10000;        //! wait interval for rs_ref_count check
 150         const static int        SCHEDULER_PRIORITY              = 20;
 151 protected:
 152
 153         struct  parameter_data{
 154                 std::string
 155                                 nic_realserver_side;
 156                 int             session_pool_size;
 157                 long    bps_interval;
 158                 long    rep_interval;
 159                 parameter_data() :      session_pool_size( SESSION_POOL_NUM_DEFAULT ),
 160                                                         bps_interval( BPS_INTERVAL_DEFAULT ),
 161                                                         rep_interval( REP_INTERVAL_DEFAULT ){}
 162         };
 163
 164         boost::thread::id                       this_id;
 165
 166         const   l7vsd&                          vsd;                    //! l7vsd reference
 167         const   replication&            rep;                    //! replication reference
 168
 169         Logger                                          log;
 170
 171         boost::asio::io_service         dispatcher;             //! dispatcer service
 172         deadline_timer_ptr_type         calc_bps_timer; //! timer object
 173         deadline_timer_ptr_type         replication_timer;      //! timer object
 174         deadline_timer_ptr_type         protomod_rep_timer;     //! timer object
 175         deadline_timer_ptr_type         schedmod_rep_timer;     //! timer object
 176
 177         parameter_data                          param_data;             //! virtual service parameter data
 178         virtualservice_element          element;                //! virtual service element
 179         boost::mutex                            element_mutex;  //! mutex for virtual service element
 180
 181         protocol_module_base*           protomod;                       //! protocol module smart pointer
 182         schedule_module_base*           schedmod;                       //! schedule module smart pointer
 183
 184         std::list<realserver>           rs_list;                                                //! realserver list
 185         AUUL                                            rs_list_ref_count;                              //! reference count of realserver list
 186         wr_mutex                                        rs_list_ref_count_inc_mutex;    //! mutex for increase reference count
 187
 188         AUUL                                            recvsize_up;                                    //! upstream total receive data size
 189         AUUL                                            current_up_recvsize;                    //! current upstream receive data size for calcurate upstream throughput
 190         AUUL                                            sendsize_up;                                    //! upstream total send data size
 191         AUUL                                            recvsize_down;                                  //! downstream total receive data size
 192         AUUL                                            current_down_recvsize;                  //! current downstream receive data size for calcurate upstream throughput
 193         AUUL                                            sendsize_down;                                  //! downstream total send data size
 194
 195         AUUL                                            throughput_up;                                  //! upstream throughput value
 196         AUUL                                            throughput_down;                                //! downstream throughput value
 197
 198         AUUL                                            wait_count_up;                                  //! upstream recv wait count
 199         AUUL                                            wait_count_down;                                //! downstream recv wait count
 200
 201         void                                            load_parameter( l7vs::error_code& );
 202
 203         virtual void                            handle_replication_interrupt( const boost::system::error_code& ) = 0;
 204         virtual void                            read_replicationdata() = 0;
 205
 206         void                                            handle_protomod_replication( const boost::system::error_code& );
 207         void                                            handle_schedmod_replication( const boost::system::error_code& );
 208
 209         void                                            handle_throughput_update( const boost::system::error_code& );
 210
 211         std::list<realserver>::iterator
 212                                                                 rs_list_begin(){
 213                 return rs_list.begin();
 214         }
 215         std::list<realserver>::iterator
 216                                                                 rs_list_end(){
 217                 return rs_list.end();
 218         }
 219         std::list<realserver>::iterator
 220                                                                 rs_list_next( std::list<realserver>::iterator in_itr ){
 221                 return ++in_itr;
 222         }
 223         void*                                           replication_pay_memory( const std::string& inid, unsigned int* outsize ){
 224                 l7vs::replication&      tmp_rep = const_cast<l7vs::replication&>( rep );
 225                 return tmp_rep.pay_memory( inid, *outsize );
 226         }
 227         void                                            replication_area_lock( const std::string& inid ){
 228                 l7vs::replication&      tmp_rep = const_cast<l7vs::replication&>( rep );
 229                 tmp_rep.lock( inid );
 230         }
 231         void                                            replication_area_unlock( const std::string& inid ){
 232                 l7vs::replication&      tmp_rep = const_cast<l7vs::replication&>( rep );
 233                 tmp_rep.unlock( inid );
 234         }
 235
 236         void                                            schedule_rs(
 237                                                                                 boost::thread::id thread_id,
 238                                                                                 rslist_iterator_begin_func_type in_begin,
 239                                                                                 rslist_iterator_end_func_type in_end,
 240                                                                                 rslist_iterator_next_func_type in_next,
 241                                                                                 boost::asio::ip::tcp::endpoint& in_ep ){
 242                 schedmod->handle_schedule( thread_id, in_begin, in_end, in_next, in_ep );
 243         }
 244
 245         cpu_set_t                                       vsnic_cpumask;
 246         cpu_set_t                                       rsnic_cpumask;
 247         void                                            get_nic_list( std::vector< std::string >& );
 248         cpu_set_t                                       get_cpu_mask( boost::asio::ip::address& );
 249         cpu_set_t                                       get_cpu_mask( std::string );
 250
 251 public:
 252         explicit virtualservice_base(   const l7vsd&,
 253                                                         const replication&,
 254                                                         const virtualservice_element& );
 255         virtual ~virtualservice_base(){};
 256
 257         virtual void                            initialize( error_code& ) = 0;
 258         virtual void                            finalize( error_code& ) = 0;
 259
 260         virtual bool                            operator==( const virtualservice_base& ) = 0;
 261         virtual bool                            operator!=( const virtualservice_base& ) = 0;
 262
 263         void                                            rs_list_lock(){
 264                                                                         rd_scoped_lock  lock( rs_list_ref_count_inc_mutex );
 265                                                                         rs_list_ref_count++;
 266                                                                 }
 267         void                                            rs_list_unlock() { rs_list_ref_count--; };
 268
 269         virtual void                            set_virtualservice( const virtualservice_element&, error_code& ) = 0;
 270         virtual void                            edit_virtualservice( const virtualservice_element&, error_code& ) = 0;
 271
 272         virtual void                            add_realserver( const virtualservice_element&, error_code& ) = 0;
 273         virtual void                            edit_realserver( const virtualservice_element&, error_code& ) = 0;
 274         virtual void                            del_realserver( const virtualservice_element&, error_code& ) = 0;
 275
 276         virtualservice_element&         get_element();
 277
 278         virtual void                            run() = 0;
 279         virtual void                            stop() = 0;
 280
 281         virtual void                            connection_active( const tcp_endpoint_type& ) = 0;
 282         virtual void                            connection_inactive( const tcp_endpoint_type& ) = 0;
 283         virtual void                            release_session( const tcp_session* session_ptr ) = 0;
 284
 285         unsigned long long                      get_qos_upstream(){ return element.qos_upstream; }
 286         unsigned long long                      get_qos_downstream(){ return element.qos_downstream; }
 287         unsigned long long                      get_throughput_upstream(){ return throughput_up.get(); }
 288         unsigned long long                      get_throughput_downstream(){ return throughput_down.get(); }
 289         unsigned long long                      get_up_recv_size(){ return recvsize_up.get(); }
 290         unsigned long long                      get_up_send_size(){ return sendsize_up.get(); }
 291         unsigned long long                      get_down_recv_size(){ return recvsize_down.get(); }
 292         unsigned long long                      get_down_send_size(){ return sendsize_down.get(); }
 293
 294         unsigned long long                      get_wait_upstream(){ return wait_count_up.get(); }
 295         unsigned long long                      get_wait_downstream(){ return wait_count_down.get(); }
 296
 297         void                                            update_up_recv_size( unsigned long long datasize ){
 298                 current_up_recvsize     += datasize;
 299                 recvsize_up             += datasize;
 300         }
 301         void                                            update_up_send_size( unsigned long long datasize ){
 302                 sendsize_up += datasize;
 303         }
 304         void                                            update_down_recv_size( unsigned long long datasize ){
 305                 current_down_recvsize += datasize;
 306                 recvsize_down += datasize;
 307         }
 308         void                                            update_down_send_size( unsigned long long datasize ){
 309                 sendsize_down += datasize;
 310         }
 311
 312         protocol_module_base*           get_protocol_module(){ return protomod; }
 313         schedule_module_base*           get_schedule_module(){ return schedmod; }
 314
 315 };
 316
 317 //!
 318 //!     @brief  virtualservice class for TCP
 319 //! @class      virtualservice_tcp is class of virtual service for TCP transfer.
 320 class   virtualservice_tcp : public virtualservice_base{
 321 public:
 322         typedef lockfree_queue< session_thread_control >
 323                                                                 session_queue_type;
 324         typedef lockfree_hashmap< tcp_session, session_thread_control >
 325                                                                 session_map_type;
 326 protected:
 327         boost::asio::ip::tcp::acceptor
 328                                                                 acceptor_;
 329
 330         session_queue_type                      pool_sessions;
 331         session_map_type                        waiting_sessions;
 332         session_map_type                        active_sessions;
 333         l7vs::atomic<unsigned long long>        sorry_count;
 334
 335         bool                                            defer_accept_opt;                               //! is set option TCP_DEFER_ACCEPT
 336         int                                             defer_accept_val;                               //! TCP_DEFER_ACCEPT option value
 337         tcp_socket_option_info                          set_sock_opt;           //! socket option for tcp_session class
 338
 339         // SSL flag
 340         bool                                            ssl_vs_flag;
 341         std::string                                     ssl_conf_filename;
 342         // SSL context
 343         boost::asio::ssl::context                       sslcontext;
 344         // SSL context parameter
 345         std::string                                     ca_dir;
 346         std::string                                     ca_file;
 347         std::string                                     cert_chain_dir;
 348         std::string                                     cert_chain_file;
 349         std::string                                     private_key_dir;
 350         std::string                                     private_key_file;
 351         boost::asio::ssl::context::file_format          private_key_filetype;
 352         std::string                                     private_key_passwd_dir;
 353         std::string                                     private_key_passwd_file;
 354         int                                             verify_options;
 355         int                                             verify_cert_depth;
 356         long int                                        ssl_options;
 357         bool                                            is_tmp_dh_use;
 358         std::string                                     tmp_dh_dir;
 359         std::string                                     tmp_dh_file;
 360         std::string                                     cipher_list;
 361         // SSL session cache parameter
 362         bool                                            is_session_cache_use;
 363         long                                            session_cache_mode;
 364         long                                            session_cache_size;
 365         long                                            session_cache_timeout;
 366         // SSL handshake timer parameter
 367         int                                             handshake_timeout;
 368         // SSL functions
 369         std::string                                     get_ssl_password();
 370         int                                             conv_verify_option(std::string);
 371         long int                                        conv_ssl_option(std::string);
 372         bool                                            get_ssl_parameter();
 373         bool                                            set_ssl_config();
 374
 375         void                                            handle_replication_interrupt( const boost::system::error_code& );
 376         void                                            read_replicationdata();
 377
 378         void                                            handle_accept( const session_thread_control*, const boost::system::error_code& );
 379
 380 public:
 381         virtualservice_tcp(             const l7vsd&,
 382                                                         const replication&,
 383                                                         const virtualservice_element& );
 384         ~virtualservice_tcp();
 385
 386         void                                            initialize( error_code& );
 387         void                                            finalize( error_code& );
 388
 389         bool                                            operator==( const virtualservice_base& );
 390         bool                                            operator!=( const virtualservice_base& );
 391
 392         void                                            set_virtualservice( const virtualservice_element&, error_code& );
 393         void                                            edit_virtualservice( const virtualservice_element&, error_code& );
 394
 395         void                                            add_realserver( const virtualservice_element&, error_code& );
 396         void                                            edit_realserver( const virtualservice_element&, error_code& );
 397         void                                            del_realserver( const virtualservice_element&, error_code& );
 398
 399         void                                            run();
 400         void                                            stop();
 401
 402         void                                            connection_active( const tcp_endpoint_type& );
 403         void                                            connection_inactive( const tcp_endpoint_type& );
 404         void                                            release_session( const tcp_session* session_ptr );
 405
 406         protocol_module_base::check_message_result parse_socket_option(std::vector<std::string>& args);
 407
 408         // SSL functions
 409         void                                            flush_ssl_session();
 410         // SSL functions (for debug)
 411         void                                            get_ssl_config(std::stringstream&);
 412         void                                            get_ssl_session_cache_info(std::stringstream&);
 413
 414
 415 ////For external SSL session cache
 416 //      //! SSL session cache table
 417 //      std::map<std::string, SSL_SESSION>              sessioncacheTable;
 418 //      boost::mutex                                    sessioncacheTable_mutex;
 419 //      //! SSL session cache callback functions
 420 //      int                                             new_session_cb(SSL *ssl, SSL_SESSION *session);
 421 //      SSL_SESSION*                                    get_session_cb(SSL *ssl, unsigned char *session_id, int session_id_len, int *ref);
 422 //      void                                            remove_session_cb(SSL_CTX *ssl_ctx, SSL_SESSION *session);
 423
 424
 425 };
 426
 427 //!
 428 //!     @brief  virtualservice class for UDP
 429 //! @class      virtualservice_udp is class of virtual service for UDP transfer.
 430 class   virtualservice_udp : public virtualservice_base{
 431 protected:
 432         boost::shared_ptr<udp_session>
 433                                                                 session;
 434
 435         void                                            handle_replication_interrupt( const boost::system::error_code& );
 436         void                                            read_replicationdata();
 437
 438 public:
 439         virtualservice_udp(             const l7vsd&,
 440                                                         const replication&,
 441                                                         const virtualservice_element& );
 442         ~virtualservice_udp();
 443
 444
 445         void                                            initialize( error_code& );
 446         void                                            finalize( error_code& );
 447
 448         bool                                            operator==( const virtualservice_base& );
 449         bool                                            operator!=( const virtualservice_base& );
 450
 451         void                                            set_virtualservice( const virtualservice_element&, error_code& );
 452         void                                            edit_virtualservice( const virtualservice_element&, error_code& );
 453
 454         void                                            add_realserver( const virtualservice_element&, error_code& );
 455         void                                            edit_realserver( const virtualservice_element&, error_code& );
 456         void                                            del_realserver( const virtualservice_element&, error_code& );
 457
 458         void                                            run();
 459         void                                            stop();
 460
 461         void                                            connection_active( const boost::asio::ip::tcp::endpoint& ){}
 462         void                                            connection_inactive( const boost::asio::ip::tcp::endpoint& ){}
 463         void                                            release_session( const tcp_session* session_ptr );
 464 };
 465
 466 //!
 467 //!     @brief  virtualservice wrapping class
 468 //! @class      virtual_service
 469 class   virtual_service{
 470 protected:
 471         boost::shared_ptr<virtualservice_base>  vs;
 472 public:
 473         virtual_service(        const l7vsd& ,
 474                                                 const replication& ,
 475                                                 const virtualservice_element& );
 476         ~virtual_service();
 477
 478         void                                            initialize( error_code& );
 479         void                                            finalize( error_code& );
 480
 481         bool                                            operator==( const virtualservice_base& );
 482         bool                                            operator!=( const virtualservice_base& );
 483
 484         void                                            set_virtualservice( const virtualservice_element& , error_code& );
 485         void                                            edit_virtualservice( const virtualservice_element& , error_code& );
 486
 487         void                                            add_realserver( const virtualservice_element& , error_code& );
 488         void                                            edit_realserver( const virtualservice_element& , error_code& );
 489         void                                            del_realserver( const virtualservice_element& , error_code& );
 490
 491         virtualservice_element&         get_element();
 492
 493         void                                            run();
 494         void                                            stop();
 495
 496         void                                            connection_active( const boost::asio::ip::tcp::endpoint&  );
 497         void                                            connection_inactive( const boost::asio::ip::tcp::endpoint&  );
 498         void                                            release_session( const tcp_session* session_ptr );
 499
 500         unsigned long long                      get_qos_upstream();
 501         unsigned long long                      get_qos_downstream();
 502         unsigned long long                      get_throughput_upstream();
 503         unsigned long long                      get_throughput_downstream();
 504         unsigned long long                      get_up_recv_size();
 505         unsigned long long                      get_up_send_size();
 506         unsigned long long                      get_down_recv_size();
 507         unsigned long long                      get_down_send_size();
 508
 509         void                                            update_up_recv_size( unsigned long long );
 510         void                                            update_up_send_size( unsigned long long );
 511         void                                            update_down_recv_size( unsigned long long );
 512         void                                            update_down_send_size( unsigned long long );
 513
 514         protocol_module_base*           get_protocol_module();
 515         schedule_module_base*           get_schedule_module();
 516 };
 517
 518 }       //namespace l7vs
 519
 520 #endif//VIRTUALSERVICE_H
 521