l7vsd/include/virtualservice.h

   1 /*!
   2  *    @file    virtualservice.h
   3  *    @brief    VirtualService classes
   4  *
   5  * L7VSD: Linux Virtual Server for Layer7 Load Balancing
   6  * Copyright (C) 2009  NTT COMWARE Corporation.
   7  *
   8  * This program is free software; you can redistribute it and/or
   9  * modify it under the terms of the GNU Lesser General Public
  10  * License as published by the Free Software Foundation; either
  11  * version 2.1 of the License, or (at your option) any later version.
  12  *
  13  * This program is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  16  * Lesser General Public License for more details.
  17  *
  18  * You should have received a copy of the GNU Lesser General Public
  19  * License along with this library; if not, write to the Free Software
  20  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
  21  * 02110-1301 USA
  22  *
  23  **********************************************************************/
  24
  25 #ifndef VIRTUALSERVICE_H
  26 #define VIRTUALSERVICE_H
  27
  28 #include <sched.h>
  29 #include <net/if.h>
  30 #include <unistd.h>
  31 #include <sched.h>
  32 #include <error.h>
  33 #include <string>
  34 #include <map>
  35 #include <vector>
  36 #include <boost/noncopyable.hpp>
  37 #include <boost/shared_ptr.hpp>
  38 #include <boost/asio.hpp>
  39 #include <boost/thread.hpp>
  40
  41 #include "error_code.h"
  42 #include "l7vsd.h"
  43 #include "realserver.h"
  44 #include "virtualservice_element.h"
  45 #include "tcp_session.h"
  46 #include "udp_session.h"
  47 #include "session_thread_control.h"
  48 #include "replication.h"
  49
  50 #include "protocol_module_base.h"
  51 #include "schedule_module_base.h"
  52
  53 #include "atomic.h"
  54 #include "lockfree_queue.h"
  55 #include "lockfree_hashmap.h"
  56
  57 #define PARAM_RS_SIDE_NIC_NAME     "nic_realserver_side"
  58 #define PARAM_POOLSIZE_KEY_NAME    "session_thread_pool_size"
  59 #define PARAM_BPS_CALC_INTERVAL    "throughput_calc_interval"
  60 #define PARAM_REP_INTERVAL         "interval"
  61
  62 #define PROTOMOD_NOTLOAD_ERROR_MSG "Protocol Module not loaded"
  63 #define SCHEDMOD_NOTLOAD_ERROR_MSG "Schedule Module not loaded"
  64 #define PROTOMOD_LOAD_ERROR_MSG    "Protocol Module load error"
  65 #define SCHEDMOD_LOAD_ERROR_MSG    "Schedule Module load error"
  66
  67 #define REP_BLOCK_SIZE_ERR_MSG     "Replication area block size error"
  68 #define REP_AREA_SIZE_ERR_MSG      "Replication area size error"
  69
  70 #define REP_AREA_NAME              "virtualservice"
  71
  72 //! SSL method default
  73 #define DEFAULT_SSL_METHOD         boost::asio::ssl::context::sslv23
  74                                                         //! SSLv23_method
  75 //! SSL context default
  76 #define DEFAULT_SSL_CA_DIR         "/etc/l7vs/sslproxy/"
  77 #define DEFAULT_SSL_CERT_CHAIN_DIR "/etc/l7vs/sslproxy/"
  78 #define DEFAULT_SSL_PRIVATE_KEY_DIR        "/etc/l7vs/sslproxy/"
  79 #define DEFAULT_SSL_PRIVATE_KEY_FILETYPE    boost::asio::ssl::context::pem        //! SSL_FILETYPE_PEM
  80 #define DEFAULT_SSL_PRIVATE_KEY_PASSWD_DIR "/etc/l7vs/sslproxy/"
  81 #define DEFAULT_SSL_VERIFY_OPTIONS         SSL_VERIFY_NONE
  82 #define DEFAULT_SSL_VERIFY_CERT_DEPTH      9
  83 #define DEFAULT_SSL_OPTIONS  (SSL_OP_ALL|SSL_OP_NO_SSLv2|SSL_OP_SINGLE_DH_USE)
  84 #define DEFAULT_SSL_TMP_DH_DIR     "/etc/l7vs/sslproxy/"
  85 #define DEFAULT_SSL_CIPHER_LIST    "ALL:!ADH:!LOW:!EXP:!MD5:@STRENGTH"
  86 #define MAX_SSL_PASSWD_SIZE        256
  87 //! SSL session cache default
  88 #define DEFAULT_SSL_SESSION_CACHE_MODE (SSL_SESS_CACHE_SERVER|SSL_SESS_CACHE_NO_AUTO_CLEAR)    //! "on"
  89 #define DEFAULT_SSL_SESSION_CACHE_SIZE SSL_SESSION_CACHE_MAX_SIZE_DEFAULT    //! 20480
  90 #define DEFAULT_SSL_SESSION_CACHE_TIMEOUT 300
  91 //! SSL handshake timeout default
  92 #define DEFAULT_SSL_HANDSHAKE_TIMEOUT     30
  93
  94 namespace l7vs{
  95
  96 class    l7vsd;
  97 class    replication;
  98 class    session_thread_control;
  99
 100 //! @class virtualservice_base
 101 //! @brief virtualservice base class is base class
 102 //!        of tcp virtual service and udp virtual service.
 103 class virtualservice_base : boost::noncopyable{
 104 public:
 105     //! shared_ptr session_thread_control typedef
 106     typedef boost::shared_ptr<boost::asio::deadline_timer>
 107                                     deadline_timer_ptr_type;
 108     //! tcp endpoint typedef
 109     typedef boost::asio::ip::tcp::endpoint
 110                                     tcp_endpoint_type;
 111     //! udp endpoint typedef
 112     typedef boost::asio::ip::udp::endpoint
 113                                     udp_endpoint_type;
 114
 115     typedef std::list<l7vs::realserver>
 116                                     rslist_type;
 117     typedef boost::function< rslist_type::iterator (void)>
 118                                     rslist_iterator_begin_func_type;
 119     typedef boost::function< rslist_type::iterator (void)>
 120                                     rslist_iterator_end_func_type;
 121     typedef boost::function< rslist_type::iterator (rslist_type::iterator)>
 122                                     rslist_iterator_next_func_type;
 123     typedef boost::function< void ( const boost::thread::id,
 124                                     rslist_iterator_begin_func_type,
 125                                     rslist_iterator_end_func_type,
 126                                     rslist_iterator_next_func_type,
 127                                     boost::asio::ip::tcp::endpoint& ) >
 128                                     tcp_schedule_func_type;
 129
 130     typedef l7vs::atomic<unsigned long long> AUUL;
 131         //typedef l7vs::atomic<unsigned int> AUI;
 132
 133
 134     //!    @struct    replication_header replication header structure
 135     struct    replication_header{
 136         unsigned int    data_num;
 137         replication_header() : data_num( 0 ) {}
 138     };
 139     //! @struct replication_data replication data structure
 140     struct replication_data{
 141         bool               udpmode;
 142         char               tcp_endpoint[48];
 143         char               udp_endpoint[48];
 144         long long          sorry_maxconnection;
 145         char               sorry_endpoint[48];
 146         bool               sorry_flag;
 147         unsigned long long qos_up;
 148         unsigned long long qos_down;
 149     };
 150
 151     //! Default count for number of session-pool size
 152     const static int  SESSION_POOL_NUM_DEFAULT = 256;
 153     //! bps calcurate interval(500ms)
 154     const static long BPS_INTERVAL_DEFAULT     = 500;
 155     //! replication-data create interval(500ms)
 156     const static long REP_INTERVAL_DEFAULT     = 500;
 157     //! Maximum count value of replication data array
 158     const static int  MAX_REPLICATION_DATA_NUM = 64;
 159     //! Operation timed out value
 160     const static int  OPERATION_TIMEOUT        = 1;
 161     //! wait interval for rs_ref_count check
 162     const static int  REFCOUNT_WAIT_INTERVAL   = 10000;
 163     const static int  SCHEDULER_PRIORITY       = 20;
 164 protected:
 165
 166     struct    parameter_data{
 167         std::string nic_realserver_side;
 168         int         session_pool_size;
 169         long        bps_interval;
 170         long        rep_interval;
 171         parameter_data() :  session_pool_size( SESSION_POOL_NUM_DEFAULT ),
 172                             bps_interval( BPS_INTERVAL_DEFAULT ),
 173                             rep_interval( REP_INTERVAL_DEFAULT ){}
 174     };
 175
 176     boost::thread::id       this_id;
 177
 178     const l7vsd&            vsd;                  //! l7vsd reference
 179     const replication&      rep;                  //! replication reference
 180
 181     Logger                  log;
 182
 183     boost::asio::io_service dispatcher;           //! dispatcer service
 184     deadline_timer_ptr_type calc_bps_timer;       //! timer object
 185     deadline_timer_ptr_type replication_timer;    //! timer object
 186     deadline_timer_ptr_type protomod_rep_timer;   //! timer object
 187     deadline_timer_ptr_type schedmod_rep_timer;   //! timer object
 188
 189     parameter_data          param_data;    //! virtual service parameter data
 190     virtualservice_element  element;       //! virtual service element
 191     boost::mutex            element_mutex; //! mutex for virtual service element
 192
 193     protocol_module_base*   protomod;      //! protocol module smart pointer
 194     schedule_module_base*   schedmod;      //! schedule module smart pointer
 195
 196     std::list<realserver>   rs_list;              //! realserver list
 197     AUUL                    rs_list_ref_count;    //! reference count of
 198                                                   //! realserver list
 199     wr_mutex                rs_list_ref_count_inc_mutex; //! mutex for
 200                                                   //! increase reference count
 201
 202     AUUL recvsize_up;           //! upstream total receive data size
 203     AUUL current_up_recvsize;   //! current upstream receive data size
 204                                 //!  for calcurate upstream throughput
 205     AUUL sendsize_up;           //! upstream total send data size
 206     AUUL recvsize_down;         //! downstream total receive data size
 207     AUUL current_down_recvsize; //! current downstream receive data size for
 208                                 //! calcurate upstream throughput
 209     AUUL sendsize_down;         //! downstream total send data size
 210
 211     AUUL throughput_up;         //! upstream throughput value
 212     AUUL throughput_down;       //! downstream throughput value
 213
 214     AUUL wait_count_up;         //! upstream recv wait count
 215     AUUL wait_count_down;       //! downstream recv wait count
 216
 217     AUUL interrupt_running_flag;    //! interrunpt routine is running
 218     AUUL virtualservice_stop_flag;  //! virtualservice is now shutting down
 219
 220     // protocol module option string
 221     std::string             protocol_module_for_indication_options;
 222
 223     void         load_parameter( l7vs::error_code& );
 224
 225     virtual void handle_replication_interrupt(
 226                      const boost::system::error_code& ) = 0;
 227     virtual void read_replicationdata() = 0;
 228
 229     void         handle_protomod_replication(const boost::system::error_code&);
 230     void         handle_schedmod_replication(const boost::system::error_code&);
 231
 232     void         handle_throughput_update( const boost::system::error_code& );
 233
 234     std::list<realserver>::iterator rs_list_begin(){ return rs_list.begin(); }
 235     std::list<realserver>::iterator rs_list_end(){ return rs_list.end(); }
 236     std::list<realserver>::iterator rs_list_next(
 237                                       std::list<realserver>::iterator in_itr ){
 238         return ++in_itr;
 239     }
 240     void* replication_pay_memory( const std::string& inid,
 241                                   unsigned int* outsize ){
 242         l7vs::replication& tmp_rep = const_cast<l7vs::replication&>( rep );
 243         return tmp_rep.pay_memory( inid, *outsize );
 244     }
 245     void  replication_area_lock( const std::string& inid ){
 246         l7vs::replication&    tmp_rep = const_cast<l7vs::replication&>( rep );
 247         tmp_rep.lock( inid );
 248     }
 249     void  replication_area_unlock( const std::string& inid ){
 250         l7vs::replication& tmp_rep = const_cast<l7vs::replication&>( rep );
 251         tmp_rep.unlock( inid );
 252     }
 253
 254     void  schedule_rs( boost::thread::id thread_id,
 255                        rslist_iterator_begin_func_type in_begin,
 256                        rslist_iterator_end_func_type in_end,
 257                        rslist_iterator_next_func_type in_next,
 258                        boost::asio::ip::tcp::endpoint& in_ep ){
 259         schedmod->handle_schedule(thread_id, in_begin, in_end, in_next, in_ep);
 260     }
 261
 262     cpu_set_t                    vsnic_cpumask;
 263     cpu_set_t                    rsnic_cpumask;
 264     void                         get_nic_list( std::vector< std::string >& );
 265     cpu_set_t                    get_cpu_mask( boost::asio::ip::address& );
 266     cpu_set_t                    get_cpu_mask( std::string );
 267
 268 public:
 269     explicit virtualservice_base( const l7vsd&,
 270                                   const replication&,
 271                                   const virtualservice_element& );
 272     virtual ~virtualservice_base(){ dispatcher.reset();dispatcher.stop(); };
 273
 274     virtual void initialize( error_code& ) = 0;
 275     virtual void finalize( error_code& ) = 0;
 276
 277     virtual bool operator==( const virtualservice_base& ) = 0;
 278     virtual bool operator!=( const virtualservice_base& ) = 0;
 279
 280     void rs_list_lock(){
 281         rd_scoped_lock lock( rs_list_ref_count_inc_mutex );
 282         rs_list_ref_count++;
 283     }
 284     void rs_list_unlock() { rs_list_ref_count--; };
 285
 286     virtual void set_virtualservice( const virtualservice_element&,
 287                                      error_code& ) = 0;
 288     virtual void edit_virtualservice( const virtualservice_element&,
 289                                       error_code& ) = 0;
 290
 291     virtual void add_realserver(const virtualservice_element&, error_code&) =0;
 292     virtual void edit_realserver(const virtualservice_element&, error_code&)=0;
 293     virtual void del_realserver(const virtualservice_element&, error_code&) =0;
 294
 295     virtualservice_element& get_element();
 296
 297     virtual void run() = 0;
 298     virtual void stop() = 0;
 299
 300     virtual void connection_active( const tcp_endpoint_type& ) = 0;
 301     virtual void connection_inactive( const tcp_endpoint_type& ) = 0;
 302     virtual void release_session( const tcp_session* session_ptr ) = 0;
 303
 304     unsigned long long get_qos_upstream(){ return element.qos_upstream; }
 305     unsigned long long get_qos_downstream(){ return element.qos_downstream; }
 306     unsigned long long get_throughput_upstream(){ return throughput_up.get(); }
 307     unsigned long long get_throughput_downstream(){
 308                                        return throughput_down.get(); }
 309     unsigned long long get_up_recv_size(){ return recvsize_up.get(); }
 310     unsigned long long get_up_send_size(){ return sendsize_up.get(); }
 311     unsigned long long get_down_recv_size(){ return recvsize_down.get(); }
 312     unsigned long long get_down_send_size(){ return sendsize_down.get(); }
 313
 314     unsigned long long get_wait_upstream(){ return wait_count_up.get(); }
 315     unsigned long long get_wait_downstream(){ return wait_count_down.get(); }
 316
 317     void update_up_recv_size( unsigned long long datasize ){
 318         current_up_recvsize    += datasize;
 319         recvsize_up        += datasize;
 320     }
 321     void update_up_send_size( unsigned long long datasize ){
 322         sendsize_up += datasize;
 323     }
 324     void update_down_recv_size( unsigned long long datasize ){
 325         current_down_recvsize += datasize;
 326         recvsize_down += datasize;
 327     }
 328     void update_down_send_size( unsigned long long datasize ){
 329         sendsize_down += datasize;
 330     }
 331
 332     protocol_module_base* get_protocol_module(){ return protomod; }
 333     schedule_module_base* get_schedule_module(){ return schedmod; }
 334
 335 };
 336
 337 //!
 338 //! @brief virtualservice class for TCP
 339 //! @class virtualservice_tcp is class of virtual service for TCP transfer.
 340 class virtualservice_tcp : public virtualservice_base{
 341 public:
 342     typedef lockfree_queue< session_thread_control >
 343                                 session_queue_type;
 344     typedef lockfree_hashmap< tcp_session, session_thread_control >
 345                                 session_map_type;
 346     typedef std::map< std::string, std::string >
 347                                 accesslog_argument_map_type;
 348 protected:
 349     boost::asio::ip::tcp::acceptor
 350                                 acceptor_;
 351
 352     session_queue_type          pool_sessions;
 353     session_map_type            waiting_sessions;
 354     session_map_type            active_sessions;
 355     l7vs::atomic<unsigned long long>
 356                                 active_count;
 357
 358     bool                        defer_accept_opt;
 359                                        //! is set option TCP_DEFER_ACCEPT
 360     int                         defer_accept_val;
 361                                        //! TCP_DEFER_ACCEPT option value
 362     tcp_socket_option_info      set_sock_opt;
 363                                        //! socket option for tcp_session class
 364
 365     // SSL flag
 366     bool                        ssl_virtualservice_mode_flag;
 367     std::string                 ssl_file_name;
 368     // SSL context
 369     boost::asio::ssl::context   sslcontext;
 370     // SSL context parameter
 371     std::string                 ca_dir;
 372     std::string                 ca_file;
 373     std::string                 cert_chain_dir;
 374     std::string                 cert_chain_file;
 375     std::string                 private_key_dir;
 376     std::string                 private_key_file;
 377     boost::asio::ssl::context::file_format
 378                                 private_key_filetype;
 379     std::string                 private_key_passwd_dir;
 380     std::string                 private_key_passwd_file;
 381     int                         verify_options;
 382     int                         verify_cert_depth;
 383     long int                    ssl_options;
 384     bool                        is_tmp_dh_use;
 385     std::string                 tmp_dh_dir;
 386     std::string                 tmp_dh_file;
 387     std::string                 cipher_list;
 388     // SSL session cache parameter
 389     bool                        is_session_cache_use;
 390     long                        session_cache_mode;
 391     long                        session_cache_size;
 392     long                        session_cache_timeout;
 393     // SSL handshake timer parameter
 394     int                         handshake_timeout;
 395     std::string                 access_log_file_name;
 396     accesslog_argument_map_type access_log_rotate_arguments;
 397     bool                        access_log_flag;
 398     // SSL functions
 399     std::string get_ssl_password();
 400     int         conv_verify_option(std::string);
 401     long int    conv_ssl_option(std::string);
 402     bool        get_ssl_parameter();
 403     bool        set_ssl_config();
 404
 405     void        handle_replication_interrupt(const boost::system::error_code&);
 406     void        read_replicationdata();
 407
 408     void        handle_accept( const session_thread_control*,
 409                                const boost::system::error_code& );
 410
 411 public:
 412     virtualservice_tcp( const l7vsd&,
 413                         const replication&,
 414                         const virtualservice_element& );
 415     ~virtualservice_tcp();
 416
 417     void initialize( error_code& );
 418     void finalize( error_code& );
 419
 420     bool operator==( const virtualservice_base& );
 421     bool operator!=( const virtualservice_base& );
 422
 423     void set_virtualservice( const virtualservice_element&, error_code& );
 424     void edit_virtualservice( const virtualservice_element&, error_code& );
 425
 426     void add_realserver( const virtualservice_element&, error_code& );
 427     void edit_realserver( const virtualservice_element&, error_code& );
 428     void del_realserver( const virtualservice_element&, error_code& );
 429
 430     void run();
 431     void stop();
 432
 433     void connection_active( const tcp_endpoint_type& );
 434     void connection_inactive( const tcp_endpoint_type& );
 435     void release_session( const tcp_session* session_ptr );
 436
 437     void set_socket_option();
 438
 439     // SSL functions
 440     void flush_ssl_session();
 441     // SSL functions (for debug)
 442     void get_ssl_config(std::stringstream&);
 443     void get_ssl_session_cache_info(std::stringstream&);
 444
 445 };
 446
 447 //!
 448 //! @brief virtualservice class for UDP
 449 //! @class virtualservice_udp is class of virtual service for UDP transfer.
 450 class virtualservice_udp : public virtualservice_base{
 451 protected:
 452     boost::shared_ptr<udp_session>
 453                                 session;
 454
 455     void handle_replication_interrupt( const boost::system::error_code& );
 456     void read_replicationdata();
 457
 458 public:
 459     virtualservice_udp( const l7vsd&,
 460                         const replication&,
 461                         const virtualservice_element& );
 462     ~virtualservice_udp();
 463
 464     void initialize( error_code& );
 465     void finalize( error_code& );
 466
 467     bool operator==( const virtualservice_base& );
 468     bool operator!=( const virtualservice_base& );
 469
 470     void set_virtualservice( const virtualservice_element&, error_code& );
 471     void edit_virtualservice( const virtualservice_element&, error_code& );
 472
 473     void add_realserver( const virtualservice_element&, error_code& );
 474     void edit_realserver( const virtualservice_element&, error_code& );
 475     void del_realserver( const virtualservice_element&, error_code& );
 476
 477     void run();
 478     void stop();
 479
 480     void connection_active( const boost::asio::ip::tcp::endpoint& ){}
 481     void connection_inactive( const boost::asio::ip::tcp::endpoint& ){}
 482     void release_session( const tcp_session* session_ptr );
 483 };
 484
 485 //!
 486 //! @brief virtualservice wrapping class
 487 //! @class virtual_service
 488 class    virtual_service{
 489 protected:
 490     boost::shared_ptr<virtualservice_base>    vs;
 491 public:
 492     virtual_service( const l7vsd& ,
 493                      const replication& ,
 494                      const virtualservice_element& );
 495     ~virtual_service();
 496
 497     void initialize( error_code& );
 498     void finalize( error_code& );
 499
 500     bool operator==( const virtualservice_base& );
 501     bool operator!=( const virtualservice_base& );
 502
 503     void set_virtualservice( const virtualservice_element& , error_code& );
 504     void edit_virtualservice( const virtualservice_element& , error_code& );
 505
 506     void add_realserver( const virtualservice_element& , error_code& );
 507     void edit_realserver( const virtualservice_element& , error_code& );
 508     void del_realserver( const virtualservice_element& , error_code& );
 509
 510     virtualservice_element& get_element();
 511
 512     void run();
 513     void stop();
 514
 515     void connection_active( const boost::asio::ip::tcp::endpoint&  );
 516     void connection_inactive( const boost::asio::ip::tcp::endpoint&  );
 517     void release_session( const tcp_session* session_ptr );
 518
 519     unsigned long long get_qos_upstream();
 520     unsigned long long get_qos_downstream();
 521     unsigned long long get_throughput_upstream();
 522     unsigned long long get_throughput_downstream();
 523     unsigned long long get_up_recv_size();
 524     unsigned long long get_up_send_size();
 525     unsigned long long get_down_recv_size();
 526     unsigned long long get_down_send_size();
 527
 528     void update_up_recv_size( unsigned long long );
 529     void update_up_send_size( unsigned long long );
 530     void update_down_recv_size( unsigned long long );
 531     void update_down_send_size( unsigned long long );
 532
 533     protocol_module_base* get_protocol_module();
 534     schedule_module_base* get_schedule_module();
 535 };
 536
 537 } //namespace l7vs
 538
 539 #endif //VIRTUALSERVICE_H
 540