src/net/nt_http.c

   1 /* Copyright 2013 Akira Ohta (akohta001@gmail.com)
   2     This file is part of ntch.
   3
   4     The ntch is free software: you can redistribute it and/or modify
   5     it under the terms of the GNU General Public License as published by
   6     the Free Software Foundation, either version 3 of the License, or
   7     (at your option) any later version.
   8
   9     The ntch is distributed in the hope that it will be useful,
  10     but WITHOUT ANY WARRANTY; without even the implied warranty of
  11     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12     GNU General Public License for more details.
  13
  14     You should have received a copy of the GNU General Public License
  15     along with ntch.  If not, see <http://www.gnu.org/licenses/>.
  16
  17 */
  18 #include <sys/types.h>
  19 #include <sys/socket.h>
  20 #include <sys/stat.h>
  21 #include <openssl/sha.h>
  22 #include <fcntl.h>
  23 #include <arpa/inet.h>
  24 #include <stdio.h>
  25 #include <string.h>
  26 #include <stdlib.h>
  27 #include <unistd.h>
  28 #include <assert.h>
  29
  30 #include "env.h"
  31 #include "utils/nt_std_t.h"
  32 #include "utils/text.h"
  33 #include "utils/zip.h"
  34 #include "utils/db.h"
  35 #include "utils/nt_mutex.h"
  36 #include "net/nt_socket.h"
  37 #include "net/nt_http.h"
  38 #include "net/nt_cookie.h"
  39
  40
  41 static nt_http_header_tp nt_http_init_header(const char *url);
  42 static void nt_http_free_header(nt_http_header_tp headerp);
  43 static BOOL nt_http_init_header2(
  44                 nt_http_header_tp headerp, const char *out_path);
  45
  46 BOOL nt_http_post(const char *url, const char *post_data,
  47                 char *out_buf, size_t out_buf_len,
  48                 const char *referer, const char *user_agent,
  49                 nt_link_tp extend_headers,
  50                 nt_cookie_tp cookiep)
  51 {
  52         int result_code;
  53         result_code = nt_http_post2(url, post_data,
  54                 out_buf, out_buf_len, referer, user_agent,
  55                 extend_headers, cookiep);
  56         return (result_code == 200) ? TRUE : FALSE;
  57 }
  58
  59 int nt_http_post2(const char *url, const char *post_data,
  60                 char *out_buf, size_t out_buf_len,
  61                 const char *referer, const char *user_agent,
  62                 nt_link_tp extend_headers,
  63                 nt_cookie_tp cookiep)
  64 {
  65         nt_socket_tp socketp = NULL;
  66         int fd = -1, sockfd = 0;
  67         char data[1024*8];
  68         char ssl_connect[512];
  69         size_t data_len;
  70         int nread;
  71         int result;
  72         nt_http_response_header_tp responsep;
  73         nt_key_value_tp kv;
  74         nt_link_tp linkp, cookies;
  75         char itoa_buf[33];
  76         char cookie_data[1024];
  77         char *cptr, *p1, *p2, *p3;
  78
  79         result = 0;
  80         nt_http_header_tp headerp = nt_http_init_header(url);
  81         if(headerp == NULL)
  82                 return result;
  83
  84         strcpy(data, "POST ");
  85         strcat(data, headerp->param);
  86         strcat(data, " HTTP/1.1\r\nHost: ");
  87         strcat(data, headerp->host);
  88         strcat(data, "\r\nAccept: */*");
  89         strcat(data, "\r\nAccept-Language: ja");
  90         strcat(data, "\r\nUser-Agent: ");
  91         if(user_agent)
  92                 strcat(data, user_agent);
  93         else
  94                 strcat(data, USER_AGENT);
  95         if(referer){
  96                 strcat(data, "\r\nReferer: ");
  97                 strcat(data, referer);
  98         }
  99
 100         if(cookiep){
 101                 cookies = nt_get_cookies(cookiep, headerp->host);
 102                 if(cookies){
 103                         cookie_data[0] = '\0';
 104                         linkp = cookies;
 105                         do{
 106                                 if(cookie_data[0] != '\0')
 107                                         strcat(cookie_data, "; ");
 108                                 strcat(cookie_data, (char*)linkp->data);
 109                                  linkp = linkp->next;
 110                         }while(linkp != cookies);
 111                         if(strlen(cookie_data) > 0){
 112                                 strcat(data, "\r\nCookie: ");
 113                                 strcat(data, cookie_data);
 114                         }
 115                         nt_all_link_free(cookies, NULL);
 116                 }
 117         }
 118
 119         if(extend_headers){
 120                 linkp = extend_headers;
 121                 do{
 122                         kv = (nt_key_value_tp)linkp->data;
 123                         strcat(data, "\r\n");
 124                         strcat(data, kv->key);
 125                         strcat(data,": ");
 126                         strcat(data, kv->value);
 127                         linkp = linkp->next;
 128                 }while(linkp != extend_headers);
 129         }
 130         strcat(data, "\r\nContent-Type: application/x-www-form-urlencoded");
 131         strcat(data, "\r\nContent-Length: ");
 132         sprintf(itoa_buf, "%u", (unsigned int)strlen(post_data));
 133         strcat(data, itoa_buf);
 134         strcat(data, "\r\nConnection: close\r\n\r\n");
 135
 136         strcat(data, post_data);
 137
 138
 139         if(IS_SET_FLAG(headerp,SSL_FLAG)){
 140                 data_len = strlen(data);
 141                 sprintf(ssl_connect, "%s:443", headerp->host);
 142                 if(!nt_ssl_connect(ssl_connect, CA_FILE_PATH, FALSE,
 143                                         data, &data_len, sizeof(data))){
 144                         nt_http_free_header(headerp);
 145                         return result;
 146                 }
 147                 responsep = nt_http_alloc_response_header();
 148                 if(responsep == NULL){
 149                         nt_http_free_header(headerp);
 150                         return result;
 151                 }
 152                 nread = nt_http_parse_response_header2(data, data_len,
 153                         responsep);
 154                 if(nread <= 0){
 155                         nt_http_free_response_header(responsep);
 156                         nt_http_free_header(headerp);
 157                         return result;
 158                 }
 159                 if(out_buf_len <= data_len - nread + 1){
 160                         nt_http_free_response_header(responsep);
 161                         nt_http_free_header(headerp);
 162                         return result;
 163                 }
 164                 memcpy(out_buf, data+nread, data_len-nread);
 165                 out_buf[data_len - nread] = '\0';
 166         }else{
 167                 if(headerp->port == -1)
 168                         headerp->port = 80;
 169
 170                 socketp = nt_socket_init(headerp->port, headerp->host);
 171                 if(socketp == NULL){
 172                         nt_http_free_header(headerp);
 173                         return result;
 174                 }
 175
 176                 sockfd = nt_socket_connect(socketp, data, strlen(data));
 177                 if(-1 == sockfd){
 178                         nt_http_free_header(headerp);
 179                         nt_socket_free(socketp);
 180                         return result;
 181                 }
 182
 183                 responsep = nt_http_alloc_response_header();
 184                 if(responsep == NULL){
 185                         close(sockfd);
 186                         nt_http_free_header(headerp);
 187                         nt_socket_free(socketp);
 188                         return result;
 189                 }
 190
 191                 nread = nt_http_parse_response_header(sockfd,
 192                                         responsep, cookiep, headerp->host);
 193                 if(nread == -1){
 194                         close(sockfd);
 195                         nt_http_free_header(headerp);
 196                         nt_socket_free(socketp);
 197                         nt_http_free_response_header(responsep);
 198                         return result;
 199                 }
 200         }
 201         result = responsep->status_code;
 202         switch(result){
 203         case 200:
 204                 if(IS_SET_FLAG(headerp,SSL_FLAG)){
 205                 }else{
 206                         cptr = out_buf;
 207                         do{
 208                                 nread = read(sockfd, data, sizeof(data));
 209                                 switch(nread){
 210                                 case -1:
 211                                         break;
 212                                 case 0:
 213                                         break;
 214                                 default:
 215                                         if(out_buf_len <= (cptr - out_buf) + nread){
 216                                                 goto ERROR_TRAP;
 217                                         }
 218                                         memcpy(cptr, data, nread);
 219                                         cptr[nread] = '\0';
 220                                         cptr += nread;
 221                                         break;
 222                                 }
 223                         }while(nread > 0);
 224                         if(nread > 0){
 225                                 result = 0;
 226                         }else if(IS_SET_FLAG(responsep, CHUNKED_FLAG)){
 227                                 p3 = p1 = out_buf;
 228                                 while(0 < (nread = strtol(p1, &p2, 16))){
 229                                         memmove(p3, p2+2, nread);
 230                                         p3 += nread;
 231                                         p1 = p2 + nread + 4;
 232                                 }
 233                                 *p3 = 0;
 234                         }
 235                 }
 236                 break;
 237         default:
 238                 break;
 239         }/* switch */
 240 ERROR_TRAP:
 241         if(fd > 0)
 242                 close(fd);
 243         if(sockfd)
 244                 close(sockfd);
 245         if(socketp)
 246                 nt_socket_free(socketp);
 247         nt_http_free_response_header(responsep);
 248         nt_http_free_header(headerp);
 249         return result;
 250 }
 251
 252 BOOL nt_http_get(const char *url, const char *out_path,
 253                 const char *referer, const char *user_agent,
 254                 nt_link_tp extend_headers, BOOL range, BOOL ignore_cache)
 255 {
 256         nt_socket_tp socketp;
 257         int fd = -1, sockfd;
 258         char data[1024*2];
 259         char itoa_buf[32];
 260         char ssl_connect[512];
 261         size_t data_len;
 262         int nread, nwrite;
 263         BOOL result, retry;
 264         nt_http_response_header_tp responsep;
 265         nt_key_value_tp kv;
 266         nt_link_tp linkp;
 267
 268         retry = FALSE;
 269
 270         nt_http_header_tp headerp = nt_http_init_header(url);
 271         if(headerp == NULL)
 272                 return FALSE;
 273         if(!ignore_cache){
 274                 if(!nt_http_init_header2(headerp, out_path))
 275                         return FALSE;
 276         }
 277
 278         strcpy(data, "GET ");
 279         strcat(data, headerp->param);
 280         strcat(data," HTTP/1.1\r\nHost: ");
 281         strcat(data, headerp->host);
 282         strcat(data, "\r\nUser-Agent: ");
 283         if(user_agent)
 284                 strcat(data, user_agent);
 285         else
 286                 strcat(data, USER_AGENT);
 287         if(referer){
 288                 strcat(data, "\r\nReferer: ");
 289                 strcat(data, referer);
 290         }
 291         if(IS_SET_FLAG(headerp, RANGE_FLAG) && range){
 292                 if(0 <= sprintf(itoa_buf, "%zu-", headerp->fsize)){
 293                         strcat(data, "\r\nRange: bytes=");
 294                         strcat(data, itoa_buf);
 295                 }
 296         }
 297         if(headerp->last_modified){
 298                 strcat(data, "\r\nIf-Modified-Since: ");
 299                 strcat(data, headerp->last_modified);
 300         }
 301         if(!IS_SET_FLAG(headerp,SSL_FLAG) &&
 302                         !IS_SET_FLAG(headerp, RANGE_FLAG))
 303                 strcat(data, "\r\nAccept-Encoding: gzip");
 304         if(extend_headers){
 305                 linkp = extend_headers;
 306                 do{
 307                         kv = (nt_key_value_tp)linkp->data;
 308                         strcat(data, "\r\n");
 309                         strcat(data, kv->key);
 310                         strcat(data,": ");
 311                         strcat(data, kv->value);
 312                         linkp = linkp->next;
 313                 }while(linkp != extend_headers);
 314         }
 315         strcat(data, "\r\nConnection: close\r\n\r\n");
 316
 317         if(IS_SET_FLAG(headerp,SSL_FLAG)){
 318                 data_len = strlen(data);
 319                 sprintf(ssl_connect, "%s:443", headerp->host);
 320                 if(!nt_ssl_connect(ssl_connect, CA_FILE_PATH, FALSE,
 321                                         data, &data_len, sizeof(data))){
 322                         nt_http_free_header(headerp);
 323                         return FALSE;
 324                 }
 325                 /* Not implemented yet. */
 326                 assert(1);
 327                 return FALSE;
 328         }else{
 329                 if(headerp->port == -1)
 330                         headerp->port = 80;
 331
 332                 socketp = nt_socket_init(headerp->port, headerp->host);
 333                 if(socketp == NULL){
 334                         nt_http_free_header(headerp);
 335                         return FALSE;
 336                 }
 337
 338                 sockfd = nt_socket_connect(socketp, data, strlen(data));
 339                 if(-1 == sockfd){
 340                         nt_socket_free(socketp);
 341                         nt_http_free_header(headerp);
 342                         return FALSE;
 343                 }
 344
 345                 responsep = nt_http_alloc_response_header();
 346                 if(responsep == NULL){
 347                         close(sockfd);
 348                         nt_socket_free(socketp);
 349                         nt_http_free_header(headerp);
 350                         return FALSE;
 351                 }
 352
 353                 nread = nt_http_parse_response_header(sockfd, responsep,
 354                                                         NULL, NULL);
 355                 if(nread == -1){
 356                         close(sockfd);
 357                         nt_socket_free(socketp);
 358                         nt_http_free_header(headerp);
 359                         nt_http_free_response_header(responsep);
 360                         return FALSE;
 361                 }
 362         }
 363
 364         result = FALSE;
 365         switch(responsep->status_code){
 366         case 200:
 367                 fd = open(out_path,
 368                         O_CREAT | O_WRONLY | O_TRUNC,
 369                         S_IRUSR | S_IWUSR | S_IROTH);
 370                 if(fd == -1){
 371                         result = FALSE;
 372                         fd = -1;
 373                         break;
 374                 }
 375                 if(IS_SET_FLAG(responsep, GZIP_FLAG)){
 376                         if(IS_SET_FLAG(responsep, CHUNKED_FLAG)){
 377                                 if(!nt_zip_inflate2(sockfd, fd))
 378                                         goto ERROR_TRAP;
 379                         }else{
 380                                 if(!nt_zip_inflate(sockfd, fd))
 381                                         goto ERROR_TRAP;
 382                         }
 383                 }else{
 384                         do{
 385                                 nread = read(sockfd, data, sizeof(data));
 386                                 switch(nread){
 387                                 case -1:
 388                                         break;
 389                                 case 0:
 390                                         break;
 391                                 default:
 392                                         nwrite = write(fd, data, nread);
 393                                         if(nwrite == -1){
 394                                                 goto ERROR_TRAP;
 395                                         }
 396                                         break;
 397                                 }
 398                         }while(nread > 0);
 399                 }
 400                 if(!ignore_cache)
 401                         nt_http_save_response_header(out_path, responsep);
 402                 result = TRUE;
 403                 break;
 404         case 206: /* Partial content */
 405                 fd = open(out_path,
 406                         O_APPEND | O_WRONLY,
 407                         S_IRUSR | S_IWUSR | S_IROTH);
 408                 if(fd == -1){
 409                         result = FALSE;
 410                         fd = -1;
 411                         break;
 412                 }
 413                 do{
 414                         nread = read(sockfd, data, sizeof(data));
 415                         switch(nread){
 416                         case -1:
 417                                 break;
 418                         case 0:
 419                                 break;
 420                         default:
 421                                 nwrite = write(fd, data, nread);
 422                                 if(nwrite == -1){
 423                                         goto ERROR_TRAP;
 424                                 }
 425                                 break;
 426                         }
 427                 }while(nread > 0);
 428                 if(!ignore_cache)
 429                         nt_http_save_response_header(out_path, responsep);
 430                 result = TRUE;
 431                 break;
 432         case 304: /* Not Modified */
 433                 result = TRUE;
 434                 break;
 435         case 416:
 436                 retry = TRUE;
 437                 break;
 438         default:
 439                 break;
 440         }/* switch */
 441 ERROR_TRAP:
 442         if(fd > 0)
 443                 close(fd);
 444         close(sockfd);
 445         nt_socket_free(socketp);
 446         nt_http_free_header(headerp);
 447         nt_http_free_response_header(responsep);
 448         if(retry && range)
 449                 return nt_http_get(url, out_path, referer, user_agent,
 450                                 extend_headers, FALSE, ignore_cache);
 451
 452         return result;
 453 }
 454
 455
 456 static BOOL nt_http_init_header2(
 457                 nt_http_header_tp headerp, const char *out_path)
 458 {
 459     char key[DB_KEY_SIZE_MAX];
 460     nt_db_log_rec_tp recp;
 461     datum d_key, d_data;
 462     const char *log_path;
 463         DBM *dbm;
 464         struct stat st;
 465         nt_mutex_handle h_mutex;
 466
 467         assert(headerp);
 468         assert(out_path);
 469
 470         if(!nt_db_cpy_key(out_path, key)){
 471                 return FALSE;
 472         }
 473
 474         h_mutex = nt_mutex_get_one_time_handle(NT_MUTEX_ROOT_NAME_FILE);
 475         if(!h_mutex){
 476                 return FALSE;
 477         }
 478         if(!nt_mutex_add_moniker(h_mutex, LOG_DB_W_NAME)){
 479                 return FALSE;
 480         }
 481         if(!nt_mutex_lock(h_mutex)){
 482                 return FALSE;
 483         }
 484         log_path        =       nt_db_get_log_path();
 485         dbm     =       dbm_open((char*)log_path, O_RDWR | O_CREAT, 0600);
 486         if(dbm == NULL){
 487                 nt_mutex_unlock(h_mutex);
 488                 return FALSE;
 489         }
 490
 491         d_key.dsize = strlen(key);
 492         d_key.dptr = (void*)key;
 493
 494         d_data = dbm_fetch(dbm, d_key);
 495         if(d_data.dptr && d_data.dsize == sizeof(nt_db_log_rec_t)){
 496                 recp = (nt_db_log_rec_tp)d_data.dptr;
 497                 headerp->last_modified = nt_trim(recp->last_modified);
 498         }else{
 499                 headerp->last_modified = NULL;
 500         }
 501         dbm_close(dbm);
 502         nt_mutex_unlock(h_mutex);
 503
 504         if(0 == stat(out_path, &st)){
 505                 headerp->fsize = st.st_size;
 506                 SET_FLAG(headerp, RANGE_FLAG);
 507         }else{
 508                 headerp->fsize = -1;
 509         }
 510
 511         return TRUE;
 512
 513 }
 514
 515 static nt_http_header_tp nt_http_init_header(const char *url)
 516 {
 517         nt_http_header_tp headerp;
 518         char *cptr, *host, *param;
 519         int port = -1;
 520         int len;
 521         BOOL ssl;
 522
 523         if(0 == strncmp(url, "http://", 7)){
 524                 url += 7;
 525                 ssl = FALSE;
 526         }else if(0 == strncmp(url, "https://", 8)){
 527                 url += 8;
 528                 port = 443;
 529                 ssl = TRUE;
 530         }else{
 531                 return NULL;
 532         }
 533
 534         cptr = strchr(url, '/');
 535         if(NULL == cptr){
 536                 param = malloc(2);
 537                 if(param == NULL){
 538                         return NULL;
 539                 }
 540                 param[0] = '/';
 541                 param[1] = '\0';
 542                 host = malloc(strlen(url)+1);
 543                 if(host == NULL){;
 544                         return NULL;
 545                 }
 546                 strcpy(host, url);
 547         }else{
 548                 param = malloc(strlen(cptr)+1);
 549                 if(param == NULL){
 550                         return NULL;
 551                 }
 552                 strcpy(param, cptr);
 553                 len = cptr - url;
 554                 host = malloc(len+1);
 555                 if(host == NULL){
 556                         return NULL;
 557                 }
 558                 memcpy(host, url, len);
 559                 host[len] = '\0';
 560         }
 561
 562 #ifdef NT_NET_IPV6
 563         if(nt_is_ipv6_addr(host, &cptr)){
 564                 if(!ssl){
 565                         *cptr = '\0';
 566                         cptr++;
 567                         memmove(host, host+1, cptr - host);
 568                 }else{
 569                         cptr++;
 570                 }
 571                 if(*cptr == ':'){
 572                         port = atoi(cptr+1);
 573                         port = (port == 0) ? -1 : port;
 574                         *cptr = '\0';
 575                 }
 576                 port = atoi(cptr);
 577                 port = (port == 0) ? -1 : port;
 578         }else{
 579                 cptr = strchr(host, ':');
 580                 if(cptr != NULL && strlen(cptr+1) > 0){
 581                         port = atoi(cptr+1);
 582                         port = (port == 0) ? -1 : port;
 583                         *cptr = '\0';
 584                 }
 585         }
 586 #else
 587         cptr = strrchr(host, ':');
 588         if(cptr != NULL && strlen(cptr) > 0){
 589                 port = atoi(cptr);
 590                 port = (port == 0) ? -1 : port;
 591                 *cptr = '\0';
 592         }
 593 #endif
 594         headerp = calloc(1, sizeof(nt_http_header_t));
 595         if(headerp == NULL){
 596                 free(host);
 597                 free(param);
 598                 return NULL;
 599         }
 600
 601         headerp->port = port;
 602         headerp->host = host;
 603         headerp->param = param;
 604         if(ssl){
 605                 SET_FLAG(headerp, SSL_FLAG);
 606         }else{
 607                 CLR_FLAG(headerp, SSL_FLAG);
 608         }
 609         return headerp;
 610 }
 611
 612 static void nt_http_free_header(nt_http_header_tp headerp)
 613 {
 614         if(headerp->host != NULL)
 615                 free(headerp->host);
 616         if(headerp->param != NULL)
 617                 free(headerp->param);
 618         if(headerp->last_modified != NULL)
 619                 free(headerp->last_modified);
 620         free(headerp);
 621 }