src/net/nt_http.c

   1 /* Copyright 2013 Akira Ohta (akohta001@gmail.com)
   2     This file is part of ntch.
   3
   4     The ntch is free software: you can redistribute it and/or modify
   5     it under the terms of the GNU General Public License as published by
   6     the Free Software Foundation, either version 3 of the License, or
   7     (at your option) any later version.
   8
   9     The ntch is distributed in the hope that it will be useful,
  10     but WITHOUT ANY WARRANTY; without even the implied warranty of
  11     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12     GNU General Public License for more details.
  13
  14     You should have received a copy of the GNU General Public License
  15     along with ntch.  If not, see <http://www.gnu.org/licenses/>.
  16
  17 */
  18 #include <sys/types.h>
  19 #include <sys/socket.h>
  20 #include <sys/stat.h>
  21 #include <openssl/sha.h>
  22 #include <fcntl.h>
  23 #include <arpa/inet.h>
  24 #include <stdio.h>
  25 #include <string.h>
  26 #include <stdlib.h>
  27 #include <unistd.h>
  28 #include <assert.h>
  29
  30 #include "env.h"
  31 #include "utils/nt_std_t.h"
  32 #include "utils/text.h"
  33 #include "utils/zip.h"
  34 #include "utils/db.h"
  35 #include "utils/nt_mutex.h"
  36 #include "net/nt_socket.h"
  37 #include "net/nt_http.h"
  38 #include "net/nt_cookie.h"
  39
  40
  41 static nt_http_header_tp nt_http_init_header(const char *url);
  42 static void nt_http_free_header(nt_http_header_tp headerp);
  43 static BOOL nt_http_init_header2(
  44                 nt_http_header_tp headerp, const char *out_path);
  45 static int nt_http_post_local(const char *url, const char *post_data,
  46                 char **out_buf, size_t out_buf_len,
  47                 const char *referer, const char *user_agent,
  48                 nt_link_tp extend_headers,
  49                 nt_cookie_tp cookiep);
  50
  51 BOOL nt_http_post(const char *url, const char *post_data,
  52                 char *out_buf, size_t out_buf_len,
  53                 const char *referer, const char *user_agent,
  54                 nt_link_tp extend_headers,
  55                 nt_cookie_tp cookiep)
  56 {
  57         int result_code;
  58         result_code = nt_http_post_local(url, post_data,
  59                 &out_buf, out_buf_len, referer, user_agent,
  60                 extend_headers, cookiep);
  61         return (result_code == 200) ? TRUE : FALSE;
  62 }
  63
  64 int nt_http_post2(const char *url, const char *post_data,
  65                 char *out_buf, size_t out_buf_len,
  66                 const char *referer, const char *user_agent,
  67                 nt_link_tp extend_headers,
  68                 nt_cookie_tp cookiep)
  69 {
  70         return nt_http_post_local(url, post_data,
  71                                         &out_buf, out_buf_len,
  72                                         referer, user_agent,
  73                                         extend_headers,
  74                                         cookiep);
  75 }
  76 int nt_http_post3(const char *url, const char *post_data,
  77                 char **out_buf,
  78                 const char *referer, const char *user_agent,
  79                 nt_link_tp extend_headers,
  80                 nt_cookie_tp cookiep)
  81 {
  82         return nt_http_post_local(url, post_data,
  83                                         out_buf, 0,
  84                                         referer, user_agent,
  85                                         extend_headers,
  86                                         cookiep);
  87 }
  88
  89 #define BUF_LEN_DELTA (1024*4)
  90 int nt_http_post_local(const char *url, const char *post_data,
  91                 char **out_buf, size_t out_buf_len,
  92                 const char *referer, const char *user_agent,
  93                 nt_link_tp extend_headers,
  94                 nt_cookie_tp cookiep)
  95 {
  96         nt_socket_tp socketp = NULL;
  97         int fd = -1, sockfd = 0;
  98         char data[1024*8];
  99         char ssl_connect[512];
 100         size_t data_len;
 101         int nread;
 102         int result;
 103         nt_http_response_header_tp responsep;
 104         nt_key_value_tp kv;
 105         nt_link_tp linkp, cookies;
 106         char itoa_buf[33];
 107         char cookie_data[1024];
 108         char *cptr, *p1, *p2, *p3;
 109         int len, new_len;
 110         char *tmp_buf;
 111         BOOL is_realloc_buf;
 112
 113
 114         result = 0;
 115         nt_http_header_tp headerp = nt_http_init_header(url);
 116         if(headerp == NULL)
 117                 return result;
 118
 119         strcpy(data, "POST ");
 120         strcat(data, headerp->param);
 121         strcat(data, " HTTP/1.1\r\nHost: ");
 122         strcat(data, headerp->host);
 123         strcat(data, "\r\nAccept: */*");
 124         strcat(data, "\r\nAccept-Language: ja");
 125         strcat(data, "\r\nUser-Agent: ");
 126         if(user_agent)
 127                 strcat(data, user_agent);
 128         else
 129                 strcat(data, USER_AGENT);
 130         if(referer){
 131                 strcat(data, "\r\nReferer: ");
 132                 strcat(data, referer);
 133         }
 134
 135         if(cookiep){
 136                 cookies = nt_get_cookies(cookiep, headerp->host);
 137                 if(cookies){
 138                         cookie_data[0] = '\0';
 139                         linkp = cookies;
 140                         do{
 141                                 if(cookie_data[0] != '\0')
 142                                         strcat(cookie_data, "; ");
 143                                 strcat(cookie_data, (char*)linkp->data);
 144                                  linkp = linkp->next;
 145                         }while(linkp != cookies);
 146                         if(strlen(cookie_data) > 0){
 147                                 strcat(data, "\r\nCookie: ");
 148                                 strcat(data, cookie_data);
 149                         }
 150                         nt_all_link_free(cookies, NULL);
 151                 }
 152         }
 153
 154         if(extend_headers){
 155                 linkp = extend_headers;
 156                 do{
 157                         kv = (nt_key_value_tp)linkp->data;
 158                         strcat(data, "\r\n");
 159                         strcat(data, kv->key);
 160                         strcat(data,": ");
 161                         strcat(data, kv->value);
 162                         linkp = linkp->next;
 163                 }while(linkp != extend_headers);
 164         }
 165         strcat(data, "\r\nContent-Type: application/x-www-form-urlencoded");
 166         strcat(data, "\r\nContent-Length: ");
 167         sprintf(itoa_buf, "%u", (unsigned int)strlen(post_data));
 168         strcat(data, itoa_buf);
 169         strcat(data, "\r\nConnection: close\r\n\r\n");
 170
 171         strcat(data, post_data);
 172
 173
 174         if(IS_SET_FLAG(headerp,SSL_FLAG)){
 175                 data_len = strlen(data);
 176                 sprintf(ssl_connect, "%s:443", headerp->host);
 177                 if(!nt_ssl_connect(ssl_connect, CA_FILE_PATH, FALSE,
 178                                         data, &data_len, sizeof(data))){
 179                         nt_http_free_header(headerp);
 180                         return result;
 181                 }
 182                 responsep = nt_http_alloc_response_header();
 183                 if(responsep == NULL){
 184                         nt_http_free_header(headerp);
 185                         return result;
 186                 }
 187                 nread = nt_http_parse_response_header2(data, data_len,
 188                         responsep);
 189                 if(nread <= 0){
 190                         nt_http_free_response_header(responsep);
 191                         nt_http_free_header(headerp);
 192                         return result;
 193                 }
 194                 if(out_buf_len <= data_len - nread + 1){
 195                         nt_http_free_response_header(responsep);
 196                         nt_http_free_header(headerp);
 197                         return result;
 198                 }
 199                 memcpy(*out_buf, data+nread, data_len-nread);
 200                 (*out_buf)[data_len - nread] = '\0';
 201         }else{
 202                 if(headerp->port == -1)
 203                         headerp->port = 80;
 204
 205                 socketp = nt_socket_init(headerp->port, headerp->host);
 206                 if(socketp == NULL){
 207                         nt_http_free_header(headerp);
 208                         return result;
 209                 }
 210
 211                 sockfd = nt_socket_connect(socketp, data, strlen(data));
 212                 if(-1 == sockfd){
 213                         nt_http_free_header(headerp);
 214                         nt_socket_free(socketp);
 215                         return result;
 216                 }
 217
 218                 responsep = nt_http_alloc_response_header();
 219                 if(responsep == NULL){
 220                         close(sockfd);
 221                         nt_http_free_header(headerp);
 222                         nt_socket_free(socketp);
 223                         return result;
 224                 }
 225
 226                 nread = nt_http_parse_response_header(sockfd,
 227                                         responsep, cookiep, headerp->host);
 228                 if(nread == -1){
 229                         close(sockfd);
 230                         nt_http_free_header(headerp);
 231                         nt_socket_free(socketp);
 232                         nt_http_free_response_header(responsep);
 233                         return result;
 234                 }
 235         }
 236
 237
 238         result = responsep->status_code;
 239         switch(result){
 240         case 200:
 241                 if(IS_SET_FLAG(headerp,SSL_FLAG)){
 242                 }else{
 243                         if(out_buf_len == 0){
 244                                 tmp_buf = malloc(BUF_LEN_DELTA);
 245                                 if(!tmp_buf)
 246                                         goto ERROR_TRAP;
 247                                 *out_buf = tmp_buf;
 248                                 out_buf_len = BUF_LEN_DELTA;
 249                                 is_realloc_buf = TRUE;
 250                         }else{
 251                                 is_realloc_buf = FALSE;
 252                         }
 253                         cptr = *out_buf;
 254                         do{
 255                                 nread = read(sockfd, data, sizeof(data));
 256                                 switch(nread){
 257                                 case -1:
 258                                         break;
 259                                 case 0:
 260                                         break;
 261                                 default:
 262                                         len = cptr - (*out_buf);
 263                                         if(out_buf_len <= len + nread){
 264                                                 if(!is_realloc_buf)
 265                                                         goto ERROR_TRAP;
 266                                                 new_len = ((len + nread) / BUF_LEN_DELTA + 1) * BUF_LEN_DELTA;
 267                                                 tmp_buf = malloc(new_len);
 268                                                 if(!tmp_buf)
 269                                                         goto ERROR_TRAP;
 270                                                 memcpy(tmp_buf, *out_buf, len);
 271                                                 free(*out_buf);
 272                                                 *out_buf = tmp_buf;
 273                                                 out_buf_len = new_len;
 274                                                 cptr = (*out_buf) + len;
 275                                         }
 276                                         memcpy(cptr, data, nread);
 277                                         cptr[nread] = '\0';
 278                                         cptr += nread;
 279                                         break;
 280                                 }
 281                         }while(nread > 0);
 282                         if(nread > 0){
 283                                 result = 0;
 284                         }else if(IS_SET_FLAG(responsep, CHUNKED_FLAG)){
 285                                 p3 = p1 = *out_buf;
 286                                 while(0 < (nread = strtol(p1, &p2, 16))){
 287                                         memmove(p3, p2+2, nread);
 288                                         p3 += nread;
 289                                         p1 = p2 + nread + 4;
 290                                 }
 291                                 *p3 = 0;
 292                         }
 293                 }
 294                 break;
 295         default:
 296                 break;
 297         }/* switch */
 298 ERROR_TRAP:
 299         if(fd > 0)
 300                 close(fd);
 301         if(sockfd)
 302                 close(sockfd);
 303         if(socketp)
 304                 nt_socket_free(socketp);
 305         nt_http_free_response_header(responsep);
 306         nt_http_free_header(headerp);
 307         return result;
 308 }
 309
 310 BOOL nt_http_get(const char *url, const char *out_path,
 311                 const char *referer, const char *user_agent,
 312                 nt_link_tp extend_headers, BOOL range, BOOL ignore_cache)
 313 {
 314         nt_socket_tp socketp;
 315         int fd = -1, sockfd;
 316         char data[1024*2];
 317         char itoa_buf[32];
 318         char ssl_connect[512];
 319         size_t data_len;
 320         int nread, nwrite;
 321         BOOL result, retry;
 322         nt_http_response_header_tp responsep;
 323         nt_key_value_tp kv;
 324         nt_link_tp linkp;
 325
 326         retry = FALSE;
 327
 328         nt_http_header_tp headerp = nt_http_init_header(url);
 329         if(headerp == NULL)
 330                 return FALSE;
 331         if(!ignore_cache){
 332                 if(!nt_http_init_header2(headerp, out_path))
 333                         return FALSE;
 334         }
 335
 336         strcpy(data, "GET ");
 337         strcat(data, headerp->param);
 338         strcat(data," HTTP/1.1\r\nHost: ");
 339         strcat(data, headerp->host);
 340         strcat(data, "\r\nUser-Agent: ");
 341         if(user_agent)
 342                 strcat(data, user_agent);
 343         else
 344                 strcat(data, USER_AGENT);
 345         if(referer){
 346                 strcat(data, "\r\nReferer: ");
 347                 strcat(data, referer);
 348         }
 349         if(IS_SET_FLAG(headerp, RANGE_FLAG) && range){
 350                 if(0 <= sprintf(itoa_buf, "%zu-", headerp->fsize)){
 351                         strcat(data, "\r\nRange: bytes=");
 352                         strcat(data, itoa_buf);
 353                 }
 354         }
 355         if(headerp->last_modified){
 356                 strcat(data, "\r\nIf-Modified-Since: ");
 357                 strcat(data, headerp->last_modified);
 358         }
 359         if(!IS_SET_FLAG(headerp,SSL_FLAG) &&
 360                         !IS_SET_FLAG(headerp, RANGE_FLAG))
 361                 strcat(data, "\r\nAccept-Encoding: gzip");
 362         if(extend_headers){
 363                 linkp = extend_headers;
 364                 do{
 365                         kv = (nt_key_value_tp)linkp->data;
 366                         strcat(data, "\r\n");
 367                         strcat(data, kv->key);
 368                         strcat(data,": ");
 369                         strcat(data, kv->value);
 370                         linkp = linkp->next;
 371                 }while(linkp != extend_headers);
 372         }
 373         strcat(data, "\r\nConnection: close\r\n\r\n");
 374
 375         if(IS_SET_FLAG(headerp,SSL_FLAG)){
 376                 data_len = strlen(data);
 377                 sprintf(ssl_connect, "%s:443", headerp->host);
 378                 if(!nt_ssl_connect(ssl_connect, CA_FILE_PATH, FALSE,
 379                                         data, &data_len, sizeof(data))){
 380                         nt_http_free_header(headerp);
 381                         return FALSE;
 382                 }
 383                 /* Not implemented yet. */
 384                 assert(1);
 385                 return FALSE;
 386         }else{
 387                 if(headerp->port == -1)
 388                         headerp->port = 80;
 389
 390                 socketp = nt_socket_init(headerp->port, headerp->host);
 391                 if(socketp == NULL){
 392                         nt_http_free_header(headerp);
 393                         return FALSE;
 394                 }
 395
 396                 sockfd = nt_socket_connect(socketp, data, strlen(data));
 397                 if(-1 == sockfd){
 398                         nt_socket_free(socketp);
 399                         nt_http_free_header(headerp);
 400                         return FALSE;
 401                 }
 402
 403                 responsep = nt_http_alloc_response_header();
 404                 if(responsep == NULL){
 405                         close(sockfd);
 406                         nt_socket_free(socketp);
 407                         nt_http_free_header(headerp);
 408                         return FALSE;
 409                 }
 410
 411                 nread = nt_http_parse_response_header(sockfd, responsep,
 412                                                         NULL, NULL);
 413                 if(nread == -1){
 414                         close(sockfd);
 415                         nt_socket_free(socketp);
 416                         nt_http_free_header(headerp);
 417                         nt_http_free_response_header(responsep);
 418                         return FALSE;
 419                 }
 420         }
 421
 422         result = FALSE;
 423         switch(responsep->status_code){
 424         case 200:
 425                 fd = open(out_path,
 426                         O_CREAT | O_WRONLY | O_TRUNC,
 427                         S_IRUSR | S_IWUSR | S_IROTH);
 428                 if(fd == -1){
 429                         result = FALSE;
 430                         fd = -1;
 431                         break;
 432                 }
 433                 if(IS_SET_FLAG(responsep, GZIP_FLAG)){
 434                         if(IS_SET_FLAG(responsep, CHUNKED_FLAG)){
 435                                 if(!nt_zip_inflate2(sockfd, fd))
 436                                         goto ERROR_TRAP;
 437                         }else{
 438                                 if(!nt_zip_inflate(sockfd, fd))
 439                                         goto ERROR_TRAP;
 440                         }
 441                 }else{
 442                         do{
 443                                 nread = read(sockfd, data, sizeof(data));
 444                                 switch(nread){
 445                                 case -1:
 446                                         break;
 447                                 case 0:
 448                                         break;
 449                                 default:
 450                                         nwrite = write(fd, data, nread);
 451                                         if(nwrite == -1){
 452                                                 goto ERROR_TRAP;
 453                                         }
 454                                         break;
 455                                 }
 456                         }while(nread > 0);
 457                 }
 458                 if(!ignore_cache)
 459                         nt_http_save_response_header(out_path, responsep);
 460                 result = TRUE;
 461                 break;
 462         case 206: /* Partial content */
 463                 fd = open(out_path,
 464                         O_APPEND | O_WRONLY,
 465                         S_IRUSR | S_IWUSR | S_IROTH);
 466                 if(fd == -1){
 467                         result = FALSE;
 468                         fd = -1;
 469                         break;
 470                 }
 471                 do{
 472                         nread = read(sockfd, data, sizeof(data));
 473                         switch(nread){
 474                         case -1:
 475                                 break;
 476                         case 0:
 477                                 break;
 478                         default:
 479                                 nwrite = write(fd, data, nread);
 480                                 if(nwrite == -1){
 481                                         goto ERROR_TRAP;
 482                                 }
 483                                 break;
 484                         }
 485                 }while(nread > 0);
 486                 if(!ignore_cache)
 487                         nt_http_save_response_header(out_path, responsep);
 488                 result = TRUE;
 489                 break;
 490         case 304: /* Not Modified */
 491                 result = TRUE;
 492                 break;
 493         case 416:
 494                 retry = TRUE;
 495                 break;
 496         default:
 497                 break;
 498         }/* switch */
 499 ERROR_TRAP:
 500         if(fd > 0)
 501                 close(fd);
 502         close(sockfd);
 503         nt_socket_free(socketp);
 504         nt_http_free_header(headerp);
 505         nt_http_free_response_header(responsep);
 506         if(retry && range)
 507                 return nt_http_get(url, out_path, referer, user_agent,
 508                                 extend_headers, FALSE, ignore_cache);
 509
 510         return result;
 511 }
 512
 513
 514 static BOOL nt_http_init_header2(
 515                 nt_http_header_tp headerp, const char *out_path)
 516 {
 517     char key[DB_KEY_SIZE_MAX];
 518     nt_db_log_rec_tp recp;
 519     datum d_key, d_data;
 520     const char *log_path;
 521         DBM *dbm;
 522         struct stat st;
 523         nt_mutex_handle h_mutex;
 524
 525         assert(headerp);
 526         assert(out_path);
 527
 528         if(!nt_db_cpy_key(out_path, key)){
 529                 return FALSE;
 530         }
 531
 532         h_mutex = nt_mutex_get_one_time_handle(NT_MUTEX_ROOT_NAME_FILE);
 533         if(!h_mutex){
 534                 return FALSE;
 535         }
 536         if(!nt_mutex_add_moniker(h_mutex, LOG_DB_W_NAME)){
 537                 return FALSE;
 538         }
 539         if(!nt_mutex_lock(h_mutex)){
 540                 return FALSE;
 541         }
 542         log_path        =       nt_db_get_log_path();
 543         dbm     =       dbm_open((char*)log_path, O_RDWR | O_CREAT, 0600);
 544         if(dbm == NULL){
 545                 nt_mutex_unlock(h_mutex);
 546                 return FALSE;
 547         }
 548
 549         d_key.dsize = strlen(key);
 550         d_key.dptr = (void*)key;
 551
 552         d_data = dbm_fetch(dbm, d_key);
 553         if(d_data.dptr && d_data.dsize == sizeof(nt_db_log_rec_t)){
 554                 recp = (nt_db_log_rec_tp)d_data.dptr;
 555                 headerp->last_modified = nt_trim(recp->last_modified);
 556         }else{
 557                 headerp->last_modified = NULL;
 558         }
 559         dbm_close(dbm);
 560         nt_mutex_unlock(h_mutex);
 561
 562         if(0 == stat(out_path, &st)){
 563                 headerp->fsize = st.st_size;
 564                 SET_FLAG(headerp, RANGE_FLAG);
 565         }else{
 566                 headerp->fsize = -1;
 567         }
 568
 569         return TRUE;
 570
 571 }
 572
 573 static nt_http_header_tp nt_http_init_header(const char *url)
 574 {
 575         nt_http_header_tp headerp;
 576         char *cptr, *host, *param;
 577         int port = -1;
 578         int len;
 579         BOOL ssl;
 580
 581         if(0 == strncmp(url, "http://", 7)){
 582                 url += 7;
 583                 ssl = FALSE;
 584         }else if(0 == strncmp(url, "https://", 8)){
 585                 url += 8;
 586                 port = 443;
 587                 ssl = TRUE;
 588         }else{
 589                 return NULL;
 590         }
 591
 592         cptr = strchr(url, '/');
 593         if(NULL == cptr){
 594                 param = malloc(2);
 595                 if(param == NULL){
 596                         return NULL;
 597                 }
 598                 param[0] = '/';
 599                 param[1] = '\0';
 600                 host = malloc(strlen(url)+1);
 601                 if(host == NULL){;
 602                         return NULL;
 603                 }
 604                 strcpy(host, url);
 605         }else{
 606                 param = malloc(strlen(cptr)+1);
 607                 if(param == NULL){
 608                         return NULL;
 609                 }
 610                 strcpy(param, cptr);
 611                 len = cptr - url;
 612                 host = malloc(len+1);
 613                 if(host == NULL){
 614                         return NULL;
 615                 }
 616                 memcpy(host, url, len);
 617                 host[len] = '\0';
 618         }
 619
 620 #ifdef NT_NET_IPV6
 621         if(nt_is_ipv6_addr(host, &cptr)){
 622                 if(!ssl){
 623                         *cptr = '\0';
 624                         cptr++;
 625                         memmove(host, host+1, cptr - host);
 626                 }else{
 627                         cptr++;
 628                 }
 629                 if(*cptr == ':'){
 630                         port = atoi(cptr+1);
 631                         port = (port == 0) ? -1 : port;
 632                         *cptr = '\0';
 633                 }
 634                 port = atoi(cptr);
 635                 port = (port == 0) ? -1 : port;
 636         }else{
 637                 cptr = strchr(host, ':');
 638                 if(cptr != NULL && strlen(cptr+1) > 0){
 639                         port = atoi(cptr+1);
 640                         port = (port == 0) ? -1 : port;
 641                         *cptr = '\0';
 642                 }
 643         }
 644 #else
 645         cptr = strrchr(host, ':');
 646         if(cptr != NULL && strlen(cptr) > 0){
 647                 port = atoi(cptr);
 648                 port = (port == 0) ? -1 : port;
 649                 *cptr = '\0';
 650         }
 651 #endif
 652         headerp = calloc(1, sizeof(nt_http_header_t));
 653         if(headerp == NULL){
 654                 free(host);
 655                 free(param);
 656                 return NULL;
 657         }
 658
 659         headerp->port = port;
 660         headerp->host = host;
 661         headerp->param = param;
 662         if(ssl){
 663                 SET_FLAG(headerp, SSL_FLAG);
 664         }else{
 665                 CLR_FLAG(headerp, SSL_FLAG);
 666         }
 667         return headerp;
 668 }
 669
 670 static void nt_http_free_header(nt_http_header_tp headerp)
 671 {
 672         if(headerp->host != NULL)
 673                 free(headerp->host);
 674         if(headerp->param != NULL)
 675                 free(headerp->param);
 676         if(headerp->last_modified != NULL)
 677                 free(headerp->last_modified);
 678         free(headerp);
 679 }