update $http-get function.

diff --git a/lib/util_string.cc b/lib/util_string.cc
index a2ed5bf..5baf957 100644 (file)
--- a/lib/util_string.cc
+++ b/lib/util_string.cc
@@ -149,7 +149,7 @@ static ustring  percentHex (int c) {
     return ans;
 }
 
-static ustring  percentHEX (int c) {
+ustring  percentHEX (int c) {
     ustring  ans (3, '%');
 
     ans[1] = hexchar_c ((c >> 4) & 0x0f);

diff --git a/lib/util_string.h b/lib/util_string.h
index 1c552c4..3897007 100644 (file)
--- a/lib/util_string.h
+++ b/lib/util_string.h
@@ -50,6 +50,7 @@ inline int32_t  to_int32 (const ustring& v) {
 inline uint32_t  to_uint32 (const ustring& v) {
     return boost::lexical_cast<uint32_t> (v);
 }
+ustring  percentHEX (int c);
 ustring  urldecode_nonul (const ustring& str);
 ustring  omitPattern (const ustring& text, uregex& re);
 ustring  omitCtrl (const ustring& str);

diff --git a/lib/util_url.cc b/lib/util_url.cc
new file mode 100644 (file)
index 0000000..c438d18
--- /dev/null
+++ b/lib/util_url.cc
@@ -0,0 +1,186 @@
+#include "util_url.h"
+#include "util_const.h"
+#include "util_string.h"
+#include "ml.h"
+#include "ustring.h"
+
+/*
+  genericurl     = scheme ":" schemepart
+  scheme         = 1*[ lowalpha | digit | "+" | "-" | "." ]
+  schemepart     = *xchar | ip-schemepart
+  ip-schemepart  = "//" login [ "/" urlpath ]
+  login          = [ user [ ":" password ] "@" ] hostport
+  hostport       = host [ ":" port ]
+  host           = hostname | hostnumber
+  hostname       = *[ domainlabel "." ] toplabel
+  domainlabel    = alphadigit | alphadigit *[ alphadigit | "-" ] alphadigit
+  toplabel       = alpha | alpha *[ alphadigit | "-" ] alphadigit
+  alphadigit     = alpha | digit
+  hostnumber     = digits "." digits "." digits "." digits
+  port           = digits
+  user           = *[ uchar | ";" | "?" | "&" | "=" ]
+  password       = *[ uchar | ";" | "?" | "&" | "=" ]
+  urlpath        = *xchar    ; depends on protocol see section 3.1
+
+  lowalpha       = "a" ... "z"
+  hialpha        = "A" ... "Z"
+  alpha          = lowalpha | hialpha
+  digit          = "0" ... "9"
+  safe           = "$" | "-" | "_" | "." | "+"
+  extra          = "!" | "*" | "'" | "(" | ")" | ","
+  national       = "{" | "}" | "|" | "\" | "^" | "~" | "[" | "]" | "`"
+  punctuation    = "<" | ">" | "#" | "%" | <">
+  reserved       = ";" | "/" | "?" | ":" | "@" | "&" | "="
+  hex            = digit | "A" ... "F" | "a" ... "f"
+  escape         = "%" hex hex
+
+  unreserved     = alpha | digit | safe | extra
+  uchar          = unreserved | escape
+  xchar          = unreserved | reserved | escape
+  digits         = 1*digit
+*/
+
+bool  checkURL (ustring& url, upair* scheme, upair* delim, upair* user, upair* pass, upair* host, upair* port, upair* rest) {
+#define CDIGIT         "0-9"
+#define CHEX           CDIGIT "a-fA-F"
+#define CALPHA         "a-zA-Z"
+#define CALPHADIGIT    CALPHA CDIGIT
+#define MSB1CHAR       "\x80-\xff"
+#define CSAFE          "$\\-_.+"
+#define CEXTRA         "!*'(),"
+#define CRESERVED      ";/?:@&="
+#define CESCAPE                "%[" CHEX "][" CHEX "]"
+#define CUNRESERVED    CALPHADIGIT CSAFE CEXTRA
+#define WUCHAR         "[" CUNRESERVED MSB1CHAR "]|" CESCAPE
+#define WXCHAR         "[" CUNRESERVED CRESERVED MSB1CHAR "]|" CESCAPE
+#define WUSER          "(" WUCHAR "|[;?&=])*"
+#define WPASSWORD      WUSER
+#define WHOST          "([" CALPHADIGIT "][" CALPHADIGIT "-]*\\.)*[" CALPHADIGIT "][" CALPHADIGIT "-]*"
+#define WHOSTPART      "(" WHOST ")(:([" CDIGIT "]+))?"
+#define WLOGIN         "((" WUSER ")(:(" WPASSWORD "))?@)?" WHOSTPART
+    uiterator  b, e;
+    umatch  m;
+    static uregex  re_scheme ("^([a-z0-9][a-z0-9+.-]*):");
+    static uregex  re_ippart ("^//" WLOGIN);
+    static uregex  re_part ("^(" WXCHAR ")*");
+
+    b = url.begin ();
+    e = url.end ();
+    if (usearch (b, e, m, re_scheme)) {
+       // 1:scheme
+       if (scheme)
+           *scheme = m[1];
+       b = m[0].second;
+       if (usearch (b, e, m, re_ippart)) {
+           // 2:user, 5:password, 7:host, 10:port
+           if (delim) {
+               delim->first = b - 1;
+               delim->second = b + 2;
+           }
+           if (user)
+               *user = m[2];
+           if (pass)
+               *pass = m[5];
+           if (host)
+               *host = m[7];
+           if (port)
+               *port = m[10];
+           if (rest) {
+               rest->first = m[0].second;
+               rest->second = e;
+           }
+           return true;
+       } else if (usearch (b, e, m, re_part)) {
+           // 0:*xchar
+           if (m[0].second == e) {
+               if (delim) {
+                   delim->first = b - 1;
+                   delim->second = b;
+               }
+               if (rest)
+                   *rest = m[0];
+               return true;
+           } else {
+               return false;
+           }
+       } else {
+           return false;
+       }
+    } else {
+       return false;
+    }
+}
+
+bool  checkURLSafe (ustring& url) {
+    umatch  m;
+    static uregex  re ("^(" WXCHAR ")*$");
+
+    return usearch (url, m, re);
+}
+
+bool  checkScheme (ustring& proto) {
+    umatch  m;
+    static uregex  re ("^([a-z0-9][a-z0-9+.-]*)$");
+
+    return  usearch (proto, m, re);
+}
+
+static ustring  urlencode (uiterator b, uiterator e, const uregex& re) {
+    umatch  m;
+    ustring  ans;
+
+    while (b < e && usearch (b, e, m, re)) {
+       while (b < m[0].first) {
+           if (*b == 0)
+               ans.append (uUScore);
+           else
+               ans.append (percentHEX (*b++));
+       }
+       ans.append (m[0].first, m[0].second);
+       b = m[0].second;
+    }
+    while (b < e) {
+       if (*b == 0)
+           ans.append (uUScore);
+       else
+           ans.append (percentHEX (*b++));
+    }
+
+    return ans;
+}
+
+ustring  urlencode (uiterator b, uiterator e) {
+    static uregex  re ("[" CUNRESERVED MSB1CHAR "]+");
+    return urlencode (b, e, re);
+}
+
+ustring  urlencode_path (uiterator b, uiterator e) {
+    static uregex  re ("[" CUNRESERVED "/" MSB1CHAR "]+");
+    return urlencode (b, e, re);
+}
+
+ustring  buildQuery (MNode* query) {
+    int  c = 0;
+    ustring  ans;
+    ustring  u;
+
+    while (query && query->isCons ()) {
+       MNode*  a = query->car ();
+       if (a->isCons ()) {
+           if (c == 0)
+               ans.append (CharConst ("?"));
+           else
+               ans.append (CharConst ("&"));
+           c ++;
+           u = to_string (a->car ());
+           ans.append (urlencode (u.begin (), u.end ()));
+           ans.append (uEq);
+           u = to_string (a->cdr ());
+           ans.append (urlencode (u.begin (), u.end ()));
+       } else {
+           throw (to_string (a) + uErrorBadValue);
+       }
+       nextNode (query);
+    }
+    return ans;
+}

diff --git a/lib/util_url.h b/lib/util_url.h
new file mode 100644 (file)
index 0000000..c680dd3
--- /dev/null
+++ b/lib/util_url.h
@@ -0,0 +1,15 @@
+#ifndef UTIL_URL_H
+#define UTIL_URL_H
+
+#include "ustring.h"
+
+class MNode;
+
+bool  checkURL (ustring& url, upair* scheme, upair* delim, upair* user, upair* pass, upair* host, upair* port, upair* rest);
+bool  checkURLSafe (ustring& url);
+bool  checkScheme (ustring& proto);
+ustring  urlencode (uiterator b, uiterator e);
+ustring  urlencode_path (uiterator b, uiterator e);
+ustring  buildQuery (MNode* query);
+
+#endif /* UTIL_URL_H */

diff --git a/ml/Makefile b/ml/Makefile
index 98b6b4d..4e0277b 100644 (file)
--- a/ml/Makefile
+++ b/ml/Makefile
@@ -34,6 +34,7 @@ SRCS += util_random.cc
 SRCS += util_string.cc
 SRCS += util_tcp.cc
 SRCS += util_time.cc
+SRCS += util_url.cc
 SRCS += wikiattrib.cc
 SRCS += wikimotor.cc
 SRCS += wikiformat.cc

diff --git a/modules/ml-http.cc b/modules/ml-http.cc
index 02493cf..2f7a4bc 100644 (file)
--- a/modules/ml-http.cc
+++ b/modules/ml-http.cc
@@ -5,19 +5,49 @@
 #include "motoroutput.h"
 #include "util_check.h"
 #include "util_string.h"
+#include "util_url.h"
 #include "http-iconv.h"
 #include "expr.h"
 #include "ustring.h"
 #include <iostream>
 #include <fstream>
 
-static void  location_sub (ustring& url, MNode* query, MNode* arg, MlEnv* mlenv) {
+static void  location_sub (ustring& url, MNode* query, MlEnv* mlenv) {
+    upair  scheme, delim, user, pass, host, port, rest;
+    ustring  ans;
+
+    if (checkURL (url, &scheme, &delim, &user, &pass, &host, &port, &rest)) {
+       ans.assign (scheme.first, scheme.second).append (delim.first, delim.second);
+       if (user.first != user.second) {
+           ans.append (user.first, user.second);
+           if (pass.first != pass.second) {
+               ans.append (uColon).append (pass.first, pass.second);
+           }
+           ans.append (CharConst ("@"));
+       }
+       ans.append (host.first, host.second);
+       if (port.first != port.second) {
+           ans.append (uColon).append (port.first, port.second);
+       }
+       ans.append (urlencode_path (pass.first, pass.second));
+    } else {
+       ans.assign (urlencode_path (url.begin (), url.end ()));
+    }
+    if (query)
+       ans.append (buildQuery (query));
+
+    url = ans;
+}
+
+#if 0
+static void  location_sub (ustring& url, MNode* query, MlEnv* mlenv) {
     MNodePtr  p;
     MNode*  a;
     int  c;
     uiterator  b, e;
     umatch  m;
-    static uregex  re ("^[a-z]{1,6}://[a-zA-Z0-9-]+(\\.[a-zA-Z0-9-]+)*(:[0-9]{1,5})?/");
+//    static uregex  re ("^[a-z]{1,6}://[a-zA-Z0-9-]+(\\.[a-zA-Z0-9-]+)*(:[0-9]{1,5})?/");
+    static uregex  re ("^[a-z]{1,16}://[a-zA-Z0-9-]+(\\.[a-zA-Z0-9-]+)*(:[0-9]{1,5})?");
 
     b = url.begin ();
     e = url.end ();
@@ -45,6 +75,7 @@ static void  location_sub (ustring& url, MNode* query, MNode* arg, MlEnv* mlenv)
        nextNode (query);
     }
 
+#if 0
     while (arg) {
        p = eval (arg->car (), mlenv);
 
@@ -70,7 +101,9 @@ static void  location_sub (ustring& url, MNode* query, MNode* arg, MlEnv* mlenv)
        }
        nextNode (arg);
     }
+#endif
 }
+#endif
 
 static void  url_sub (const ustring& url, HTTPSend* http, bool noencode) {
     ustring  str;
@@ -217,6 +250,8 @@ MNode*  ml_error (MNode* cell, MlEnv* mlenv) {
 /*DOC:
 ===forbidden===
  (forbidden [#continue | :continue BOOL]) -> NIL
+403 Forbiddenレスポンスを出力する。
+continueオプションを付けない場合，以降のファンクションの実行を中断する。
 
 */
 //#AFUNC       forbidden       ml_forbidden
@@ -246,8 +281,10 @@ MNode*  ml_forbidden (MNode* cell, MlEnv* mlenv) {
 
 /*DOC:
 ===location===
- (location [#continue | :continue BOOL] URL ['(NAME VALUE)...]) -> NIL
- (location [#continue | :continue BOOL] URL [:query '((NAME VALUE)...)]) -> NIL
+// (location [#continue | :continue BOOL] URL ['(NAME VALUE)...]) -> NIL
+ (location [#continue | :continue BOOL] URL [:query '((NAME VALUE)...) | #raw]) -> NIL
+
+Locationヘッダを含むレスポンスを出力する。
 
 */
 //#AFUNC       location        ml_location
@@ -256,23 +293,34 @@ MNode*  ml_location (MNode* cell, MlEnv* mlenv) {
     ustring  url;
     MNodePtr  query;
     bool  cflag = false;
+    bool  fraw = false;
     std::vector<MNode*>  params;
     std::vector<MNode*>  keywords;
-    MNode*  rest;
+//    MNode*  rest;
     static paramList  kwlist[] = {
        {CharConst ("continue"), true},
        {CharConst ("query"), false},
+       {CharConst ("raw"), true},
        {NULL, 0, 0}
     };
 
-    setParams (arg, 1, &params, kwlist, &keywords, &rest);
+//    setParams (arg, 1, &params, kwlist, &keywords, &rest);
+    setParams (arg, 1, &params, kwlist, &keywords, NULL);
     url = eval_text1 (params[0], mlenv);
     if (keywords[0] && eval_bool (keywords[0], mlenv))
        cflag = true;
     if (keywords[1])           // query
        query = eval (keywords[1], mlenv);
+    if (keywords[2] && eval_bool (keywords[2], mlenv)) // raw
+       fraw = true;
 
-    location_sub (url, query (), rest, mlenv);
+    if (fraw) {
+       if (! checkURLSafe (url))
+           throw (url + ": bad URL.");
+    } else {
+//     location_sub (url, query (), rest, mlenv);
+       location_sub (url, query (), mlenv);
+    }
     mlenv->env->location (url);
 
     if (! cflag)
@@ -283,8 +331,8 @@ MNode*  ml_location (MNode* cell, MlEnv* mlenv) {
 
 /*DOC:
 ===location-html===
- (location-html URL [#continue | :continue BOOL] ['(NAME VALUE)...]) -> NIL
- (location-html URL [#continue | :continue BOOL] [:query '((NAME VALUE)...)]) -> NIL
+// (location-html URL [#continue | :continue BOOL] ['(NAME VALUE)...]) -> NIL
+ (location-html URL [#continue | :continue BOOL] [:query '((NAME VALUE)...) | #raw]) -> NIL
 
 */
 //#AFUNC       location-html   ml_location_html
@@ -293,23 +341,34 @@ MNode*  ml_location_html (MNode* cell, MlEnv* mlenv) {
     ustring  url;
     MNodePtr  query;
     bool  cflag = false;
+    bool  fraw = false;
     std::vector<MNode*>  params;
     std::vector<MNode*>  keywords;
-    MNode*  rest;
+//    MNode*  rest;
     static paramList  kwlist[] = {
        {CharConst ("continue"), true},
        {CharConst ("query"), false},
+       {CharConst ("raw"), true},
        {NULL, 0, 0}
     };
 
-    setParams (arg, 1, &params, kwlist, &keywords, &rest);
+//    setParams (arg, 1, &params, kwlist, &keywords, &rest);
+    setParams (arg, 1, &params, kwlist, &keywords, NULL);
     url = eval_text1 (params[0], mlenv);
     if (keywords[0] && eval_bool (keywords[0], mlenv))
        cflag = true;
     if (keywords[1])           // query
        query = eval (keywords[1], mlenv);
+    if (keywords[2] && eval_bool (keywords[2], mlenv)) // raw
+       fraw = true;
 
-    location_sub (url, query (), rest, mlenv);
+    if (fraw) {
+       if (! checkURLSafe (url))
+           throw (url + ": bad URL.");
+    } else {
+//     location_sub (url, query (), rest, mlenv);
+       location_sub (url, query (), mlenv);
+    }
     mlenv->env->location_html (url);
 
     if (! cflag)
@@ -336,6 +395,105 @@ MNode*  ml_response_no_cache (MNode* cell, MlEnv* mlenv) {
 }
 
 /*DOC:
+===build-url===
+ (build-url SCHEME HOST PATH [:user NAME] [:password TEXT | :pw TEXT] [:port NUMBER] [:query '((NAME . VALUE) ...)]) -> STRING
+
+*/
+//#AFUNC       build-url       ml_build_url
+MNode*  ml_build_url (MNode* cell, MlEnv* mlenv) {
+    MNode*  arg = cell->cdr ();
+    ustring  scheme;
+    ustring  host;
+    ustring  path;
+    MNodePtr  user;
+    MNodePtr  pwd;
+    int  port = 0;
+    MNodePtr  query;
+    AutoDelete<ustring>  ans;
+    ustring  u;
+    std::vector<MNode*>  params;
+    std::vector<MNode*>  keywords;
+    static paramList  kwlist[] = {
+       {CharConst ("user"), false},
+       {CharConst ("password"), false},
+       {CharConst ("pw"), false},
+       {CharConst ("port"), false},
+       {CharConst ("query"), false},
+       {NULL, 0, 0}
+    };
+
+    setParams (arg, 3, &params, kwlist, &keywords, NULL);
+    scheme = eval_text1 (params[0], mlenv);
+    host = eval_text1 (params[1], mlenv);
+    path = eval_text1 (params[2], mlenv);
+    if (keywords[0])           // user
+       user = eval (keywords[0], mlenv);
+    if (keywords[1])           // password
+       pwd = eval (keywords[1], mlenv);
+    if (keywords[2])           // pw
+       pwd = eval (keywords[2], mlenv);
+    if (keywords[3])           // port
+       port = eval_int (keywords[3], mlenv);
+    if (keywords[4])           // query
+       query = eval (keywords[4], mlenv);
+
+    if (! checkScheme (scheme))
+       throw (scheme + ": bad scheme.");
+    if (! checkHostname (host))
+       throw (host + ": bad hostname.");
+
+    ans = new ustring;
+    ans ()->append (scheme).append (CharConst ("://"));
+    if (user ()) {
+       u = to_string (user ());
+       omitCtrl (u);
+       ans ()->append (urlencode (u.begin (), u.end ()));
+       if (pwd ()) {
+           ans ()->append (uColon);
+           u = to_string (pwd ());
+           omitCtrl (u);
+           ans ()->append (urlencode (u.begin (), u.end ()));
+       }
+       ans ()->append (CharConst ("@"));
+    }
+    ans ()->append (host);
+    if (port > 0) {
+       ans ()->append (uColon).append (to_ustring (port));
+    }
+    if (path[0] != '/') {
+       ans ()->append (uSlash);
+    }
+    ans ()->append (urlencode_path (path.begin (), path.end ()));
+    if (query ()) {
+       ans ()->append (buildQuery (query ()));
+    }
+
+    return newMNode_str (ans.release ());
+}
+
+/*DOC:
+===build-url-path===
+ (build-url-path ELEMENT...) -> STRING
+
+*/
+//#AFUNC       build-url-path  ml_build_url_path
+MNode*  ml_build_url_path (MNode* cell, MlEnv* mlenv) {
+    MNode*  arg = cell->cdr ();
+    AutoDelete<ustring>  ans;
+    ustring  u;
+
+    ans = new ustring;
+    while (arg) {
+       u = eval_str (arg->car (), mlenv);
+       if (u.length () > 0)
+           ans ()->append (uSlash).append (urlencode (u.begin (), u.end ()));
+       nextNode (arg);
+    }
+
+    return newMNode_str (ans.release ());
+}
+
+/*DOC:
 ===$http-get==
  ($http-get URL [#get] [#post] [#put] [#delete] [#file]
        [:id STRING] [:password STRING | :pw STRING]

diff --git a/modules/ml-http.h b/modules/ml-http.h
index 85407ae..82e47f8 100644 (file)
--- a/modules/ml-http.h
+++ b/modules/ml-http.h
@@ -36,6 +36,8 @@ MNode*  ml_forbidden (MNode* cell, MlEnv* mlenv);
 MNode*  ml_location (MNode* cell, MlEnv* mlenv);
 MNode*  ml_location_html (MNode* cell, MlEnv* mlenv);
 MNode*  ml_response_no_cache (MNode* cell, MlEnv* mlenv);
+MNode*  ml_build_url (MNode* cell, MlEnv* mlenv);
+MNode*  ml_build_url_path (MNode* cell, MlEnv* mlenv);
 MNode*  ml_http_get (MNode* cell, MlEnv* mlenv);
 MNode*  ml_http_get_http_status (MNode* cell, MlEnv* mlenv, MLFunc* mobj);
 MNode*  ml_http_get_http_response (MNode* cell, MlEnv* mlenv, MLFunc* mobj);

diff --git a/modules/ml-string.cc b/modules/ml-string.cc
index aaab8a1..0a8d40b 100644 (file)
--- a/modules/ml-string.cc
+++ b/modules/ml-string.cc
@@ -34,7 +34,7 @@ MNode*  ml_concat (MNode* cell, MlEnv* mlenv) {
     AutoDelete<ustring>  a1;
 
     a1 = new ustring;
-    a1 ()->reserve (256);
+//    a1 ()->reserve (256);
 
     while (arg) {
        a1 ()->append (eval_str (arg->car (), mlenv));