OpenTween/ShortUrl.cs

   1 // OpenTween - Client of Twitter
   2 // Copyright (c) 2007-2011 kiri_feather (@kiri_feather) <kiri.feather@gmail.com>
   3 //           (c) 2008-2011 Moz (@syo68k)
   4 //           (c) 2008-2011 takeshik (@takeshik) <http://www.takeshik.org/>
   5 //           (c) 2010-2011 anis774 (@anis774) <http://d.hatena.ne.jp/anis774/>
   6 //           (c) 2010-2011 fantasticswallow (@f_swallow) <http://twitter.com/f_swallow>
   7 //           (c) 2011      kim_upsilon (@kim_upsilon) <https://upsilo.net/~upsilon/>
   8 // All rights reserved.
   9 //
  10 // This file is part of OpenTween.
  11 //
  12 // This program is free software; you can redistribute it and/or modify it
  13 // under the terms of the GNU General public License as published by the Free
  14 // Software Foundation; either version 3 of the License, or (at your option)
  15 // any later version.
  16 //
  17 // This program is distributed in the hope that it will be useful, but
  18 // WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
  19 // or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General public License
  20 // for more details.
  21 //
  22 // You should have received a copy of the GNU General public License along
  23 // with this program. If not, see <http://www.gnu.org/licenses/>, or write to
  24 // the Free Software Foundation, Inc., 51 Franklin Street - Fifth Floor,
  25 // Boston, MA 02110-1301, USA.
  26
  27 using System;
  28 using System.Collections.Concurrent;
  29 using System.Collections.Generic;
  30 using System.Diagnostics.CodeAnalysis;
  31 using System.Linq;
  32 using System.Net.Http;
  33 using System.Text;
  34 using System.Text.RegularExpressions;
  35 using System.Threading;
  36 using System.Threading.Tasks;
  37 using System.Web;
  38 using OpenTween.Connection;
  39
  40 namespace OpenTween
  41 {
  42     /// <summary>
  43     /// 短縮 URL サービスによる URL の展開・短縮を行うクラス
  44     /// </summary>
  45     public class ShortUrl
  46     {
  47         private static Lazy<ShortUrl> _instance;
  48
  49         /// <summary>
  50         /// ShortUrl のインスタンスを取得します
  51         /// </summary>
  52         public static ShortUrl Instance
  53         {
  54             get { return _instance.Value; }
  55         }
  56
  57         /// <summary>
  58         /// 短縮 URL の展開を無効にするか否か
  59         /// </summary>
  60         public bool DisableExpanding { get; set; }
  61
  62         /// <summary>
  63         /// 短縮 URL のキャッシュを定期的にクリアする回数
  64         /// </summary>
  65         public int PurgeCount { get; set; }
  66
  67         public string BitlyId { get; set; }
  68         public string BitlyKey { get; set; }
  69
  70         private HttpClient http;
  71         private ConcurrentDictionary<Uri, Uri> urlCache = new ConcurrentDictionary<Uri, Uri>();
  72
  73         private static readonly Regex HtmlLinkPattern = new Regex(@"(<a href="")(.+?)("")");
  74
  75         private static readonly HashSet<string> ShortUrlHosts = new HashSet<string>()
  76         {
  77             "4sq.com",
  78             "airme.us",
  79             "amzn.to",
  80             "bctiny.com",
  81             "bit.ly",
  82             "bkite.com",
  83             "blip.fm",
  84             "budurl.com",
  85             "cli.gs",
  86             "digg.com",
  87             "dlvr.it",
  88             "fb.me",
  89             "feeds.feedburner.com",
  90             "ff.im",
  91             "flic.kr",
  92             "goo.gl",
  93             "ht.ly",
  94             "htn.to",
  95             "icanhaz.com",
  96             "is.gd",
  97             "j.mp",
  98             "linkbee.com",
  99             "moi.st",
 100             "nico.ms",
 101             "nsfw.in",
 102             "on.fb.me",
 103             "ow.ly",
 104             "p.tl",
 105             "pic.gd",
 106             "qurl.com",
 107             "rubyurl.com",
 108             "snipurl.com",
 109             "snurl.com",
 110             "t.co",
 111             "tinami.jp",
 112             "tiny.cc",
 113             "tinyurl.com",
 114             "tl.gd",
 115             "tmblr.co",
 116             "traceurl.com",
 117             "tumblr.com",
 118             "twitthis.com",
 119             "twurl.nl",
 120             "urlenco.de",
 121             "ustre.am",
 122             "ux.nu",
 123             "www.qurl.com",
 124             "www.tumblr.com",
 125             "youtu.be",
 126         };
 127
 128         static ShortUrl()
 129         {
 130             _instance = new Lazy<ShortUrl>(() => new ShortUrl(), true);
 131         }
 132
 133         [SuppressMessage("Microsoft.Reliability", "CA2000:DisposeObjectsBeforeLosingScope")]
 134         internal ShortUrl()
 135             : this(CreateDefaultHttpClient())
 136         {
 137             Networking.WebProxyChanged += (o, e) =>
 138             {
 139                 var newClient = CreateDefaultHttpClient();
 140                 var oldClient = Interlocked.Exchange(ref this.http, newClient);
 141                 oldClient.Dispose();
 142             };
 143         }
 144
 145         internal ShortUrl(HttpClient http)
 146         {
 147             this.DisableExpanding = false;
 148             this.PurgeCount = 500;
 149             this.BitlyId = "";
 150             this.BitlyKey = "";
 151
 152             this.http = http;
 153         }
 154
 155         [Obsolete]
 156         public string ExpandUrl(string uri)
 157         {
 158             try
 159             {
 160                 return this.ExpandUrlAsync(new Uri(uri), 10).Result.ToString();
 161             }
 162             catch (UriFormatException)
 163             {
 164                 return uri;
 165             }
 166         }
 167
 168         /// <summary>
 169         /// 短縮 URL を非同期に展開します
 170         /// </summary>
 171         /// <param name="uri">展開するURL</param>
 172         /// <returns>URLの展開を行うタスク</returns>
 173         public Task<Uri> ExpandUrlAsync(Uri uri)
 174         {
 175             return this.ExpandUrlAsync(uri, 10);
 176         }
 177
 178         /// <summary>
 179         /// 短縮 URL を非同期に展開します
 180         /// </summary>
 181         /// <param name="uri">展開するURL</param>
 182         /// <param name="redirectLimit">再帰的に展開を試みる上限</param>
 183         /// <returns>URLの展開を行うタスク</returns>
 184         public async Task<Uri> ExpandUrlAsync(Uri uri, int redirectLimit)
 185         {
 186             if (this.DisableExpanding)
 187                 return uri;
 188
 189             if (redirectLimit <= 0)
 190                 return uri;
 191
 192             if (!uri.IsAbsoluteUri)
 193                 return uri;
 194
 195             try
 196             {
 197                 if (!ShortUrlHosts.Contains(uri.Host) && !IsIrregularShortUrl(uri))
 198                     return uri;
 199
 200                 Uri expanded;
 201                 if (this.urlCache.TryGetValue(uri, out expanded))
 202                     return expanded;
 203
 204                 if (this.urlCache.Count > this.PurgeCount)
 205                     this.urlCache.Clear();
 206
 207                 expanded = null;
 208                 try
 209                 {
 210                     expanded = await this.GetRedirectTo(uri)
 211                         .ConfigureAwait(false);
 212                 }
 213                 catch (TaskCanceledException) { }
 214                 catch (HttpRequestException) { }
 215
 216                 if (expanded == null || expanded == uri)
 217                     return uri;
 218
 219                 this.urlCache[uri] = expanded;
 220
 221                 var recursiveExpanded = await this.ExpandUrlAsync(expanded, --redirectLimit)
 222                     .ConfigureAwait(false);
 223
 224                 // URL1 -> URL2 -> URL3 のように再帰的に展開されたURLを URL1 -> URL3 としてキャッシュに格納する
 225                 if (recursiveExpanded != expanded)
 226                     this.urlCache[uri] = recursiveExpanded;
 227
 228                 return recursiveExpanded;
 229             }
 230             catch (UriFormatException)
 231             {
 232                 return uri;
 233             }
 234         }
 235
 236         /// <summary>
 237         /// 短縮 URL を非同期に展開します
 238         /// </summary>
 239         /// <remarks>
 240         /// 不正なURLが渡された場合は例外を投げず uriStr をそのまま返します
 241         /// </remarks>
 242         /// <param name="uriStr">展開するURL</param>
 243         /// <returns>URLの展開を行うタスク</returns>
 244         public Task<string> ExpandUrlAsync(string uriStr)
 245         {
 246             return this.ExpandUrlAsync(uriStr, 10);
 247         }
 248
 249         /// <summary>
 250         /// 短縮 URL を非同期に展開します
 251         /// </summary>
 252         /// <remarks>
 253         /// 不正なURLが渡された場合は例外を投げず uriStr をそのまま返します
 254         /// </remarks>
 255         /// <param name="uriStr">展開するURL</param>
 256         /// <param name="redirectLimit">再帰的に展開を試みる上限</param>
 257         /// <returns>URLの展開を行うタスク</returns>
 258         public async Task<string> ExpandUrlAsync(string uriStr, int redirectLimit)
 259         {
 260             Uri uri;
 261
 262             try
 263             {
 264                 if (!uriStr.StartsWith("http", StringComparison.OrdinalIgnoreCase))
 265                     uri = new Uri("http://" + uriStr);
 266                 else
 267                     uri = new Uri(uriStr);
 268             }
 269             catch (UriFormatException)
 270             {
 271                 return uriStr;
 272             }
 273
 274             var expandedUri = await this.ExpandUrlAsync(uri, redirectLimit)
 275                 .ConfigureAwait(false);
 276
 277             return expandedUri.OriginalString;
 278         }
 279
 280         [Obsolete]
 281         public string ExpandUrlHtml(string html)
 282         {
 283             return this.ExpandUrlHtmlAsync(html, 10).Result;
 284         }
 285
 286         /// <summary>
 287         /// HTML内に含まれるリンクのURLを非同期に展開する
 288         /// </summary>
 289         /// <param name="html">処理対象のHTML</param>
 290         /// <returns>URLの展開を行うタスク</returns>
 291         public Task<string> ExpandUrlHtmlAsync(string html)
 292         {
 293             return this.ExpandUrlHtmlAsync(html, 10);
 294         }
 295
 296         /// <summary>
 297         /// HTML内に含まれるリンクのURLを非同期に展開する
 298         /// </summary>
 299         /// <param name="html">処理対象のHTML</param>
 300         /// <param name="redirectLimit">再帰的に展開を試みる上限</param>
 301         /// <returns>URLの展開を行うタスク</returns>
 302         public Task<string> ExpandUrlHtmlAsync(string html, int redirectLimit)
 303         {
 304             if (this.DisableExpanding)
 305                 return Task.FromResult(html);
 306
 307             return HtmlLinkPattern.ReplaceAsync(html, async m =>
 308                 m.Groups[1].Value + await this.ExpandUrlAsync(m.Groups[2].Value, redirectLimit).ConfigureAwait(false) + m.Groups[3].Value);
 309         }
 310
 311         /// <summary>
 312         /// 指定された短縮URLサービスを使用してURLを短縮します
 313         /// </summary>
 314         /// <param name="shortenerType">使用する短縮URLサービス</param>
 315         /// <param name="srcUri">短縮するURL</param>
 316         /// <returns>短縮されたURL</returns>
 317         public Task<Uri> ShortenUrlAsync(MyCommon.UrlConverter shortenerType, Uri srcUri)
 318         {
 319             // 既に短縮されている状態のURLであれば短縮しない
 320             if (ShortUrlHosts.Contains(srcUri.Host))
 321                 return Task.FromResult(srcUri);
 322
 323             switch (shortenerType)
 324             {
 325                 case MyCommon.UrlConverter.TinyUrl:
 326                     return this.ShortenByTinyUrlAsync(srcUri);
 327                 case MyCommon.UrlConverter.Isgd:
 328                     return this.ShortenByIsgdAsync(srcUri);
 329                 case MyCommon.UrlConverter.Twurl:
 330                     return this.ShortenByTwurlAsync(srcUri);
 331                 case MyCommon.UrlConverter.Bitly:
 332                     return this.ShortenByBitlyAsync(srcUri, "bit.ly");
 333                 case MyCommon.UrlConverter.Jmp:
 334                     return this.ShortenByBitlyAsync(srcUri, "j.mp");
 335                 case MyCommon.UrlConverter.Uxnu:
 336                     return this.ShortenByUxnuAsync(srcUri);
 337                 default:
 338                     throw new ArgumentException("Unknown shortener.", "shortenerType");
 339             }
 340         }
 341
 342         private async Task<Uri> ShortenByTinyUrlAsync(Uri srcUri)
 343         {
 344             // 明らかに長くなると推測できる場合は短縮しない
 345             if ("http://tinyurl.com/xxxxxx".Length > srcUri.OriginalString.Length)
 346                 return srcUri;
 347
 348             var content = new FormUrlEncodedContent(new[]
 349             {
 350                 new KeyValuePair<string, string>("url", srcUri.OriginalString),
 351             });
 352
 353             using (var response = await this.http.PostAsync("http://tinyurl.com/api-create.php", content).ConfigureAwait(false))
 354             {
 355                 response.EnsureSuccessStatusCode();
 356
 357                 var result = await response.Content.ReadAsStringAsync()
 358                     .ConfigureAwait(false);
 359
 360                 if (!Regex.IsMatch(result, @"^https?://"))
 361                     throw new WebApiException("Failed to create URL.", result);
 362
 363                 return new Uri(result.TrimEnd());
 364             }
 365         }
 366
 367         private async Task<Uri> ShortenByIsgdAsync(Uri srcUri)
 368         {
 369             // 明らかに長くなると推測できる場合は短縮しない
 370             if ("http://is.gd/xxxx".Length > srcUri.OriginalString.Length)
 371                 return srcUri;
 372
 373             var content = new FormUrlEncodedContent(new[]
 374             {
 375                 new KeyValuePair<string, string>("format", "simple"),
 376                 new KeyValuePair<string, string>("url", srcUri.OriginalString),
 377             });
 378
 379             using (var response = await this.http.PostAsync("http://is.gd/create.php", content).ConfigureAwait(false))
 380             {
 381                 response.EnsureSuccessStatusCode();
 382
 383                 var result = await response.Content.ReadAsStringAsync()
 384                     .ConfigureAwait(false);
 385
 386                 if (!Regex.IsMatch(result, @"^https?://"))
 387                     throw new WebApiException("Failed to create URL.", result);
 388
 389                 return new Uri(result.TrimEnd());
 390             }
 391         }
 392
 393         private async Task<Uri> ShortenByTwurlAsync(Uri srcUri)
 394         {
 395             // 明らかに長くなると推測できる場合は短縮しない
 396             if ("http://twurl.nl/xxxxxx".Length > srcUri.OriginalString.Length)
 397                 return srcUri;
 398
 399             var content = new FormUrlEncodedContent(new[]
 400             {
 401                 new KeyValuePair<string, string>("link[url]", srcUri.OriginalString),
 402             });
 403
 404             using (var response = await this.http.PostAsync("http://tweetburner.com/links", content).ConfigureAwait(false))
 405             {
 406                 response.EnsureSuccessStatusCode();
 407
 408                 var result = await response.Content.ReadAsStringAsync()
 409                     .ConfigureAwait(false);
 410
 411                 if (!Regex.IsMatch(result, @"^https?://"))
 412                     throw new WebApiException("Failed to create URL.", result);
 413
 414                 return new Uri(result.TrimEnd());
 415             }
 416         }
 417
 418         private async Task<Uri> ShortenByBitlyAsync(Uri srcUri, string domain = "bit.ly")
 419         {
 420             // 明らかに長くなると推測できる場合は短縮しない
 421             if ("http://bit.ly/xxxx".Length > srcUri.OriginalString.Length)
 422                 return srcUri;
 423
 424             // bit.ly 短縮機能実装のプライバシー問題の暫定対応
 425             // ログインIDとAPIキーが指定されていない場合は短縮せずにPOSTする
 426             // 参照: http://sourceforge.jp/projects/opentween/lists/archive/dev/2012-January/000020.html
 427             if (string.IsNullOrEmpty(this.BitlyId) || string.IsNullOrEmpty(this.BitlyKey))
 428                 return srcUri;
 429
 430             var query = new Dictionary<string, string>
 431             {
 432                 {"login", this.BitlyId},
 433                 {"apiKey", this.BitlyKey},
 434                 {"format", "txt"},
 435                 {"domain", domain},
 436                 {"longUrl", srcUri.OriginalString},
 437             };
 438
 439             var uri = new Uri("https://api-ssl.bitly.com/v3/shorten?" + MyCommon.BuildQueryString(query));
 440             using (var response = await this.http.GetAsync(uri).ConfigureAwait(false))
 441             {
 442                 response.EnsureSuccessStatusCode();
 443
 444                 var result = await response.Content.ReadAsStringAsync()
 445                     .ConfigureAwait(false);
 446
 447                 if (!Regex.IsMatch(result, @"^https?://"))
 448                     throw new WebApiException("Failed to create URL.", result);
 449
 450                 return new Uri(result.TrimEnd());
 451             }
 452         }
 453
 454         private async Task<Uri> ShortenByUxnuAsync(Uri srcUri)
 455         {
 456             // 明らかに長くなると推測できる場合は短縮しない
 457             if ("http://ux.nx/xxxxxx".Length > srcUri.OriginalString.Length)
 458                 return srcUri;
 459
 460             var query = new Dictionary<string, string>
 461             {
 462                 {"format", "plain"},
 463                 {"url", srcUri.OriginalString},
 464             };
 465
 466             var uri = new Uri("http://ux.nu/api/short?" + MyCommon.BuildQueryString(query));
 467             using (var response = await this.http.GetAsync(uri).ConfigureAwait(false))
 468             {
 469                 response.EnsureSuccessStatusCode();
 470
 471                 var result = await response.Content.ReadAsStringAsync()
 472                     .ConfigureAwait(false);
 473
 474                 if (!Regex.IsMatch(result, @"^https?://"))
 475                     throw new WebApiException("Failed to create URL.", result);
 476
 477                 return new Uri(result.TrimEnd());
 478             }
 479         }
 480
 481         private bool IsIrregularShortUrl(Uri uri)
 482         {
 483             // Flickrの https://www.flickr.com/photo.gne?short=... 形式のURL
 484             // flic.kr ドメインのURLを展開する途中に経由する
 485             if (uri.Host.EndsWith("flickr.com", StringComparison.OrdinalIgnoreCase) &&
 486                 uri.PathAndQuery.StartsWith("/photo.gne", StringComparison.OrdinalIgnoreCase))
 487                 return true;
 488
 489             return false;
 490         }
 491
 492         private async Task<Uri> GetRedirectTo(Uri url)
 493         {
 494             var request = new HttpRequestMessage(HttpMethod.Head, url);
 495
 496             using (var response = await this.http.SendAsync(request).ConfigureAwait(false))
 497             {
 498                 if (!response.IsSuccessStatusCode)
 499                 {
 500                     // ステータスコードが 3xx であれば例外を発生させない
 501                     if ((int)response.StatusCode / 100 != 3)
 502                         response.EnsureSuccessStatusCode();
 503                 }
 504
 505                 var redirectedUrl = response.Headers.Location;
 506
 507                 if (redirectedUrl == null)
 508                     return null;
 509
 510                 if (redirectedUrl.IsAbsoluteUri)
 511                     return redirectedUrl;
 512                 else
 513                     return new Uri(url, redirectedUrl);
 514             }
 515         }
 516
 517         [SuppressMessage("Microsoft.Reliability", "CA2000:DisposeObjectsBeforeLosingScope")]
 518         private static HttpClient CreateDefaultHttpClient()
 519         {
 520             var handler = new HttpClientHandler
 521             {
 522                 AllowAutoRedirect = false,
 523             };
 524
 525             var http = Networking.CreateHttpClient(handler);
 526             http.Timeout = TimeSpan.FromSeconds(5);
 527
 528             return http;
 529         }
 530     }
 531 }