OpenTween/ShortUrl.cs

   1 // OpenTween - Client of Twitter
   2 // Copyright (c) 2007-2011 kiri_feather (@kiri_feather) <kiri.feather@gmail.com>
   3 //           (c) 2008-2011 Moz (@syo68k)
   4 //           (c) 2008-2011 takeshik (@takeshik) <http://www.takeshik.org/>
   5 //           (c) 2010-2011 anis774 (@anis774) <http://d.hatena.ne.jp/anis774/>
   6 //           (c) 2010-2011 fantasticswallow (@f_swallow) <http://twitter.com/f_swallow>
   7 //           (c) 2011      kim_upsilon (@kim_upsilon) <https://upsilo.net/~upsilon/>
   8 // All rights reserved.
   9 //
  10 // This file is part of OpenTween.
  11 //
  12 // This program is free software; you can redistribute it and/or modify it
  13 // under the terms of the GNU General public License as published by the Free
  14 // Software Foundation; either version 3 of the License, or (at your option)
  15 // any later version.
  16 //
  17 // This program is distributed in the hope that it will be useful, but
  18 // WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
  19 // or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General public License
  20 // for more details.
  21 //
  22 // You should have received a copy of the GNU General public License along
  23 // with this program. If not, see <http://www.gnu.org/licenses/>, or write to
  24 // the Free Software Foundation, Inc., 51 Franklin Street - Fifth Floor,
  25 // Boston, MA 02110-1301, USA.
  26
  27 using System;
  28 using System.Collections.Concurrent;
  29 using System.Collections.Generic;
  30 using System.Diagnostics.CodeAnalysis;
  31 using System.Linq;
  32 using System.Net.Http;
  33 using System.Text;
  34 using System.Text.RegularExpressions;
  35 using System.Threading;
  36 using System.Threading.Tasks;
  37 using System.Web;
  38 using OpenTween.Api;
  39 using OpenTween.Connection;
  40
  41 namespace OpenTween
  42 {
  43     /// <summary>
  44     /// 短縮 URL サービスによる URL の展開・短縮を行うクラス
  45     /// </summary>
  46     public class ShortUrl
  47     {
  48         private static Lazy<ShortUrl> _instance;
  49
  50         /// <summary>
  51         /// ShortUrl のインスタンスを取得します
  52         /// </summary>
  53         public static ShortUrl Instance
  54         {
  55             get { return _instance.Value; }
  56         }
  57
  58         /// <summary>
  59         /// 短縮 URL の展開を無効にするか否か
  60         /// </summary>
  61         public bool DisableExpanding { get; set; }
  62
  63         /// <summary>
  64         /// 短縮 URL のキャッシュを定期的にクリアする回数
  65         /// </summary>
  66         public int PurgeCount { get; set; }
  67
  68         public string BitlyAccessToken { get; set; }
  69         public string BitlyId { get; set; }
  70         public string BitlyKey { get; set; }
  71
  72         private HttpClient http;
  73         private ConcurrentDictionary<Uri, Uri> urlCache = new ConcurrentDictionary<Uri, Uri>();
  74
  75         private static readonly Regex HtmlLinkPattern = new Regex(@"(<a href="")(.+?)("")");
  76
  77         private static readonly HashSet<string> ShortUrlHosts = new HashSet<string>
  78         {
  79             "4sq.com",
  80             "airme.us",
  81             "amzn.to",
  82             "bctiny.com",
  83             "bit.ly",
  84             "bkite.com",
  85             "blip.fm",
  86             "budurl.com",
  87             "buff.ly",
  88             "cli.gs",
  89             "digg.com",
  90             "disq.us",
  91             "dlvr.it",
  92             "fb.me",
  93             "feedly.com",
  94             "feeds.feedburner.com",
  95             "ff.im",
  96             "flic.kr",
  97             "goo.gl",
  98             "ht.ly",
  99             "htn.to",
 100             "icanhaz.com",
 101             "ift.tt",
 102             "is.gd",
 103             "j.mp",
 104             "linkbee.com",
 105             "moby.to",
 106             "moi.st",
 107             "nico.ms",
 108             "nsfw.in",
 109             "on.fb.me",
 110             "ow.ly",
 111             "p.tl",
 112             "pic.gd",
 113             "qurl.com",
 114             "rubyurl.com",
 115             "snipurl.com",
 116             "snurl.com",
 117             "t.co",
 118             "tinami.jp",
 119             "tiny.cc",
 120             "tinyurl.com",
 121             "tl.gd",
 122             "tmblr.co",
 123             "traceurl.com",
 124             "tumblr.com",
 125             "twitthis.com",
 126             "twme.jp",
 127             "twurl.nl",
 128             "u-rl.jp",
 129             "urlenco.de",
 130             "urx2.nu",
 131             "ustre.am",
 132             "ux.nu",
 133             "wp.me",
 134             "www.qurl.com",
 135             "www.tumblr.com",
 136             "youtu.be",
 137         };
 138
 139         static ShortUrl()
 140         {
 141             _instance = new Lazy<ShortUrl>(() => new ShortUrl(), true);
 142         }
 143
 144         [SuppressMessage("Microsoft.Reliability", "CA2000:DisposeObjectsBeforeLosingScope")]
 145         internal ShortUrl()
 146             : this(CreateDefaultHttpClient())
 147         {
 148             Networking.WebProxyChanged += (o, e) =>
 149             {
 150                 var newClient = CreateDefaultHttpClient();
 151                 var oldClient = Interlocked.Exchange(ref this.http, newClient);
 152                 oldClient.Dispose();
 153             };
 154         }
 155
 156         internal ShortUrl(HttpClient http)
 157         {
 158             this.DisableExpanding = false;
 159             this.PurgeCount = 500;
 160             this.BitlyId = "";
 161             this.BitlyKey = "";
 162
 163             this.http = http;
 164         }
 165
 166         [Obsolete]
 167         public string ExpandUrl(string uri)
 168         {
 169             try
 170             {
 171                 return this.ExpandUrlAsync(new Uri(uri), 10).Result.AbsoluteUri;
 172             }
 173             catch (UriFormatException)
 174             {
 175                 return uri;
 176             }
 177         }
 178
 179         /// <summary>
 180         /// 短縮 URL を非同期に展開します
 181         /// </summary>
 182         /// <param name="uri">展開するURL</param>
 183         /// <returns>URLの展開を行うタスク</returns>
 184         public Task<Uri> ExpandUrlAsync(Uri uri)
 185         {
 186             return this.ExpandUrlAsync(uri, 10);
 187         }
 188
 189         /// <summary>
 190         /// 短縮 URL を非同期に展開します
 191         /// </summary>
 192         /// <param name="uri">展開するURL</param>
 193         /// <param name="redirectLimit">再帰的に展開を試みる上限</param>
 194         /// <returns>URLの展開を行うタスク</returns>
 195         public async Task<Uri> ExpandUrlAsync(Uri uri, int redirectLimit)
 196         {
 197             if (this.DisableExpanding)
 198                 return uri;
 199
 200             if (redirectLimit <= 0)
 201                 return uri;
 202
 203             if (!uri.IsAbsoluteUri)
 204                 return uri;
 205
 206             try
 207             {
 208                 if (!ShortUrlHosts.Contains(uri.Host) && !IsIrregularShortUrl(uri))
 209                     return uri;
 210
 211                 if (this.urlCache.TryGetValue(uri, out var expanded))
 212                     return expanded;
 213
 214                 if (this.urlCache.Count > this.PurgeCount)
 215                     this.urlCache.Clear();
 216
 217                 expanded = null;
 218                 try
 219                 {
 220                     expanded = await this.GetRedirectTo(uri)
 221                         .ConfigureAwait(false);
 222                 }
 223                 catch (TaskCanceledException) { }
 224                 catch (HttpRequestException) { }
 225
 226                 if (expanded == null || expanded == uri)
 227                     return uri;
 228
 229                 this.urlCache[uri] = expanded;
 230
 231                 var recursiveExpanded = await this.ExpandUrlAsync(expanded, --redirectLimit)
 232                     .ConfigureAwait(false);
 233
 234                 // URL1 -> URL2 -> URL3 のように再帰的に展開されたURLを URL1 -> URL3 としてキャッシュに格納する
 235                 if (recursiveExpanded != expanded)
 236                     this.urlCache[uri] = recursiveExpanded;
 237
 238                 return recursiveExpanded;
 239             }
 240             catch (UriFormatException)
 241             {
 242                 return uri;
 243             }
 244         }
 245
 246         /// <summary>
 247         /// 短縮 URL を非同期に展開します
 248         /// </summary>
 249         /// <remarks>
 250         /// 不正なURLが渡された場合は例外を投げず uriStr をそのまま返します
 251         /// </remarks>
 252         /// <param name="uriStr">展開するURL</param>
 253         /// <returns>URLの展開を行うタスク</returns>
 254         public Task<string> ExpandUrlAsync(string uriStr)
 255         {
 256             return this.ExpandUrlAsync(uriStr, 10);
 257         }
 258
 259         /// <summary>
 260         /// 短縮 URL を非同期に展開します
 261         /// </summary>
 262         /// <remarks>
 263         /// 不正なURLが渡された場合は例外を投げず uriStr をそのまま返します
 264         /// </remarks>
 265         /// <param name="uriStr">展開するURL</param>
 266         /// <param name="redirectLimit">再帰的に展開を試みる上限</param>
 267         /// <returns>URLの展開を行うタスク</returns>
 268         public async Task<string> ExpandUrlAsync(string uriStr, int redirectLimit)
 269         {
 270             Uri uri;
 271
 272             try
 273             {
 274                 if (!uriStr.StartsWith("http", StringComparison.OrdinalIgnoreCase))
 275                     uri = new Uri("http://" + uriStr);
 276                 else
 277                     uri = new Uri(uriStr);
 278             }
 279             catch (UriFormatException)
 280             {
 281                 return uriStr;
 282             }
 283
 284             var expandedUri = await this.ExpandUrlAsync(uri, redirectLimit)
 285                 .ConfigureAwait(false);
 286
 287             return expandedUri.OriginalString;
 288         }
 289
 290         [Obsolete]
 291         public string ExpandUrlHtml(string html)
 292         {
 293             return this.ExpandUrlHtmlAsync(html, 10).Result;
 294         }
 295
 296         /// <summary>
 297         /// HTML内に含まれるリンクのURLを非同期に展開する
 298         /// </summary>
 299         /// <param name="html">処理対象のHTML</param>
 300         /// <returns>URLの展開を行うタスク</returns>
 301         public Task<string> ExpandUrlHtmlAsync(string html)
 302         {
 303             return this.ExpandUrlHtmlAsync(html, 10);
 304         }
 305
 306         /// <summary>
 307         /// HTML内に含まれるリンクのURLを非同期に展開する
 308         /// </summary>
 309         /// <param name="html">処理対象のHTML</param>
 310         /// <param name="redirectLimit">再帰的に展開を試みる上限</param>
 311         /// <returns>URLの展開を行うタスク</returns>
 312         public Task<string> ExpandUrlHtmlAsync(string html, int redirectLimit)
 313         {
 314             if (this.DisableExpanding)
 315                 return Task.FromResult(html);
 316
 317             return HtmlLinkPattern.ReplaceAsync(html, async m =>
 318                 m.Groups[1].Value + await this.ExpandUrlAsync(m.Groups[2].Value, redirectLimit).ConfigureAwait(false) + m.Groups[3].Value);
 319         }
 320
 321         /// <summary>
 322         /// 指定された短縮URLサービスを使用してURLを短縮します
 323         /// </summary>
 324         /// <param name="shortenerType">使用する短縮URLサービス</param>
 325         /// <param name="srcUri">短縮するURL</param>
 326         /// <returns>短縮されたURL</returns>
 327         public async Task<Uri> ShortenUrlAsync(MyCommon.UrlConverter shortenerType, Uri srcUri)
 328         {
 329             // 既に短縮されている状態のURLであれば短縮しない
 330             if (ShortUrlHosts.Contains(srcUri.Host))
 331                 return srcUri;
 332
 333             try
 334             {
 335                 switch (shortenerType)
 336                 {
 337                     case MyCommon.UrlConverter.TinyUrl:
 338                         return await this.ShortenByTinyUrlAsync(srcUri)
 339                             .ConfigureAwait(false);
 340                     case MyCommon.UrlConverter.Isgd:
 341                         return await this.ShortenByIsgdAsync(srcUri)
 342                             .ConfigureAwait(false);
 343                     case MyCommon.UrlConverter.Twurl:
 344                         return await this.ShortenByTwurlAsync(srcUri)
 345                             .ConfigureAwait(false);
 346                     case MyCommon.UrlConverter.Bitly:
 347                         return await this.ShortenByBitlyAsync(srcUri, "bit.ly")
 348                             .ConfigureAwait(false);
 349                     case MyCommon.UrlConverter.Jmp:
 350                         return await this.ShortenByBitlyAsync(srcUri, "j.mp")
 351                             .ConfigureAwait(false);
 352                     case MyCommon.UrlConverter.Uxnu:
 353                         return await this.ShortenByUxnuAsync(srcUri)
 354                             .ConfigureAwait(false);
 355                     default:
 356                         throw new ArgumentException("Unknown shortener.", nameof(shortenerType));
 357                 }
 358             }
 359             catch (OperationCanceledException)
 360             {
 361                 // 短縮 URL の API がタイムアウトした場合
 362                 return srcUri;
 363             }
 364         }
 365
 366         private async Task<Uri> ShortenByTinyUrlAsync(Uri srcUri)
 367         {
 368             // 明らかに長くなると推測できる場合は短縮しない
 369             if ("http://tinyurl.com/xxxxxx".Length > srcUri.OriginalString.Length)
 370                 return srcUri;
 371
 372             var content = new FormUrlEncodedContent(new[]
 373             {
 374                 new KeyValuePair<string, string>("url", srcUri.OriginalString),
 375             });
 376
 377             using (var response = await this.http.PostAsync("http://tinyurl.com/api-create.php", content).ConfigureAwait(false))
 378             {
 379                 response.EnsureSuccessStatusCode();
 380
 381                 var result = await response.Content.ReadAsStringAsync()
 382                     .ConfigureAwait(false);
 383
 384                 if (!Regex.IsMatch(result, @"^https?://"))
 385                     throw new WebApiException("Failed to create URL.", result);
 386
 387                 return new Uri(result.TrimEnd());
 388             }
 389         }
 390
 391         private async Task<Uri> ShortenByIsgdAsync(Uri srcUri)
 392         {
 393             // 明らかに長くなると推測できる場合は短縮しない
 394             if ("http://is.gd/xxxx".Length > srcUri.OriginalString.Length)
 395                 return srcUri;
 396
 397             var content = new FormUrlEncodedContent(new[]
 398             {
 399                 new KeyValuePair<string, string>("format", "simple"),
 400                 new KeyValuePair<string, string>("url", srcUri.OriginalString),
 401             });
 402
 403             using (var response = await this.http.PostAsync("http://is.gd/create.php", content).ConfigureAwait(false))
 404             {
 405                 response.EnsureSuccessStatusCode();
 406
 407                 var result = await response.Content.ReadAsStringAsync()
 408                     .ConfigureAwait(false);
 409
 410                 if (!Regex.IsMatch(result, @"^https?://"))
 411                     throw new WebApiException("Failed to create URL.", result);
 412
 413                 return new Uri(result.TrimEnd());
 414             }
 415         }
 416
 417         private async Task<Uri> ShortenByTwurlAsync(Uri srcUri)
 418         {
 419             // 明らかに長くなると推測できる場合は短縮しない
 420             if ("http://twurl.nl/xxxxxx".Length > srcUri.OriginalString.Length)
 421                 return srcUri;
 422
 423             var content = new FormUrlEncodedContent(new[]
 424             {
 425                 new KeyValuePair<string, string>("link[url]", srcUri.OriginalString),
 426             });
 427
 428             using (var response = await this.http.PostAsync("http://tweetburner.com/links", content).ConfigureAwait(false))
 429             {
 430                 response.EnsureSuccessStatusCode();
 431
 432                 var result = await response.Content.ReadAsStringAsync()
 433                     .ConfigureAwait(false);
 434
 435                 if (!Regex.IsMatch(result, @"^https?://"))
 436                     throw new WebApiException("Failed to create URL.", result);
 437
 438                 return new Uri(result.TrimEnd());
 439             }
 440         }
 441
 442         private async Task<Uri> ShortenByBitlyAsync(Uri srcUri, string domain = "bit.ly")
 443         {
 444             // 明らかに長くなると推測できる場合は短縮しない
 445             if ("http://bit.ly/xxxx".Length > srcUri.OriginalString.Length)
 446                 return srcUri;
 447
 448             // OAuth2 アクセストークンまたは API キー (旧方式) のいずれも設定されていなければ短縮しない
 449             if (string.IsNullOrEmpty(this.BitlyAccessToken) && (string.IsNullOrEmpty(this.BitlyId) || string.IsNullOrEmpty(this.BitlyKey)))
 450                 return srcUri;
 451
 452             var bitly = new BitlyApi
 453             {
 454                 EndUserAccessToken = this.BitlyAccessToken,
 455                 EndUserLoginName = this.BitlyId,
 456                 EndUserApiKey = this.BitlyKey,
 457             };
 458
 459             return await bitly.ShortenAsync(srcUri, domain)
 460                 .ConfigureAwait(false);
 461         }
 462
 463         private async Task<Uri> ShortenByUxnuAsync(Uri srcUri)
 464         {
 465             // 明らかに長くなると推測できる場合は短縮しない
 466             if ("http://ux.nx/xxxxxx".Length > srcUri.OriginalString.Length)
 467                 return srcUri;
 468
 469             var query = new Dictionary<string, string>
 470             {
 471                 ["format"] = "plain",
 472                 ["url"] = srcUri.OriginalString,
 473             };
 474
 475             var uri = new Uri("http://ux.nu/api/short?" + MyCommon.BuildQueryString(query));
 476             using (var response = await this.http.GetAsync(uri).ConfigureAwait(false))
 477             {
 478                 response.EnsureSuccessStatusCode();
 479
 480                 var result = await response.Content.ReadAsStringAsync()
 481                     .ConfigureAwait(false);
 482
 483                 if (!Regex.IsMatch(result, @"^https?://"))
 484                     throw new WebApiException("Failed to create URL.", result);
 485
 486                 return new Uri(result.TrimEnd());
 487             }
 488         }
 489
 490         private bool IsIrregularShortUrl(Uri uri)
 491         {
 492             // Flickrの https://www.flickr.com/photo.gne?short=... 形式のURL
 493             // flic.kr ドメインのURLを展開する途中に経由する
 494             if (uri.Host.EndsWith("flickr.com", StringComparison.OrdinalIgnoreCase) &&
 495                 uri.PathAndQuery.StartsWith("/photo.gne", StringComparison.OrdinalIgnoreCase))
 496                 return true;
 497
 498             return false;
 499         }
 500
 501         private async Task<Uri> GetRedirectTo(Uri url)
 502         {
 503             var request = new HttpRequestMessage(HttpMethod.Head, url);
 504
 505             using (var response = await this.http.SendAsync(request).ConfigureAwait(false))
 506             {
 507                 if (!response.IsSuccessStatusCode)
 508                 {
 509                     // ステータスコードが 3xx であれば例外を発生させない
 510                     if ((int)response.StatusCode / 100 != 3)
 511                         response.EnsureSuccessStatusCode();
 512                 }
 513
 514                 var redirectedUrl = response.Headers.Location;
 515
 516                 if (redirectedUrl == null)
 517                     return null;
 518
 519                 // サーバーが URL を適切にエンコードしていない場合、OriginalString に非 ASCII 文字が含まれる。
 520                 // その場合、redirectedUrl は文字化けしている可能性があるため使用しない
 521                 // 参照: http://stackoverflow.com/questions/1888933
 522                 if (redirectedUrl.OriginalString.Any(x => x < ' ' || x > '~'))
 523                     return null;
 524
 525                 if (redirectedUrl.IsAbsoluteUri)
 526                     return redirectedUrl;
 527                 else
 528                     return new Uri(url, redirectedUrl);
 529             }
 530         }
 531
 532         [SuppressMessage("Microsoft.Reliability", "CA2000:DisposeObjectsBeforeLosingScope")]
 533         private static HttpClient CreateDefaultHttpClient()
 534         {
 535             var handler = Networking.CreateHttpClientHandler();
 536             handler.AllowAutoRedirect = false;
 537
 538             var http = Networking.CreateHttpClient(handler);
 539             http.Timeout = TimeSpan.FromSeconds(30);
 540
 541             return http;
 542         }
 543     }
 544 }