OpenTween/ShortUrl.cs

   1 // OpenTween - Client of Twitter
   2 // Copyright (c) 2007-2011 kiri_feather (@kiri_feather) <kiri.feather@gmail.com>
   3 //           (c) 2008-2011 Moz (@syo68k)
   4 //           (c) 2008-2011 takeshik (@takeshik) <http://www.takeshik.org/>
   5 //           (c) 2010-2011 anis774 (@anis774) <http://d.hatena.ne.jp/anis774/>
   6 //           (c) 2010-2011 fantasticswallow (@f_swallow) <http://twitter.com/f_swallow>
   7 //           (c) 2011      kim_upsilon (@kim_upsilon) <https://upsilo.net/~upsilon/>
   8 // All rights reserved.
   9 //
  10 // This file is part of OpenTween.
  11 //
  12 // This program is free software; you can redistribute it and/or modify it
  13 // under the terms of the GNU General public License as published by the Free
  14 // Software Foundation; either version 3 of the License, or (at your option)
  15 // any later version.
  16 //
  17 // This program is distributed in the hope that it will be useful, but
  18 // WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
  19 // or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General public License
  20 // for more details.
  21 //
  22 // You should have received a copy of the GNU General public License along
  23 // with this program. If not, see <http://www.gnu.org/licenses/>, or write to
  24 // the Free Software Foundation, Inc., 51 Franklin Street - Fifth Floor,
  25 // Boston, MA 02110-1301, USA.
  26
  27 using System;
  28 using System.Collections.Concurrent;
  29 using System.Collections.Generic;
  30 using System.Diagnostics.CodeAnalysis;
  31 using System.Linq;
  32 using System.Net.Http;
  33 using System.Text;
  34 using System.Text.RegularExpressions;
  35 using System.Threading;
  36 using System.Threading.Tasks;
  37 using System.Web;
  38 using OpenTween.Api;
  39 using OpenTween.Connection;
  40
  41 namespace OpenTween
  42 {
  43     /// <summary>
  44     /// 短縮 URL サービスによる URL の展開・短縮を行うクラス
  45     /// </summary>
  46     public class ShortUrl
  47     {
  48         private static Lazy<ShortUrl> _instance;
  49
  50         /// <summary>
  51         /// ShortUrl のインスタンスを取得します
  52         /// </summary>
  53         public static ShortUrl Instance
  54             => _instance.Value;
  55
  56         /// <summary>
  57         /// 短縮 URL の展開を無効にするか否か
  58         /// </summary>
  59         public bool DisableExpanding { get; set; }
  60
  61         /// <summary>
  62         /// 短縮 URL のキャッシュを定期的にクリアする回数
  63         /// </summary>
  64         public int PurgeCount { get; set; }
  65
  66         public string BitlyAccessToken { get; set; }
  67         public string BitlyId { get; set; }
  68         public string BitlyKey { get; set; }
  69
  70         private HttpClient http;
  71         private ConcurrentDictionary<Uri, Uri> urlCache = new ConcurrentDictionary<Uri, Uri>();
  72
  73         private static readonly Regex HtmlLinkPattern = new Regex(@"(<a href="")(.+?)("")");
  74
  75         private static readonly HashSet<string> ShortUrlHosts = new HashSet<string>
  76         {
  77             "4sq.com",
  78             "airme.us",
  79             "amzn.to",
  80             "bctiny.com",
  81             "bit.ly",
  82             "bkite.com",
  83             "blip.fm",
  84             "budurl.com",
  85             "buff.ly",
  86             "cli.gs",
  87             "digg.com",
  88             "disq.us",
  89             "dlvr.it",
  90             "fb.me",
  91             "feedly.com",
  92             "feeds.feedburner.com",
  93             "ff.im",
  94             "flic.kr",
  95             "goo.gl",
  96             "ht.ly",
  97             "htn.to",
  98             "icanhaz.com",
  99             "ift.tt",
 100             "is.gd",
 101             "j.mp",
 102             "linkbee.com",
 103             "moby.to",
 104             "moi.st",
 105             "nico.ms",
 106             "nsfw.in",
 107             "on.fb.me",
 108             "ow.ly",
 109             "p.tl",
 110             "pic.gd",
 111             "qurl.com",
 112             "rubyurl.com",
 113             "snipurl.com",
 114             "snurl.com",
 115             "t.co",
 116             "tinami.jp",
 117             "tiny.cc",
 118             "tinyurl.com",
 119             "tl.gd",
 120             "tmblr.co",
 121             "traceurl.com",
 122             "tumblr.com",
 123             "twitthis.com",
 124             "twme.jp",
 125             "twurl.nl",
 126             "u-rl.jp",
 127             "urlenco.de",
 128             "urx2.nu",
 129             "ustre.am",
 130             "ux.nu",
 131             "wp.me",
 132             "www.qurl.com",
 133             "www.tumblr.com",
 134             "youtu.be",
 135         };
 136
 137         static ShortUrl()
 138             => _instance = new Lazy<ShortUrl>(() => new ShortUrl(), true);
 139
 140         [SuppressMessage("Microsoft.Reliability", "CA2000:DisposeObjectsBeforeLosingScope")]
 141         internal ShortUrl()
 142             : this(CreateDefaultHttpClient())
 143         {
 144             Networking.WebProxyChanged += (o, e) =>
 145             {
 146                 var newClient = CreateDefaultHttpClient();
 147                 var oldClient = Interlocked.Exchange(ref this.http, newClient);
 148                 oldClient.Dispose();
 149             };
 150         }
 151
 152         internal ShortUrl(HttpClient http)
 153         {
 154             this.DisableExpanding = false;
 155             this.PurgeCount = 500;
 156             this.BitlyId = "";
 157             this.BitlyKey = "";
 158
 159             this.http = http;
 160         }
 161
 162         [Obsolete]
 163         public string ExpandUrl(string uri)
 164         {
 165             try
 166             {
 167                 return this.ExpandUrlAsync(new Uri(uri), 10).Result.AbsoluteUri;
 168             }
 169             catch (UriFormatException)
 170             {
 171                 return uri;
 172             }
 173         }
 174
 175         /// <summary>
 176         /// 短縮 URL を非同期に展開します
 177         /// </summary>
 178         /// <param name="uri">展開するURL</param>
 179         /// <returns>URLの展開を行うタスク</returns>
 180         public Task<Uri> ExpandUrlAsync(Uri uri)
 181             => this.ExpandUrlAsync(uri, 10);
 182
 183         /// <summary>
 184         /// 短縮 URL を非同期に展開します
 185         /// </summary>
 186         /// <param name="uri">展開するURL</param>
 187         /// <param name="redirectLimit">再帰的に展開を試みる上限</param>
 188         /// <returns>URLの展開を行うタスク</returns>
 189         public async Task<Uri> ExpandUrlAsync(Uri uri, int redirectLimit)
 190         {
 191             if (this.DisableExpanding)
 192                 return uri;
 193
 194             if (redirectLimit <= 0)
 195                 return uri;
 196
 197             if (!uri.IsAbsoluteUri)
 198                 return uri;
 199
 200             try
 201             {
 202                 if (!ShortUrlHosts.Contains(uri.Host) && !IsIrregularShortUrl(uri))
 203                     return uri;
 204
 205                 if (this.urlCache.TryGetValue(uri, out var expanded))
 206                     return expanded;
 207
 208                 if (this.urlCache.Count > this.PurgeCount)
 209                     this.urlCache.Clear();
 210
 211                 expanded = null;
 212                 try
 213                 {
 214                     expanded = await this.GetRedirectTo(uri)
 215                         .ConfigureAwait(false);
 216                 }
 217                 catch (TaskCanceledException) { }
 218                 catch (HttpRequestException) { }
 219
 220                 if (expanded == null || expanded == uri)
 221                     return uri;
 222
 223                 this.urlCache[uri] = expanded;
 224
 225                 var recursiveExpanded = await this.ExpandUrlAsync(expanded, --redirectLimit)
 226                     .ConfigureAwait(false);
 227
 228                 // URL1 -> URL2 -> URL3 のように再帰的に展開されたURLを URL1 -> URL3 としてキャッシュに格納する
 229                 if (recursiveExpanded != expanded)
 230                     this.urlCache[uri] = recursiveExpanded;
 231
 232                 return recursiveExpanded;
 233             }
 234             catch (UriFormatException)
 235             {
 236                 return uri;
 237             }
 238         }
 239
 240         /// <summary>
 241         /// 短縮 URL を非同期に展開します
 242         /// </summary>
 243         /// <remarks>
 244         /// 不正なURLが渡された場合は例外を投げず uriStr をそのまま返します
 245         /// </remarks>
 246         /// <param name="uriStr">展開するURL</param>
 247         /// <returns>URLの展開を行うタスク</returns>
 248         public Task<string> ExpandUrlAsync(string uriStr)
 249             => this.ExpandUrlAsync(uriStr, 10);
 250
 251         /// <summary>
 252         /// 短縮 URL を非同期に展開します
 253         /// </summary>
 254         /// <remarks>
 255         /// 不正なURLが渡された場合は例外を投げず uriStr をそのまま返します
 256         /// </remarks>
 257         /// <param name="uriStr">展開するURL</param>
 258         /// <param name="redirectLimit">再帰的に展開を試みる上限</param>
 259         /// <returns>URLの展開を行うタスク</returns>
 260         public async Task<string> ExpandUrlAsync(string uriStr, int redirectLimit)
 261         {
 262             Uri uri;
 263
 264             try
 265             {
 266                 if (!uriStr.StartsWith("http", StringComparison.OrdinalIgnoreCase))
 267                     uri = new Uri("http://" + uriStr);
 268                 else
 269                     uri = new Uri(uriStr);
 270             }
 271             catch (UriFormatException)
 272             {
 273                 return uriStr;
 274             }
 275
 276             var expandedUri = await this.ExpandUrlAsync(uri, redirectLimit)
 277                 .ConfigureAwait(false);
 278
 279             return expandedUri.OriginalString;
 280         }
 281
 282         [Obsolete]
 283         public string ExpandUrlHtml(string html)
 284             => this.ExpandUrlHtmlAsync(html, 10).Result;
 285
 286         /// <summary>
 287         /// HTML内に含まれるリンクのURLを非同期に展開する
 288         /// </summary>
 289         /// <param name="html">処理対象のHTML</param>
 290         /// <returns>URLの展開を行うタスク</returns>
 291         public Task<string> ExpandUrlHtmlAsync(string html)
 292             => this.ExpandUrlHtmlAsync(html, 10);
 293
 294         /// <summary>
 295         /// HTML内に含まれるリンクのURLを非同期に展開する
 296         /// </summary>
 297         /// <param name="html">処理対象のHTML</param>
 298         /// <param name="redirectLimit">再帰的に展開を試みる上限</param>
 299         /// <returns>URLの展開を行うタスク</returns>
 300         public Task<string> ExpandUrlHtmlAsync(string html, int redirectLimit)
 301         {
 302             if (this.DisableExpanding)
 303                 return Task.FromResult(html);
 304
 305             return HtmlLinkPattern.ReplaceAsync(html, async m =>
 306                 m.Groups[1].Value + await this.ExpandUrlAsync(m.Groups[2].Value, redirectLimit).ConfigureAwait(false) + m.Groups[3].Value);
 307         }
 308
 309         /// <summary>
 310         /// 指定された短縮URLサービスを使用してURLを短縮します
 311         /// </summary>
 312         /// <param name="shortenerType">使用する短縮URLサービス</param>
 313         /// <param name="srcUri">短縮するURL</param>
 314         /// <returns>短縮されたURL</returns>
 315         public async Task<Uri> ShortenUrlAsync(MyCommon.UrlConverter shortenerType, Uri srcUri)
 316         {
 317             // 既に短縮されている状態のURLであれば短縮しない
 318             if (ShortUrlHosts.Contains(srcUri.Host))
 319                 return srcUri;
 320
 321             try
 322             {
 323                 switch (shortenerType)
 324                 {
 325                     case MyCommon.UrlConverter.TinyUrl:
 326                         return await this.ShortenByTinyUrlAsync(srcUri)
 327                             .ConfigureAwait(false);
 328                     case MyCommon.UrlConverter.Isgd:
 329                         return await this.ShortenByIsgdAsync(srcUri)
 330                             .ConfigureAwait(false);
 331                     case MyCommon.UrlConverter.Twurl:
 332                         return await this.ShortenByTwurlAsync(srcUri)
 333                             .ConfigureAwait(false);
 334                     case MyCommon.UrlConverter.Bitly:
 335                         return await this.ShortenByBitlyAsync(srcUri, "bit.ly")
 336                             .ConfigureAwait(false);
 337                     case MyCommon.UrlConverter.Jmp:
 338                         return await this.ShortenByBitlyAsync(srcUri, "j.mp")
 339                             .ConfigureAwait(false);
 340                     case MyCommon.UrlConverter.Uxnu:
 341                         return await this.ShortenByUxnuAsync(srcUri)
 342                             .ConfigureAwait(false);
 343                     default:
 344                         throw new ArgumentException("Unknown shortener.", nameof(shortenerType));
 345                 }
 346             }
 347             catch (OperationCanceledException)
 348             {
 349                 // 短縮 URL の API がタイムアウトした場合
 350                 return srcUri;
 351             }
 352         }
 353
 354         private async Task<Uri> ShortenByTinyUrlAsync(Uri srcUri)
 355         {
 356             // 明らかに長くなると推測できる場合は短縮しない
 357             if ("http://tinyurl.com/xxxxxx".Length > srcUri.OriginalString.Length)
 358                 return srcUri;
 359
 360             var content = new FormUrlEncodedContent(new[]
 361             {
 362                 new KeyValuePair<string, string>("url", srcUri.OriginalString),
 363             });
 364
 365             using (var response = await this.http.PostAsync("http://tinyurl.com/api-create.php", content).ConfigureAwait(false))
 366             {
 367                 response.EnsureSuccessStatusCode();
 368
 369                 var result = await response.Content.ReadAsStringAsync()
 370                     .ConfigureAwait(false);
 371
 372                 if (!Regex.IsMatch(result, @"^https?://"))
 373                     throw new WebApiException("Failed to create URL.", result);
 374
 375                 return new Uri(result.TrimEnd());
 376             }
 377         }
 378
 379         private async Task<Uri> ShortenByIsgdAsync(Uri srcUri)
 380         {
 381             // 明らかに長くなると推測できる場合は短縮しない
 382             if ("http://is.gd/xxxx".Length > srcUri.OriginalString.Length)
 383                 return srcUri;
 384
 385             var content = new FormUrlEncodedContent(new[]
 386             {
 387                 new KeyValuePair<string, string>("format", "simple"),
 388                 new KeyValuePair<string, string>("url", srcUri.OriginalString),
 389             });
 390
 391             using (var response = await this.http.PostAsync("http://is.gd/create.php", content).ConfigureAwait(false))
 392             {
 393                 response.EnsureSuccessStatusCode();
 394
 395                 var result = await response.Content.ReadAsStringAsync()
 396                     .ConfigureAwait(false);
 397
 398                 if (!Regex.IsMatch(result, @"^https?://"))
 399                     throw new WebApiException("Failed to create URL.", result);
 400
 401                 return new Uri(result.TrimEnd());
 402             }
 403         }
 404
 405         private async Task<Uri> ShortenByTwurlAsync(Uri srcUri)
 406         {
 407             // 明らかに長くなると推測できる場合は短縮しない
 408             if ("http://twurl.nl/xxxxxx".Length > srcUri.OriginalString.Length)
 409                 return srcUri;
 410
 411             var content = new FormUrlEncodedContent(new[]
 412             {
 413                 new KeyValuePair<string, string>("link[url]", srcUri.OriginalString),
 414             });
 415
 416             using (var response = await this.http.PostAsync("http://tweetburner.com/links", content).ConfigureAwait(false))
 417             {
 418                 response.EnsureSuccessStatusCode();
 419
 420                 var result = await response.Content.ReadAsStringAsync()
 421                     .ConfigureAwait(false);
 422
 423                 if (!Regex.IsMatch(result, @"^https?://"))
 424                     throw new WebApiException("Failed to create URL.", result);
 425
 426                 return new Uri(result.TrimEnd());
 427             }
 428         }
 429
 430         private async Task<Uri> ShortenByBitlyAsync(Uri srcUri, string domain = "bit.ly")
 431         {
 432             // 明らかに長くなると推測できる場合は短縮しない
 433             if ("http://bit.ly/xxxx".Length > srcUri.OriginalString.Length)
 434                 return srcUri;
 435
 436             // OAuth2 アクセストークンまたは API キー (旧方式) のいずれも設定されていなければ短縮しない
 437             if (string.IsNullOrEmpty(this.BitlyAccessToken) && (string.IsNullOrEmpty(this.BitlyId) || string.IsNullOrEmpty(this.BitlyKey)))
 438                 return srcUri;
 439
 440             var bitly = new BitlyApi
 441             {
 442                 EndUserAccessToken = this.BitlyAccessToken,
 443                 EndUserLoginName = this.BitlyId,
 444                 EndUserApiKey = this.BitlyKey,
 445             };
 446
 447             return await bitly.ShortenAsync(srcUri, domain)
 448                 .ConfigureAwait(false);
 449         }
 450
 451         private async Task<Uri> ShortenByUxnuAsync(Uri srcUri)
 452         {
 453             // 明らかに長くなると推測できる場合は短縮しない
 454             if ("http://ux.nx/xxxxxx".Length > srcUri.OriginalString.Length)
 455                 return srcUri;
 456
 457             var query = new Dictionary<string, string>
 458             {
 459                 ["format"] = "plain",
 460                 ["url"] = srcUri.OriginalString,
 461             };
 462
 463             var uri = new Uri("http://ux.nu/api/short?" + MyCommon.BuildQueryString(query));
 464             using (var response = await this.http.GetAsync(uri).ConfigureAwait(false))
 465             {
 466                 response.EnsureSuccessStatusCode();
 467
 468                 var result = await response.Content.ReadAsStringAsync()
 469                     .ConfigureAwait(false);
 470
 471                 if (!Regex.IsMatch(result, @"^https?://"))
 472                     throw new WebApiException("Failed to create URL.", result);
 473
 474                 return new Uri(result.TrimEnd());
 475             }
 476         }
 477
 478         private bool IsIrregularShortUrl(Uri uri)
 479         {
 480             // Flickrの https://www.flickr.com/photo.gne?short=... 形式のURL
 481             // flic.kr ドメインのURLを展開する途中に経由する
 482             if (uri.Host.EndsWith("flickr.com", StringComparison.OrdinalIgnoreCase) &&
 483                 uri.PathAndQuery.StartsWith("/photo.gne", StringComparison.OrdinalIgnoreCase))
 484                 return true;
 485
 486             return false;
 487         }
 488
 489         private async Task<Uri> GetRedirectTo(Uri url)
 490         {
 491             var request = new HttpRequestMessage(HttpMethod.Head, url);
 492
 493             using (var response = await this.http.SendAsync(request).ConfigureAwait(false))
 494             {
 495                 if (!response.IsSuccessStatusCode)
 496                 {
 497                     // ステータスコードが 3xx であれば例外を発生させない
 498                     if ((int)response.StatusCode / 100 != 3)
 499                         response.EnsureSuccessStatusCode();
 500                 }
 501
 502                 var redirectedUrl = response.Headers.Location;
 503
 504                 if (redirectedUrl == null)
 505                     return null;
 506
 507                 // サーバーが URL を適切にエンコードしていない場合、OriginalString に非 ASCII 文字が含まれる。
 508                 // その場合、redirectedUrl は文字化けしている可能性があるため使用しない
 509                 // 参照: http://stackoverflow.com/questions/1888933
 510                 if (redirectedUrl.OriginalString.Any(x => x < ' ' || x > '~'))
 511                     return null;
 512
 513                 if (redirectedUrl.IsAbsoluteUri)
 514                     return redirectedUrl;
 515                 else
 516                     return new Uri(url, redirectedUrl);
 517             }
 518         }
 519
 520         [SuppressMessage("Microsoft.Reliability", "CA2000:DisposeObjectsBeforeLosingScope")]
 521         private static HttpClient CreateDefaultHttpClient()
 522         {
 523             var handler = Networking.CreateHttpClientHandler();
 524             handler.AllowAutoRedirect = false;
 525
 526             var http = Networking.CreateHttpClient(handler);
 527             http.Timeout = TimeSpan.FromSeconds(30);
 528
 529             return http;
 530         }
 531     }
 532 }