OpenTween/ShortUrl.cs

   1 // OpenTween - Client of Twitter
   2 // Copyright (c) 2007-2011 kiri_feather (@kiri_feather) <kiri.feather@gmail.com>
   3 //           (c) 2008-2011 Moz (@syo68k)
   4 //           (c) 2008-2011 takeshik (@takeshik) <http://www.takeshik.org/>
   5 //           (c) 2010-2011 anis774 (@anis774) <http://d.hatena.ne.jp/anis774/>
   6 //           (c) 2010-2011 fantasticswallow (@f_swallow) <http://twitter.com/f_swallow>
   7 //           (c) 2011      kim_upsilon (@kim_upsilon) <https://upsilo.net/~upsilon/>
   8 // All rights reserved.
   9 //
  10 // This file is part of OpenTween.
  11 //
  12 // This program is free software; you can redistribute it and/or modify it
  13 // under the terms of the GNU General public License as published by the Free
  14 // Software Foundation; either version 3 of the License, or (at your option)
  15 // any later version.
  16 //
  17 // This program is distributed in the hope that it will be useful, but
  18 // WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
  19 // or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General public License
  20 // for more details.
  21 //
  22 // You should have received a copy of the GNU General public License along
  23 // with this program. If not, see <http://www.gnu.org/licenses/>, or write to
  24 // the Free Software Foundation, Inc., 51 Franklin Street - Fifth Floor,
  25 // Boston, MA 02110-1301, USA.
  26
  27 using System;
  28 using System.Collections.Concurrent;
  29 using System.Collections.Generic;
  30 using System.Diagnostics.CodeAnalysis;
  31 using System.Linq;
  32 using System.Net.Http;
  33 using System.Text;
  34 using System.Text.RegularExpressions;
  35 using System.Threading;
  36 using System.Threading.Tasks;
  37 using System.Web;
  38 using OpenTween.Api;
  39 using OpenTween.Connection;
  40
  41 namespace OpenTween
  42 {
  43     /// <summary>
  44     /// 短縮 URL サービスによる URL の展開・短縮を行うクラス
  45     /// </summary>
  46     public class ShortUrl
  47     {
  48         private static Lazy<ShortUrl> _instance;
  49
  50         /// <summary>
  51         /// ShortUrl のインスタンスを取得します
  52         /// </summary>
  53         public static ShortUrl Instance
  54             => _instance.Value;
  55
  56         /// <summary>
  57         /// 短縮 URL の展開を無効にするか否か
  58         /// </summary>
  59         public bool DisableExpanding { get; set; }
  60
  61         /// <summary>
  62         /// 短縮 URL のキャッシュを定期的にクリアする回数
  63         /// </summary>
  64         public int PurgeCount { get; set; }
  65
  66         public string BitlyAccessToken { get; set; }
  67         public string BitlyId { get; set; }
  68         public string BitlyKey { get; set; }
  69
  70         private HttpClient http;
  71         private ConcurrentDictionary<Uri, Uri> urlCache = new ConcurrentDictionary<Uri, Uri>();
  72
  73         private static readonly Regex HtmlLinkPattern = new Regex(@"(<a href="")(.+?)("")");
  74
  75         private static readonly HashSet<string> ShortUrlHosts = new HashSet<string>
  76         {
  77             "4sq.com",
  78             "amzn.to",
  79             "bit.ly",
  80             "blip.fm",
  81             "budurl.com",
  82             "budurl.me",
  83             "buff.ly",
  84             "disq.us",
  85             "dlvr.it",
  86             "fb.me",
  87             "feedly.com",
  88             "feeds.feedburner.com",
  89             "ff.im",
  90             "flic.kr",
  91             "goo.gl",
  92             "ht.ly",
  93             "htn.to",
  94             "ift.tt",
  95             "is.gd",
  96             "j.mp",
  97             "moby.to",
  98             "moi.st",
  99             "nico.ms",
 100             "on.digg.com",
 101             "on.fb.me",
 102             "ow.ly",
 103             "qurl.com",
 104             "t.co",
 105             "tinami.jp",
 106             "tiny.cc",
 107             "tinyurl.com",
 108             "tl.gd",
 109             "tmblr.co",
 110             "tumblr.com",
 111             "twme.jp",
 112             "urx2.nu",
 113             "ux.nu",
 114             "wp.me",
 115             "www.qurl.com",
 116             "www.tumblr.com",
 117             "youtu.be",
 118         };
 119
 120         static ShortUrl()
 121             => _instance = new Lazy<ShortUrl>(() => new ShortUrl(), true);
 122
 123         [SuppressMessage("Microsoft.Reliability", "CA2000:DisposeObjectsBeforeLosingScope")]
 124         internal ShortUrl()
 125             : this(CreateDefaultHttpClient())
 126         {
 127             Networking.WebProxyChanged += (o, e) =>
 128             {
 129                 var newClient = CreateDefaultHttpClient();
 130                 var oldClient = Interlocked.Exchange(ref this.http, newClient);
 131                 oldClient.Dispose();
 132             };
 133         }
 134
 135         internal ShortUrl(HttpClient http)
 136         {
 137             this.DisableExpanding = false;
 138             this.PurgeCount = 500;
 139             this.BitlyId = "";
 140             this.BitlyKey = "";
 141
 142             this.http = http;
 143         }
 144
 145         [Obsolete]
 146         public string ExpandUrl(string uri)
 147         {
 148             try
 149             {
 150                 return this.ExpandUrlAsync(new Uri(uri), 10).Result.AbsoluteUri;
 151             }
 152             catch (UriFormatException)
 153             {
 154                 return uri;
 155             }
 156         }
 157
 158         /// <summary>
 159         /// 短縮 URL を非同期に展開します
 160         /// </summary>
 161         /// <param name="uri">展開するURL</param>
 162         /// <returns>URLの展開を行うタスク</returns>
 163         public Task<Uri> ExpandUrlAsync(Uri uri)
 164             => this.ExpandUrlAsync(uri, 10);
 165
 166         /// <summary>
 167         /// 短縮 URL を非同期に展開します
 168         /// </summary>
 169         /// <param name="uri">展開するURL</param>
 170         /// <param name="redirectLimit">再帰的に展開を試みる上限</param>
 171         /// <returns>URLの展開を行うタスク</returns>
 172         public async Task<Uri> ExpandUrlAsync(Uri uri, int redirectLimit)
 173         {
 174             if (this.DisableExpanding)
 175                 return uri;
 176
 177             if (redirectLimit <= 0)
 178                 return uri;
 179
 180             if (!uri.IsAbsoluteUri)
 181                 return uri;
 182
 183             try
 184             {
 185                 if (!ShortUrlHosts.Contains(uri.Host) && !IsIrregularShortUrl(uri))
 186                     return uri;
 187
 188                 if (this.urlCache.TryGetValue(uri, out var expanded))
 189                     return expanded;
 190
 191                 if (this.urlCache.Count > this.PurgeCount)
 192                     this.urlCache.Clear();
 193
 194                 expanded = null;
 195                 try
 196                 {
 197                     expanded = await this.GetRedirectTo(uri)
 198                         .ConfigureAwait(false);
 199                 }
 200                 catch (TaskCanceledException) { }
 201                 catch (HttpRequestException) { }
 202
 203                 if (expanded == null || expanded == uri)
 204                     return uri;
 205
 206                 this.urlCache[uri] = expanded;
 207
 208                 var recursiveExpanded = await this.ExpandUrlAsync(expanded, --redirectLimit)
 209                     .ConfigureAwait(false);
 210
 211                 // URL1 -> URL2 -> URL3 のように再帰的に展開されたURLを URL1 -> URL3 としてキャッシュに格納する
 212                 if (recursiveExpanded != expanded)
 213                     this.urlCache[uri] = recursiveExpanded;
 214
 215                 return recursiveExpanded;
 216             }
 217             catch (UriFormatException)
 218             {
 219                 return uri;
 220             }
 221         }
 222
 223         /// <summary>
 224         /// 短縮 URL を非同期に展開します
 225         /// </summary>
 226         /// <remarks>
 227         /// 不正なURLが渡された場合は例外を投げず uriStr をそのまま返します
 228         /// </remarks>
 229         /// <param name="uriStr">展開するURL</param>
 230         /// <returns>URLの展開を行うタスク</returns>
 231         public Task<string> ExpandUrlAsync(string uriStr)
 232             => this.ExpandUrlAsync(uriStr, 10);
 233
 234         /// <summary>
 235         /// 短縮 URL を非同期に展開します
 236         /// </summary>
 237         /// <remarks>
 238         /// 不正なURLが渡された場合は例外を投げず uriStr をそのまま返します
 239         /// </remarks>
 240         /// <param name="uriStr">展開するURL</param>
 241         /// <param name="redirectLimit">再帰的に展開を試みる上限</param>
 242         /// <returns>URLの展開を行うタスク</returns>
 243         public async Task<string> ExpandUrlAsync(string uriStr, int redirectLimit)
 244         {
 245             Uri uri;
 246
 247             try
 248             {
 249                 if (!uriStr.StartsWith("http", StringComparison.OrdinalIgnoreCase))
 250                     uri = new Uri("http://" + uriStr);
 251                 else
 252                     uri = new Uri(uriStr);
 253             }
 254             catch (UriFormatException)
 255             {
 256                 return uriStr;
 257             }
 258
 259             var expandedUri = await this.ExpandUrlAsync(uri, redirectLimit)
 260                 .ConfigureAwait(false);
 261
 262             return expandedUri.OriginalString;
 263         }
 264
 265         [Obsolete]
 266         public string ExpandUrlHtml(string html)
 267             => this.ExpandUrlHtmlAsync(html, 10).Result;
 268
 269         /// <summary>
 270         /// HTML内に含まれるリンクのURLを非同期に展開する
 271         /// </summary>
 272         /// <param name="html">処理対象のHTML</param>
 273         /// <returns>URLの展開を行うタスク</returns>
 274         public Task<string> ExpandUrlHtmlAsync(string html)
 275             => this.ExpandUrlHtmlAsync(html, 10);
 276
 277         /// <summary>
 278         /// HTML内に含まれるリンクのURLを非同期に展開する
 279         /// </summary>
 280         /// <param name="html">処理対象のHTML</param>
 281         /// <param name="redirectLimit">再帰的に展開を試みる上限</param>
 282         /// <returns>URLの展開を行うタスク</returns>
 283         public Task<string> ExpandUrlHtmlAsync(string html, int redirectLimit)
 284         {
 285             if (this.DisableExpanding)
 286                 return Task.FromResult(html);
 287
 288             return HtmlLinkPattern.ReplaceAsync(html, async m =>
 289                 m.Groups[1].Value + await this.ExpandUrlAsync(m.Groups[2].Value, redirectLimit).ConfigureAwait(false) + m.Groups[3].Value);
 290         }
 291
 292         /// <summary>
 293         /// 指定された短縮URLサービスを使用してURLを短縮します
 294         /// </summary>
 295         /// <param name="shortenerType">使用する短縮URLサービス</param>
 296         /// <param name="srcUri">短縮するURL</param>
 297         /// <returns>短縮されたURL</returns>
 298         public async Task<Uri> ShortenUrlAsync(MyCommon.UrlConverter shortenerType, Uri srcUri)
 299         {
 300             // 既に短縮されている状態のURLであれば短縮しない
 301             if (ShortUrlHosts.Contains(srcUri.Host))
 302                 return srcUri;
 303
 304             try
 305             {
 306                 switch (shortenerType)
 307                 {
 308                     case MyCommon.UrlConverter.TinyUrl:
 309                         return await this.ShortenByTinyUrlAsync(srcUri)
 310                             .ConfigureAwait(false);
 311                     case MyCommon.UrlConverter.Isgd:
 312                         return await this.ShortenByIsgdAsync(srcUri)
 313                             .ConfigureAwait(false);
 314                     case MyCommon.UrlConverter.Twurl:
 315                         return await this.ShortenByTwurlAsync(srcUri)
 316                             .ConfigureAwait(false);
 317                     case MyCommon.UrlConverter.Bitly:
 318                         return await this.ShortenByBitlyAsync(srcUri, "bit.ly")
 319                             .ConfigureAwait(false);
 320                     case MyCommon.UrlConverter.Jmp:
 321                         return await this.ShortenByBitlyAsync(srcUri, "j.mp")
 322                             .ConfigureAwait(false);
 323                     case MyCommon.UrlConverter.Uxnu:
 324                         return await this.ShortenByUxnuAsync(srcUri)
 325                             .ConfigureAwait(false);
 326                     default:
 327                         throw new ArgumentException("Unknown shortener.", nameof(shortenerType));
 328                 }
 329             }
 330             catch (OperationCanceledException)
 331             {
 332                 // 短縮 URL の API がタイムアウトした場合
 333                 return srcUri;
 334             }
 335         }
 336
 337         private async Task<Uri> ShortenByTinyUrlAsync(Uri srcUri)
 338         {
 339             // 明らかに長くなると推測できる場合は短縮しない
 340             if ("http://tinyurl.com/xxxxxx".Length > srcUri.OriginalString.Length)
 341                 return srcUri;
 342
 343             var content = new FormUrlEncodedContent(new[]
 344             {
 345                 new KeyValuePair<string, string>("url", srcUri.OriginalString),
 346             });
 347
 348             using (var response = await this.http.PostAsync("http://tinyurl.com/api-create.php", content).ConfigureAwait(false))
 349             {
 350                 response.EnsureSuccessStatusCode();
 351
 352                 var result = await response.Content.ReadAsStringAsync()
 353                     .ConfigureAwait(false);
 354
 355                 if (!Regex.IsMatch(result, @"^https?://"))
 356                     throw new WebApiException("Failed to create URL.", result);
 357
 358                 return new Uri(result.TrimEnd());
 359             }
 360         }
 361
 362         private async Task<Uri> ShortenByIsgdAsync(Uri srcUri)
 363         {
 364             // 明らかに長くなると推測できる場合は短縮しない
 365             if ("http://is.gd/xxxx".Length > srcUri.OriginalString.Length)
 366                 return srcUri;
 367
 368             var content = new FormUrlEncodedContent(new[]
 369             {
 370                 new KeyValuePair<string, string>("format", "simple"),
 371                 new KeyValuePair<string, string>("url", srcUri.OriginalString),
 372             });
 373
 374             using (var response = await this.http.PostAsync("http://is.gd/create.php", content).ConfigureAwait(false))
 375             {
 376                 response.EnsureSuccessStatusCode();
 377
 378                 var result = await response.Content.ReadAsStringAsync()
 379                     .ConfigureAwait(false);
 380
 381                 if (!Regex.IsMatch(result, @"^https?://"))
 382                     throw new WebApiException("Failed to create URL.", result);
 383
 384                 return new Uri(result.TrimEnd());
 385             }
 386         }
 387
 388         private async Task<Uri> ShortenByTwurlAsync(Uri srcUri)
 389         {
 390             // 明らかに長くなると推測できる場合は短縮しない
 391             if ("http://twurl.nl/xxxxxx".Length > srcUri.OriginalString.Length)
 392                 return srcUri;
 393
 394             var content = new FormUrlEncodedContent(new[]
 395             {
 396                 new KeyValuePair<string, string>("link[url]", srcUri.OriginalString),
 397             });
 398
 399             using (var response = await this.http.PostAsync("http://tweetburner.com/links", content).ConfigureAwait(false))
 400             {
 401                 response.EnsureSuccessStatusCode();
 402
 403                 var result = await response.Content.ReadAsStringAsync()
 404                     .ConfigureAwait(false);
 405
 406                 if (!Regex.IsMatch(result, @"^https?://"))
 407                     throw new WebApiException("Failed to create URL.", result);
 408
 409                 return new Uri(result.TrimEnd());
 410             }
 411         }
 412
 413         private async Task<Uri> ShortenByBitlyAsync(Uri srcUri, string domain = "bit.ly")
 414         {
 415             // 明らかに長くなると推測できる場合は短縮しない
 416             if ("http://bit.ly/xxxx".Length > srcUri.OriginalString.Length)
 417                 return srcUri;
 418
 419             // OAuth2 アクセストークンまたは API キー (旧方式) のいずれも設定されていなければ短縮しない
 420             if (string.IsNullOrEmpty(this.BitlyAccessToken) && (string.IsNullOrEmpty(this.BitlyId) || string.IsNullOrEmpty(this.BitlyKey)))
 421                 return srcUri;
 422
 423             var bitly = new BitlyApi
 424             {
 425                 EndUserAccessToken = this.BitlyAccessToken,
 426                 EndUserLoginName = this.BitlyId,
 427                 EndUserApiKey = this.BitlyKey,
 428             };
 429
 430             return await bitly.ShortenAsync(srcUri, domain)
 431                 .ConfigureAwait(false);
 432         }
 433
 434         private async Task<Uri> ShortenByUxnuAsync(Uri srcUri)
 435         {
 436             // 明らかに長くなると推測できる場合は短縮しない
 437             if ("http://ux.nx/xxxxxx".Length > srcUri.OriginalString.Length)
 438                 return srcUri;
 439
 440             var query = new Dictionary<string, string>
 441             {
 442                 ["format"] = "plain",
 443                 ["url"] = srcUri.OriginalString,
 444             };
 445
 446             var uri = new Uri("http://ux.nu/api/short?" + MyCommon.BuildQueryString(query));
 447             using (var response = await this.http.GetAsync(uri).ConfigureAwait(false))
 448             {
 449                 response.EnsureSuccessStatusCode();
 450
 451                 var result = await response.Content.ReadAsStringAsync()
 452                     .ConfigureAwait(false);
 453
 454                 if (!Regex.IsMatch(result, @"^https?://"))
 455                     throw new WebApiException("Failed to create URL.", result);
 456
 457                 return new Uri(result.TrimEnd());
 458             }
 459         }
 460
 461         private bool IsIrregularShortUrl(Uri uri)
 462         {
 463             // Flickrの https://www.flickr.com/photo.gne?short=... 形式のURL
 464             // flic.kr ドメインのURLを展開する途中に経由する
 465             if (uri.Host.EndsWith("flickr.com", StringComparison.OrdinalIgnoreCase) &&
 466                 uri.PathAndQuery.StartsWith("/photo.gne", StringComparison.OrdinalIgnoreCase))
 467                 return true;
 468
 469             return false;
 470         }
 471
 472         private async Task<Uri> GetRedirectTo(Uri url)
 473         {
 474             var request = new HttpRequestMessage(HttpMethod.Head, url);
 475
 476             using (var response = await this.http.SendAsync(request).ConfigureAwait(false))
 477             {
 478                 if (!response.IsSuccessStatusCode)
 479                 {
 480                     // ステータスコードが 3xx であれば例外を発生させない
 481                     if ((int)response.StatusCode / 100 != 3)
 482                         response.EnsureSuccessStatusCode();
 483                 }
 484
 485                 var redirectedUrl = response.Headers.Location;
 486
 487                 if (redirectedUrl == null)
 488                     return null;
 489
 490                 // サーバーが URL を適切にエンコードしていない場合、OriginalString に非 ASCII 文字が含まれる。
 491                 // その場合、redirectedUrl は文字化けしている可能性があるため使用しない
 492                 // 参照: http://stackoverflow.com/questions/1888933
 493                 if (redirectedUrl.OriginalString.Any(x => x < ' ' || x > '~'))
 494                     return null;
 495
 496                 if (redirectedUrl.IsAbsoluteUri)
 497                     return redirectedUrl;
 498                 else
 499                     return new Uri(url, redirectedUrl);
 500             }
 501         }
 502
 503         [SuppressMessage("Microsoft.Reliability", "CA2000:DisposeObjectsBeforeLosingScope")]
 504         private static HttpClient CreateDefaultHttpClient()
 505         {
 506             var handler = Networking.CreateHttpClientHandler();
 507             handler.AllowAutoRedirect = false;
 508
 509             var http = Networking.CreateHttpClient(handler);
 510             http.Timeout = TimeSpan.FromSeconds(30);
 511
 512             return http;
 513         }
 514     }
 515 }