OpenTween/ShortUrl.cs

   1 // OpenTween - Client of Twitter
   2 // Copyright (c) 2007-2011 kiri_feather (@kiri_feather) <kiri.feather@gmail.com>
   3 //           (c) 2008-2011 Moz (@syo68k)
   4 //           (c) 2008-2011 takeshik (@takeshik) <http://www.takeshik.org/>
   5 //           (c) 2010-2011 anis774 (@anis774) <http://d.hatena.ne.jp/anis774/>
   6 //           (c) 2010-2011 fantasticswallow (@f_swallow) <http://twitter.com/f_swallow>
   7 //           (c) 2011      kim_upsilon (@kim_upsilon) <https://upsilo.net/~upsilon/>
   8 // All rights reserved.
   9 //
  10 // This file is part of OpenTween.
  11 //
  12 // This program is free software; you can redistribute it and/or modify it
  13 // under the terms of the GNU General public License as published by the Free
  14 // Software Foundation; either version 3 of the License, or (at your option)
  15 // any later version.
  16 //
  17 // This program is distributed in the hope that it will be useful, but
  18 // WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
  19 // or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General public License
  20 // for more details.
  21 //
  22 // You should have received a copy of the GNU General public License along
  23 // with this program. If not, see <http://www.gnu.org/licenses/>, or write to
  24 // the Free Software Foundation, Inc., 51 Franklin Street - Fifth Floor,
  25 // Boston, MA 02110-1301, USA.
  26
  27 #nullable enable
  28
  29 using System;
  30 using System.Collections.Concurrent;
  31 using System.Collections.Generic;
  32 using System.Diagnostics.CodeAnalysis;
  33 using System.Linq;
  34 using System.Net.Http;
  35 using System.Text;
  36 using System.Text.RegularExpressions;
  37 using System.Threading;
  38 using System.Threading.Tasks;
  39 using System.Web;
  40 using OpenTween.Api;
  41 using OpenTween.Connection;
  42
  43 namespace OpenTween
  44 {
  45     /// <summary>
  46     /// 短縮 URL サービスによる URL の展開・短縮を行うクラス
  47     /// </summary>
  48     public class ShortUrl
  49     {
  50         private static readonly Lazy<ShortUrl> _instance;
  51
  52         /// <summary>
  53         /// ShortUrl のインスタンスを取得します
  54         /// </summary>
  55         public static ShortUrl Instance
  56             => _instance.Value;
  57
  58         /// <summary>
  59         /// 短縮 URL の展開を無効にするか否か
  60         /// </summary>
  61         public bool DisableExpanding { get; set; }
  62
  63         /// <summary>
  64         /// 短縮 URL のキャッシュを定期的にクリアする回数
  65         /// </summary>
  66         public int PurgeCount { get; set; }
  67
  68         public string BitlyAccessToken { get; set; } = "";
  69         public string BitlyId { get; set; } = "";
  70         public string BitlyKey { get; set; } = "";
  71
  72         private HttpClient http;
  73         private readonly ConcurrentDictionary<Uri, Uri> urlCache = new ConcurrentDictionary<Uri, Uri>();
  74
  75         private static readonly Regex HtmlLinkPattern = new Regex(@"(<a href="")(.+?)("")");
  76
  77         private static readonly HashSet<string> ShortUrlHosts = new HashSet<string>
  78         {
  79             "4sq.com",
  80             "amzn.to",
  81             "bit.ly",
  82             "blip.fm",
  83             "budurl.com",
  84             "budurl.me",
  85             "buff.ly",
  86             "disq.us",
  87             "dlvr.it",
  88             "fb.me",
  89             "feedly.com",
  90             "feeds.feedburner.com",
  91             "ff.im",
  92             "flic.kr",
  93             "goo.gl",
  94             "ht.ly",
  95             "htn.to",
  96             "ift.tt",
  97             "is.gd",
  98             "j.mp",
  99             "moby.to",
 100             "moi.st",
 101             "nico.ms",
 102             "on.digg.com",
 103             "on.fb.me",
 104             "ow.ly",
 105             "qurl.com",
 106             "t.co",
 107             "tinami.jp",
 108             "tiny.cc",
 109             "tinyurl.com",
 110             "tl.gd",
 111             "tmblr.co",
 112             "tumblr.com",
 113             "twme.jp",
 114             "urx2.nu",
 115             "ux.nu",
 116             "wp.me",
 117             "www.qurl.com",
 118             "www.tumblr.com",
 119             "youtu.be",
 120         };
 121
 122         /// <summary>
 123         /// HTTPS非対応の短縮URLサービス
 124         /// </summary>
 125         private static readonly HashSet<string> InsecureDomains = new HashSet<string>
 126         {
 127             "budurl.com",
 128             "ff.im",
 129             "ht.ly",
 130             "moi.st",
 131             "ow.ly",
 132             "tinami.jp",
 133             "tl.gd",
 134             "twme.jp",
 135             "urx2.nu",
 136         };
 137
 138         static ShortUrl()
 139             => _instance = new Lazy<ShortUrl>(() => new ShortUrl(), true);
 140
 141         [SuppressMessage("Microsoft.Reliability", "CA2000:DisposeObjectsBeforeLosingScope")]
 142         internal ShortUrl()
 143             : this(CreateDefaultHttpClient())
 144         {
 145             Networking.WebProxyChanged += (o, e) =>
 146             {
 147                 var newClient = CreateDefaultHttpClient();
 148                 var oldClient = Interlocked.Exchange(ref this.http, newClient);
 149                 oldClient.Dispose();
 150             };
 151         }
 152
 153         internal ShortUrl(HttpClient http)
 154         {
 155             this.DisableExpanding = false;
 156             this.PurgeCount = 500;
 157             this.BitlyId = "";
 158             this.BitlyKey = "";
 159
 160             this.http = http;
 161         }
 162
 163         [Obsolete]
 164         public string ExpandUrl(string uri)
 165         {
 166             try
 167             {
 168                 return this.ExpandUrlAsync(new Uri(uri), 10).Result.AbsoluteUri;
 169             }
 170             catch (UriFormatException)
 171             {
 172                 return uri;
 173             }
 174         }
 175
 176         /// <summary>
 177         /// 短縮 URL を非同期に展開します
 178         /// </summary>
 179         /// <param name="uri">展開するURL</param>
 180         /// <returns>URLの展開を行うタスク</returns>
 181         public Task<Uri> ExpandUrlAsync(Uri uri)
 182             => this.ExpandUrlAsync(uri, 10);
 183
 184         /// <summary>
 185         /// 短縮 URL を非同期に展開します
 186         /// </summary>
 187         /// <param name="uri">展開するURL</param>
 188         /// <param name="redirectLimit">再帰的に展開を試みる上限</param>
 189         /// <returns>URLの展開を行うタスク</returns>
 190         public async Task<Uri> ExpandUrlAsync(Uri uri, int redirectLimit)
 191         {
 192             if (this.DisableExpanding)
 193                 return uri;
 194
 195             if (redirectLimit <= 0)
 196                 return uri;
 197
 198             if (!uri.IsAbsoluteUri)
 199                 return uri;
 200
 201             try
 202             {
 203                 if (!ShortUrlHosts.Contains(uri.Host) && !IsIrregularShortUrl(uri))
 204                     return uri;
 205
 206                 Uri? expanded;
 207                 if (this.urlCache.TryGetValue(uri, out expanded))
 208                     return expanded;
 209
 210                 if (this.urlCache.Count > this.PurgeCount)
 211                     this.urlCache.Clear();
 212
 213                 expanded = null;
 214                 try
 215                 {
 216                     expanded = await this.GetRedirectTo(uri)
 217                         .ConfigureAwait(false);
 218                 }
 219                 catch (TaskCanceledException) { }
 220                 catch (HttpRequestException) { }
 221
 222                 if (expanded == null || expanded == uri)
 223                     return uri;
 224
 225                 this.urlCache[uri] = expanded;
 226
 227                 var recursiveExpanded = await this.ExpandUrlAsync(expanded, --redirectLimit)
 228                     .ConfigureAwait(false);
 229
 230                 // URL1 -> URL2 -> URL3 のように再帰的に展開されたURLを URL1 -> URL3 としてキャッシュに格納する
 231                 if (recursiveExpanded != expanded)
 232                     this.urlCache[uri] = recursiveExpanded;
 233
 234                 return recursiveExpanded;
 235             }
 236             catch (UriFormatException)
 237             {
 238                 return uri;
 239             }
 240         }
 241
 242         /// <summary>
 243         /// 短縮 URL を非同期に展開します
 244         /// </summary>
 245         /// <remarks>
 246         /// 不正なURLが渡された場合は例外を投げず uriStr をそのまま返します
 247         /// </remarks>
 248         /// <param name="uriStr">展開するURL</param>
 249         /// <returns>URLの展開を行うタスク</returns>
 250         public Task<string> ExpandUrlAsync(string uriStr)
 251             => this.ExpandUrlAsync(uriStr, 10);
 252
 253         /// <summary>
 254         /// 短縮 URL を非同期に展開します
 255         /// </summary>
 256         /// <remarks>
 257         /// 不正なURLが渡された場合は例外を投げず uriStr をそのまま返します
 258         /// </remarks>
 259         /// <param name="uriStr">展開するURL</param>
 260         /// <param name="redirectLimit">再帰的に展開を試みる上限</param>
 261         /// <returns>URLの展開を行うタスク</returns>
 262         public async Task<string> ExpandUrlAsync(string uriStr, int redirectLimit)
 263         {
 264             Uri uri;
 265
 266             try
 267             {
 268                 if (!uriStr.StartsWith("http", StringComparison.OrdinalIgnoreCase))
 269                     uri = new Uri("http://" + uriStr);
 270                 else
 271                     uri = new Uri(uriStr);
 272             }
 273             catch (UriFormatException)
 274             {
 275                 return uriStr;
 276             }
 277
 278             var expandedUri = await this.ExpandUrlAsync(uri, redirectLimit)
 279                 .ConfigureAwait(false);
 280
 281             return expandedUri.OriginalString;
 282         }
 283
 284         [Obsolete]
 285         public string ExpandUrlHtml(string html)
 286             => this.ExpandUrlHtmlAsync(html, 10).Result;
 287
 288         /// <summary>
 289         /// HTML内に含まれるリンクのURLを非同期に展開する
 290         /// </summary>
 291         /// <param name="html">処理対象のHTML</param>
 292         /// <returns>URLの展開を行うタスク</returns>
 293         public Task<string> ExpandUrlHtmlAsync(string html)
 294             => this.ExpandUrlHtmlAsync(html, 10);
 295
 296         /// <summary>
 297         /// HTML内に含まれるリンクのURLを非同期に展開する
 298         /// </summary>
 299         /// <param name="html">処理対象のHTML</param>
 300         /// <param name="redirectLimit">再帰的に展開を試みる上限</param>
 301         /// <returns>URLの展開を行うタスク</returns>
 302         public Task<string> ExpandUrlHtmlAsync(string html, int redirectLimit)
 303         {
 304             if (this.DisableExpanding)
 305                 return Task.FromResult(html);
 306
 307             return HtmlLinkPattern.ReplaceAsync(html, async m =>
 308                 m.Groups[1].Value + await this.ExpandUrlAsync(m.Groups[2].Value, redirectLimit).ConfigureAwait(false) + m.Groups[3].Value);
 309         }
 310
 311         /// <summary>
 312         /// 指定された短縮URLサービスを使用してURLを短縮します
 313         /// </summary>
 314         /// <param name="shortenerType">使用する短縮URLサービス</param>
 315         /// <param name="srcUri">短縮するURL</param>
 316         /// <returns>短縮されたURL</returns>
 317         public async Task<Uri> ShortenUrlAsync(MyCommon.UrlConverter shortenerType, Uri srcUri)
 318         {
 319             // 既に短縮されている状態のURLであれば短縮しない
 320             if (ShortUrlHosts.Contains(srcUri.Host))
 321                 return srcUri;
 322
 323             try
 324             {
 325                 switch (shortenerType)
 326                 {
 327                     case MyCommon.UrlConverter.TinyUrl:
 328                         return await this.ShortenByTinyUrlAsync(srcUri)
 329                             .ConfigureAwait(false);
 330                     case MyCommon.UrlConverter.Isgd:
 331                         return await this.ShortenByIsgdAsync(srcUri)
 332                             .ConfigureAwait(false);
 333                     case MyCommon.UrlConverter.Bitly:
 334                         return await this.ShortenByBitlyAsync(srcUri, "bit.ly")
 335                             .ConfigureAwait(false);
 336                     case MyCommon.UrlConverter.Jmp:
 337                         return await this.ShortenByBitlyAsync(srcUri, "j.mp")
 338                             .ConfigureAwait(false);
 339                     case MyCommon.UrlConverter.Uxnu:
 340                         return await this.ShortenByUxnuAsync(srcUri)
 341                             .ConfigureAwait(false);
 342                     default:
 343                         throw new ArgumentException("Unknown shortener.", nameof(shortenerType));
 344                 }
 345             }
 346             catch (OperationCanceledException)
 347             {
 348                 // 短縮 URL の API がタイムアウトした場合
 349                 return srcUri;
 350             }
 351         }
 352
 353         private async Task<Uri> ShortenByTinyUrlAsync(Uri srcUri)
 354         {
 355             // 明らかに長くなると推測できる場合は短縮しない
 356             if ("https://tinyurl.com/xxxxxxxx".Length > srcUri.OriginalString.Length)
 357                 return srcUri;
 358
 359             var content = new FormUrlEncodedContent(new[]
 360             {
 361                 new KeyValuePair<string, string>("url", srcUri.OriginalString),
 362             });
 363
 364             using var response = await this.http.PostAsync("https://tinyurl.com/api-create.php", content)
 365                 .ConfigureAwait(false);
 366
 367             response.EnsureSuccessStatusCode();
 368
 369             var result = await response.Content.ReadAsStringAsync()
 370                 .ConfigureAwait(false);
 371
 372             if (!Regex.IsMatch(result, @"^https?://"))
 373                 throw new WebApiException("Failed to create URL.", result);
 374
 375             return this.UpgradeToHttpsIfAvailable(new Uri(result.TrimEnd()));
 376         }
 377
 378         private async Task<Uri> ShortenByIsgdAsync(Uri srcUri)
 379         {
 380             // 明らかに長くなると推測できる場合は短縮しない
 381             if ("https://is.gd/xxxxxx".Length > srcUri.OriginalString.Length)
 382                 return srcUri;
 383
 384             var content = new FormUrlEncodedContent(new[]
 385             {
 386                 new KeyValuePair<string, string>("format", "simple"),
 387                 new KeyValuePair<string, string>("url", srcUri.OriginalString),
 388             });
 389
 390             using var response = await this.http.PostAsync("https://is.gd/create.php", content)
 391                 .ConfigureAwait(false);
 392
 393             response.EnsureSuccessStatusCode();
 394
 395             var result = await response.Content.ReadAsStringAsync()
 396                 .ConfigureAwait(false);
 397
 398             if (!Regex.IsMatch(result, @"^https?://"))
 399                 throw new WebApiException("Failed to create URL.", result);
 400
 401             return new Uri(result.TrimEnd());
 402         }
 403
 404         private async Task<Uri> ShortenByBitlyAsync(Uri srcUri, string domain = "bit.ly")
 405         {
 406             // 明らかに長くなると推測できる場合は短縮しない
 407             if ("https://bit.ly/xxxxxxx".Length > srcUri.OriginalString.Length)
 408                 return srcUri;
 409
 410             // OAuth2 アクセストークンまたは API キー (旧方式) のいずれも設定されていなければ短縮しない
 411             if (string.IsNullOrEmpty(this.BitlyAccessToken) && (string.IsNullOrEmpty(this.BitlyId) || string.IsNullOrEmpty(this.BitlyKey)))
 412                 return srcUri;
 413
 414             var bitly = new BitlyApi
 415             {
 416                 EndUserAccessToken = this.BitlyAccessToken,
 417                 EndUserLoginName = this.BitlyId,
 418                 EndUserApiKey = this.BitlyKey,
 419             };
 420
 421             var result = await bitly.ShortenAsync(srcUri, domain)
 422                 .ConfigureAwait(false);
 423
 424             return this.UpgradeToHttpsIfAvailable(result);
 425         }
 426
 427         private async Task<Uri> ShortenByUxnuAsync(Uri srcUri)
 428         {
 429             // 明らかに長くなると推測できる場合は短縮しない
 430             if ("https://ux.nu/xxxxx".Length > srcUri.OriginalString.Length)
 431                 return srcUri;
 432
 433             var query = new Dictionary<string, string>
 434             {
 435                 ["format"] = "plain",
 436                 ["url"] = srcUri.OriginalString,
 437             };
 438
 439             var uri = new Uri("https://ux.nu/api/short?" + MyCommon.BuildQueryString(query));
 440             using var response = await this.http.GetAsync(uri)
 441                 .ConfigureAwait(false);
 442
 443             response.EnsureSuccessStatusCode();
 444
 445             var result = await response.Content.ReadAsStringAsync()
 446                 .ConfigureAwait(false);
 447
 448             if (!Regex.IsMatch(result, @"^https?://"))
 449                 throw new WebApiException("Failed to create URL.", result);
 450
 451             return new Uri(result.TrimEnd());
 452         }
 453
 454         private bool IsIrregularShortUrl(Uri uri)
 455         {
 456             // Flickrの https://www.flickr.com/photo.gne?short=... 形式のURL
 457             // flic.kr ドメインのURLを展開する途中に経由する
 458             if (uri.Host.EndsWith("flickr.com", StringComparison.OrdinalIgnoreCase) &&
 459                 uri.PathAndQuery.StartsWith("/photo.gne", StringComparison.OrdinalIgnoreCase))
 460                 return true;
 461
 462             return false;
 463         }
 464
 465         private async Task<Uri?> GetRedirectTo(Uri url)
 466         {
 467             url = this.UpgradeToHttpsIfAvailable(url);
 468
 469             var request = new HttpRequestMessage(HttpMethod.Head, url);
 470
 471             using var response = await this.http.SendAsync(request)
 472                 .ConfigureAwait(false);
 473
 474             if (!response.IsSuccessStatusCode)
 475             {
 476                 // ステータスコードが 3xx であれば例外を発生させない
 477                 if ((int)response.StatusCode / 100 != 3)
 478                     response.EnsureSuccessStatusCode();
 479             }
 480
 481             var redirectedUrl = response.Headers.Location;
 482
 483             if (redirectedUrl == null)
 484                 return null;
 485
 486             // サーバーが URL を適切にエンコードしていない場合、OriginalString に非 ASCII 文字が含まれる。
 487             // その場合、redirectedUrl は文字化けしている可能性があるため使用しない
 488             // 参照: http://stackoverflow.com/questions/1888933
 489             if (redirectedUrl.OriginalString.Any(x => x < ' ' || x > '~'))
 490                 return null;
 491
 492             if (redirectedUrl.IsAbsoluteUri)
 493                 return redirectedUrl;
 494             else
 495                 return new Uri(url, redirectedUrl);
 496         }
 497
 498         /// <summary>
 499         /// 指定されたURLのスキームを https:// に書き換える
 500         /// </summary>
 501         private Uri UpgradeToHttpsIfAvailable(Uri original)
 502         {
 503             if (original.Scheme != "http")
 504                 return original;
 505
 506             if (InsecureDomains.Contains(original.Host))
 507                 return original;
 508
 509             var builder = new UriBuilder(original);
 510             builder.Scheme = "https";
 511             builder.Port = 443;
 512
 513             return builder.Uri;
 514         }
 515
 516         [SuppressMessage("Microsoft.Reliability", "CA2000:DisposeObjectsBeforeLosingScope")]
 517         private static HttpClient CreateDefaultHttpClient()
 518         {
 519             var handler = Networking.CreateHttpClientHandler();
 520             handler.AllowAutoRedirect = false;
 521
 522             var http = Networking.CreateHttpClient(handler);
 523             http.Timeout = TimeSpan.FromSeconds(30);
 524
 525             return http;
 526         }
 527     }
 528 }