X-Git-Url: http://git.osdn.net/view?a=blobdiff_plain;f=ItemDownload.pas;h=36b54972965113506209d93c719b56cb180609e4;hb=refs%2Fheads%2Fmaster;hp=9d7c155a8dec1858e5959c9a0bd9e7c03a671781;hpb=09876d43bac960b9dfffb7251cf59b2cd1a6f71c;p=gikonavigoeson%2Fgikonavi.git diff --git a/ItemDownload.pas b/ItemDownload.pas index 9d7c155..36b5497 100644 --- a/ItemDownload.pas +++ b/ItemDownload.pas @@ -5,8 +5,8 @@ interface uses Windows, SysUtils, Classes, ComCtrls, Controls, Forms, IdHTTP, {HTTPApp,} YofUtils, IdGlobal, IdException, IdComponent, IniFiles, {DateUtils,} - GikoSystem, BoardGroup, MonaUtils, ExternalBoardManager, ExternalBoardPlugInMain, - Sort; + GikoSystem, BoardGroup, ExternalBoardManager, ExternalBoardPlugInMain, + Sort, SyncObjs, bmRegExp; type TDownloadItem = class; @@ -27,7 +27,17 @@ type FErrText: string; end; - + TWorkData = record + //FWorkCS: TCriticalSection; + //IdHttp‚ÌOnWorkAOnWorkBegin,OnWorkEnd‚͕ʂ̃XƒŒƒbƒh‚©‚ç + //ŒÄ‚΂ê‚éˆ×ASynchronize‚Å“¯Šú‚·‚é•K—v‚ª‚ ‚éB + //ƒNƒŠƒeƒBƒJƒ‹ƒZƒNƒVƒ‡ƒ“‚Í–³‚­‚Ä‚à‚½‚Ô‚ñ•½‹C‚¾‚¯‚Ç—pS‚ׁ̈B by eggcake + FWorkCS: TCriticalSection; + FSender: TObject; + FAWorkMode: TWorkMode; + FAWorkCount: Integer; + FAWorkCountMax: Integer + end; TDownloadThread = class(TThread) private FIndy: TIdHttp; @@ -43,6 +53,7 @@ type FOnDownloadEnd: TDownloadEndEvent; FOnDownloadMsg: TDownloadMsgEvent; FDownloadTitle: string; + FWorkData: TWorkData; procedure FireDownloadEnd; procedure FireDownloadMsg; @@ -51,10 +62,17 @@ type procedure WorkEnd(Sender: TObject; AWorkMode: TWorkMode); procedure Work(Sender: TObject; AWorkMode: TWorkMode; const AWorkCount: Integer); function ParseCgiStatus(Content: string): TCgiStatus; + function ParseRokkaStatus(Content: string): TCgiStatus; function CgiDownload(ItemType: TGikoDownloadType; URL: string; Modified: TDateTime): Boolean; function DatDownload(ItemType: TGikoDownloadType; URL: string; Modified: TDateTime; RangeStart: Integer; AdjustLen: Integer): Boolean; - function DeleteStatusLine(Content: string): string; - protected + procedure DeleteStatusLine(Item: TDownloadItem); + procedure InitHttpClient(client: TIdHttp); + procedure ClearHttpClient(client: TIdHttp); + procedure FireWork; + procedure FireWorkBegin; + procedure FireWorkEnd; + procedure GetLastModified; + protected procedure Execute; override; public property Item: TDownloadItem read FItem write FItem; @@ -103,9 +121,14 @@ type implementation +uses + Y_TextConverter, MojuUtils, HTMLCreate, ReplaceDataModule; + constructor TDownloadThread.Create(CreateSuspended: Boolean); begin inherited Create(CreateSuspended); + FWorkData.FWorkCS := TCriticalSection.Create; + FIndy := TIdHttp.Create(nil); FIndy.OnWorkBegin := WorkBegin; @@ -115,14 +138,9 @@ end; destructor TDownloadThread.Destroy; begin - FIndy.Request.CustomHeaders.Clear; - FIndy.Request.RawHeaders.Clear; - FIndy.Request.Clear; - FIndy.Response.CustomHeaders.Clear; - FIndy.Response.RawHeaders.Clear; - FIndy.Response.Clear; - FIndy.ProxyParams.Clear; + ClearHttpClient(FIndy); FIndy.Free; + FWorkData.FWorkCS.Free; inherited; end; @@ -143,7 +161,66 @@ begin [Day, Copy(StrMonth, 1 + 3 * (Month - 1), 3), Year, Hour, Min, Sec]); end; - +// ****************************************************************** +// HTTPClient‚̏‰Šú‰» +// ****************************************************************** +procedure TDownloadThread.InitHttpClient(client: TIdHttp); +begin + ClearHttpClient(client); + client.Disconnect; + client.Request.UserAgent := GikoSys.GetUserAgent; + client.RecvBufferSize := Gikosys.Setting.RecvBufferSize; + client.ProxyParams.BasicAuthentication := False; + client.ReadTimeout := GikoSys.Setting.ReadTimeOut; + client.ConnectTimeout := GikoSys.Setting.ReadTimeOut; + {$IFDEF DEBUG} + Writeln('------------------------------------------------------------'); + {$ENDIF} + //FIndy.AllowCookies := False; + if GikoSys.Setting.ReadProxy then begin + if GikoSys.Setting.ProxyProtocol then + client.ProtocolVersion := pv1_1 + else + client.ProtocolVersion := pv1_0; + client.ProxyParams.ProxyServer := GikoSys.Setting.ReadProxyAddress; + client.ProxyParams.ProxyPort := GikoSys.Setting.ReadProxyPort; + client.ProxyParams.ProxyUsername := GikoSys.Setting.ReadProxyUserID; + client.ProxyParams.ProxyPassword := GikoSys.Setting.ReadProxyPassword; + if GikoSys.Setting.ReadProxyUserID <> '' then + client.ProxyParams.BasicAuthentication := True; + {$IFDEF DEBUG} + Writeln('ƒvƒƒLƒVÝ’è‚ ‚è'); + Writeln('ƒzƒXƒg: ' + GikoSys.Setting.ReadProxyAddress); + Writeln('ƒ|[ƒg: ' + IntToStr(GikoSys.Setting.ReadProxyPort)); + {$ENDIF} + end else begin + if GikoSys.Setting.Protocol then + client.ProtocolVersion := pv1_1 + else + client.ProtocolVersion := pv1_0; + client.ProxyParams.ProxyServer := ''; + client.ProxyParams.ProxyPort := 80; + client.ProxyParams.ProxyUsername := ''; + client.ProxyParams.ProxyPassword := ''; + {$IFDEF DEBUG} + Writeln('ƒvƒƒLƒVÝ’è‚È‚µ'); + {$ENDIF} + end; +end; +// ****************************************************************** +// HTTPClient‚̃ŠƒNƒGƒXƒg‚ƃŒƒXƒ|ƒ“ƒX‚̃f[ƒ^‚̏Á‹Ž +// ****************************************************************** +procedure TDownloadThread.ClearHttpClient(client: TIdHttp); +begin + client.Request.CustomHeaders.Clear; + client.Request.RawHeaders.Clear; + client.Request.Clear; + client.Response.CustomHeaders.Clear; + client.Response.RawHeaders.Clear; + client.Response.Clear; + + client.ProxyParams.Clear; +end; procedure TDownloadThread.Execute; var ResStream: TMemoryStream; @@ -160,6 +237,8 @@ var lastContent : string; logFile : TFileStream; adjustMargin : Integer; + Host: String; + Sep: Integer; const ADJUST_MARGIN = 16; begin @@ -187,10 +266,14 @@ begin begin FDownloadTitle := FItem.FThreadItem.Title; if FItem.FThreadItem <> nil then begin - if FItem.FThreadItem.IsBoardPlugInAvailable then begin - boardPlugIn := FItem.FThreadItem.BoardPlugIn; + if FItem.FThreadItem.ParentBoard.IsBoardPlugInAvailable then begin + boardPlugIn := FItem.FThreadItem.ParentBoard.BoardPlugIn; Item.State := TGikoDownloadState( boardPlugIn.DownloadThread( DWORD( FItem.FThreadItem ) ) ); end; + //if FItem.FThreadItem.IsBoardPlugInAvailable then begin + // boardPlugIn := FItem.FThreadItem.BoardPlugIn; + // Item.State := TGikoDownloadState( boardPlugIn.DownloadThread( DWORD( FItem.FThreadItem ) ) ); + //end; end; end; end; @@ -210,51 +293,9 @@ begin Continue; end; - //===== ƒvƒ‰ƒOƒCƒ“‚ðŽg—p‚µ‚È‚¢ê‡ FAbort := False; - FIndy.Request.CustomHeaders.Clear; - FIndy.Response.Clear; - FIndy.Request.Clear; - FIndy.ProxyParams.Clear; - FIndy.Disconnect; - FIndy.Request.UserAgent := GikoSys.GetUserAgent; - FIndy.RecvBufferSize := Gikosys.Setting.RecvBufferSize; - FIndy.ProxyParams.BasicAuthentication := False; - FIndy.ReadTimeout := GikoSys.Setting.ReadTimeOut; - {$IFDEF DEBUG} - Writeln('------------------------------------------------------------'); - {$ENDIF} - //FIndy.AllowCookies := False; - if GikoSys.Setting.ReadProxy then begin - if GikoSys.Setting.ProxyProtocol then - FIndy.ProtocolVersion := pv1_1 - else - FIndy.ProtocolVersion := pv1_0; - FIndy.ProxyParams.ProxyServer := GikoSys.Setting.ReadProxyAddress; - FIndy.ProxyParams.ProxyPort := GikoSys.Setting.ReadProxyPort; - FIndy.ProxyParams.ProxyUsername := GikoSys.Setting.ReadProxyUserID; - FIndy.ProxyParams.ProxyPassword := GikoSys.Setting.ReadProxyPassword; - if GikoSys.Setting.ReadProxyUserID <> '' then - FIndy.ProxyParams.BasicAuthentication := True; - {$IFDEF DEBUG} - Writeln('ƒvƒƒLƒVÝ’è‚ ‚è'); - Writeln('ƒzƒXƒg: ' + GikoSys.Setting.ReadProxyAddress); - Writeln('ƒ|[ƒg: ' + IntToStr(GikoSys.Setting.ReadProxyPort)); - {$ENDIF} - end else begin - if GikoSys.Setting.Protocol then - FIndy.ProtocolVersion := pv1_1 - else - FIndy.ProtocolVersion := pv1_0; - FIndy.ProxyParams.ProxyServer := ''; - FIndy.ProxyParams.ProxyPort := 80; - FIndy.ProxyParams.ProxyUsername := ''; - FIndy.ProxyParams.ProxyPassword := ''; - {$IFDEF DEBUG} - Writeln('ƒvƒƒLƒVÝ’è‚È‚µ'); - {$ENDIF} - end; - + //===== ƒvƒ‰ƒOƒCƒ“‚ðŽg—p‚µ‚È‚¢ê‡ + InitHttpClient(FIndy); adjustMargin := 0; if Item.DownType = gdtThread then begin if FileExists( Item.ThreadItem.GetThreadFileName ) then begin @@ -356,9 +397,9 @@ begin if Trim(FIndy.Response.RawHeaders.Values['Date']) <> '' then begin if Item.DownType = gdtBoard then - Item.Board.LastGetTime := MonaUtils.DateStrToDateTime(FIndy.Response.RawHeaders.Values['Date']) + Item.Board.LastGetTime := GikoSys.DateStrToDateTime(FIndy.Response.RawHeaders.Values['Date']) else - Item.ThreadItem.ParentBoard.LastGetTime := MonaUtils.DateStrToDateTime(FIndy.Response.RawHeaders.Values['Date']); + Item.ThreadItem.ParentBoard.LastGetTime := GikoSys.DateStrToDateTime(FIndy.Response.RawHeaders.Values['Date']); end; if DownloadResult then begin @@ -427,6 +468,28 @@ begin end; //******************** + //dat.gz@¨@dat‚̎擾@2005”N6ŒŽ’ljÁ@by‚à‚¶‚ã + //******************** + if (Item.DownType = gdtThread) and (Item.ResponseCode = 302) then begin + {$IFDEF DEBUG} + Writeln('datŽæ“¾'); + {$ENDIF} + FMsg := '‰ß‹ŽƒƒO(dat.gz)‚ª‘¶Ý‚µ‚È‚¢‚½‚߉ߋŽƒƒO(dat)‚ð’T‚µ‚Ü‚· - [' + ATitle + ']'; + FIcon := gmiWhat; + if Assigned(OnDownloadMsg) then + Synchronize(FireDownloadMsg); + URL := ChangeFileExt(URL, ''); + Modified := Item.ThreadItem.LastModified; + RangeStart := 0; + AdjustLen := 0; + if not DatDownload(Item.DownType, URL, Modified, RangeStart, AdjustLen) then + Item.State := gdsError; + {$IFDEF DEBUG} + Writeln('ResponseCode: ' + IntToStr(Item.ResponseCode)); + {$ENDIF} + end; + + //******************** //dat.gzŽæ“¾(2) //******************** { @@ -447,6 +510,148 @@ begin end; } + if (Item.DownType = gdtThread) and (Item.ResponseCode = 302) then begin + {$IFDEF DEBUG} + Writeln('offlaw2.so‚Ŏ擾'); + {$ENDIF} + ATitle := Item.ThreadItem.Title; + if ATitle = '' then + ATitle := 'i–¼Ì•s–¾j'; + FMsg := 'šdat.gz‚ª‘¶Ý‚µ‚È‚¢‚½‚ßofflaw2.so‚ð—˜—p‚µ‚Ü‚· - [' + ATitle + ']'; + FIcon := gmiWhat; + if Assigned(OnDownloadMsg) then + Synchronize(FireDownloadMsg); + URL := Item.ThreadItem.GetOfflaw2SoURL; + Modified := Item.ThreadItem.LastModified; + RangeStart := 0; + AdjustLen := 0; + if not DatDownload(Item.DownType, URL, Modified, RangeStart, AdjustLen) then begin + {$IFDEF DEBUG} + Writeln('ResponseCode: ' + IntToStr(Item.ResponseCode)); + {$ENDIF} + Item.State := gdsError; + + if (Item.DownType = gdtThread) and (Item.ResponseCode = 302) then begin + FMsg := '”‚ªˆÚ“]‚µ‚½‚©‚à‚µ‚ê‚È‚¢‚̂ŔXV‚ðs‚Á‚Ä‚­‚¾‚³‚¢B'; + FIcon := gmiNG; + if Assigned(OnDownloadMsg) then + Synchronize(FireDownloadMsg); + end; + + end else begin + {$IFDEF DEBUG} + Writeln('ResponseCode: ' + IntToStr(Item.ResponseCode)); + {$ENDIF} + if Item.ResponseCode = 200 then begin + {$IFDEF DEBUG} + Writeln('CGIStatus: OK'); + {$ENDIF} + if Copy(Item.Content, 1, 5) = 'ERROR' then begin + {$IFDEF DEBUG} + Writeln('Offlow2Error'); + {$ENDIF} + Item.ResponseCode := 404; + Item.State := gdsError; + Item.ErrText := 'ƒXƒŒ‚Í‘¶Ý‚µ‚È‚¢‚悤‚Å‚·B' + Item.Content; + end else begin + GetLastModified; + end; + end else begin + {$IFDEF DEBUG} + Writeln('CGIStatus: 404(ERROR)'); + {$ENDIF} + Item.ResponseCode := 404; + Item.State := gdsError; + Item.ErrText := CgiStatus.FErrText; + end; + end; + end; + + if (Item.DownType = gdtThread) and ((Item.ResponseCode = 302) or (Item.ResponseCode = 404)) then begin + FSessionID := ''; + Synchronize(GetSessionID); + if (FSessionID <> '') then begin + {$IFDEF DEBUG} + Writeln('Rokka‚Ŏ擾'); + {$ENDIF} + ATitle := Item.ThreadItem.Title; + if ATitle = '' then + ATitle := 'i–¼Ì•s–¾j'; + FMsg := 'šofflow2.so‚É‘¶Ý‚µ‚È‚¢‚½‚ßRokka‚ð—˜—p‚µ‚Ü‚· - [' + ATitle + ']'; + FIcon := gmiWhat; + if Assigned(OnDownloadMsg) then + Synchronize(FireDownloadMsg); + URL := Item.ThreadItem.GetRokkaURL(FSessionID); + Modified := Item.ThreadItem.LastModified; + RangeStart := 0; + AdjustLen := 0; + + if not DatDownload(Item.DownType, URL, Modified, RangeStart, AdjustLen) then begin + {$IFDEF DEBUG} + Writeln('ResponseCode: ' + IntToStr(Item.ResponseCode)); + {$ENDIF} + Item.State := gdsError; + end else begin + CgiStatus := ParseRokkaStatus(Item.Content); + {$IFDEF DEBUG} + Writeln('ResponseCode: ' + IntToStr(Item.ResponseCode)); + {$ENDIF} + case CgiStatus.FStatus of + gcsOK: begin + {$IFDEF DEBUG} + Writeln('CGIStatus: OK'); + {$ENDIF} + Item.ResponseCode := 200; + DeleteStatusLine(Item); + end; + gcsERR: begin + {$IFDEF DEBUG} + Writeln('CGIStatus: 404(ERROR)'); + {$ENDIF} + Item.ResponseCode := 404; + Item.State := gdsError; + Item.ErrText := CgiStatus.FErrText; + end; + end; + end; + end; + end; + + //******************** + // 2chŠO•”” + //******************** +// if not GikoSys.Is2chHost(GikoSys.UrlToServer(URL)) then begin + Host := URL; + Sep := Pos('://', Host); + if (Sep > 0) then + Delete(Host, 1, Sep + 2); + Sep := Pos('/', Host); + if (Sep > 0) then + SetLength(Host, Sep - 1); + if not GikoSys.Is2chHost(Host) then begin + if (Item.DownType = gdtThread) and (Item.ResponseCode = 404) then begin + {$IFDEF DEBUG} + Writeln('ŠO•””‰ߋŽƒƒOŽæ“¾'); + {$ENDIF} + ATitle := Item.ThreadItem.Title; + if ATitle = '' then + ATitle := 'i–¼Ì•s–¾j'; + FMsg := 'š‚Q‚¿‚á‚ñ‚Ë‚éŠO•””‚̉ߋŽƒƒOŽæ“¾‚ðs‚¢‚Ü‚· - [' + ATitle + ']'; + FIcon := gmiWhat; + if Assigned(OnDownloadMsg) then + Synchronize(FireDownloadMsg); + URL := Item.ThreadItem.GetExternalBoardKakoDatURL; + Modified := Item.ThreadItem.LastModified; + RangeStart := 0; + AdjustLen := 0; + if not DatDownload(Item.DownType, URL, Modified, RangeStart, AdjustLen) then + Item.State := gdsError; + {$IFDEF DEBUG} + Writeln('ResponseCode: ' + IntToStr(Item.ResponseCode)); + {$ENDIF} + end; + end; +(* //******************** //offlaw.cgi‚Ŏ擾 //******************** @@ -491,8 +696,7 @@ begin Writeln('CGIStatus: OK'); {$ENDIF} Item.ResponseCode := 200; - Item.Content := DeleteStatusLine(Item.Content); - Item.ContentLength := CgiStatus.FSize; + DeleteStatusLine(Item); end; gcsINCR: begin //¡‚Í‚ ‚肦‚È‚¢ @@ -500,8 +704,7 @@ begin Writeln('CGIStatus: 206'); {$ENDIF} Item.ResponseCode := 206; - Item.Content := DeleteStatusLine(Item.Content); - Item.ContentLength := CgiStatus.FSize; + DeleteStatusLine(Item); end; gcsERR: begin {$IFDEF DEBUG} @@ -558,7 +761,7 @@ begin Synchronize(FireDownloadMsg); end; end; - +*) case Item.ResponseCode of 200: Item.State := gdsComplete; 206: Item.State := gdsDiffComplete; @@ -566,68 +769,9 @@ begin else Item.State := gdsError; end; -{ - //–³‚¢‚ÆŽv‚¤‚¯‚ǁBBB - if (Item.ResponseCode in [200, 206]) and (Item.Content = '') then - Item.State := gdsError; - Item.LastModified := FIndy.Response.LastModified; - //·•ªŽæ“¾‚Å‚PƒoƒCƒg‘O‚©‚ç‚Æ‚Á‚Ä‚«‚½‚Æ‚«‚̓}ƒCƒiƒX‚·‚é - Item.ContentLength := FIndy.Response.ContentLength + AdjustLen; - try - ResStream.Clear; - FIndy.Get(URL, ResStream); - Item.Content := GikoSys.GzipDecompress(ResStream, FIndy.Response.ContentEncoding); - if (Item.DownType = gdtThread) and (AdjustLen = -1) and (Item.Content[1] <> #10) then begin - //·•ªŽæ“¾‚©‚‚PƒoƒCƒg–Ú‚ªLF‚Å‚È‚¢ê‡‚́u‚ ‚ځ[‚ñv‚³‚ê‚Ä‚¢‚é‚©‚à‚µ‚ê‚ñ‚̂ōĎ擾 - //‚±‚±‚сƒbƒZ[ƒW•\Ž¦ƒCƒxƒ“ƒg - //event - FMsg := 'u‚ ‚ځ[‚ñv‚ðŒŸo‚µ‚½‚̂ōĎ擾‚ðs‚¢‚Ü‚·'; - if Assigned(OnDownloadMsg) then - Synchronize(FireDownloadMsg); - FIndy.Request.ContentRangeStart := 0; - FIndy.Request.ContentRangeEnd := 0; - AdjustLen := 0; - ResStream.Clear; - FIndy.Get(URL, ResStream); - Item.Content := GikoSys.GzipDecompress(ResStream, FIndy.Response.ContentEncoding); - end else if (Item.DownType = gdtThread) and (AdjustLen = -1) and (Item.Content[1] = #10) then begin - //·•ªŽæ“¾‚©‚‚PƒoƒCƒg–Ú‚ªLF‚̏ꍇi³íŽæ“¾j‚Í“ª‚ÌLF‚ðíœ - Item.Content := Copy(Item.Content, 2, Length(Item.Content)); - end; - except - Item.State := gdsError; - end; - Item.ResponseCode := FIndy.ResponseCode; -} -{ - try - ResStream.Clear; - FIndy.Get(URL, ResStream); - Item.Content := GikoSys.GzipDecompress(ResStream, FIndy.Response.ContentEncoding); - except - Item.State := gdsError; - end; - - CgiStatus := ParseCgiStatus(Item.Content); - if CgiStatus.FStatus = gcsOK then begin - if CgiStatus.FSize = 0 then - Item.State := gdsNotModify - else if Item.DownType = gdtBoard then - Item.State := gdsComplete - else - Item.State := gdsDiffComplete; - end else if CgiStatus.FStatus = gcsINCR then begin - Item.State := gdsComplete; - end else if CgiStatus.FStatus = gcsERR then begin - Item.State := gdsError; - Item.ErrText := CgiStatus.FErrText; - end; - Item.ContentLength := CgiStatus.FSize; - } except Item.State := gdsError; end; - //Item.ResponseCode := FIndy.ResponseCode; if FAbort then Item.State := gdsAbort; finally @@ -636,13 +780,7 @@ begin ResStream.Free; end; - FIndy.Request.CustomHeaders.Clear; - FIndy.Request.RawHeaders.Clear; - FIndy.Request.Clear; - FIndy.Response.CustomHeaders.Clear; - FIndy.Response.RawHeaders.Clear; - FIndy.Response.Clear; - FIndy.ProxyParams.Clear; + ClearHttpClient(FIndy); if Terminated then Break; Suspend; @@ -734,8 +872,6 @@ var begin ResponseCode := -1; if (ItemType = gdtThread) and (RangeStart > 0) then begin -// if (ItemType = gdtThread) and (Item.ThreadItem.Size > 0) then begin -// FIndy.Request.ContentRangeStart := Item.ThreadItem.Size + AdjustLen; FIndy.Request.ContentRangeStart := RangeStart + AdjustLen; FIndy.Request.ContentRangeEnd := 0; end else begin @@ -748,9 +884,7 @@ begin FIndy.Request.CustomHeaders.Add('Pragma: no-cache'); if (Modified <> 0) and (Modified <> ZERO_DATE) then begin FIndy.Request.LastModified := modified - OffsetFromUTC; - //FIndy.Request.CustomHeaders.Add('If-Modified-Since: ' + RFC1123_Date(modified - OffsetFromUTC) + ' GMT'); end; -// FIndy.Request.AcceptEncoding := 'gzip'; if RangeStart = 0 then FIndy.Request.AcceptEncoding := 'gzip' else @@ -763,14 +897,13 @@ begin Writeln('URL: ' + URL); {$ENDIF} FIndy.Get(URL, ResStream); - Item.Content := GikoSys.GzipDecompress(ResStream, FIndy.Response.ContentEncoding); - Item.LastModified := FIndy.Response.LastModified; - //·•ªŽæ“¾‚Å‚PƒoƒCƒg‘O‚©‚ç‚Æ‚Á‚Ä‚«‚½‚Æ‚«‚̓}ƒCƒiƒX‚·‚é -// Item.ContentLength := FIndy.Response.ContentLength + AdjustLen; + Item.Content := GikoSys.GzipDecompress(ResStream, FIndy.Response.ContentEncoding); Item.ContentLength := Length(Item.Content) + AdjustLen; - //–³‚¢‚ÆŽv‚¤‚¯‚ǁBBB -// if (FIndy.ResponseCode in [200, 206]) and (Item.Content = '') then -// Result := False + // ’uŠ·‚·‚é + if GikoSys.Setting.ReplaceDat then begin + Item.Content := ReplaceDM.Replace(Item.Content); + end; + Item.LastModified := FIndy.Response.LastModified; if Item.Content = '' then Result := False else @@ -845,22 +978,72 @@ begin end; end; -procedure TDownloadThread.WorkBegin(Sender: TObject; AWorkMode: TWorkMode; const AWorkCountMax: Integer); +procedure TDownloadThread.WorkBegin(Sender: TObject; +AWorkMode: TWorkMode; const AWorkCountMax: Integer); begin - if Assigned(OnWorkBegin) then - OnWorkBegin(Sender, AWorkMode, AWorkCountMax, FNumber, FDownloadTitle); + if Assigned(OnWorkBegin) then begin + FWorkData.FWorkCS.Acquire; + try + FWorkData.FSender := Sender; + FWorkData.FAWorkMode := AWorkMode; + FWorkData.FAWorkCountMax := AWorkCountMax; + Synchronize(FireWorkBegin); + finally + FWorkData.FWorkCS.Release; + end; + end; end; -procedure TDownloadThread.WorkEnd(Sender: TObject; AWorkMode: TWorkMode); +procedure TDownloadThread.WorkEnd(Sender: TObject; +AWorkMode: TWorkMode); begin - if Assigned(OnWorkEnd) then - OnWorkEnd(Sender, AWorkMode, FNumber); + if Assigned(OnWorkEnd) then begin; + FWorkData.FWorkCS.Acquire; + try + FWorkData.FSender := Sender; + FWorkData.FAWorkMode := AWorkMode; + Synchronize(FireWorkEnd); + finally + FWorkData.FWorkCS.Release; + end; + end; +end; + +procedure TDownloadThread.Work(Sender: TObject; AWorkMode: +TWorkMode; const AWorkCount: Integer); +begin + if Assigned(OnWork) then begin + FWorkData.FWorkCS.Acquire; + try + FWorkData.FSender := Sender; + FWorkData.FAWorkMode := AWorkMode; + FWorkData.FAWorkCount := AWorkCount; + Synchronize(FireWork); + finally + FWorkData.FWorkCS.Release; + end; + end; end; -procedure TDownloadThread.Work(Sender: TObject; AWorkMode: TWorkMode; const AWorkCount: Integer); +//‚±‚±‚©‚çV‹Kƒƒ\ƒbƒh +procedure TDownloadThread.FireWorkBegin; begin - if Assigned(OnWork) then - OnWork(Sender, AWorkMode, AWorkCount, FNumber); + OnWorkBegin(FWorkData.FSender, FWorkData.FAWorkMode, + FWorkData.FAWorkCountMax, + FNumber, FDownloadTitle); +end; + +procedure TDownloadThread.FireWorkEnd; +begin + OnWorkEnd(FWorkData.FSender, FWorkData.FAWorkMode, + FNumber); +end; + +procedure TDownloadThread.FireWork; +begin + OnWork(FWorkData.FSender, FWorkData.FAWorkMode, + FWorkData.FAWorkCount, + FNumber); end; function TDownloadThread.ParseCgiStatus(Content: string): TCgiStatus; @@ -919,17 +1102,84 @@ begin end; end; -//Žè”²‚«‚ȏˆ—‚Å1s–Ú‚ðÁ‚· -function TDownloadThread.DeleteStatusLine(Content: string): string; +function TDownloadThread.ParseRokkaStatus(Content: string): TCgiStatus; +var + StatusLine: string; + Idx: Integer; + Status: string; +begin +// @ƒŒƒXƒ|ƒ“ƒX : 1s–Ú‚Érokka‚̏ˆ—Œ‹‰Ê‚ª‹Lq‚³‚ê‚Ü‚· +// @@"Success XXX"@- ¬Œ÷@XXX‚Édat‚̏ó‘ԁiŽæ“¾Œ³j‚ª‹Lq‚³‚ê‚Ü‚· +// @@@@@@@@@@@Live@@@@ƒ‰ƒCƒuƒXƒŒƒbƒh +// @@@@@@@@@@@Pool@@@@dat—Ž‚¿ƒXƒŒƒbƒh +// @@@@@@@@@@@Archive @@‰ß‹ŽƒƒO +// @@@@@@@@@@ ˆÈ~‚̍s‚ÉDATŒ`Ž®(name<>email<>datetime<>body<>[title])‚эƒO‚ª‹Lq‚³‚ê‚Ä‚¢‚Ü‚· +// @@"Error XXX"@@- ‰½‚ç‚©‚̃Gƒ‰[‚Å‚·@XXX ‚ªƒGƒ‰[ƒR[ƒh‚Å‚·B +// @@@@@@@@@@@13 @@@not found@@@@@@—v‹‚³‚ꂽdat‚ªŒ©‚‚©‚è‚Ü‚¹‚ñ‚Å‚µ‚½ +// @@@@@@@@@@@8008135@inputError @@@@@ƒŠƒNƒGƒXƒgURL‚ÌSERVER‚©BOARD‚ª³‚µ‚­‚È‚¢‚Å‚· +// @@@@@@@@@@@666@@@urlError @@@@@@OPTIONS‚Ü‚½‚ÍQueryString‚ª³‚µ‚­‚È‚¢‚Å‚· +// @@@@@@@@@@@69 @@@authenticationError@KAGI‚ª•s³i—LŒøŠúŒÀØ‚ê‚»‚Ì‘¼j +// @@@@@@@@@@@420@@@timeLimitError @@@ƒAƒNƒZƒXŠÔŠu‚ª’Z‚·‚¬‚Ü‚· +// @@@@@@@@@@@42 @@@methodError@@@@@‚»‚ÌHTTPƒƒ\ƒbƒh‚Í‹–‰Â‚³‚ê‚Ä‚¢‚Ü‚¹‚ñ + Idx := AnsiPos(#10, Content); + if (Idx > 0) then + StatusLine := Copy(Content, 0, Idx) + else + StatusLine := Content; + + if (AnsiPos('Success', StatusLine) = 1) then begin + Result.FStatus := gcsOK; + Delete(StatusLine, 1, 7); + Status := Trim(StatusLine); + if (Status = 'Live') then // ‘½•ª‚±‚ê‚Í—ˆ‚È‚¢ + Result.FErrText := 'Žæ“¾¬Œ÷iƒ‰ƒCƒuƒXƒŒƒbƒhj' + else if (Status = 'Pool') then + Result.FErrText := 'Žæ“¾¬Œ÷idat—Ž‚¿ƒXƒŒƒbƒhj' + else if (Status = 'Archive') then + Result.FErrText := 'Žæ“¾¬Œ÷i‰ß‹ŽƒƒOj' + else // ??? + Result.FErrText := 'Žæ“¾¬Œ÷'; + end + else if (AnsiPos('Error', StatusLine) = 1) then begin + Result.FStatus := gcsERR; + Delete(StatusLine, 1, 5); + Status := Trim(StatusLine); + if (Status = '13') then + Result.FErrText := '—v‹‚³‚ꂽdat‚ªŒ©‚‚©‚è‚Ü‚¹‚ñ‚Å‚µ‚½' + else if (Status = '8008135') then + Result.FErrText := 'ƒŠƒNƒGƒXƒgURL‚ÌSERVER‚©BOARD‚ª³‚µ‚­‚È‚¢‚Å‚·' + else if (Status = '666') then + Result.FErrText := 'OPTIONS‚Ü‚½‚ÍQueryString‚ª³‚µ‚­‚È‚¢‚Å‚·' + else if (Status = '69') then + Result.FErrText := 'KAGI‚ª•s³i—LŒøŠúŒÀØ‚ê‚»‚Ì‘¼j' + else if (Status = '20') then + Result.FErrText := 'ƒAƒNƒZƒXŠÔŠu‚ª’Z‚·‚¬‚Ü‚·' + else if (Status = '42') then + Result.FErrText := '‚»‚ÌHTTPƒƒ\ƒbƒh‚Í‹–‰Â‚³‚ê‚Ä‚¢‚Ü‚¹‚ñ' + else // ??? + Result.FErrText := 'Žæ“¾ƒGƒ‰[[' + Status + ']'; + end + else begin + Result.FStatus := gcsERR; + Result.FErrText := 'ƒXƒe[ƒ^ƒX‰ðÍŽ¸”s[' + StatusLine + ']'; + end; + Result.FSize := 0; +end; + +//‚Ps–Ú‚ðÁ‚µ‚āAƒRƒ“ƒeƒ“ƒcƒTƒCƒY‚ðÝ’è‚·‚é +procedure TDownloadThread.DeleteStatusLine(Item: TDownloadItem); var SList: TStringList; begin SList := TStringList.Create; try - SList.Text := Content; + SList.Text := Item.Content; + //1s–Ú‚ðíœ if SList.Count > 1 then SList.Delete(0); - Result := SList.Text; + Item.Content := SList.Text; + //‰üsƒR[ƒh‚ðCRLF -> LF‚ƍl‚¦‚āAs”•ª‚¾‚¯ƒ}ƒCƒiƒX + Item.ContentLength := Length(SList.Text) - SList.Count; finally SList.Free; end; @@ -943,6 +1193,9 @@ var NumCount: Integer; Body: TStringList; Rec: TSubjectRec; + {$IFDEF DEBUG} + st, rt : Cardinal; + {$ENDIF} function MakeThreadCallBack( inInstance : DWORD; // TBoardItem ‚̃Cƒ“ƒXƒ^ƒ“ƒX inURL : PChar; // ƒXƒŒƒbƒh‚Ì URL @@ -957,10 +1210,13 @@ var boardItem := TBoard( inInstance ); boardItem.IntData := boardItem.IntData + 1; - index := boardItem.GetIndexFromURL( string( inURL ) ); + if boardItem.IntData < (boardItem.Count shr 2) then + index := boardItem.GetIndexFromURL( string( inURL ) ) + else + index := boardItem.GetIndexFromURL( string( inURL ), True ); if index = -1 then begin //V‚µ‚¢ƒXƒŒƒbƒh - _ThreadItem := TThreadItem.Create( boardItem.BoardPlugIn, string( inURL ) ); + _ThreadItem := TThreadItem.Create( boardItem.BoardPlugIn, boardItem, string( inURL ) ); _ThreadItem.Title := string( inTitle ); _ThreadItem.AllResCount := inCount; @@ -971,18 +1227,26 @@ var _ThreadItem.AgeSage := gasNew; boardItem.Add(_ThreadItem); end else begin + //‡ˆÊ‚ªã‚ª‚Á‚Ä‚¢‚ê‚ÎAge‚É‚·‚é if boardItem.Items[index].No > boardItem.IntData then boardItem.Items[index].AgeSage := gasAge - else if boardItem.Items[index].AllResCount < inCount then + //‡ˆÊ‚ªã‚ª‚Á‚Ä‚È‚¢‚¯‚ǁAƒŒƒX‚ª‚‚¢‚Ä‚½‚çASage‚É + else if boardItem.Items[index].AllResCount <> inCount then boardItem.Items[index].AgeSage := gasSage + //‡ˆÊã‚ª‚Á‚Ä‚È‚¢‚µAƒŒƒX‚Ì‘Œ¸‚à–³‚¯‚ê‚΁ANone else boardItem.Items[index].AgeSage := gasNone; boardItem.Items[index].No := boardItem.IntData; boardItem.Items[index].AllResCount := inCount; end; + end; begin +{$IFDEF DEBUG} + st := GetTickCount; + Writeln('SAVELIST'); +{$ENDIF} //Board.ListData := TList.Create; Body := TStringList.Create; try @@ -991,15 +1255,21 @@ begin //ƒT[ƒoãƒtƒ@ƒCƒ‹‚̍XVŽžÝ’è Board.LastModified := LastModified; + //dat—Ž‚¿ƒXƒŒ‚̃\[ƒg‡‚ðŒˆ’è‚·‚邽‚߂Ƀ\[ƒg‚·‚é if GikoSys.Setting.DatOchiSortIndex >= 0 then begin - Sort.SortNoFlag := true; - Sort.SortOrder := GikoSys.Setting.DatOchiSortOrder; - Sort.SortIndex := GikoSys.Setting.DatOchiSortIndex; + Sort.SetSortNoFlag(true); + Sort.SetSortOrder(GikoSys.Setting.DatOchiSortOrder); + Sort.SetSortIndex(GikoSys.Setting.DatOchiSortIndex); //Sort.SortNonAcquiredCountFlag := GikoSys.Setting.NonAcquiredCount; Board.CustomSort(ThreadItemSortProc); end; +{$IFDEF DEBUG} + rt := GetTickCount - st; + Writeln('END Sortd' + IntToStr(rt) + ' ms'); +{$ENDIF} + for i := Board.Count - 1 downto 0 do Board.Items[i].AgeSage := gasNull; @@ -1008,9 +1278,22 @@ begin // V‚µ‚¢ƒŠƒXƒg‚Ɍ¢ƒŠƒXƒg‚̃ƒO‚ª‚ ‚é‚È‚ç‚»‚ê‚ðV‚µ‚¢ƒŠƒXƒg‚ɒljÁ // ŒÃ‚¢ƒƒO‚ª‚È‚¯‚ê‚΁AV‚½‚ɃXƒŒƒIƒuƒWƒFƒNƒg‚ðì¬ Board.IntData := 0; +{$IFDEF DEBUG} + rt := GetTickCount - st; + Writeln('Start Enum' + IntToStr(rt) + ' ms'); +{$ENDIF} + + //‚±‚ꂪ’x‚¢@—v‰ü‘P + Board.BeginUpdate; Board.BoardPlugIn.EnumThread( DWORD( Board ), @MakeThreadCallBack ); + Board.EndUpdate; - //ŒÃ‚¢ƒŠƒXƒg‚É‚µ‚©‚È‚¢‚â‚‚ç‚ðíœ +{$IFDEF DEBUG} + rt := GetTickCount - st; + Writeln('End Enum' + IntToStr(rt) + ' ms'); +{$ENDIF} + + //ŒÃ‚¢ƒŠƒXƒg‚É‚µ‚©‚È‚¢‚â‚‚ç‚ðíœ for i := Board.Count - 1 downto 0 do begin if( Board.Items[i].AgeSage = gasNull )and not (Board.Items[i].IsLogFile) then Board.Delete(i); @@ -1023,9 +1306,10 @@ begin Board.Items[i].No := Board.IntData; Board.Items[i].AllResCount := Board.Items[i].Count; Board.Items[i].NewResCount := 0; - Board.Items[i].AgeSage := gasNone; + Board.Items[i].AgeSage := gasArch; end; end; + end else begin //V‚µ‚¢ƒŠƒXƒg‚ðì¬‚·‚é //V‚µ‚¢ƒŠƒXƒg‚Ɍ¢ƒŠƒXƒg‚̃ƒO‚ª‚ ‚é‚È‚ç‚»‚ê‚ðV‚µ‚¢ƒŠƒXƒg‚ɒljÁ @@ -1041,7 +1325,9 @@ begin if index = -1 then begin //V‚µ‚¢ƒXƒŒƒbƒh NewItem := TThreadItem.Create( - nil, GikoSys.Get2chBoard2ThreadURL( Board, ChangeFileExt( Rec.FFileName, '' ) ) ); + nil, + Board, + GikoSys.Get2chBoard2ThreadURL( Board, ChangeFileExt( Rec.FFileName, '' ) ) ); NewItem.Title := Rec.FTitle; NewItem.AllResCount := Rec.FCount; NewItem.ParentBoard := Board; @@ -1075,11 +1361,12 @@ begin Board.Items[i].No := NumCount; Board.Items[i].AllResCount := Board.Items[i].Count; Board.Items[i].NewResCount := 0; - Board.Items[i].AgeSage := gasNone; + Board.Items[i].AgeSage := gasArch; end; end; //ƒŠƒXƒg(subject.txt)‚ð•Û‘¶ GikoSys.ForceDirectoriesEx(ExtractFilePath(Board.GetSubjectFileName)); + Body.Text := MojuUtils.Sanitize(Body.Text); Body.SaveToFile(Board.GetSubjectFileName); end; finally @@ -1185,16 +1472,12 @@ var NewRes: Integer; finish : Boolean; loopCnt : Integer; -// KokoTxt : string; -// KokoIdx : Integer; -// NewTxt : string; -// NewIdx : Integer; -// LastTxt : string; LastIdx : Integer; begin FileName := ThreadItem.GetThreadFileName; - if not ThreadItem.IsBoardPlugInAvailable then begin + //if not ThreadItem.IsBoardPlugInAvailable then begin + if not ThreadItem.ParentBoard.IsBoardPlugInAvailable then begin if Trim(Content) = '' then Exit; @@ -1287,7 +1570,7 @@ begin Body.SaveToFile(FileName); if ThreadItem.Title = '' then begin - Res := GikoSys.DivideStrLine(Body[0]); + THTMLCreate.DivideStrLine(Body[0], @Res); ThreadItem.Title := Res.FTitle; end; ThreadItem.Size := 0; @@ -1309,7 +1592,6 @@ begin ThreadItem.RoundDate := Now; if not ThreadItem.UnRead then begin ThreadItem.UnRead := True; - ThreadItem.ParentBoard.UnRead := ThreadItem.ParentBoard.UnRead + 1; end; // if ThreadItem.RoundNo = 6 then // ThreadItem.RoundNo := 0; @@ -1327,7 +1609,7 @@ begin ini.WriteInteger('Setting', 'NewResCount', ThreadItem.NewResCount); ini.WriteInteger('Setting', 'NewReceive', ThreadItem.NewReceive); // ini.WriteInteger('Setting', 'RoundNo', ThreadItem.RoundNo); - ini.WriteBool('Setting', 'Round', ThreadItem.Round); +// ini.WriteBool('Setting', 'Round', ThreadItem.Round); ini.WriteBool('Setting', 'UnRead', ThreadItem.UnRead); ini.UpdateFile; finally @@ -1335,4 +1617,66 @@ begin end; end; +procedure TDownloadThread.GetLastModified; +var + ResultDate: TDateTime; + ResList: TStringList; + LastRes: String; + KwPos: Integer; + ResRow: Integer; + DTIdx: Integer; + Ok: Boolean; + AWKStr: TAWKStr; + RStart: Integer; + RLength: Integer; +begin + AWKStr := TAWKStr.Create(nil); + Ok := False; + ResultDate := Item.LastModified; + ResList := TStringList.Create; + try + ResList.Text := Item.Content; + for ResRow := ResList.Count - 1 downto 0 do begin + if (ResRow > 999) then + continue; + LastRes := ResList.Strings[ResRow]; + KwPos := Pos('<>', LastRes); + if (KwPos < 1) then + continue; + Delete(LastRes, 1, KwPos + 1); + KwPos := Pos('<>', LastRes); + if (KwPos < 1) then + continue; + Delete(LastRes, 1, KwPos + 1); + // '2013/04/22(ŒŽ) 02:32:36' + SetLength(LastRes, 23); + Delete(LastRes, 11, 4); // —j“úíœ + + // “ú•tŠm”F +// AWKStr.RegExp := '(\d{4})/(0[1-9]|1[0-2])/(0[1-9]|[12][0-9]|3[01])'; + AWKStr.RegExp := '(19|20)([0-9][0-9])/(0[1-9]|1[0-2])/(0[1-9]|[12][0-9]|3[01])'; + if (AWKStr.Match(AWKStr.ProcessEscSeq(LastRes), RStart, RLength) = 1) then begin + + AWKStr.RegExp := '([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9])'; + if (AWKStr.Match(AWKStr.ProcessEscSeq(LastRes), RStart, RLength) < 1) then begin + SetLength(LastRes, 10); + LastRes := LastRes + ' 00:00:00'; + end; + + try + ResultDate := StrToDateTime(LastRes); + Ok := True; + except + end; + end; + if (Ok = True) then + break; + end; + finally + ResList.Free; + end; + Item.LastModified :=ResultDate; + FreeAndNil(AWKStr); +end; + end.