OSDN Git Service

BugTrack/560 tracker_list speeding up with cache bugtrack_560_tracker_list2
authorumorigu <umorigu@gmail.com>
Mon, 26 Feb 2018 15:10:34 +0000 (00:10 +0900)
committerumorigu <umorigu@gmail.com>
Mon, 26 Feb 2018 15:10:34 +0000 (00:10 +0900)
* Implement 2-layer caching
  * Converted HTML cache
  * Contents cache for list pages
* Get only updated/deleted page contents
* Detect page updates by file or page timestamps
* Refer cache/recent.dat file to detect updated pages
* Refer RecentDeleted page to detect deleted pages
* Handle page links properly on tracker_list table

plugin/tracker.inc.php
plugin/tracker_list.inc.php

index 33f8a53..874d0eb 100644 (file)
@@ -1,7 +1,7 @@
 <?php
 // PukiWiki - Yet another WikiWikiWeb clone
 // tracker.inc.php
-// Copyright 2003-2017 PukiWiki Development Team
+// Copyright 2003-2018 PukiWiki Development Team
 // License: GPL v2 or (at your option) any later version
 //
 // Issue tracker plugin (See Also bugtrack plugin)
@@ -15,6 +15,9 @@ define('TRACKER_LIST_EXCLUDE_PATTERN','#^SubMenu$|/#');
 // 項目の取り出しに失敗したページを一覧に表示する
 define('TRACKER_LIST_SHOW_ERROR_PAGE',TRUE);
 
+// Use cache
+define('TRACKER_LIST_USE_CACHE', TRUE);
+
 function plugin_tracker_convert()
 {
        global $vars;
@@ -684,12 +687,15 @@ function plugin_tracker_list_action()
 }
 function plugin_tracker_getlist($page,$refer,$config_name,$list,$order='',$limit=NULL)
 {
-       $config = new Config('plugin/tracker/'.$config_name);
+       global $whatsdeleted;
 
+       $config = new Config('plugin/tracker/'.$config_name);
        if (!$config->read())
        {
                return "<p>config file '".htmlsc($config_name)."' is not exist.</p>";
        }
+       //pkwk_log('AAAA');
+       //pkwk_log(print_r(['page:' => $page, 'refer' => $refer, 'config_name' => $config_name, 'list' => $list, 'order' => $order], true));
 
        $config->config_name = $config_name;
 
@@ -698,9 +704,79 @@ function plugin_tracker_getlist($page,$refer,$config_name,$list,$order='',$limit
                return "<p>config file '".make_pagelink($config->page.'/'.$list)."' not found.</p>";
        }
 
-       $list = new Tracker_list($page,$refer,$config,$list);
-       $list->sort($order);
-       return $list->toString($limit);
+       $cache_enabled = defined('TRACKER_LIST_USE_CACHE') && TRACKER_LIST_USE_CACHE &&
+               defined('JSON_UNESCAPED_UNICODE') && defined('PKWK_UTF8_ENABLE');
+       $cache_filepath = CACHE_DIR . encode($page) . '.tracker';
+       $cachedata = null;
+       $cache_format_version = 1;
+       if ($cache_enabled && file_exists($cache_filepath)) {
+               $config_filetime = get_filetime($config->page);
+               $config_list_filetime = get_filetime($config->page.'/'. $list);
+               $json_cached = pkwk_file_get_contents($cache_filepath);
+               if ($json_cached) {
+                       $wrapdata = json_decode($json_cached, true);
+                       if (is_array($wrapdata)) {
+                               if (isset($wrapdata['version'], $wrapdata['html'], $wrapdata['refreshed_at'])) {
+                                       $cache_time_prev = $wrapdata['refreshed_at'];
+                                       if ($cache_format_version === $wrapdata['version']) {
+                                               if ($config_filetime === $wrapdata['config_updated_at'] &&
+                                                       $config_list_filetime === $wrapdata['config_list_updated_at']) {
+                                                       $cachedata = $wrapdata;
+                                               } else {
+                                                       // (Ignore) delete file
+                                                       unlink($cache_filepath);
+                                               }
+                                       }
+                               }
+                       }
+               }
+       }
+       // Check recent.dat timestamp
+       $recent_dat_filemtime = filemtime(CACHE_DIR . PKWK_MAXSHOW_CACHE);
+       // Check RecentDeleted timestamp
+       $recent_deleted_filetime = get_filetime($whatsdeleted);
+       if (is_null($cachedata)) {
+               $cachedata = array();
+       } else {
+               if ($recent_dat_filemtile !== false) {
+                       if ($recent_dat_filemtime === $cachedata['recent_dat_filemtime'] &&
+                               $recent_deleted_filetime === $cachedata['recent_deleted_filetime'] &&
+                               $order === $cachedata['order']) {
+                               // recent.dat is unchanged
+                               // RecentDeleted is unchanged
+                               // order is unchanged
+                               return $cachedata['html'];
+                       }
+               }
+       }
+       $cache_holder = $cachedata;
+       $tracker_list = new Tracker_list($page,$refer,$config,$list,$cache_holder);
+       if ($order === $cache_holder['order'] &&
+               empty($tracker_list->newly_deleted_pages) &&
+               empty($tracker_list->newly_updated_pages) &&
+               !$tracker_list->link_update_required) {
+               $result = $cache_holder['html'];
+       } else {
+               $tracker_list->sort($order);
+               $result = $tracker_list->toString($limit);
+       }
+       if ($cache_enabled) {
+               $refreshed_at = time();
+               $json = array(
+                       'refreshed_at' => $refreshed_at,
+                       'rows' => $tracker_list->rows,
+                       'html' => $result,
+                       'order' => $order,
+                       'config_updated_at' => $config_filetime,
+                       'config_list_updated_at' => $config_list_filetime,
+                       'recent_dat_filemtime' => $recent_dat_filemtime,
+                       'recent_deleted_filetime' => $recent_deleted_filetime,
+                       'link_pages' => $tracker_list->link_pages,
+                       'version' => $cache_format_version);
+               $cache_body = json_encode($json, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES);
+               file_put_contents($cache_filepath, $cache_body, LOCK_EX);
+       }
+       return $result;
 }
 
 // 一覧クラス
@@ -715,13 +791,16 @@ class Tracker_list
        var $rows;
        var $order;
        var $sort_keys;
+       var $newly_deleted_pages = array();
+       var $newly_updated_pages = array();
 
-       function Tracker_list($page,$refer,&$config,$list)
+       function Tracker_list($page,$refer,&$config,$list,&$cache_holder)
        {
-               $this->__construct($page, $refer, $config, $list);
+               $this->__construct($page, $refer, $config, $list, $cache_holder);
        }
-       function __construct($page,$refer,&$config,$list)
+       function __construct($page,$refer,&$config,$list,&$cache_holder)
        {
+               global $whatsdeleted;
                $this->page = $page;
                $this->config = &$config;
                $this->list = $list;
@@ -746,22 +825,190 @@ class Tracker_list
                                $this->pattern .= '(.*?)';
                        }
                }
-               // ページの列挙と取り込み
-               $this->rows = array();
-               $pattern = "$page/";
-               $pattern_len = strlen($pattern);
-               foreach (get_existpages() as $_page)
-               {
-                       if (strpos($_page,$pattern) === 0)
+               if (empty($cache_holder)) {
+                       // List pages and get contents (non-cache behavior)
+                       $this->rows = array();
+                       $pattern = "$page/";
+                       $pattern_len = strlen($pattern);
+                       foreach (get_existpages() as $_page)
                        {
-                               $name = substr($_page,$pattern_len);
-                               if (preg_match(TRACKER_LIST_EXCLUDE_PATTERN,$name))
+                               if (substr($_page, 0, $pattern_len) === $pattern)
                                {
-                                       continue;
+                                       $name = substr($_page,$pattern_len);
+                                       if (preg_match(TRACKER_LIST_EXCLUDE_PATTERN,$name))
+                                       {
+                                               continue;
+                                       }
+                                       $this->add($_page,$name);
+                               }
+                       }
+                       $this->link_pages = $this->get_filetimes($this->get_all_links());
+               } else {
+                       // Cache-available behavior
+                       // Check RecentDeleted timestamp
+                       $cached_rows = $this->decode_cached_rows($cache_holder['rows']);
+                       $updated_linked_pages = array();
+                       $newly_deleted_pages = array();
+                       $pattern = "$page/";
+                       $pattern_len = strlen($pattern);
+                       $recent_deleted_filetime = get_filetime($whatsdeleted);
+                       $deleted_page_list = array();
+                       if ($recent_deleted_filetime !== $cache_holder['recent_deleted_filetime']) {
+                               //pkwk_log(print_r());
+                               foreach (plugin_tracker_get_source($whatsdeleted) as $line) {
+                                       $m = null;
+                                       if (preg_match('#\[\[([^\]]+)\]\]#', $line, $m)) {
+                                               $_page = $m[1];
+                                               if (is_pagename($_page)) {
+                                                       $deleted_page_list[] = $m[1];
+                                               }
+                                       }
+                               }
+                               foreach ($deleted_page_list as $_page) {
+                                       if (substr($_page, 0, $pattern_len) === $pattern) {
+                                               $name = substr($_page, $pattern_len);
+                                               if (!is_page($_page) && isset($cached_rows[$name]) &&
+                                                       !preg_match(TRACKER_LIST_EXCLUDE_PATTERN, $name)) {
+                                                       // This page was just deleted
+                                                       array_push($newly_deleted_pages, $_page);
+                                                       unset($cached_rows[$name]);
+                                               }
+                                       }
                                }
-                               $this->add($_page,$name);
                        }
+                       $this->newly_deleted_pages = $newly_deleted_pages;
+                       $updated_pages = array();
+                       $this->rows = $cached_rows;
+                       // Check recent.dat timestamp
+                       $recent_dat_filemtime = filemtime(CACHE_DIR . PKWK_MAXSHOW_CACHE);
+                       $updated_page_list = array();
+                       if ($recent_dat_filemtime !== $cache_holder['recent_dat_filemtime']) {
+                               // recent.dat was updated. Search which page was updated.
+                               $target_pages = array();
+                               // Active page file time (1 hour before timestamp of recent.dat)
+                               $target_filetime = $cache_holder['recent_dat_filemtime'] - LOCALZONE - 60 * 60;
+                               foreach (get_recent_files() as $_page=>$time) {
+                                       if ($time <= $target_filetime) {
+                                               // Older updated pages
+                                               break;
+                                       }
+                                       $updated_page_list[$_page] = $time;
+                                       $name = substr($_page, $pattern_len);
+                                       if (substr($_page, 0, $pattern_len) === $pattern) {
+                                               $name = substr($_page, $pattern_len);
+                                               if (preg_match(TRACKER_LIST_EXCLUDE_PATTERN, $name)) {
+                                                       continue;
+                                               }
+                                               // Tracker target page
+                                               if (isset($this->rows[$name])) {
+                                                       // Existing page
+                                                       $row = $this->rows[$name];
+                                                       if ($row['_update'] === get_filetime($_page)) {
+                                                               // Same as cache
+                                                               continue;
+                                                       } else {
+                                                               // Found updated page
+                                                               $updated_pages[] = $_page;
+                                                               unset($this->rows[$name]);
+                                                               $this->add($_page, $name);
+                                                       }
+                                               } else {
+                                                       // Add new page
+                                                       $updated_pages[] = $_page;
+                                                       $this->add($_page, $name);
+                                               }
+                                       }
+                               }
+                       }
+                       $this->newly_updated_pages = $updated_pages;
+                       $new_link_names = $this->get_all_links();
+                       $old_link_map = array();
+                       foreach ($cache_holder['link_pages'] as $link_page) {
+                               $old_link_map[$link_page['page']] = $link_page['filetime'];
+                       }
+                       $new_link_map = $old_link_map;
+                       $link_update_required = false;
+                       foreach ($deleted_page_list as $_page) {
+                               if (in_array($_page, $new_link_names)) {
+                                       if (isset($old_link_map[$_page])) {
+                                               // This link keeps existing
+                                               if (!is_page($_page)) {
+                                                       // OK. Confirmed the page doesn't exist
+                                                       if ($old_link_map[$_page] === 0) {
+                                                               // Do nothing (From no-page to no-page)
+                                                       } else {
+                                                               // This page was just deleted
+                                                               $new_link_map[$_page] = get_filetime($_page);
+                                                               $link_update_required = true;
+                                                       }
+                                               }
+                                       } else {
+                                               // This link was just added
+                                               $new_link_map[$_page] = get_filetime($_page);
+                                               $link_update_required = true;
+                                       }
+                               }
+                       }
+                       foreach ($updated_page_list as $_page=>$time) {
+                               if (in_array($_page, $new_link_names)) {
+                                       if (isset($old_link_map[$_page])) {
+                                               // This link keeps existing
+                                               if (is_page($_page)) {
+                                                       // OK. Confirmed the page now exists
+                                                       if ($old_link_map[$_page] === 0) {
+                                                               // This page was just added
+                                                               $new_link_map[$_page] = get_filetime($_page);
+                                                               $link_update_required = true;
+                                                       } else {
+                                                               // Do nothing (existing-page to existing-page)
+                                                       }
+                                               }
+                                       } else {
+                                               // This link was just added
+                                               $new_link_map[$_page] = get_filetime($_page);
+                                               $link_update_required = true;
+                                       }
+                               }
+                       }
+                       $new_link_pages = array();
+                       foreach ($new_link_map as $_page => $time) {
+                               $new_link_pages[] = array(
+                                       'page' => $_page,
+                                       'filetime' => $time,
+                               );
+                       }
+                       $this->link_pages = $new_link_pages;
+                       $this->link_update_required = $link_update_required;
+               }
+       }
+       function decode_cached_rows($decoded_rows)
+       {
+               $ar = array();
+               foreach ($decoded_rows as $row) {
+                       $ar[$row['_real']] = $row;
+               }
+               return $ar;
+       }
+       function get_all_links() {
+               $ar = array();
+               foreach ($this->rows as $row) {
+                       foreach ($row['_links'] as $link) {
+                               $ar[$link] = 0;
+                       }
+               }
+               pkwk_log('LINK');
+               pkwk_log(print_r(array_keys($ar), true));
+               return array_keys($ar);
+       }
+       function get_filetimes($pages) {
+               $filetimes = array();
+               foreach ($pages as $page) {
+                       $filetimes[] = array(
+                               'page' => $page,
+                               'filetime' => get_filetime($page),
+                       );
                }
+               return $filetimes;
        }
        function add($page,$name)
        {
@@ -786,22 +1033,38 @@ class Tracker_list
                }
                $source = join('',preg_replace('/^(\*{1,3}.*)\[#[A-Za-z][\w-]+\](.*)$/','$1$2',$source));
 
-               // デフォルト値
-               $this->rows[$name] = array(
+               // Default value
+               $page_filetime = get_filetime($page);
+               $row = array(
                        '_page'  => "[[$page]]",
                        '_refer' => $this->page,
                        '_real'  => $name,
-                       '_update'=> get_filetime($page),
-                       '_past'  => get_filetime($page)
+                       '_update'=> $page_filetime,
+                       '_past'  => $page_filetime,
                );
-               if ($this->rows[$name]['_match'] = preg_match("/{$this->pattern}/s",$source,$matches))
+               $links = array();
+               if ($row['_match'] = preg_match("/{$this->pattern}/s",$source,$matches))
                {
                        array_shift($matches);
                        foreach ($this->pattern_fields as $key=>$field)
                        {
-                               $this->rows[$name][$field] = trim($matches[$key]);
+                               $row[$field] = trim($matches[$key]);
+                               if ($field === '_refer') {
+                                       continue;
+                               }
+                               $lmatch = null;
+                               if (preg_match('/\[\[([^\]\]]+)\]/', $row[$field], $lmatch)) {
+                                       $link = $lmatch[1];
+                                       if (is_pagename($link) && $link !== $this->page && $link !== $page) {
+                                               if (!in_array($link, $links)) {
+                                                       $links[] = $link;
+                                               }
+                                       }
+                               }
                        }
                }
+               $row['_links'] = $links;
+               $this->rows[$name] = $row;
        }
        function compare($a, $b)
        {
index f69ee83..26ad65a 100644 (file)
@@ -1,7 +1,7 @@
 <?php
 // PukiWiki - Yet another WikiWikiWeb clone
 // tracker_list.inc.php
-// Copyright 2003-2017 PukiWiki Development Team
+// Copyright 2003-2018 PukiWiki Development Team
 // License: GPL v2 or (at your option) any later version
 //
 // Issue tracker list plugin (a part of tracker plugin)