From c01968f857e0c50d59d87d403f1a53a9c5c39d89 Mon Sep 17 00:00:00 2001 From: hylom Date: Wed, 27 Feb 2019 20:23:26 +0900 Subject: [PATCH] Plugin::Preprocessor: fix cache related problem --- .../lib/Newslash/Plugin/CompositeCache.pm | 21 +++ .../lib/Newslash/Plugin/Preprocessor.pm | 179 +++++++++++---------- src/newslash_web/t/plugins/preprocessor.t | 63 ++++++++ src/newslash_web/templates/common/header.html.tt2 | 2 +- 4 files changed, 181 insertions(+), 84 deletions(-) create mode 100644 src/newslash_web/t/plugins/preprocessor.t diff --git a/src/newslash_web/lib/Newslash/Plugin/CompositeCache.pm b/src/newslash_web/lib/Newslash/Plugin/CompositeCache.pm index 227dfe10..6d1977ad 100644 --- a/src/newslash_web/lib/Newslash/Plugin/CompositeCache.pm +++ b/src/newslash_web/lib/Newslash/Plugin/CompositeCache.pm @@ -133,6 +133,27 @@ sub get { return; } +sub expire { + my ($self, $base_key, $variable_key, $target) = @_; + $target ||= "all"; + my $key = $base_key . ":" . $variable_key; + + if ($target ne "all" && $target ne "inmemory" && $target ne "kvs") { + $self->app->log->error("CompositeCache::expire: invalid taget $target. $base_key - $variable_key"); + return; + } + + # expire inmemory cache + if ($self->inmemory && ($target eq "all" || $target eq "inmemory")) { + $self->inmemory->del($key); + } + + # expire KVS cache + if ($self->kvs && ($target eq "all" || $target eq "kvs")) { + $self->kvs->del($key); + } +} + sub set { my ($self, $base_key, $variable_key, $value) = @_; if (!$base_key) { diff --git a/src/newslash_web/lib/Newslash/Plugin/Preprocessor.pm b/src/newslash_web/lib/Newslash/Plugin/Preprocessor.pm index 70916cf9..537381b0 100644 --- a/src/newslash_web/lib/Newslash/Plugin/Preprocessor.pm +++ b/src/newslash_web/lib/Newslash/Plugin/Preprocessor.pm @@ -17,8 +17,8 @@ use constant CACHE_KEY => "Preprocessor"; use constant DEFAULT_MEM_EXPIRE => 60; use constant DEFAULT_KVS_EXPIRE => 60 * 60 * 24; -sub _absolute_path { - my ($self, $rpath) = @_; +sub _get_absolute_path { + my ($self, $rel_path) = @_; my $base_dir = $self->{conf}->{base_directory}; if (!$base_dir) { $base_dir = $self->{app}->home; @@ -26,91 +26,99 @@ sub _absolute_path { if (!$base_dir) { # fallback $base_dir = "./"; } - my $path = File::Spec->catdir($base_dir, $rpath); + my $path = File::Spec->catdir($base_dir, $rel_path); $path = File::Spec->rel2abs($path); return $path; } -sub _generate { - my ($self, $target) = @_; - $target = $self->_strip_md5($target); +sub _strip_md5_from_path { + my ($self, $pathname) = @_; + if ($pathname =~ m/^(.+)_[0-9a-f]+(\.\w+)$/) { + return $1 . $2; + } + return $pathname; +} + +sub _add_md5_to_path { + my ($self, $pathname, $md5) = @_; + my ($base, $dir, $ext) = fileparse($pathname, qw/\.[^.]*/); + return "${dir}${base}_$md5$ext"; +} + +sub _generate_content { + my ($self, $pathname) = @_; + $pathname = $self->_strip_md5_from_path($pathname); my $targets = $self->{conf}->{targets} || {}; - my $params = $targets->{$target}; + my $params = $targets->{$pathname}; if (!$params) { - $self->{app}->log->debug("Preprocessor: cannot generate '$target' - target not defined"); + $self->{app}->log->error("Preprocessor: cannot generate '$pathname' - target not defined"); return; } my $gen_conf = $self->{conf}->{$params->{type}}; if (!defined $gen_conf) { - $self->{app}->log->debug("Preprocessor: cannot generate '$target' - generate config not found for '$params->{type}'"); + $self->{app}->log->error("Preprocessor: cannot generate '$pathname' - generate config not found for '$params->{type}'"); return; } my $cmd = $gen_conf->{command}; if (!$cmd) { - $self->{app}->log->error("Preprocessor: cannot generate '$target' - command '$gen_conf->{command}' not found"); + $self->{app}->log->error("Preprocessor: cannot generate '$pathname' - command '$gen_conf->{command}' not found"); return; } my $content = ''; if ($params->{source}) { - my $pathname = $self->_absolute_path($params->{source}); - if (! -f $pathname) { - $self->{app}->log->error("Preprocessor: cannot generate '$target' - pathname '$pathname' not found"); + my $abs_pathname = $self->_get_absolute_path($params->{source}); + if (! -f $abs_pathname) { + $self->{app}->log->error("Preprocessor: cannot generate '$pathname' - '$abs_pathname' not found"); return; } my $fh = FileHandle->new; - if ($fh->open($pathname, "<:utf8")) { + if ($fh->open($abs_pathname, "<:utf8")) { $content = do { local $/; <$fh> }; $fh->close; } } my $opts = $self->_build_options($gen_conf->{options}); - my $result = $self->_execute_cmd($cmd, $content, $opts); + my $content_body = $self->_execute_cmd($cmd, $content, $opts); if ($self->{last_execute_status}) { # error occured return; } - return $self->add_content($target, $result, $gen_conf->{"content-type"}); + return $self->_build_content_object($content_body, $pathname, $gen_conf->{content_type}); } -sub _strip_md5 { - my ($self, $path) = @_; - if ($path =~ m/^(.+)_[0-9a-f]+(\.\w+)$/) { - return $1 . $2; +sub _build_content_object { + my ($self, $content_body, $pathname, $content_type) = @_; + my $rs = { md5 => md5_hex($content_body), + content => $content_body, + path => $pathname, + type => $content_type }; + + if ($self->{compress}) { + gzip(\$content_body, \my $compressed); + $rs->{gz_content} = $compressed; } - return $path; + + return $rs; } -sub add_content { - my ($self, $pathname, $content, $content_type) = @_; +sub _cache_content { + my ($self, $content) = @_; - if (utf8::is_utf8($content)) { - $content = Encode::encode('utf8', $content); - } - my $md5 = md5_hex($content); - my ($base, $dir, $ext) = fileparse($pathname, qw/\.[^.]*/); - my $static_dir = $self->static_dir; - my $path = "$static_dir/$dir${base}_$md5$ext"; - my $value = { - md5 => $md5, - content => $content, - path => $path, - type => $content_type, - }; - if ($self->{compress}) { - gzip(\$content, \my $compressed); - $value->{gz_content} = $compressed; - } - my $rpath = "$dir${base}_$md5$ext"; - $self->{app}->ccache->set(CACHE_KEY, $pathname, $value); - $self->{app}->ccache->set(CACHE_KEY, $rpath, $value); - return $value; + my $md5_url = $self->_add_md5_to_path($content->{path}, $content->{md5}); + $self->{app}->ccache->set(CACHE_KEY, $content->{path}, $content); + $self->{app}->ccache->set(CACHE_KEY, $md5_url, $content); +} + +sub expire_cache { + my ($self, $key, $target) = @_; + $self->{app}->ccache->expire(CACHE_KEY, $key, $target); } sub static_dir { @@ -125,7 +133,7 @@ sub _hash_to_options { for my $name (keys %$hash) { my $value = $hash->{$name}; if ($value =~ m/^\.\//) { - $value = $self->_absolute_path($value); + $value = $self->_get_absolute_path($value); } my $opt = $name; if ($value) { @@ -178,11 +186,13 @@ sub generate_all { my $err = 0; my @err_msgs; for my $target (keys %$targets) { - if (!$self->_generate($target)) { + my $content = $self->_generate_content($target); + if (!$content) { $err = 1; push @err_msgs, $self->{last_error_msg}; push @err_msgs, $self->{last_result}; } + $self->_cache_content($content); } if ($err) { $self->{last_error_msg} = join("\n", @err_msgs); @@ -195,16 +205,31 @@ sub rebuild { return $self->generate_all; } +sub get_content { + my ($self, $path) = @_; + my $content = $self->{app}->ccache->get(CACHE_KEY, $path); + + if (!$content) { + # regenerate contents + $content = $self->_generate_content($path); + if ($content) { + $self->_cache_content($content); + } + } + return $content; +} + sub register { my ($self, $app, $conf) = @_; # set default config value my $cnf = $app->config->{Preprocessor} ||= {}; + $cnf->{base_url} ||= "/css"; $cnf->{targets} ||= { "css/newslash.css" => { source => "css/newslash.less", type => "less" }, }; $cnf->{less} ||= { command => "/usr/bin/nodejs", - "content-type" => "text/css", + "content_type" => "text/css", options => [ "--no-deprecation", "/usr/bin/lessc", "--include-path=./css", @@ -220,26 +245,17 @@ sub register { $self->{last_execute_status} = 0; $self->{compress} = $cnf->{use_compression}; - #$self->generate_all; - my $r = $app->routes; - $r->any('/static/*content_path' => sub { + my $base_url = $self->{conf}->{base_url}; + if (substr($base_url, -1, 1) ne "/") { + $base_url = $base_url . "/"; + } + + $r->any("${base_url}*content_path" => sub { my $c = shift; - my $path = $c->stash('content_path'); + my $path = $base_url . $c->stash('content_path'); if ($path) { - my $content = $self->{app}->ccache->get(CACHE_KEY, $path); - - if (!$content) { - # regenerate contents - my $targets = $self->{conf}->{targets} || {}; - if ($targets->{$path}) { - my $rs = $self->_generate($path); - if ($rs) { - $content = $rs->{content}; - } - } - } - + my $content = $self->get_content($path); if ($content) { # check Etag if ($c->req->headers->if_none_match @@ -248,9 +264,10 @@ sub register { $c->rendered(304); return; } - + $c->res->headers->content_type($content->{type}); my $output = $content->{content}; + # check compressable if ($self->{compress} && $content->{type} =~ m{^(text/|application/json|application/javascript)} && ($c->req->headers->accept_encoding // '') =~ /gzip/i @@ -260,14 +277,16 @@ sub register { # Add header $c->res->headers->append(Vary => 'Accept-Encoding'); $c->res->headers->content_encoding('gzip'); - $output = $content->{gz_content}; } - $c->res->headers->content_type($content->{type}); - if ($content->{md5}) { + + # add Etag + if ($content->{md5}) { $c->res->headers->etag("\"$content->{md5}\""); } - $c->res->body($output); + + # send result + $c->res->body($output); $c->rendered(200); return; } @@ -278,24 +297,15 @@ sub register { $app->helper(preprocessor => sub { state $preprocessor = $self; }); } -sub get_path { +sub get_md5_path { my ($self, $path) = @_; - my $content = $self->{app}->ccache->get(CACHE_KEY, $path); - - if (!$content) { - # regenerate contents - my $targets = $self->{conf}->{targets} || {}; - if ($targets->{$path}) { - $content = $self->_generate($path); - } - } + my $content = $self->get_content($path); if ($content && $content->{path}) { - return $content->{path}; - } - else { - $self->{app}->log->debug("Preprocessor: '$path' not found."); + return $self->_add_md5_to_path($content->{path}, $content->{md5}); } + + $self->{app}->log->error("Preprocessor: '$path' not found."); return; } @@ -367,6 +377,9 @@ sub _execute_cmd { } return; } + # decode result to Perl's internal format + $result = Encode::decode('utf8', $result); + return $result; } diff --git a/src/newslash_web/t/plugins/preprocessor.t b/src/newslash_web/t/plugins/preprocessor.t new file mode 100644 index 00000000..73273103 --- /dev/null +++ b/src/newslash_web/t/plugins/preprocessor.t @@ -0,0 +1,63 @@ +# -*-Perl-*- +# Preprocessor plugin tests + +use Mojo::Base -strict; +use Test::More; +use Test::Mojo; +use IO::Uncompress::Gunzip qw(gunzip); +use Newslash::Util::TestMan; +use Data::Dumper; +use Newslash::Eventd::Handlers; + +my $t = Test::Mojo->new('Newslash::Web'); + +my $pp = $t->app->preprocessor; +ok($pp, "get preprocessor instance"); + +# check add_md5_to_path +is($pp->_add_md5_to_path("/css/newslash.css", "1234abcd"), "/css/newslash_1234abcd.css", "_add_md5_to_path ok"); + +# check strip_md5_from_path +is($pp->_strip_md5_from_path("/css/newslash_1234abcd.css"), "/css/newslash.css", "_strip_md5_from_path ok"); + +# get css/newslash.css +my $path = "css/newslash.css"; + + +my $content = $pp->get_content("/css/newslash.css"); +ok($content, "get content"); +ok(($content && $content->{content}), "get valid content"); +ok(($content && $content->{type} eq "text/css"), "get valid type"); + + +if ($content) { + is($content->{path}, "/css/newslash.css", "get valid path"); + my $md5_path = $pp->get_md5_path($content->{path}); + is($md5_path, $pp->_add_md5_to_path($content->{path}, $content->{md5}), "get valid md5path"); + + # expire all cache + $pp->expire_cache($path, "all"); + my $content_nocache = $pp->get_content("$md5_path"); + + # cached read + my $content_cached = $pp->get_content("$md5_path"); + + # expire inmemory cache + $pp->expire_cache($path, "inmemory"); + my $content_kvs = $pp->get_content("$md5_path"); + + ok($content_nocache, "nocache read succeed"); + ok($content_cached, "cached read succeed"); + ok($content_kvs, "kvscached read succeed"); + + is_deeply($content_nocache, $content, "valid nocache read"); + is_deeply($content_cached, $content, "valid cached read"); + is_deeply($content_kvs, $content, "valid kvs cached read"); + + # check gziped content + my $c = $content_kvs->{gz_content}; + gunzip(\$c, \my $gunziped); + is($gunziped, $content_kvs->{content}, "valid gzipped content"); +} + +done_testing(); diff --git a/src/newslash_web/templates/common/header.html.tt2 b/src/newslash_web/templates/common/header.html.tt2 index ec8360c7..658766fb 100644 --- a/src/newslash_web/templates/common/header.html.tt2 +++ b/src/newslash_web/templates/common/header.html.tt2 @@ -17,7 +17,7 @@ [% INCLUDE common/title %] - + [%- IF production -%] -- 2.11.0