From d68644437289f768f06e4d0372243f1df34c73fd Mon Sep 17 00:00:00 2001 From: hylom Date: Fri, 2 Mar 2018 19:51:12 +0900 Subject: [PATCH] ns_searchd: fix Searchd to use __iter__() instead of __call__() --- src/ns_search/newslash_searchd/__init__.py | 2 +- src/ns_search/newslash_searchd/router.py | 31 +++++-- src/ns_search/newslash_searchd/searchd.py | 78 ------------------ src/ns_search/ns_searchd.py | 12 +-- src/ns_search/searchd.py | 127 +++++++++++++++++++++++++++++ 5 files changed, 157 insertions(+), 93 deletions(-) delete mode 100644 src/ns_search/newslash_searchd/searchd.py create mode 100644 src/ns_search/searchd.py diff --git a/src/ns_search/newslash_searchd/__init__.py b/src/ns_search/newslash_searchd/__init__.py index 9692fcd8..6004e720 100644 --- a/src/ns_search/newslash_searchd/__init__.py +++ b/src/ns_search/newslash_searchd/__init__.py @@ -3,6 +3,6 @@ __author__ = 'Hiromichi Matsushima ' __version__ = "0.1.0" -from searchd import Searchd +from router import Router, Route __all__ = [] diff --git a/src/ns_search/newslash_searchd/router.py b/src/ns_search/newslash_searchd/router.py index ac513b22..f76e2e7b 100644 --- a/src/ns_search/newslash_searchd/router.py +++ b/src/ns_search/newslash_searchd/router.py @@ -64,26 +64,33 @@ class Request(object): raw_json = raw_req.read(body_len) return json.loads(raw_json) return None - + + +class Route(object): + pass + class Router(object): - def __init__(self): + def __init__(self, environ, start_response): self.routes = [] + self.environ = environ + self.start_response = start_response - def route(self, rex, receiver): - self.routes.append((rex, receiver)) + def __iter__(self): + environ = self.environ + start_response = self.start_response - def __call__(self, environ, start_response): req = Request(environ) resp = Response(start_response) # find route path_info = environ['PATH_INFO'] - receiver = self + receiver = self._default_route for (rex, recv) in self.routes: if rex.search(path_info): receiver = recv - + + # routing by method method = environ['REQUEST_METHOD'] try: if method == 'GET': @@ -98,7 +105,15 @@ class Router(object): if not resp.finished: resp.finish() - return resp.response_bodies + # done + for res in resp.response_bodies: + yield res + + def route(self, rex, receiver): + self.routes.append((rex, receiver)) + + def default_route(self, receiver): + self._default_route = receiver def fallback(self, req, resp): resp.finish(500, "Internal Server Error") diff --git a/src/ns_search/newslash_searchd/searchd.py b/src/ns_search/newslash_searchd/searchd.py deleted file mode 100644 index adf33215..00000000 --- a/src/ns_search/newslash_searchd/searchd.py +++ /dev/null @@ -1,78 +0,0 @@ -# -*- coding: utf-8 -''' searchd: Search daemon for Newslash''' - -import re -import sys - -from router import Router -import lucene_wrapper - - -class Searchd(Router): - def __init__(self, config={}): - super(Searchd, self).__init__() - self.route(re.compile(r'^/admin'), SearchdAdmin()) - self._config = config - - def get(self, req, resp): - resp.render(200, json={"error": 0}) - - def post(self, req, resp): - # check request body is valid - if req.body is None: - resp.render(400) - return - - # start query - query = req.body.get('query', '') - limit = req.body.get('limit', 10) - try: - offset = int(req.body.get('offset')) - except ValueError: - offset = 0 - except TypeError: - offset = 0 - - # log - #errors = req.environ['wsgi.errors'] - #errors.write(query.encode('utf-8')) - - searcher = lucene_wrapper.Searcher(index_directory=self._config.get('index_directory')) - try: - query = lucene_wrapper.Query("content_text", query) - except lucene_wrapper.QueryParseError as e: - resp.render(400, json={"error": { "message": e.message }}) - return - except: - resp.render(500, json={"error": { "message": "query_error" }}) - return - - - #result = searcher.search(query, limit) - result = searcher.search2(query, limit, offset) - - resp_body = { - "total_hits": result.total_hits, - "hits": [], - "start": offset, - } - - for item in result: - resp_body["hits"].append({ "number": item.number, - "type": item.type, - "id": item.id, - "title": item.title, - "content_text": item.content_text}) - - resp.render(200, json=resp_body) - - -class SearchdAdmin(Router): - def __init__(self): - super(SearchdAdmin, self).__init__() - - def get(self, req, resp): - resp.render(200, json={"error": 0}) - - def post(self, req, resp): - pass diff --git a/src/ns_search/ns_searchd.py b/src/ns_search/ns_searchd.py index 1f001384..6cf6a272 100755 --- a/src/ns_search/ns_searchd.py +++ b/src/ns_search/ns_searchd.py @@ -21,23 +21,23 @@ def args_to_conf(): conf = {} # parse command line option parser = argparse.ArgumentParser(description='search daemon for Newslash') - parser.add_argument('-i', '--index-dir', required=True) + #parser.add_argument('-i', '--index-dir', required=True) args = parser.parse_args() # index directory - conf["index_directory"] = os.path.abspath(args.index_dir) + #conf["index_directory"] = os.path.abspath(args.index_dir) return conf if __name__ == '__main__': # parse argument and build config dict - conf = args_to_conf() + #conf = args_to_conf() # start Java VM - lucene_wrapper.init_vm() + #lucene_wrapper.init_vm() - app = Searchd(conf) - server = make_server('', 6000, app, handler_class=CustomRequestHandler) + #app = Searchd() + server = make_server('', 6000, Searchd, handler_class=CustomRequestHandler) print("starting server at localhost:6000...") server.serve_forever() diff --git a/src/ns_search/searchd.py b/src/ns_search/searchd.py new file mode 100644 index 00000000..3feeafa1 --- /dev/null +++ b/src/ns_search/searchd.py @@ -0,0 +1,127 @@ +# -*- coding: utf-8 +''' searchd: Search daemon for Newslash''' + +import re +import sys +import os +import os.path + +from yaml import load +try: + from yaml import CLoader as Loader, CDumper as Dumper +except ImportError: + from yaml import Loader, Dumper + +from newslash_searchd import Router, Route +import lucene_wrapper + +lucene_wrapper.init_vm() + +class SearchdError(Exception): + def __init__(self, message): + self.message = message + + +# load config file +def _load_config(pathname): + try: + fh = open(pathname) + except IOError: + raise SearchdError("config file not found") + + config = load(fh, Loader=Loader) + fh.close() + return config + +config_path = os.environ.get("SEARCHD_CONFIG", "/etc/newslash/searchd.conf") +if not os.path.exists(config_path): + config_path = './searchd.conf' + if not os.path.exists(config_path): + raise SearchdError("config file not found") + +config = _load_config(config_path) + + +class Searchd(Router): + def __init__(self, environ, start_response): + super(Searchd, self).__init__(environ, start_response) + self.default_route(Root()) + self.route(re.compile(r'^/admin'), SearchdAdmin()) + + +class Root(Route): + def __init__(self): + self._config = config + + def config(self, section, key=None, default=None): + if key is None: + return self._config.get(section, default) + d = self._config.get(section, {}) + return d.get(key, default) + + def get(self, req, resp): + resp.render(200, json={"error": 0}) + + def post(self, req, resp): + # check request body is valid + if req.body is None: + resp.render(400) + return + + # start query + query_text = req.body.get('query', '') + limit = req.body.get('limit', 10) + try: + offset = int(req.body.get('offset')) + except ValueError: + offset = 0 + except TypeError: + offset = 0 + + # log + # req.environ['wsgi.errors'].write(query.encode('utf-8')) + + searcher = lucene_wrapper.Searcher(index_directory=self.config('Index', 'path')) + try: + query = lucene_wrapper.Query("content_text", query_text) + except lucene_wrapper.QueryParseError as e: + resp.render(400, json={"error": { "message": e.message }}) + return + except: + req.environ['wsgi.errors'].write("query error - query is {}".format(query_text)) + resp.render(500, json={"error": { "message": "query_error" }}) + return + + result = searcher.search(query, limit, offset) + + resp_body = { + "total_hits": result.total_hits, + "hits": [], + "start": offset, + } + + for item in result: + resp_body["hits"].append({ "number": item.number, + "type": item.type, + "id": item.id, + "title": item.title, + "content_text": item.content_text}) + + resp.render(200, json=resp_body) + + +class SearchdAdmin(Root): + def get(self, req, resp): + resp.render(200, json={"error": 0}) + + def post(self, req, resp): + pass + + +if __name__ == '__main__': + from wsgiref.simple_server import make_server, WSGIRequestHandler + + server = make_server('', 6000, Searchd) + + print("starting server at localhost:6000...") + server.serve_forever() -- 2.11.0