2 <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
3 "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
5 <html xmlns="http://www.w3.org/1999/xhtml">
7 <meta http-equiv="X-UA-Compatible" content="IE=Edge" />
8 <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
9 <title>joy.parser — Thun 0.4.1 documentation</title>
10 <link rel="stylesheet" href="../../_static/alabaster.css" type="text/css" />
11 <link rel="stylesheet" href="../../_static/pygments.css" type="text/css" />
12 <script type="text/javascript" src="../../_static/documentation_options.js"></script>
13 <script type="text/javascript" src="../../_static/jquery.js"></script>
14 <script type="text/javascript" src="../../_static/underscore.js"></script>
15 <script type="text/javascript" src="../../_static/doctools.js"></script>
16 <script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML"></script>
17 <link rel="index" title="Index" href="../../genindex.html" />
18 <link rel="search" title="Search" href="../../search.html" />
20 <link rel="stylesheet" href="../../_static/custom.css" type="text/css" />
23 <meta name="viewport" content="width=device-width, initial-scale=0.9, maximum-scale=0.9" />
28 <div class="document">
29 <div class="documentwrapper">
30 <div class="bodywrapper">
31 <div class="body" role="main">
33 <h1>Source code for joy.parser</h1><div class="highlight"><pre>
34 <span></span><span class="c1"># -*- coding: utf-8 -*-</span>
35 <span class="c1">#</span>
36 <span class="c1"># Copyright © 2014, 2015, 2016, 2017 Simon Forman</span>
37 <span class="c1">#</span>
38 <span class="c1"># This file is part of Thun.</span>
39 <span class="c1">#</span>
40 <span class="c1"># Thun is free software: you can redistribute it and/or modify</span>
41 <span class="c1"># it under the terms of the GNU General Public License as published by</span>
42 <span class="c1"># the Free Software Foundation, either version 3 of the License, or</span>
43 <span class="c1"># (at your option) any later version.</span>
44 <span class="c1">#</span>
45 <span class="c1"># Thun is distributed in the hope that it will be useful,</span>
46 <span class="c1"># but WITHOUT ANY WARRANTY; without even the implied warranty of</span>
47 <span class="c1"># MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the</span>
48 <span class="c1"># GNU General Public License for more details.</span>
49 <span class="c1">#</span>
50 <span class="c1"># You should have received a copy of the GNU General Public License</span>
51 <span class="c1"># along with Thun. If not see <http://www.gnu.org/licenses/>.</span>
52 <span class="c1">#</span>
53 <span class="sd">'''</span>
54 <span class="sd">This module exports a single function for converting text to a joy</span>
55 <span class="sd">expression as well as a single Symbol class and a single Exception type.</span>
57 <span class="sd">The Symbol string class is used by the interpreter to recognize literals</span>
58 <span class="sd">by the fact that they are not Symbol objects.</span>
60 <span class="sd">A crude grammar::</span>
62 <span class="sd"> joy = term*</span>
63 <span class="sd"> term = int | float | string | '[' joy ']' | symbol</span>
65 <span class="sd">A Joy expression is a sequence of zero or more terms. A term is a</span>
66 <span class="sd">literal value (integer, float, string, or Joy expression) or a function</span>
67 <span class="sd">symbol. Function symbols are unquoted strings and cannot contain square</span>
68 <span class="sd">brackets. Terms must be separated by blanks, which can be omitted</span>
69 <span class="sd">around square brackets.</span>
71 <span class="sd">'''</span>
72 <span class="kn">from</span> <span class="nn">re</span> <span class="k">import</span> <span class="n">Scanner</span>
73 <span class="kn">from</span> <span class="nn">.utils.stack</span> <span class="k">import</span> <span class="n">list_to_stack</span>
76 <span class="c1">#TODO: explain the details of float lits and strings.</span>
77 <span class="n">FLOAT</span> <span class="o">=</span> <span class="sa">r</span><span class="s1">'-?\d+\.\d*(e(-|\+)\d+)?'</span>
78 <span class="n">INT</span> <span class="o">=</span> <span class="sa">r</span><span class="s1">'-?\d+'</span>
79 <span class="n">SYMBOL</span> <span class="o">=</span> <span class="sa">r</span><span class="s1">'[•\w!@$%^&*()_+<>?|\/;:`~,.=-]+'</span>
80 <span class="n">BRACKETS</span> <span class="o">=</span> <span class="sa">r</span><span class="s1">'\[|\]'</span>
81 <span class="n">STRING_DOUBLE_QUOTED</span> <span class="o">=</span> <span class="sa">r</span><span class="s1">'"(?:[^"</span><span class="se">\\</span><span class="s1">]|</span><span class="se">\\</span><span class="s1">.)*"'</span>
82 <span class="n">STRING_SINGLE_QUOTED</span> <span class="o">=</span> <span class="sa">r</span><span class="s2">"'(?:[^'</span><span class="se">\\</span><span class="s2">]|</span><span class="se">\\</span><span class="s2">.)*'"</span>
83 <span class="n">BLANKS</span> <span class="o">=</span> <span class="sa">r</span><span class="s1">'\s+'</span>
86 <div class="viewcode-block" id="Symbol"><a class="viewcode-back" href="../../parser.html#joy.parser.Symbol">[docs]</a><span class="k">class</span> <span class="nc">Symbol</span><span class="p">(</span><span class="nb">str</span><span class="p">):</span>
87 <span class="sd">'''A string class that represents Joy function names.'''</span>
88 <span class="fm">__repr__</span> <span class="o">=</span> <span class="nb">str</span><span class="o">.</span><span class="fm">__str__</span></div>
91 <div class="viewcode-block" id="text_to_expression"><a class="viewcode-back" href="../../parser.html#joy.parser.text_to_expression">[docs]</a><span class="k">def</span> <span class="nf">text_to_expression</span><span class="p">(</span><span class="n">text</span><span class="p">):</span>
92 <span class="sd">'''Convert a string to a Joy expression.</span>
94 <span class="sd"> When supplied with a string this function returns a Python datastructure</span>
95 <span class="sd"> that represents the Joy datastructure described by the text expression.</span>
96 <span class="sd"> Any unbalanced square brackets will raise a ParseError.</span>
98 <span class="sd"> :param str text: Text to convert.</span>
99 <span class="sd"> :rtype: stack</span>
100 <span class="sd"> :raises ParseError: if the parse fails.</span>
101 <span class="sd"> '''</span>
102 <span class="k">return</span> <span class="n">_parse</span><span class="p">(</span><span class="n">_tokenize</span><span class="p">(</span><span class="n">text</span><span class="p">))</span></div>
105 <div class="viewcode-block" id="ParseError"><a class="viewcode-back" href="../../parser.html#joy.parser.ParseError">[docs]</a><span class="k">class</span> <span class="nc">ParseError</span><span class="p">(</span><span class="ne">ValueError</span><span class="p">):</span>
106 <span class="sd">'''Raised when there is a error while parsing text.'''</span></div>
109 <span class="k">def</span> <span class="nf">_tokenize</span><span class="p">(</span><span class="n">text</span><span class="p">):</span>
110 <span class="sd">'''Convert a text into a stream of tokens.</span>
112 <span class="sd"> Converts function names to Symbols.</span>
114 <span class="sd"> Raise ParseError (with some of the failing text) if the scan fails.</span>
115 <span class="sd"> '''</span>
116 <span class="n">tokens</span><span class="p">,</span> <span class="n">rest</span> <span class="o">=</span> <span class="n">_scanner</span><span class="o">.</span><span class="n">scan</span><span class="p">(</span><span class="n">text</span><span class="p">)</span>
117 <span class="k">if</span> <span class="n">rest</span><span class="p">:</span>
118 <span class="k">raise</span> <span class="n">ParseError</span><span class="p">(</span>
119 <span class="s1">'Scan failed at position </span><span class="si">%i</span><span class="s1">, </span><span class="si">%r</span><span class="s1">'</span>
120 <span class="o">%</span> <span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">text</span><span class="p">)</span> <span class="o">-</span> <span class="nb">len</span><span class="p">(</span><span class="n">rest</span><span class="p">),</span> <span class="n">rest</span><span class="p">[:</span><span class="mi">10</span><span class="p">])</span>
121 <span class="p">)</span>
122 <span class="k">return</span> <span class="n">tokens</span>
125 <span class="k">def</span> <span class="nf">_parse</span><span class="p">(</span><span class="n">tokens</span><span class="p">):</span>
126 <span class="sd">'''</span>
127 <span class="sd"> Return a stack/list expression of the tokens.</span>
128 <span class="sd"> '''</span>
129 <span class="n">frame</span> <span class="o">=</span> <span class="p">[]</span>
130 <span class="n">stack</span> <span class="o">=</span> <span class="p">[]</span>
131 <span class="k">for</span> <span class="n">tok</span> <span class="ow">in</span> <span class="n">tokens</span><span class="p">:</span>
132 <span class="k">if</span> <span class="n">tok</span> <span class="o">==</span> <span class="s1">'['</span><span class="p">:</span>
133 <span class="n">stack</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">frame</span><span class="p">)</span>
134 <span class="n">frame</span> <span class="o">=</span> <span class="p">[]</span>
135 <span class="n">stack</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">frame</span><span class="p">)</span>
136 <span class="k">elif</span> <span class="n">tok</span> <span class="o">==</span> <span class="s1">']'</span><span class="p">:</span>
137 <span class="k">try</span><span class="p">:</span>
138 <span class="n">frame</span> <span class="o">=</span> <span class="n">stack</span><span class="o">.</span><span class="n">pop</span><span class="p">()</span>
139 <span class="k">except</span> <span class="ne">IndexError</span><span class="p">:</span>
140 <span class="k">raise</span> <span class="n">ParseError</span><span class="p">(</span><span class="s1">'Extra closing bracket.'</span><span class="p">)</span>
141 <span class="n">frame</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span> <span class="o">=</span> <span class="n">list_to_stack</span><span class="p">(</span><span class="n">frame</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">])</span>
142 <span class="k">else</span><span class="p">:</span>
143 <span class="n">frame</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">tok</span><span class="p">)</span>
144 <span class="k">if</span> <span class="n">stack</span><span class="p">:</span>
145 <span class="k">raise</span> <span class="n">ParseError</span><span class="p">(</span><span class="s1">'Unclosed bracket.'</span><span class="p">)</span>
146 <span class="k">return</span> <span class="n">list_to_stack</span><span class="p">(</span><span class="n">frame</span><span class="p">)</span>
149 <span class="n">_scanner</span> <span class="o">=</span> <span class="n">Scanner</span><span class="p">([</span>
150 <span class="p">(</span><span class="n">FLOAT</span><span class="p">,</span> <span class="k">lambda</span> <span class="n">_</span><span class="p">,</span> <span class="n">token</span><span class="p">:</span> <span class="nb">float</span><span class="p">(</span><span class="n">token</span><span class="p">)),</span>
151 <span class="p">(</span><span class="n">INT</span><span class="p">,</span> <span class="k">lambda</span> <span class="n">_</span><span class="p">,</span> <span class="n">token</span><span class="p">:</span> <span class="nb">int</span><span class="p">(</span><span class="n">token</span><span class="p">)),</span>
152 <span class="p">(</span><span class="n">SYMBOL</span><span class="p">,</span> <span class="k">lambda</span> <span class="n">_</span><span class="p">,</span> <span class="n">token</span><span class="p">:</span> <span class="n">Symbol</span><span class="p">(</span><span class="n">token</span><span class="p">)),</span>
153 <span class="p">(</span><span class="n">BRACKETS</span><span class="p">,</span> <span class="k">lambda</span> <span class="n">_</span><span class="p">,</span> <span class="n">token</span><span class="p">:</span> <span class="n">token</span><span class="p">),</span>
154 <span class="p">(</span><span class="n">STRING_DOUBLE_QUOTED</span><span class="p">,</span> <span class="k">lambda</span> <span class="n">_</span><span class="p">,</span> <span class="n">token</span><span class="p">:</span> <span class="n">token</span><span class="p">[</span><span class="mi">1</span><span class="p">:</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">'</span><span class="se">\\</span><span class="s1">"'</span><span class="p">,</span> <span class="s1">'"'</span><span class="p">)),</span>
155 <span class="p">(</span><span class="n">STRING_SINGLE_QUOTED</span><span class="p">,</span> <span class="k">lambda</span> <span class="n">_</span><span class="p">,</span> <span class="n">token</span><span class="p">:</span> <span class="n">token</span><span class="p">[</span><span class="mi">1</span><span class="p">:</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">"</span><span class="se">\\</span><span class="s2">'"</span><span class="p">,</span> <span class="s2">"'"</span><span class="p">)),</span>
156 <span class="p">(</span><span class="n">BLANKS</span><span class="p">,</span> <span class="kc">None</span><span class="p">),</span>
157 <span class="p">])</span>
163 <div class="sphinxsidebar" role="navigation" aria-label="main navigation">
164 <div class="sphinxsidebarwrapper"><div class="relations">
165 <h3>Related Topics</h3>
167 <li><a href="../../index.html">Documentation overview</a><ul>
168 <li><a href="../index.html">Module code</a><ul>
173 <div id="searchbox" style="display: none" role="search">
174 <h3>Quick search</h3>
175 <div class="searchformwrapper">
176 <form class="search" action="../../search.html" method="get">
177 <input type="text" name="q" />
178 <input type="submit" value="Go" />
179 <input type="hidden" name="check_keywords" value="yes" />
180 <input type="hidden" name="area" value="default" />
184 <script type="text/javascript">$('#searchbox').show(0);</script>
187 <div class="clearer"></div>
189 <div class="footer" role="contentinfo">
190 <a rel="license" href="http://creativecommons.org/licenses/by-nc-sa/4.0/">
191 <img alt="Creative Commons License" style="border-width:0" src="https://i.creativecommons.org/l/by-nc-sa/4.0/88x31.png" />
194 <span xmlns:dct="http://purl.org/dc/terms/" property="dct:title">Thun Documentation</span> by <a xmlns:cc="http://creativecommons.org/ns#" href="https://joypy.osdn.io/" property="cc:attributionName" rel="cc:attributionURL">Simon Forman</a> is licensed under a <a rel="license" href="http://creativecommons.org/licenses/by-nc-sa/4.0/">Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International License</a>.<br />Based on a work at <a xmlns:dct="http://purl.org/dc/terms/" href="https://osdn.net/projects/joypy/" rel="dct:source">https://osdn.net/projects/joypy/</a>.
195 Created using <a href="http://sphinx-doc.org/">Sphinx</a> 1.7.3.