1.1 --- a/moinformat/parsing.py Tue Dec 12 22:53:20 2017 +0100
1.2 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000
1.3 @@ -1,328 +0,0 @@
1.4 -#!/usr/bin/env python
1.5 -
1.6 -"""
1.7 -Moin wiki parsing functionality.
1.8 -
1.9 -Copyright (C) 2017 Paul Boddie <paul@boddie.org.uk>
1.10 -
1.11 -This program is free software; you can redistribute it and/or modify it under
1.12 -the terms of the GNU General Public License as published by the Free Software
1.13 -Foundation; either version 3 of the License, or (at your option) any later
1.14 -version.
1.15 -
1.16 -This program is distributed in the hope that it will be useful, but WITHOUT
1.17 -ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
1.18 -FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
1.19 -details.
1.20 -
1.21 -You should have received a copy of the GNU General Public License along with
1.22 -this program. If not, see <http://www.gnu.org/licenses/>.
1.23 -"""
1.24 -
1.25 -from moinformat.tree import Block, Region, Text
1.26 -import re
1.27 -
1.28 -# Pattern management.
1.29 -
1.30 -ws_excl_nl = r"[ \f\r\t\v]"
1.31 -
1.32 -def get_patterns(syntax):
1.33 -
1.34 - """
1.35 - Define patterns for the regular expressions in the 'syntax' mapping. In each
1.36 - pattern, replace \N with a pattern for matching whitespace excluding
1.37 - newlines.
1.38 - """
1.39 -
1.40 - patterns = {}
1.41 - for name, value in syntax.items():
1.42 - value = value.replace(r"\N", ws_excl_nl)
1.43 - patterns[name] = re.compile(value, re.UNICODE | re.MULTILINE)
1.44 - return patterns
1.45 -
1.46 -def get_subset(d, keys):
1.47 -
1.48 - "Return a subset of 'd' having the given 'keys'."
1.49 -
1.50 - subset = {}
1.51 - for key in keys:
1.52 - subset[key] = d[key]
1.53 - return subset
1.54 -
1.55 -
1.56 -
1.57 -# Tokenising functions.
1.58 -
1.59 -class TokenStream:
1.60 -
1.61 - "A stream of tokens taken from a string."
1.62 -
1.63 - def __init__(self, s, pos=0):
1.64 - self.s = s
1.65 - self.pos = pos
1.66 - self.match = None
1.67 - self.matching = None
1.68 -
1.69 - def rewind(self, length):
1.70 -
1.71 - "Rewind in the string by 'length'."
1.72 -
1.73 - self.pos -= min(length, self.pos)
1.74 -
1.75 - def read_until(self, patterns, remaining=True):
1.76 -
1.77 - """
1.78 - Find the first match for the given 'patterns'. Return the text preceding
1.79 - any match, the remaining text if no match was found, or None if no match
1.80 - was found and 'remaining' is given as a false value.
1.81 - """
1.82 -
1.83 - first = None
1.84 - self.matching = None
1.85 -
1.86 - # Find the first matching pattern.
1.87 -
1.88 - for pattern_name, pattern in patterns.items():
1.89 - match = pattern.search(self.s, self.pos)
1.90 - if match:
1.91 - start, end = match.span()
1.92 - if self.matching is None or start < first:
1.93 - first = start
1.94 - self.matching = pattern_name
1.95 - self.match = match
1.96 -
1.97 - if self.matching is None:
1.98 - if remaining:
1.99 - return self.s[self.pos:]
1.100 - else:
1.101 - return None
1.102 - else:
1.103 - return self.s[self.pos:first]
1.104 -
1.105 - def read_match(self, group=1):
1.106 -
1.107 - """
1.108 - Return the matched text, updating the position in the stream. If 'group'
1.109 - is specified, the indicated group in a match will be returned.
1.110 - Typically, group 1 should contain all pertinent data, but groups defined
1.111 - within group 1 can provide sections of the data.
1.112 - """
1.113 -
1.114 - if self.match:
1.115 - _start, self.pos = self.match.span()
1.116 - try:
1.117 - return self.match.group(group)
1.118 - except IndexError:
1.119 - return ""
1.120 - else:
1.121 - self.pos = len(self.s)
1.122 - return None
1.123 -
1.124 -
1.125 -
1.126 -# Utility functions.
1.127 -
1.128 -def new_block(region):
1.129 -
1.130 - "Start a new block in 'region'."
1.131 -
1.132 - region.add(Block([]))
1.133 -
1.134 -
1.135 -
1.136 -# Parser abstractions.
1.137 -
1.138 -class ParserBase:
1.139 -
1.140 - "Common parsing methods."
1.141 -
1.142 - region_pattern_names = None
1.143 -
1.144 - def __init__(self, formats=None):
1.145 -
1.146 - """
1.147 - Initialise the parser with any given 'formats' mapping from region type
1.148 - names to parser objects.
1.149 - """
1.150 -
1.151 - self.formats = formats
1.152 -
1.153 - def get_parser(self, format_type):
1.154 -
1.155 - """
1.156 - Return a parser for 'format_type' or None if no suitable parser is found.
1.157 - """
1.158 -
1.159 - if not self.formats:
1.160 - return None
1.161 -
1.162 - cls = self.formats.get(format_type)
1.163 - if cls:
1.164 - return cls(self.formats)
1.165 - else:
1.166 - return None
1.167 -
1.168 - def get_patterns(self, pattern_names):
1.169 -
1.170 - "Return a mapping of the given 'pattern_names' to patterns."
1.171 -
1.172 - return get_subset(self.patterns, pattern_names)
1.173 -
1.174 - def get_items(self, s, pos=0):
1.175 -
1.176 - "Return a sequence of token items for 's' and 'pos'."
1.177 -
1.178 - return TokenStream(s, pos)
1.179 -
1.180 - def set_region(self, items, region):
1.181 -
1.182 - "Set the 'items' used to populate the given 'region'."
1.183 -
1.184 - self.items = items
1.185 - self.region = region
1.186 -
1.187 - def read_until(self, pattern_names, remaining=True):
1.188 -
1.189 - """
1.190 - Read the next portion of input, matching using 'pattern_names'. Return
1.191 - the text preceding any match, the remaining text if no match was found,
1.192 - or None if no match was found and 'remaining' is given as a false value.
1.193 - """
1.194 -
1.195 - return self.items.read_until(self.get_patterns(pattern_names))
1.196 -
1.197 - def read_match(self, group=1):
1.198 -
1.199 - """
1.200 - Return the group of the matching pattern with the given 'group' number.
1.201 - """
1.202 -
1.203 - return self.items.read_match(group)
1.204 -
1.205 - def read_matching(self):
1.206 -
1.207 - "Return the name of the matching pattern."
1.208 -
1.209 - return self.items.matching
1.210 -
1.211 - # Parser methods invoked from other objects.
1.212 -
1.213 - def parse(self, s):
1.214 -
1.215 - """
1.216 - Parse page text 's'. Pages consist of regions delimited by markers.
1.217 - """
1.218 -
1.219 - self.items = self.get_items(s)
1.220 - self.region = self.parse_region()
1.221 - return self.region
1.222 -
1.223 - def parse_region_content(self, items, region):
1.224 -
1.225 - "Parse the data provided by 'items' to populate a 'region'."
1.226 -
1.227 - self.set_region(items, region)
1.228 -
1.229 - # Define a block to hold text and start parsing.
1.230 -
1.231 - new_block(region)
1.232 -
1.233 - if self.region_pattern_names:
1.234 - self.parse_region_details(region, self.region_pattern_names)
1.235 -
1.236 - # Top-level parser handler methods.
1.237 -
1.238 - def parse_region(self, level=0, indent=0):
1.239 -
1.240 - """
1.241 - Parse the data to populate a region with the given 'level' at the given
1.242 - 'indent'.
1.243 - """
1.244 -
1.245 - region = Region([], level, indent)
1.246 -
1.247 - # Parse section headers, then parse according to region type.
1.248 -
1.249 - self.parse_region_header(region)
1.250 - self.parse_region_type(region)
1.251 -
1.252 - return region
1.253 -
1.254 - def parse_region_type(self, region):
1.255 -
1.256 - """
1.257 - Use configured parsers to parse 'region' based on its type.
1.258 - """
1.259 -
1.260 - # Find an appropriate parser given the type.
1.261 -
1.262 - parser = self.get_parser(region.type)
1.263 -
1.264 - if parser:
1.265 - parser.parse_region_content(self.items, region)
1.266 -
1.267 - # Otherwise, treat the section as opaque.
1.268 -
1.269 - else:
1.270 - self.parse_region_opaque(region)
1.271 -
1.272 - def parse_region_header(self, region):
1.273 -
1.274 - """
1.275 - Parse the region header, setting it on the 'region' object.
1.276 - """
1.277 -
1.278 - if self.read_until(["header"], False) == "": # None means no header
1.279 - region.type = self.read_match()
1.280 -
1.281 - def parse_region_opaque(self, region):
1.282 -
1.283 - "Parse the data to populate an opaque 'region'."
1.284 -
1.285 - region.transparent = False
1.286 - self.parse_region_details(region, ["regionend"])
1.287 -
1.288 - # Parsing utilities.
1.289 -
1.290 - def parse_region_details(self, region, pattern_names):
1.291 -
1.292 - "Search 'region' using the 'pattern_names'."
1.293 -
1.294 - try:
1.295 - while True:
1.296 -
1.297 - # Obtain text before any marker or the end of the input.
1.298 -
1.299 - preceding = self.read_until(pattern_names)
1.300 - if preceding:
1.301 - region.append_inline(Text(preceding))
1.302 -
1.303 - # End of input.
1.304 -
1.305 - if not self.read_matching():
1.306 - break
1.307 -
1.308 - # Obtain any feature.
1.309 -
1.310 - feature = self.read_match()
1.311 - handler = self.handlers.get(self.read_matching())
1.312 -
1.313 - # Handle each feature or add text to the region.
1.314 -
1.315 - if handler:
1.316 - handler(self, region)
1.317 - else:
1.318 - region.append_inline(Text(feature))
1.319 -
1.320 - except StopIteration:
1.321 - pass
1.322 -
1.323 - region.normalise()
1.324 -
1.325 - def end_region(self, region):
1.326 -
1.327 - "End the parsing of 'region', breaking out of the parsing loop."
1.328 -
1.329 - raise StopIteration
1.330 -
1.331 -# vim: tabstop=4 expandtab shiftwidth=4