author | Oleksandr Gavenko <gavenkoa@gmail.com> |
Sun, 11 Sep 2016 22:48:16 +0300 | |
changeset 550 | 6728ffd79d0c |
parent 542 | b5197c70972c |
child 552 | 7398bc1829d6 |
permissions | -rw-r--r-- |
406
f0ac87e10d9a
Fix warnings from pylint/pyflakes.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
402
diff
changeset
|
1 |
""" |
f0ac87e10d9a
Fix warnings from pylint/pyflakes.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
402
diff
changeset
|
2 |
gadict dictionary format parser. |
f0ac87e10d9a
Fix warnings from pylint/pyflakes.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
402
diff
changeset
|
3 |
""" |
385 | 4 |
|
5 |
import regex |
|
6 |
||
7 |
||
402
b47698d5ccab
Parse dictionary metainfo.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
400
diff
changeset
|
8 |
class Prelude: |
406
f0ac87e10d9a
Fix warnings from pylint/pyflakes.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
402
diff
changeset
|
9 |
"""Dictionary metainfo structure.""" |
402
b47698d5ccab
Parse dictionary metainfo.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
400
diff
changeset
|
10 |
name = None |
b47698d5ccab
Parse dictionary metainfo.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
400
diff
changeset
|
11 |
about = "" |
b47698d5ccab
Parse dictionary metainfo.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
400
diff
changeset
|
12 |
urls = [] |
b47698d5ccab
Parse dictionary metainfo.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
400
diff
changeset
|
13 |
authors = [] |
b47698d5ccab
Parse dictionary metainfo.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
400
diff
changeset
|
14 |
licences = [] |
b47698d5ccab
Parse dictionary metainfo.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
400
diff
changeset
|
15 |
|
b47698d5ccab
Parse dictionary metainfo.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
400
diff
changeset
|
16 |
|
399
a6a7036f3c6f
File name is not available in parser. Move error printing to writer.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
394
diff
changeset
|
17 |
class ParseException(BaseException): |
385 | 18 |
|
406
f0ac87e10d9a
Fix warnings from pylint/pyflakes.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
402
diff
changeset
|
19 |
def __init__(self, msg, lineno=None, line=None): |
f0ac87e10d9a
Fix warnings from pylint/pyflakes.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
402
diff
changeset
|
20 |
super().__init__() |
385 | 21 |
self.msg = msg |
399
a6a7036f3c6f
File name is not available in parser. Move error printing to writer.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
394
diff
changeset
|
22 |
self.lineno = lineno |
a6a7036f3c6f
File name is not available in parser. Move error printing to writer.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
394
diff
changeset
|
23 |
self.line = line |
385 | 24 |
|
25 |
def __repr__(self): |
|
399
a6a7036f3c6f
File name is not available in parser. Move error printing to writer.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
394
diff
changeset
|
26 |
if self.lineno is None: |
a6a7036f3c6f
File name is not available in parser. Move error printing to writer.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
394
diff
changeset
|
27 |
return self.msg |
a6a7036f3c6f
File name is not available in parser. Move error printing to writer.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
394
diff
changeset
|
28 |
elif self.line is None: |
a6a7036f3c6f
File name is not available in parser. Move error printing to writer.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
394
diff
changeset
|
29 |
return ":{:d}:{:s}".format(self.lineno, self.msg) |
a6a7036f3c6f
File name is not available in parser. Move error printing to writer.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
394
diff
changeset
|
30 |
else: |
a6a7036f3c6f
File name is not available in parser. Move error printing to writer.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
394
diff
changeset
|
31 |
return ":{:d}: {:s}\nLINE: {:s}".format(self.lineno, self.msg, self.line) |
385 | 32 |
|
530
91771594bc8b
Make storage for topics, antonyms and synonyms. Require pos marker.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
527
diff
changeset
|
33 |
class Sense: |
91771594bc8b
Make storage for topics, antonyms and synonyms. Require pos marker.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
527
diff
changeset
|
34 |
|
91771594bc8b
Make storage for topics, antonyms and synonyms. Require pos marker.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
527
diff
changeset
|
35 |
def __init__(self, pos, tr_list = None, ex_list = None, syn_list = None, ant_list = None, topic_list = None): |
91771594bc8b
Make storage for topics, antonyms and synonyms. Require pos marker.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
527
diff
changeset
|
36 |
if not pos: |
91771594bc8b
Make storage for topics, antonyms and synonyms. Require pos marker.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
527
diff
changeset
|
37 |
raise ParseException("Part of speech expected...\n") |
91771594bc8b
Make storage for topics, antonyms and synonyms. Require pos marker.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
527
diff
changeset
|
38 |
self.pos = pos |
91771594bc8b
Make storage for topics, antonyms and synonyms. Require pos marker.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
527
diff
changeset
|
39 |
if tr_list: |
91771594bc8b
Make storage for topics, antonyms and synonyms. Require pos marker.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
527
diff
changeset
|
40 |
self.tr_list = tr_list |
91771594bc8b
Make storage for topics, antonyms and synonyms. Require pos marker.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
527
diff
changeset
|
41 |
else: |
91771594bc8b
Make storage for topics, antonyms and synonyms. Require pos marker.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
527
diff
changeset
|
42 |
self.tr_list = [] |
91771594bc8b
Make storage for topics, antonyms and synonyms. Require pos marker.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
527
diff
changeset
|
43 |
self.ex_list = ex_list |
91771594bc8b
Make storage for topics, antonyms and synonyms. Require pos marker.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
527
diff
changeset
|
44 |
self.syn_list = syn_list |
91771594bc8b
Make storage for topics, antonyms and synonyms. Require pos marker.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
527
diff
changeset
|
45 |
self.ant_list = ant_list |
91771594bc8b
Make storage for topics, antonyms and synonyms. Require pos marker.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
527
diff
changeset
|
46 |
self.topic_list = topic_list |
91771594bc8b
Make storage for topics, antonyms and synonyms. Require pos marker.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
527
diff
changeset
|
47 |
|
91771594bc8b
Make storage for topics, antonyms and synonyms. Require pos marker.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
527
diff
changeset
|
48 |
def add_tr(self, tr): |
91771594bc8b
Make storage for topics, antonyms and synonyms. Require pos marker.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
527
diff
changeset
|
49 |
self.tr_list.append(tr) |
91771594bc8b
Make storage for topics, antonyms and synonyms. Require pos marker.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
527
diff
changeset
|
50 |
|
91771594bc8b
Make storage for topics, antonyms and synonyms. Require pos marker.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
527
diff
changeset
|
51 |
def add_ex(self, ex): |
91771594bc8b
Make storage for topics, antonyms and synonyms. Require pos marker.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
527
diff
changeset
|
52 |
if not self.ex_list: |
91771594bc8b
Make storage for topics, antonyms and synonyms. Require pos marker.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
527
diff
changeset
|
53 |
self.ex_list = [ex] |
91771594bc8b
Make storage for topics, antonyms and synonyms. Require pos marker.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
527
diff
changeset
|
54 |
else: |
91771594bc8b
Make storage for topics, antonyms and synonyms. Require pos marker.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
527
diff
changeset
|
55 |
self.ex_list.append(ex) |
91771594bc8b
Make storage for topics, antonyms and synonyms. Require pos marker.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
527
diff
changeset
|
56 |
|
91771594bc8b
Make storage for topics, antonyms and synonyms. Require pos marker.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
527
diff
changeset
|
57 |
def add_syn(self, syn): |
91771594bc8b
Make storage for topics, antonyms and synonyms. Require pos marker.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
527
diff
changeset
|
58 |
if not self.syn_list: |
91771594bc8b
Make storage for topics, antonyms and synonyms. Require pos marker.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
527
diff
changeset
|
59 |
self.syn_list = [syn] |
91771594bc8b
Make storage for topics, antonyms and synonyms. Require pos marker.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
527
diff
changeset
|
60 |
else: |
91771594bc8b
Make storage for topics, antonyms and synonyms. Require pos marker.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
527
diff
changeset
|
61 |
self.syn_list.append(syn) |
91771594bc8b
Make storage for topics, antonyms and synonyms. Require pos marker.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
527
diff
changeset
|
62 |
|
91771594bc8b
Make storage for topics, antonyms and synonyms. Require pos marker.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
527
diff
changeset
|
63 |
def add_ant(self, ant): |
91771594bc8b
Make storage for topics, antonyms and synonyms. Require pos marker.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
527
diff
changeset
|
64 |
if not self.ant_list: |
91771594bc8b
Make storage for topics, antonyms and synonyms. Require pos marker.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
527
diff
changeset
|
65 |
self.ant_list = [ant] |
91771594bc8b
Make storage for topics, antonyms and synonyms. Require pos marker.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
527
diff
changeset
|
66 |
else: |
91771594bc8b
Make storage for topics, antonyms and synonyms. Require pos marker.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
527
diff
changeset
|
67 |
self.ant_list.append(ant) |
91771594bc8b
Make storage for topics, antonyms and synonyms. Require pos marker.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
527
diff
changeset
|
68 |
|
91771594bc8b
Make storage for topics, antonyms and synonyms. Require pos marker.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
527
diff
changeset
|
69 |
def add_topic(self, topic): |
91771594bc8b
Make storage for topics, antonyms and synonyms. Require pos marker.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
527
diff
changeset
|
70 |
if not self.topic_list: |
91771594bc8b
Make storage for topics, antonyms and synonyms. Require pos marker.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
527
diff
changeset
|
71 |
self.topic_list = [topic] |
91771594bc8b
Make storage for topics, antonyms and synonyms. Require pos marker.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
527
diff
changeset
|
72 |
else: |
91771594bc8b
Make storage for topics, antonyms and synonyms. Require pos marker.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
527
diff
changeset
|
73 |
self.topic_list.append(topic) |
385 | 74 |
|
75 |
class Parser: |
|
406
f0ac87e10d9a
Fix warnings from pylint/pyflakes.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
402
diff
changeset
|
76 |
"""gadict dictionary format parser.""" |
385 | 77 |
|
432
b3a78fc20b31
Add parsing comment syntax.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
422
diff
changeset
|
78 |
COMMENT_RE = regex.compile(r"^# ") |
b3a78fc20b31
Add parsing comment syntax.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
422
diff
changeset
|
79 |
|
385 | 80 |
SEPARATOR_RE = regex.compile(r"^__$") |
81 |
HEADWORD_RE = regex.compile(r"^(\p{L}.*)$") |
|
527
0a31299fad70
Add support for country spelling variant.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
454
diff
changeset
|
82 |
HEADWORD_VAR_RE = regex.compile(r"^ +(s|pl|v[123]|male|female|comp|super|abbr|Am|Br|Au)$") |
385 | 83 |
HEADWORD_PRON_RE = regex.compile(r"^ +\[([\p{L}' ]+)\]$") |
542
b5197c70972c
Add commonly used contractions.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
530
diff
changeset
|
84 |
TRANSL_POS_RE = regex.compile(r"^n|det|pron|adj|v|adv|prep|conj|num|int|phr|phr\.v|contr|abbr|prefix$") |
412
ece60575a96a
Adopt to parse VOA dictionary: add topics support and translation continuation.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
406
diff
changeset
|
85 |
TRANSL_RE = regex.compile(r"^(ru|uk|la|en): ([\p{L}(].*)$") |
454
462bc6140f5b
Allow dash in example to mark dialogs.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
446
diff
changeset
|
86 |
TRANSL_EX_RE = regex.compile(r"^(ru|uk|la|en)> ([-\p{L}].*)$") |
530
91771594bc8b
Make storage for topics, antonyms and synonyms. Require pos marker.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
527
diff
changeset
|
87 |
TOPIC_RE = regex.compile(r"^topic: (\p{L}.*)$") |
91771594bc8b
Make storage for topics, antonyms and synonyms. Require pos marker.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
527
diff
changeset
|
88 |
SYN_RE = regex.compile(r"^syn: (\p{L}.*)$") |
91771594bc8b
Make storage for topics, antonyms and synonyms. Require pos marker.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
527
diff
changeset
|
89 |
ANT_RE = regex.compile(r"^ant: (\p{L}.*)$") |
385 | 90 |
|
402
b47698d5ccab
Parse dictionary metainfo.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
400
diff
changeset
|
91 |
CONT_RE = regex.compile(r"^ +(.*)") |
b47698d5ccab
Parse dictionary metainfo.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
400
diff
changeset
|
92 |
|
385 | 93 |
TRAILING_SPACES_RE = regex.compile(r"\p{Z}+$") |
94 |
||
402
b47698d5ccab
Parse dictionary metainfo.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
400
diff
changeset
|
95 |
PRELUDE_NAME_RE = regex.compile(r"^name: (.*)") |
b47698d5ccab
Parse dictionary metainfo.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
400
diff
changeset
|
96 |
PRELUDE_URL_RE = regex.compile(r"^url: (.*)") |
b47698d5ccab
Parse dictionary metainfo.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
400
diff
changeset
|
97 |
PRELUDE_AUTHOR_RE = regex.compile(r"^by: (.*)") |
b47698d5ccab
Parse dictionary metainfo.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
400
diff
changeset
|
98 |
PRELUDE_LICENSE_RE = regex.compile(r"^term: (.*)") |
b47698d5ccab
Parse dictionary metainfo.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
400
diff
changeset
|
99 |
PRELUDE_ABOUT_RE = regex.compile(r"^about: ?(.*)") |
b47698d5ccab
Parse dictionary metainfo.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
400
diff
changeset
|
100 |
|
385 | 101 |
def __init__(self): |
102 |
pass |
|
103 |
||
104 |
def readline(self): |
|
432
b3a78fc20b31
Add parsing comment syntax.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
422
diff
changeset
|
105 |
while True: |
b3a78fc20b31
Add parsing comment syntax.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
422
diff
changeset
|
106 |
self.line = self.stream.readline() |
b3a78fc20b31
Add parsing comment syntax.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
422
diff
changeset
|
107 |
self.eof = len(self.line) == 0 |
b3a78fc20b31
Add parsing comment syntax.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
422
diff
changeset
|
108 |
if not self.eof: |
b3a78fc20b31
Add parsing comment syntax.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
422
diff
changeset
|
109 |
self.lineno += 1 |
b3a78fc20b31
Add parsing comment syntax.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
422
diff
changeset
|
110 |
if self.TRAILING_SPACES_RE.search(self.line): |
b3a78fc20b31
Add parsing comment syntax.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
422
diff
changeset
|
111 |
raise ParseException("Traling spaces detected...\n") |
b3a78fc20b31
Add parsing comment syntax.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
422
diff
changeset
|
112 |
if self.COMMENT_RE.search(self.line): |
b3a78fc20b31
Add parsing comment syntax.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
422
diff
changeset
|
113 |
continue |
b3a78fc20b31
Add parsing comment syntax.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
422
diff
changeset
|
114 |
break |
385 | 115 |
|
116 |
def parse(self, stream): |
|
117 |
self.lineno = 0 |
|
118 |
self.stream = stream |
|
119 |
self.dom = [] |
|
120 |
try: |
|
121 |
self.parse_prelude() |
|
122 |
while not self.eof: |
|
123 |
self.parse_article() |
|
124 |
except ParseException as ex: |
|
406
f0ac87e10d9a
Fix warnings from pylint/pyflakes.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
402
diff
changeset
|
125 |
raise ParseException(ex.msg, self.lineno, self.line) |
385 | 126 |
return self.dom |
127 |
||
412
ece60575a96a
Adopt to parse VOA dictionary: add topics support and translation continuation.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
406
diff
changeset
|
128 |
def parse_prelude_continuation(self): |
402
b47698d5ccab
Parse dictionary metainfo.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
400
diff
changeset
|
129 |
string = "" |
b47698d5ccab
Parse dictionary metainfo.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
400
diff
changeset
|
130 |
while True: |
b47698d5ccab
Parse dictionary metainfo.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
400
diff
changeset
|
131 |
self.readline() |
b47698d5ccab
Parse dictionary metainfo.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
400
diff
changeset
|
132 |
if self.eof: |
b47698d5ccab
Parse dictionary metainfo.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
400
diff
changeset
|
133 |
return string |
406
f0ac87e10d9a
Fix warnings from pylint/pyflakes.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
402
diff
changeset
|
134 |
m = self.CONT_RE.match(self.line) |
402
b47698d5ccab
Parse dictionary metainfo.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
400
diff
changeset
|
135 |
if m is not None: |
b47698d5ccab
Parse dictionary metainfo.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
400
diff
changeset
|
136 |
string += "\n" + m.group(1) |
b47698d5ccab
Parse dictionary metainfo.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
400
diff
changeset
|
137 |
elif len(self.line) == 1: |
b47698d5ccab
Parse dictionary metainfo.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
400
diff
changeset
|
138 |
string += "\n" |
b47698d5ccab
Parse dictionary metainfo.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
400
diff
changeset
|
139 |
else: |
b47698d5ccab
Parse dictionary metainfo.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
400
diff
changeset
|
140 |
return string |
b47698d5ccab
Parse dictionary metainfo.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
400
diff
changeset
|
141 |
|
385 | 142 |
def parse_prelude(self): |
143 |
"""Read dictionary prelude until first "__" delimiter.""" |
|
402
b47698d5ccab
Parse dictionary metainfo.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
400
diff
changeset
|
144 |
pre = Prelude() |
385 | 145 |
while True: |
146 |
self.readline() |
|
147 |
if self.eof: |
|
148 |
raise ParseException("There are no articles...") |
|
402
b47698d5ccab
Parse dictionary metainfo.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
400
diff
changeset
|
149 |
m = self.PRELUDE_ABOUT_RE.match(self.line) |
b47698d5ccab
Parse dictionary metainfo.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
400
diff
changeset
|
150 |
if m: |
412
ece60575a96a
Adopt to parse VOA dictionary: add topics support and translation continuation.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
406
diff
changeset
|
151 |
pre.about += m.group(1) + self.parse_prelude_continuation() |
402
b47698d5ccab
Parse dictionary metainfo.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
400
diff
changeset
|
152 |
if self.eof: |
b47698d5ccab
Parse dictionary metainfo.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
400
diff
changeset
|
153 |
raise ParseException("There are no articles...") |
385 | 154 |
if self.SEPARATOR_RE.match(self.line): |
155 |
break |
|
402
b47698d5ccab
Parse dictionary metainfo.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
400
diff
changeset
|
156 |
m = self.PRELUDE_NAME_RE.match(self.line) |
b47698d5ccab
Parse dictionary metainfo.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
400
diff
changeset
|
157 |
if m: |
b47698d5ccab
Parse dictionary metainfo.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
400
diff
changeset
|
158 |
pre.name = m.group(1) |
b47698d5ccab
Parse dictionary metainfo.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
400
diff
changeset
|
159 |
continue |
b47698d5ccab
Parse dictionary metainfo.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
400
diff
changeset
|
160 |
m = self.PRELUDE_URL_RE.match(self.line) |
b47698d5ccab
Parse dictionary metainfo.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
400
diff
changeset
|
161 |
if m: |
b47698d5ccab
Parse dictionary metainfo.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
400
diff
changeset
|
162 |
pre.urls.append(m.group(1)) |
b47698d5ccab
Parse dictionary metainfo.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
400
diff
changeset
|
163 |
continue |
b47698d5ccab
Parse dictionary metainfo.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
400
diff
changeset
|
164 |
m = self.PRELUDE_AUTHOR_RE.match(self.line) |
b47698d5ccab
Parse dictionary metainfo.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
400
diff
changeset
|
165 |
if m: |
b47698d5ccab
Parse dictionary metainfo.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
400
diff
changeset
|
166 |
pre.authors.append(m.group(1)) |
b47698d5ccab
Parse dictionary metainfo.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
400
diff
changeset
|
167 |
continue |
b47698d5ccab
Parse dictionary metainfo.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
400
diff
changeset
|
168 |
m = self.PRELUDE_LICENSE_RE.match(self.line) |
b47698d5ccab
Parse dictionary metainfo.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
400
diff
changeset
|
169 |
if m: |
b47698d5ccab
Parse dictionary metainfo.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
400
diff
changeset
|
170 |
pre.licences.append(m.group(1)) |
b47698d5ccab
Parse dictionary metainfo.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
400
diff
changeset
|
171 |
continue |
b47698d5ccab
Parse dictionary metainfo.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
400
diff
changeset
|
172 |
self.dom.append(pre) |
385 | 173 |
|
174 |
def parse_article(self): |
|
175 |
"""Try to match article until next "__" delimiter. Assume that `self.line` point to "__" delimiter.""" |
|
176 |
self.words = None |
|
177 |
self.tran = None |
|
178 |
self.parse_empty_line() |
|
179 |
self.parse_headlines() |
|
180 |
self.parse_translation() |
|
181 |
self.dom.append((self.words, self.tran)) |
|
182 |
||
183 |
def parse_empty_line(self): |
|
184 |
self.readline() |
|
185 |
if self.eof or len(self.line) != 1: |
|
186 |
raise ParseException(""""__" delimiter should followed by empty line...""") |
|
187 |
||
188 |
def parse_headlines(self): |
|
189 |
"""Try to match word variations with attributed. Assume that `self.line` on preceding empty line.""" |
|
190 |
self.words = {} |
|
191 |
self.readline() |
|
192 |
if self.eof: |
|
193 |
raise ParseException("""There are no definition after "__" delimiter...""") |
|
194 |
m = self.HEADWORD_RE.match(self.line) |
|
195 |
if m is None: |
|
196 |
raise ParseException("""There are no headword after "__" delimiter...""") |
|
197 |
word = m.group(1) |
|
198 |
pron = None |
|
199 |
attrs = set() |
|
200 |
while True: |
|
201 |
self.readline() |
|
202 |
if self.eof or len(self.line) == 1: |
|
203 |
break |
|
204 |
m = self.HEADWORD_RE.match(self.line) |
|
205 |
if m is not None: |
|
206 |
if word is None: |
|
207 |
raise ParseException("""Didn't match previous headword...""") |
|
208 |
self.words[word] = (pron, attrs) |
|
209 |
word = m.group(1) |
|
210 |
pron = None |
|
211 |
attrs = set() |
|
212 |
continue |
|
213 |
m = self.HEADWORD_PRON_RE.match(self.line) |
|
214 |
if m is not None: |
|
215 |
if pron is not None: |
|
216 |
raise ParseException("""Pronunciation is redefined...""") |
|
217 |
pron = m.group(1) |
|
218 |
continue |
|
219 |
m = self.HEADWORD_VAR_RE.match(self.line) |
|
220 |
if m is not None: |
|
221 |
attrs.add(m.group(1)) |
|
222 |
continue |
|
400
aa03182d2e26
Proper check for trailing spaces.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
399
diff
changeset
|
223 |
raise ParseException("""Line is not a headword or translation or headword attribute...""") |
385 | 224 |
self.words[word] = (pron, attrs) |
225 |
||
412
ece60575a96a
Adopt to parse VOA dictionary: add topics support and translation continuation.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
406
diff
changeset
|
226 |
def parse_translation_continuation(self): |
ece60575a96a
Adopt to parse VOA dictionary: add topics support and translation continuation.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
406
diff
changeset
|
227 |
string = "" |
ece60575a96a
Adopt to parse VOA dictionary: add topics support and translation continuation.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
406
diff
changeset
|
228 |
while True: |
ece60575a96a
Adopt to parse VOA dictionary: add topics support and translation continuation.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
406
diff
changeset
|
229 |
self.readline() |
ece60575a96a
Adopt to parse VOA dictionary: add topics support and translation continuation.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
406
diff
changeset
|
230 |
if self.eof: |
ece60575a96a
Adopt to parse VOA dictionary: add topics support and translation continuation.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
406
diff
changeset
|
231 |
return string |
ece60575a96a
Adopt to parse VOA dictionary: add topics support and translation continuation.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
406
diff
changeset
|
232 |
m = self.CONT_RE.match(self.line) |
ece60575a96a
Adopt to parse VOA dictionary: add topics support and translation continuation.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
406
diff
changeset
|
233 |
if m is not None: |
ece60575a96a
Adopt to parse VOA dictionary: add topics support and translation continuation.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
406
diff
changeset
|
234 |
string += "\n" + m.group(1) |
ece60575a96a
Adopt to parse VOA dictionary: add topics support and translation continuation.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
406
diff
changeset
|
235 |
else: |
ece60575a96a
Adopt to parse VOA dictionary: add topics support and translation continuation.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
406
diff
changeset
|
236 |
return string |
ece60575a96a
Adopt to parse VOA dictionary: add topics support and translation continuation.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
406
diff
changeset
|
237 |
|
385 | 238 |
def parse_translation(self): |
239 |
senses = [] |
|
530
91771594bc8b
Make storage for topics, antonyms and synonyms. Require pos marker.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
527
diff
changeset
|
240 |
sense = None |
412
ece60575a96a
Adopt to parse VOA dictionary: add topics support and translation continuation.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
406
diff
changeset
|
241 |
read = True |
385 | 242 |
while True: |
412
ece60575a96a
Adopt to parse VOA dictionary: add topics support and translation continuation.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
406
diff
changeset
|
243 |
if read: |
ece60575a96a
Adopt to parse VOA dictionary: add topics support and translation continuation.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
406
diff
changeset
|
244 |
self.readline() |
ece60575a96a
Adopt to parse VOA dictionary: add topics support and translation continuation.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
406
diff
changeset
|
245 |
read = True |
385 | 246 |
if self.eof: |
247 |
break |
|
248 |
m = self.SEPARATOR_RE.match(self.line) |
|
249 |
if m is not None: |
|
530
91771594bc8b
Make storage for topics, antonyms and synonyms. Require pos marker.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
527
diff
changeset
|
250 |
if sense: |
91771594bc8b
Make storage for topics, antonyms and synonyms. Require pos marker.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
527
diff
changeset
|
251 |
senses.append(sense) |
385 | 252 |
break |
253 |
if len(self.line) == 1: |
|
530
91771594bc8b
Make storage for topics, antonyms and synonyms. Require pos marker.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
527
diff
changeset
|
254 |
if sense: |
91771594bc8b
Make storage for topics, antonyms and synonyms. Require pos marker.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
527
diff
changeset
|
255 |
senses.append(sense) |
91771594bc8b
Make storage for topics, antonyms and synonyms. Require pos marker.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
527
diff
changeset
|
256 |
sense = None |
385 | 257 |
continue |
258 |
m = self.TRANSL_POS_RE.match(self.line) |
|
259 |
if m is not None: |
|
530
91771594bc8b
Make storage for topics, antonyms and synonyms. Require pos marker.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
527
diff
changeset
|
260 |
if sense is not None: |
385 | 261 |
raise ParseException("""Each translation should have only one part of speech marker...""") |
262 |
pos = m.group(0) |
|
530
91771594bc8b
Make storage for topics, antonyms and synonyms. Require pos marker.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
527
diff
changeset
|
263 |
sense = Sense(pos) |
385 | 264 |
continue |
530
91771594bc8b
Make storage for topics, antonyms and synonyms. Require pos marker.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
527
diff
changeset
|
265 |
if not sense: |
91771594bc8b
Make storage for topics, antonyms and synonyms. Require pos marker.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
527
diff
changeset
|
266 |
raise ParseException("""Missing part of speech marker...""") |
412
ece60575a96a
Adopt to parse VOA dictionary: add topics support and translation continuation.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
406
diff
changeset
|
267 |
m = self.TOPIC_RE.match(self.line) |
ece60575a96a
Adopt to parse VOA dictionary: add topics support and translation continuation.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
406
diff
changeset
|
268 |
if m is not None: |
530
91771594bc8b
Make storage for topics, antonyms and synonyms. Require pos marker.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
527
diff
changeset
|
269 |
topics = m.group(1).split(";") |
91771594bc8b
Make storage for topics, antonyms and synonyms. Require pos marker.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
527
diff
changeset
|
270 |
for topic in topics: |
91771594bc8b
Make storage for topics, antonyms and synonyms. Require pos marker.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
527
diff
changeset
|
271 |
topic = topic.strip() |
91771594bc8b
Make storage for topics, antonyms and synonyms. Require pos marker.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
527
diff
changeset
|
272 |
if len(topic) == 0: |
91771594bc8b
Make storage for topics, antonyms and synonyms. Require pos marker.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
527
diff
changeset
|
273 |
raise ParseException("""Empty topic...""") |
91771594bc8b
Make storage for topics, antonyms and synonyms. Require pos marker.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
527
diff
changeset
|
274 |
sense.add_topic(topic) |
91771594bc8b
Make storage for topics, antonyms and synonyms. Require pos marker.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
527
diff
changeset
|
275 |
continue |
91771594bc8b
Make storage for topics, antonyms and synonyms. Require pos marker.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
527
diff
changeset
|
276 |
m = self.SYN_RE.match(self.line) |
91771594bc8b
Make storage for topics, antonyms and synonyms. Require pos marker.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
527
diff
changeset
|
277 |
if m is not None: |
91771594bc8b
Make storage for topics, antonyms and synonyms. Require pos marker.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
527
diff
changeset
|
278 |
syns = m.group(1).split(";") |
91771594bc8b
Make storage for topics, antonyms and synonyms. Require pos marker.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
527
diff
changeset
|
279 |
for syn in syns: |
91771594bc8b
Make storage for topics, antonyms and synonyms. Require pos marker.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
527
diff
changeset
|
280 |
syn = syn.strip() |
91771594bc8b
Make storage for topics, antonyms and synonyms. Require pos marker.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
527
diff
changeset
|
281 |
if len(syn) == 0: |
91771594bc8b
Make storage for topics, antonyms and synonyms. Require pos marker.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
527
diff
changeset
|
282 |
raise ParseException("""Empty synonym...""") |
91771594bc8b
Make storage for topics, antonyms and synonyms. Require pos marker.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
527
diff
changeset
|
283 |
sense.add_syn(syn) |
91771594bc8b
Make storage for topics, antonyms and synonyms. Require pos marker.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
527
diff
changeset
|
284 |
continue |
91771594bc8b
Make storage for topics, antonyms and synonyms. Require pos marker.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
527
diff
changeset
|
285 |
m = self.ANT_RE.match(self.line) |
91771594bc8b
Make storage for topics, antonyms and synonyms. Require pos marker.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
527
diff
changeset
|
286 |
if m is not None: |
91771594bc8b
Make storage for topics, antonyms and synonyms. Require pos marker.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
527
diff
changeset
|
287 |
ants = m.group(1).split(";") |
91771594bc8b
Make storage for topics, antonyms and synonyms. Require pos marker.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
527
diff
changeset
|
288 |
for ant in ants: |
91771594bc8b
Make storage for topics, antonyms and synonyms. Require pos marker.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
527
diff
changeset
|
289 |
ant = ant.strip() |
91771594bc8b
Make storage for topics, antonyms and synonyms. Require pos marker.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
527
diff
changeset
|
290 |
if len(ant) == 0: |
91771594bc8b
Make storage for topics, antonyms and synonyms. Require pos marker.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
527
diff
changeset
|
291 |
raise ParseException("""Empty antonym...""") |
91771594bc8b
Make storage for topics, antonyms and synonyms. Require pos marker.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
527
diff
changeset
|
292 |
sense.add_ant(ant) |
412
ece60575a96a
Adopt to parse VOA dictionary: add topics support and translation continuation.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
406
diff
changeset
|
293 |
continue |
385 | 294 |
m = self.TRANSL_RE.match(self.line) |
295 |
if m is not None: |
|
530
91771594bc8b
Make storage for topics, antonyms and synonyms. Require pos marker.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
527
diff
changeset
|
296 |
sense.add_tr((m.group(1), m.group(2) + self.parse_translation_continuation())) |
412
ece60575a96a
Adopt to parse VOA dictionary: add topics support and translation continuation.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
406
diff
changeset
|
297 |
read = False |
385 | 298 |
continue |
299 |
m = self.TRANSL_EX_RE.match(self.line) |
|
300 |
if m is not None: |
|
530
91771594bc8b
Make storage for topics, antonyms and synonyms. Require pos marker.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
527
diff
changeset
|
301 |
sense.add_ex((m.group(1), m.group(2) + self.parse_translation_continuation())) |
412
ece60575a96a
Adopt to parse VOA dictionary: add topics support and translation continuation.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
406
diff
changeset
|
302 |
read = False |
385 | 303 |
continue |
304 |
raise ParseException("""Uknown syntax...""") |
|
305 |
self.tran = senses |