author | Oleksandr Gavenko <gavenkoa@gmail.com> |
Fri, 11 Nov 2016 00:19:47 +0200 | |
changeset 667 | 5f69f0776c37 |
parent 643 | c2c32f45dde6 |
child 720 | b5a4b476eddf |
permissions | -rw-r--r-- |
643
c2c32f45dde6
Search rare words in gadict.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
diff
changeset
|
1 |
|
c2c32f45dde6
Search rare words in gadict.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
diff
changeset
|
2 |
import sys |
c2c32f45dde6
Search rare words in gadict.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
diff
changeset
|
3 |
import codecs |
c2c32f45dde6
Search rare words in gadict.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
diff
changeset
|
4 |
import io |
c2c32f45dde6
Search rare words in gadict.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
diff
changeset
|
5 |
import regex |
c2c32f45dde6
Search rare words in gadict.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
diff
changeset
|
6 |
|
c2c32f45dde6
Search rare words in gadict.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
diff
changeset
|
7 |
FINAME = None |
c2c32f45dde6
Search rare words in gadict.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
diff
changeset
|
8 |
FONAME = None |
c2c32f45dde6
Search rare words in gadict.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
diff
changeset
|
9 |
if len(sys.argv) >= 2: |
c2c32f45dde6
Search rare words in gadict.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
diff
changeset
|
10 |
FINAME = sys.argv[1] |
c2c32f45dde6
Search rare words in gadict.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
diff
changeset
|
11 |
if len(sys.argv) >= 3: |
c2c32f45dde6
Search rare words in gadict.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
diff
changeset
|
12 |
FONAME = sys.argv[2] |
c2c32f45dde6
Search rare words in gadict.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
diff
changeset
|
13 |
|
c2c32f45dde6
Search rare words in gadict.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
diff
changeset
|
14 |
FIN = io.open(FINAME, mode='r', buffering=1, encoding="utf-8") |
c2c32f45dde6
Search rare words in gadict.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
diff
changeset
|
15 |
if FONAME is None: |
c2c32f45dde6
Search rare words in gadict.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
diff
changeset
|
16 |
FOUT = sys.stdout |
c2c32f45dde6
Search rare words in gadict.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
diff
changeset
|
17 |
else: |
c2c32f45dde6
Search rare words in gadict.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
diff
changeset
|
18 |
FOUT = codecs.open(FONAME, "w", "utf-8") |
c2c32f45dde6
Search rare words in gadict.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
diff
changeset
|
19 |
|
c2c32f45dde6
Search rare words in gadict.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
diff
changeset
|
20 |
|
c2c32f45dde6
Search rare words in gadict.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
diff
changeset
|
21 |
class GadictParser: |
c2c32f45dde6
Search rare words in gadict.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
diff
changeset
|
22 |
|
c2c32f45dde6
Search rare words in gadict.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
diff
changeset
|
23 |
SEPARATOR_RE = regex.compile(u"^__$") |
c2c32f45dde6
Search rare words in gadict.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
diff
changeset
|
24 |
EMPTY_RE = regex.compile( u"^$" ) |
c2c32f45dde6
Search rare words in gadict.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
diff
changeset
|
25 |
HEADWORD_RE = regex.compile( u"^(\\p{L}.*)$" ) |
c2c32f45dde6
Search rare words in gadict.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
diff
changeset
|
26 |
|
c2c32f45dde6
Search rare words in gadict.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
diff
changeset
|
27 |
def __init__(self, stream): |
c2c32f45dde6
Search rare words in gadict.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
diff
changeset
|
28 |
self.stream = stream |
c2c32f45dde6
Search rare words in gadict.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
diff
changeset
|
29 |
self.lineno = 0 |
c2c32f45dde6
Search rare words in gadict.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
diff
changeset
|
30 |
|
c2c32f45dde6
Search rare words in gadict.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
diff
changeset
|
31 |
def parse(self): |
c2c32f45dde6
Search rare words in gadict.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
diff
changeset
|
32 |
wlist = [] |
c2c32f45dde6
Search rare words in gadict.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
diff
changeset
|
33 |
while True: |
c2c32f45dde6
Search rare words in gadict.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
diff
changeset
|
34 |
line = self.stream.readline() |
c2c32f45dde6
Search rare words in gadict.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
diff
changeset
|
35 |
if len(line) == 0: |
c2c32f45dde6
Search rare words in gadict.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
diff
changeset
|
36 |
break |
c2c32f45dde6
Search rare words in gadict.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
diff
changeset
|
37 |
self.lineno += 1 |
c2c32f45dde6
Search rare words in gadict.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
diff
changeset
|
38 |
m = self.SEPARATOR_RE.match(line) |
c2c32f45dde6
Search rare words in gadict.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
diff
changeset
|
39 |
if not m: |
c2c32f45dde6
Search rare words in gadict.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
diff
changeset
|
40 |
continue |
c2c32f45dde6
Search rare words in gadict.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
diff
changeset
|
41 |
|
c2c32f45dde6
Search rare words in gadict.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
diff
changeset
|
42 |
line = self.stream.readline() |
c2c32f45dde6
Search rare words in gadict.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
diff
changeset
|
43 |
if len(line) == 0: |
c2c32f45dde6
Search rare words in gadict.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
diff
changeset
|
44 |
break |
c2c32f45dde6
Search rare words in gadict.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
diff
changeset
|
45 |
self.lineno += 1 |
c2c32f45dde6
Search rare words in gadict.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
diff
changeset
|
46 |
m = self.EMPTY_RE.match(line) |
c2c32f45dde6
Search rare words in gadict.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
diff
changeset
|
47 |
if not m: |
c2c32f45dde6
Search rare words in gadict.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
diff
changeset
|
48 |
raise Exception("Line {:d}: '{:s}' is not empty line\n".format(self.lineno, line)) |
c2c32f45dde6
Search rare words in gadict.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
diff
changeset
|
49 |
|
c2c32f45dde6
Search rare words in gadict.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
diff
changeset
|
50 |
line = self.stream.readline() |
c2c32f45dde6
Search rare words in gadict.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
diff
changeset
|
51 |
if len(line) == 0: |
c2c32f45dde6
Search rare words in gadict.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
diff
changeset
|
52 |
break |
c2c32f45dde6
Search rare words in gadict.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
diff
changeset
|
53 |
line = line.strip() |
c2c32f45dde6
Search rare words in gadict.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
diff
changeset
|
54 |
self.lineno += 1 |
c2c32f45dde6
Search rare words in gadict.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
diff
changeset
|
55 |
m = self.HEADWORD_RE.match(line) |
c2c32f45dde6
Search rare words in gadict.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
diff
changeset
|
56 |
if not m: |
c2c32f45dde6
Search rare words in gadict.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
diff
changeset
|
57 |
raise Exception("Line {:d}: '{:s}' is not a headword\n".format(self.lineno, line)) |
c2c32f45dde6
Search rare words in gadict.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
diff
changeset
|
58 |
|
c2c32f45dde6
Search rare words in gadict.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
diff
changeset
|
59 |
wlist.append(line) |
c2c32f45dde6
Search rare words in gadict.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
diff
changeset
|
60 |
return wlist |
c2c32f45dde6
Search rare words in gadict.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
diff
changeset
|
61 |
|
c2c32f45dde6
Search rare words in gadict.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
diff
changeset
|
62 |
try: |
c2c32f45dde6
Search rare words in gadict.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
diff
changeset
|
63 |
parser = GadictParser(FIN) |
c2c32f45dde6
Search rare words in gadict.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
diff
changeset
|
64 |
for headword in parser.parse(): |
c2c32f45dde6
Search rare words in gadict.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
diff
changeset
|
65 |
FOUT.write(headword) |
c2c32f45dde6
Search rare words in gadict.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
diff
changeset
|
66 |
FOUT.write("\n") |
c2c32f45dde6
Search rare words in gadict.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
diff
changeset
|
67 |
finally: |
c2c32f45dde6
Search rare words in gadict.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
diff
changeset
|
68 |
FIN.close() |
c2c32f45dde6
Search rare words in gadict.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
diff
changeset
|
69 |
FOUT.close() |
c2c32f45dde6
Search rare words in gadict.
Oleksandr Gavenko <gavenkoa@gmail.com>
parents:
diff
changeset
|
70 |