script-to-data-analysis/vectorize.py at main · cuban-digital-language/script-to-data-analysis · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
from submodulos.tokenizer.custom_tokenizer import SpacyCustomTokenizer, get_progressbar
from submodulos.data import get_all_text
import sys
import os
import ast


def word():
    bar = get_progressbar(len(texts), ' Tokenizer ')
    bar.start()
    for i, text in enumerate(texts):
        for _ in nlp(text[0]):
            pass
        bar.update(i+1)
    bar.finish()

    bar = get_progressbar(len(nlp.embedding), ' Save ')
    bar.start()
    i = 0
    for key, value in nlp.embedding.items():
        with open(f'results/words/{hash(key)}.txt', 'w+') as f:
            f.write(str((key, value)))
            f.close()

        i += 1
        bar.update(i)
    bar.finish()


def tokens():
    bar = get_progressbar(len(texts), ' Tokenizer ')
    bar.start()
    for i, text in enumerate(texts):
        for _ in nlp(text[0]):
            pass
        bar.update(i+1)
    bar.finish()

    bar = get_progressbar(len(texts), ' Save ')
    bar.start()
    i = 0
    for t, _ in texts:
        hsh = hash(t)
        with open(f'results/tokens/{hsh}.txt', 'w+') as f:
            f.write(str((t, nlp.memory[str(hsh)])))
            f.close()

        i += 1
        bar.update(i)
    bar.finish()


def length(type):
    return len(os.listdir(f'results/{type}'))


def load(type='words'):
    for filename in os.listdir(f'results/{type}'):
        with open(f'results/{type}/{filename}', 'r') as f:
            text = f.read()
            f.close()
            yield ast.literal_eval(text)


def sent():
    bar = get_progressbar(len(texts), ' Tokenizer ')
    bar.start()
    embedding = {}
    for i, text in enumerate(texts):
        for token in nlp.nlp(text[0]).sents:
            embedding[token.text] = tuple(token.vector)
        bar.update(i+1)
    bar.finish()

    bar = get_progressbar(len(embedding), ' Save ')
    bar.start()
    i = 0
    for key, value in embedding.items():
        with open(f'results/sentences/{hash(key)}.txt', 'w+') as f:
            f.write(str((key, value)))
            f.close()

        i += 1
        bar.update(i)
    bar.finish()


def text():
    bar = get_progressbar(len(texts), ' Tokenizer ')
    bar.start()
    embedding = {}
    for i, text in enumerate(texts):
        embedding[text[0]] = tuple(nlp.nlp(text[0]).vector)
        bar.update(i+1)
    bar.finish()

    bar = get_progressbar(len(texts), ' Save ')
    bar.start()
    i = 0
    for key, value in embedding.items():
        with open(f'results/texts/{hash(key)}.txt', 'w+') as f:
            f.write(str((key, value)))
            f.close()
        i += 1
        bar.update(i)
    bar.finish()


if __name__ == '__main__':
    texts = get_all_text('submodulos/data/')
    nlp = SpacyCustomTokenizer()

    cmd = sys.argv[1]

    if cmd == word.__name__:
        print('word command')
        word()

    if cmd == tokens.__name__:
        print('tokens command')
        tokens()

    if cmd == sent.__name__:
        print('sent command')
        sent()

    if cmd == text.__name__:
        print('text command')
        text()