-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathdemo_main.py
161 lines (131 loc) · 5.76 KB
/
demo_main.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
"""
OHTM-Pipeline:
With this board you can select every option for your topic model.
You can set the different options on and off via "True" and "False".
First set the required paths.
- the first "mallet_path" to your mallet directory.
(see https://programminghistorian.org/en/lessons/topic-modeling-and-mallet)
- the second one to your working folder. This folder is your working environment.
All models are saved there and can be loaded from this folder.
- your stopword file must be stored in this working folder.
"""
"""
This demo is used to test, if the code runs proper and is linked to the demo folder and its demo files.
- You can test importing the text files and preprocessing them.
- You can print the prepared ohmt_files or create the graphs for this file or use the search functions
- Topic modeling is not able, because you need to install mallet first.
"""
from ohtm.pipeline import ohtm_pipeline_function
import os
''' Path Settings: '''
# Path to your mallet folder.
os.environ['MALLET_HOME'] = r'C:\\mallet-2.0.8' # does not work in the demo_main, just leave it, as it is
mallet_path: str = r'C:\mallet-2.0.8\bin\mallet' # does not work in the demo_main, just leave it, as it is
# Path to your output_folder.
output_folder: str = (r"Demo")
# Set the path for your stop_word list.
stopword_file_name = "german_stopwords_standard.txt"
# Set the path to your sources. This must be the folder, where your documents are stored in another folder.
source_path: str = (r"Demo\demo_interviews")
source_folder = ["archive_1", "archive_2", "txt_files"]
""" Topic Modeling Settings: """
load_ohtm_file = True
load_file_name = "demo_ohtm_file_trained"
create_ohtm_file = False
save_ohtm_file = False
save_name = "demo_ohtm_file_trained"
# You need to set a save_name and set the option save_json to True to save the model
save_model = False # does not work in the demo_main, just leave it, as it is
use_preprocessing = False
# If you don't want to chunk your documents, set use_chunking to True and chunk_setting to 0
use_chunking = False
chunk_setting = 20
use_topic_modeling = False # does not work in the demo_main, just leave it, as it is
topics = 10 # does not work in the demo_main, just leave it, as it is
save_top_words = True
number_of_words = 20
print_ohtm_file = True
print_ohtm_file_settings = True
show_bar_graph_corpus = True
show_heatmap_corpus = True
interview_id = "abc0001"
chunk_number = 0
show_heatmap_interview = False
print_interview_chunk = True
search_for_topics_in_chunks = True
topic_search = 0
chunk_weight = 0.2
search_for_topics_in_interview = True
''' advanced options: '''
# topic_modeling
optimize_interval_mallet = 50
iterations_mallet = 500
alpha = 5
random_seed = 80
# ohtm_file creation
speaker_txt = True
folder_as_archive = False
# preprocessing
# Select just one of those two:
stopword_removal_by_stop_list = True
# To use this options, you have to select a valid spacy model:
stopword_removal_by_spacy = False # does not work in the demo_main, just leave it, as it is
use_lemmatization = False # does not work in the demo_main, just leave it, as it is
lemmatization_model_spacy = "de_core_news_lg" # does not work in the demo_main, just leave it, as it is
use_pos_filter = False # does not work in the demo_main, just leave it, as it is
# possible settings: 'NOUN', 'PROPN', 'VERB', 'ADJ', 'ADV', 'PRON', 'ADP', 'DET', 'AUX', 'NUM', 'SCONJ', 'CCONJ', 'X'
allowed_postags_settings_lemmatization = ['NOUN', 'PROPN', 'VERB', 'ADJ', 'NUM', 'ADV']
''' Inferring new documents with an trained topic model'''
infer_new_documents = False
trained_ohtm_file = "...." # load the trained json and the model to train that json
save_separate_ohtm_file = False # save the inferred documents as a new json
separate_ohtm_file_name = "...."
if __name__ == "__main__":
ohtm_pipeline_function(
output_folder=output_folder,
source_folder=source_folder,
source_path=source_path,
stopword_file_name=stopword_file_name,
allowed_postags_settings=allowed_postags_settings_lemmatization,
save_name=save_name,
load_file_name=load_file_name,
mallet_path= mallet_path,
interview_id=interview_id,
chunk_setting=chunk_setting,
topics=topics,
number_of_words=number_of_words,
chunk_number=chunk_number,
topic_search=topic_search,
chunk_weight=chunk_weight,
optimize_interval_mallet=optimize_interval_mallet,
iterations_mallet=iterations_mallet,
alpha=alpha,
random_seed=random_seed,
save_ohtm_file=save_ohtm_file,
create_ohtm_file=create_ohtm_file,
load_ohtm_file=load_ohtm_file,
use_preprocessing=use_preprocessing,
use_chunking=use_chunking,
use_topic_modeling=use_topic_modeling,
save_top_words=save_top_words,
print_ohtm_file=print_ohtm_file,
show_bar_graph_corpus=show_bar_graph_corpus,
show_heatmap_corpus=show_heatmap_corpus,
show_heatmap_interview=show_heatmap_interview,
print_interview_chunk=print_interview_chunk,
search_for_topics_in_chunks=search_for_topics_in_chunks,
search_for_topics_in_interview=search_for_topics_in_interview,
by_list=stopword_removal_by_stop_list,
pos_filter_setting=use_pos_filter,
lemma=use_lemmatization,
save_model=save_model,
infer_new_documents=infer_new_documents,
trained_ohtm_file=trained_ohtm_file,
save_separate_ohtm_file=save_separate_ohtm_file,
separate_ohtm_file_name=separate_ohtm_file_name,
speaker_txt=speaker_txt,
folder_as_archive=folder_as_archive,
print_ohtm_file_settings=print_ohtm_file_settings,
spacy_model_name=lemmatization_model_spacy,
stopword_removal_by_spacy=stopword_removal_by_spacy
)