|
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 |
from nltk.tokenize import RegexpTokenizer from nltk.stem.snowball import SnowballStemmer from gensim import models, corpora from nltk.corpus import stopwords # Load input data def load_data(input_file): data = [] with open(input_file, 'r') as f: for line in f.readlines(): data.append(line[:-1]) return data # Class to preprocess text class Preprocessor(object): # Initialize various operators def __init__(self): # Create a regular expression tokenizer self.tokenizer = RegexpTokenizer(r'\w+') # get the list of stop words self.stop_words_english = stopwords.words('english') # Create a Snowball stemmer self.stemmer = SnowballStemmer('english') # Tokenizing, stop word removal, and stemming def process(self, input_text): # Tokenize the string tokens = self.tokenizer.tokenize(input_text.lower()) # Remove the stop words tokens_stopwords = [x for x in tokens if not x in self.stop_words_english] # Perform stemming on the tokens tokens_stemmed = [self.stemmer.stem(x) for x in tokens_stopwords] return tokens_stemmed if __name__=='__main__': # File containing linewise input data input_file = 'data_topic_modeling.txt' # Load data data = load_data(input_file) # Create a preprocessor object preprocessor = Preprocessor() # Create a list for processed documents processed_tokens = [preprocessor.process(x) for x in data] # Create a dictionary based on the tokenized documents dict_tokens = corpora.Dictionary(processed_tokens) # Create a document-term matrix corpus = [dict_tokens.doc2bow(text) for text in processed_tokens] # Generate the LDA model based on the corpus we just created num_topics = 2 num_words = 4 ldamodel = models.ldamodel.LdaModel(corpus, num_topics=num_topics, id2word=dict_tokens, passes=25) print("\nMost contributing words to the topics:") for item in ldamodel.print_topics(num_topics=num_topics, num_words=num_words): print("\nTopic", item[0], "==>", item[1]) |
