feature_array.py

from mongo import load_from_mongo
import numpy as np
import re
import lda
feature_array=[]
result=load_from_mongo("hindu_modified","docs1")
f=open('vocab.txt','r')
feature_vector=f.read().split('\n')
#print len(feature_vector)
len_of_feature_vector=len(feature_vector)
#print feature_vector[:10]

for each in result:
   
    text=each["text"]
    #print text
    text = re.sub(r'[^a-zA-Z0-9 ]',' ',text)
    text_tokens=text.split(" ")
    
    text_dist={}
    for each in text_tokens:
        if each.lower() in text_dist.keys():
            text_dist[each.lower()] =text_dist[each.lower()]+1
        else:
            text_dist[each.lower()] = 1
    vector=[0]*len_of_feature_vector
    for each in text_dist.keys():
      
        #print feature_vector.index(each)
        
        vector[feature_vector.index(each)]=text_dist[each]
    feature_array.append(vector)
    

print len(feature_array)
model=lda.LDA(n_topics=10,n_iter=500,random_state=1)
model.fit(feature_array)
topic_word=model.topic_word_
n_top_words=7
for i,topic_dist in enumerate(topic_word):
    topic_words=np.array(tuple(feature_vector))[np.argsort(topic_dist)][:-n_top_words:-1]
    print ("topic {}:{}".format(i, " ".join(topic_words)))


##titles=[each['HD'] for each in result]
##titles=tuple(titles)
##
##for i in range(5):
##	print("{} (index-{}))".format(titles[i],i))