main_run.py

"""
Online LDA with a soft alignment to integrate previous states.
"""

import os
import gc
#import resource
import logging
import itertools
import time
import json
import _pickle as cPickle
#import cPickle
import re
import numpy as np
from scipy import sparse
from scipy.stats import entropy
from datetime import datetime, timedelta
import nltk
from nltk.corpus import stopwords
from gensim import corpora
from collections import defaultdict
from gensim.models import Word2Vec, LdaMulticore, TfidfModel
from extractSentenceWords import *
from onlineLDA import *
from config import Config
from extract_phrase import extract_phrases

bigram = None
trigram = None
wv_model = None
my_stoplst = [
    "app", "good", "excellent", "awesome", "please", "they", "very", "too",
    "like", "love", "nice", "yeah", "amazing", "lovely", "perfect", "much",
    "bad", "best", "yup", "suck", "super", "thank", "great", "really", "omg",
    "gud", "yes", "cool", "fine", "hello", "alright", "poor", "plz", "pls",
    "google", "facebook", "three", "ones", "one", "two", "five", "four", "old",
    "new", "asap", "version", "times", "update", "star", "first", "rid", "bit",
    "annoying", "beautiful", "dear", "master", "evernote", "per", "line", "oh",
    "ah", "cannot", "doesnt", "won't", "dont", "unless", "you're", "aren't",
    "i'd", "can't", "wouldn't", "around", "i've", "i'll", "gonna", "ago",
    "you'll", "you'd", "28th", "gen", "it'll", "vice", "would've", "wasn't",
    "year", "boy", "they'd", "isnt", "1st", "i'm", "nobody", "youtube",
    "isn't", "don't", "2016", "2017", "since", "near", "god"
]

# dataset
#app_files = Config.get_datasets()
app_files_pre = {}
validate_files = Config.get_validate_files()
candidate_num = Config.get_candidate_num()
topic_num = Config.get_topic_num()
win_size = Config.get_window_size()
bigram_min = Config.get_bigram_min()
trigram_min = Config.get_trigram_min()
info_num = Config.get_info_num()
store_num = Config.get_store_num()
val_index = Config.get_validate_or_not()


def extract_review(app_files):
    """
    Extract reviews with time and version stamp
    :return:
    """
    timed_reviews = {}
    num_docs = 0
    num_words = 0
    #for apk, app in app_files:
    apk = "youtube"
    app = app_files

    timed_reviews[apk] = []
    with open(app) as fin:
        lines = fin.readlines()
    for l_id, line in enumerate(lines):
        line = line.strip()
        terms = line.split("******")
        if len(terms) != info_num:
            logging.error("review format error at %s in %s" % (apk, line))
            continue
        if not store_num:  ## for ios
            date = terms[3]
            version = terms[4]
        else:  ## for android
            date = terms[2]
            version = terms[3]
        review_o = terms[1]
        review_p, wc = extractSentenceWords(review_o)
        review = list(build_phrase(review_p))
        review = [list(replace_digit(s)) for s in review]
        rate = float(terms[0]) if re.match(
            r'\d*\.?\d+', terms[0]) else 2.0  # 2.0 is the average rate

        timed_reviews[apk].append({
            "review": review,
            "date": date,
            "rate": rate,
            "version": version
        })
        num_docs += 1
        num_words += wc
        if l_id % 1000 == 0:
            logging.info("processed %d docs of %s" % (l_id, apk))
    logging.info("total read %d reviews, %d words." % (num_docs, num_words))
    return timed_reviews


# def extract_review():
#     """
#     Extract reviews with time and version stamp
#     :return:
#     """
#     timed_reviews = {}
#     num_docs = 0
#     num_words = 0
#     for apk, app in app_files:
#         timed_reviews[apk] = []
#         with open(app) as fin:
#             lines = fin.readlines()
#         for l_id, line in enumerate(lines):
#             line = line.strip()
#             terms = line.split("******")
#             if len(terms) != info_num:
#                 logging.error("review format error at %s in %s" % (apk, line))
#                 continue
#             if not store_num: ## for ios
#                 date = terms[3]
#                 version = terms[4]
#             else:             ## for android
#                 date = terms[2]
#                 version = terms[3]
#             review_o = terms[1]
#             review_p, wc = extractSentenceWords(review_o)
#             review = list(build_phrase(review_p))
#             review = [list(replace_digit(s)) for s in review]
#             rate = float(terms[0]) if re.match(r'\d*\.?\d+', terms[0]) else 2.0  # 2.0 is the average rate
#
#             timed_reviews[apk].append({"review": review, "date": date, "rate": rate, "version": version})
#             num_docs += 1
#             num_words += wc
#             if l_id % 1000 == 0:
#                 logging.info("processed %d docs of %s" % (l_id, apk))
#     logging.info("total read %d reviews, %d words."%(num_docs, num_words))
#     return timed_reviews


def replace_digit(sent):
    for w in sent:
        if w.isdigit():
            yield '<digit>'
        else:
            yield w


def build_phrase(doc):
    # load phrase model
    return trigram[bigram[doc]]


def update_phrase():
    """
    Update bigram and trigram model
    :return:
    """
    apk = "youtube"
    app = app_files

    with open(app) as fin:
        lines = fin.readlines()
    for line in lines:
        line = line.strip()
        terms = line.split("******")
        if len(terms) != info_num:
            logging.error("review format error at %s in %s" % (apk, line))
            continue
        review_o = terms[1]
        review_p, wc = extractSentenceWords(review_o)
        bigram.add_vocab(review_p)
        trigram.add_vocab(bigram[review_p])
    # update
    bigram.save("model/bigram.model")
    trigram.save("model/trigram.model")


# def update_phrase():
#     """
#     Update bigram and trigram model
#     :return:
#     """
#     for apk, app in app_files:
#         with open(app) as fin:
#             lines = fin.readlines()
#         for line in lines:
#             line = line.strip()
#             terms = line.split("******")
#             if len(terms) != info_num:
#                 logging.error("review format error at %s in %s" % (apk, line))
#                 continue
#             review_o = terms[1]
#             review_p, wc = extractSentenceWords(review_o)
#             bigram.add_vocab(review_p)
#             trigram.add_vocab(bigram[review_p])
#         # update
#         bigram.save("model/bigram.model")
#         trigram.save("model/trigram.model")


def load_phrase():
    global bigram
    global trigram
    bigram = Phrases.load(os.path.join("model", "bigram.model"))
    trigram = Phrases.load(os.path.join("model", "trigram.model"))


def save_obj(filename, rst):
    with open(filename, 'w') as fout:
        cPickle.dump(rst, fout)


def load_obj(filename):
    with open(filename) as fin:
        return cPickle.load(fin)


def build_AOLDA_input_version(timed_reviews=None):
    """
    build version-aligned input for AOLDA
    :param timed_reviews:
    :return:
    """
    if timed_reviews is None:
        with open("result/timed_reviews") as fin:
            timed_reviews = json.load(fin)
    stoplist = stopwords.words('english') + my_stoplst

    OLDA_input = {}
    for apk, reviews in timed_reviews.items():
        # build a dictionary to store the version and review
        version_dict = {}
        input = []
        rate = []
        tag = []
        for review in reviews:
            review_ver = review['version']
            if review_ver == "Unknown":
                continue
            if review_ver not in version_dict:
                version_dict[review_ver] = ([], [])
            version_dict[review_ver][0].append(review['review'])
            version_dict[review_ver][1].append(review['rate'])

        # re-arrange the version sequence
        #for ver in sorted(version_dict.iterkeys(), key=lambda s: map(int, s.split('.'))):
        version_dict_keys = []
        version_dict_keys = list(version_dict.keys())
        version_dict_keys.sort(key=lambda s: list(map(int, s.split('.'))))

        for ver in version_dict_keys:
            if len(version_dict[ver]
                   [0]) > 50:  # skip versions with not enough reviews
                tag.append(ver)
                input.append(version_dict[ver][0])
                rate.append(version_dict[ver][1])

        dict_input = list(
            itertools.chain.from_iterable(
                list(itertools.chain.from_iterable(input))))
        dictionary = corpora.Dictionary(dict_input)
        dictionary.filter_tokens(map(dictionary.token2id.get, stoplist))
        dictionary.compactify()
        dictionary.filter_extremes(no_below=2, keep_n=None)
        dictionary.compactify()

        # for each interval, build bow
        input_X = []
        for t_i, text_period in enumerate(input):
            # construct sparse matrix
            text_period = list(itertools.chain.from_iterable(
                text_period))  # sentence level to doc level
            row = []
            col = []
            value = []
            r_id = 0
            for k, text in enumerate(text_period):
                empty = True
                for i, j in dictionary.doc2bow(text):
                    row.append(r_id)
                    col.append(i)
                    value.append(j)
                    empty = False
                if not empty:
                    r_id += 1
            input_X.append(
                sparse.coo_matrix((value, (row, col)),
                                  shape=(r_id, len(dictionary)),
                                  dtype=np.integer))
        OLDA_input[apk] = (dictionary, input_X, input, rate, tag
                           )  # input: raw input, with time and sent
    return OLDA_input


def generate_labeling_candidates(OLDA_input):
    """
    Filter phrase labels and choose for candidates
    :param OLDA_input:
    :return:
    """
    phrases = {}
    for apk, item in OLDA_input.items():
        dic, _, _1, _2, _3 = item
        phrases[apk] = defaultdict(int)
        # filter bigram and trigram
        for word in dic.values():
            if '_' in word:
                phrase = word
                #words, tags = zip(*nltk.pos_tag(phrase.split(b'_')))
                words, tags = zip(*nltk.pos_tag(phrase.split('_')))
                match = False
                for tag in tags:
                    if re.match(r"^NN", tag):
                        match = True
                        continue
                    if re.match(r"DT", tag):
                        match = False
                        break
                    if re.match(r"RB", tag):
                        match = False
                        break
                for word in words:
                    if word in stopwords.words(
                            'english') + my_stoplst:  # remove stop word
                        match = False
                        break
                    if len(word) < 3:
                        match = False
                        break
                    if "\\'" in word:
                        match = False
                        break
                if match:
                    # keep phrase
                    phrases[apk][phrase] = 1
    return phrases


def OLDA_fit(OLDA_input, n_topics, win_size):
    phis = {}
    theta = {}
    for apk, item in OLDA_input.items():
        dictionary, input_X, _, _1, _2 = item
        olda_model = OLDA(n_topics=n_topics,
                          n_iter=1000,
                          refresh=500,
                          window_size=win_size)
        olda_model.fit(input_X)
        phis[apk] = olda_model.B
        theta[apk] = olda_model.A
        fout = open("result/topic_words_%s_%s_%s" % (apk, n_topics, win_size),
                    'w')
        for t_i, phi in enumerate(phis[apk]):
            fout.write("time slice %s\n" % t_i)
            for i, topic_dist in enumerate(phi):
                topic_words = [
                    dictionary[w_id]
                    for w_id in np.argsort(topic_dist)[:-10:-1]
                ]
                fout.write('Topic {}: {}\n'.format(i, ' '.join(topic_words)))
            fout.write('\n')
        fout.close()
    return phis


def count_occurence(dic, rawinput, label_ids):
    count = []
    for d_i, rawinput_i in enumerate(rawinput):
        count_i = defaultdict(int)
        for input in list(itertools.chain.from_iterable(rawinput_i)):
            bow = dic.doc2bow(input)
            for id, value in bow:
                count_i[id] += value
                if id in label_ids[d_i]:
                    for idx, valuex in bow:
                        count_i[id, idx] += min(value,
                                                valuex)  # label always first
        count.append(count_i)
    return count


def total_count_(dic, rawinput):
    total_count = []
    for rawinput_i in rawinput:
        total_count_i = 0
        for input in list(itertools.chain.from_iterable(rawinput_i)):
            bow = dic.doc2bow(input)
            for id, value in bow:
                total_count_i += value
        total_count.append(total_count_i)
    return total_count


def get_candidate_label_ids(dic, labels, rawinput):
    all_label_ids = list(map(dic.token2id.get, labels))
    label_ids = []

    for rawinput_i in rawinput:
        count = defaultdict(int)

        for input in list(itertools.chain.from_iterable(rawinput_i)):
            bow = dic.doc2bow(input)
            for id, value in bow:
                if id in all_label_ids:
                    count[id] += value

        label_ids.append(list(count.keys()))
    return label_ids


def get_candidate_sentences_ids(rawinput, rates):
    sent_ids = []
    sent_rates = []
    index = 0
    for t_i, rawinput_i in enumerate(rawinput):
        sent_id = []
        sent_rate = []
        for i_d, input_d in enumerate(rawinput_i):
            for i_s, input_s in enumerate(input_d):
                if len(input_s) < 5:  # length should be bigger than 5
                    continue
                sent_id.append(index + i_s)
                sent_rate.append(rates[t_i][i_d])
            index += len(input_d)
        sent_ids.append(sent_id)
        sent_rates.append(sent_rate)
    return sent_ids, sent_rates


def get_sensitivities(dic, rawinput, rates, label_ids):
    sensi = []
    for t_i, rawinput_i in enumerate(rawinput):
        sensi_t = []
        label_sensi = [[] for _ in label_ids[t_i]]
        for d_i, input in enumerate(rawinput_i):
            doc_input = list(itertools.chain.from_iterable(input))
            bow = dic.doc2bow(doc_input)
            for id, value in bow:
                if id in label_ids[t_i]:
                    label_sensi[label_ids[t_i].index(id)].append(
                        [rates[t_i][d_i],
                         len(doc_input)])  # record the rate and length
        for rl in label_sensi:
            rl = np.array(rl)
            m_rl = np.mean(rl, 0)
            sensi_t.append(np.exp(-m_rl[0] / np.log(1 + m_rl[1])))
        sensi.append(np.array(sensi_t))
    return sensi


def get_sensitivities_sent(rawinput_sent, sent_rates, sent_ids):
    sensi = []
    for t_i, sent_id in enumerate(sent_ids):
        sensi_i = []
        for id, s_id in enumerate(sent_id):
            r = sent_rates[t_i][id]
            l = len(rawinput_sent[s_id])
            sensi_i.append(np.exp(-r / float(np.log(l))))
        sensi.append(np.array(sensi_i))
    return sensi


def JSD(P, Q):
    """
    Jensen-Shannon divergence
    :param P:
    :param Q:
    :return:
    """
    _M = 0.5 * (P + Q)
    return 0.5 * (entropy(P, _M) + entropy(Q, _M))


def sim_topic_word(phi, label_id, count):
    # sim = 0
    c_l = np.array([
        np.log((count[label_id, w_id] + 1) / float(
            (count[w_id] + 1) * (count[label_id] + 1)))
        for w_id in range(len(phi))
    ])
    return np.dot(phi, c_l)


def topic_labeling(OLDA_input, apk_phis, phrases, mu, lam, theta, save=True):
    """
    Topic labeling for phrase and sentence
    :param OLDA_input:
    :param apk_phis:
    :param phrases:
    :param mu:
    :param lam:
    :param theta:
    :param save:
    :return:
    """
    logging.info("labeling topics(mu: %f, lam: %f, theta: %f)......" %
                 (mu, lam, theta))
    apk_jsds = {}
    for apk, item in OLDA_input.items():
        dictionary, _, rawinput, rates, tag = item
        phis = apk_phis[apk]
        labels = phrases[apk].keys()
        # label_ids = map(dictionary.token2id.get, labels)
        label_ids = get_candidate_label_ids(dictionary, labels, rawinput)
        count = count_occurence(dictionary, rawinput, label_ids)
        total_count = total_count_(dictionary, rawinput)
        sensi_label = get_sensitivities(dictionary, rawinput, rates, label_ids)
        rawinput_sent = list(
            itertools.chain.from_iterable(
                list(itertools.chain.from_iterable(rawinput))))
        sent_ids, sent_rates = get_candidate_sentences_ids(rawinput, rates)
        sensi_sent = get_sensitivities_sent(rawinput_sent, sent_rates,
                                            sent_ids)
        jsds = []
        label_phrases = []
        label_sents = []
        emerge_phrases = []
        emerge_sents = []
        if save:
            result_path = "result/%s" % apk
            if not os.path.exists(result_path):
                os.makedirs(result_path)
            fout_labels = open(os.path.join(result_path, "topic_labels"), 'w')
            fout_emerging = open(
                os.path.join(result_path, "emerging_topic_labels"), 'w')
            fout_sents = open(os.path.join(result_path, "topic_sents"), "w")
            fout_emerging_sent = open(
                os.path.join(result_path, "emerging_topic_sents"), 'w')
            fout_topic_width = open(os.path.join(result_path, "topic_width"),
                                    'w')

        for t_i, phi in enumerate(phis):
            # label topic
            logging.info("labeling topic at %s slice of %s" % (t_i, apk))
            topic_label_scores = topic_labeling_(count[t_i], total_count[t_i],
                                                 label_ids[t_i],
                                                 sensi_label[t_i], phi, mu,
                                                 lam)
            topic_label_sent_score = topic_label_sent(dictionary, phi,
                                                      rawinput_sent,
                                                      sent_ids[t_i],
                                                      sensi_sent[t_i], mu, lam)

            # write to file: topic phrase
            if save:
                fout_labels.write("time slice %s, tag: %s\n" % (t_i, tag[t_i]))
                for tp_i, label_scores in enumerate(topic_label_scores):
                    fout_labels.write("Topic %d:" % tp_i)
                    for w_id in np.argsort(label_scores)[:-candidate_num -
                                                         1:-1]:
                        fout_labels.write("%s\t%f\t" %
                                          (dictionary[label_ids[t_i][w_id]],
                                           label_scores[w_id]))
                    fout_labels.write('\n')

                fout_sents.write("time slice %s, tag: %s\n" % (t_i, tag[t_i]))
                for tp_i, sent_scores in enumerate(topic_label_sent_score):
                    fout_sents.write("Topic %d:" % tp_i)
                    for s_id in np.argsort(sent_scores)[:-candidate_num -
                                                        1:-1]:
                        fout_sents.write(
                            "%s\t%f\t" %
                            (" ".join(rawinput_sent[sent_ids[t_i][s_id]]),
                             sent_scores[s_id]))
                    fout_sents.write('\n')

            # store for verification
            label_phrases_ver = []
            label_sents_ver = []
            for tp_i, label_scores in enumerate(topic_label_scores):
                label_phrases_ver.append([
                    dictionary[label_ids[t_i][w_id]]
                    for w_id in np.argsort(label_scores)[:-candidate_num -
                                                         1:-1]
                ])
            label_phrases.append(
                list(itertools.chain.from_iterable(label_phrases_ver)))
            for tp_i, sent_scores in enumerate(topic_label_sent_score):
                label_sents_ver.append([
                    rawinput_sent[sent_ids[t_i][s_id]]
                    for s_id in np.argsort(sent_scores)[:-candidate_num - 1:-1]
                ])
            label_sents.append(
                list(itertools.chain.from_iterable(label_sents_ver)))

            # detect emerging topic
            logging.info("detecting topic at %s slice of %s" % (t_i, apk))
            if save and t_i == 0:
                topic_width = count_width(dictionary, label_phrases_ver,
                                          count[t_i], sensi_label[t_i],
                                          label_ids[t_i])
                for theta in topic_width:
                    fout_topic_width.write("%f\t" % theta)
                fout_topic_width.write("\n")
                continue  # skip the first epoch
            emerging_label_scores, emerging_sent_scores = topic_detect(
                rawinput_sent, dictionary, phi, phis[t_i - 1], count[t_i],
                count[t_i - 1], total_count[t_i], total_count[t_i - 1],
                label_ids[t_i], sent_ids[t_i], sensi_label[t_i],
                sensi_sent[t_i], jsds, theta, mu, lam)
            # write to file
            if save:
                fout_emerging.write("time slice %s, tag: %s\n" %
                                    (t_i, tag[t_i]))
                for tp_i, label_scores in enumerate(emerging_label_scores):
                    fout_emerging.write("Topic %d: " % tp_i)
                    if np.sum(label_scores) == 0:
                        fout_emerging.write('None\n')
                    else:
                        for w_id in np.argsort(label_scores)[:-4:-1]:
                            fout_emerging.write(
                                "%s\t%f\t" % (dictionary[label_ids[t_i][w_id]],
                                              label_scores[w_id]))
                        fout_emerging.write('\n')
                fout_emerging_sent.write("time slice %s, tag: %s\n" %
                                         (t_i, tag[t_i]))
                for tp_i, sent_scores in enumerate(emerging_sent_scores):
                    fout_emerging_sent.write("Topic %d: " % tp_i)
                    if np.sum(sent_scores) == 0:
                        fout_emerging_sent.write('None\n')
                    else:
                        for s_id in np.argsort(sent_scores)[:-4:-1]:
                            fout_emerging_sent.write(
                                "%s\t%f\t" %
                                (" ".join(rawinput_sent[sent_ids[t_i][s_id]]),
                                 sent_scores[s_id]))
                        fout_emerging_sent.write('\n')
            # store for verification
            emerge_phrases_ver = []
            emerge_sents_ver = []
            emerge_phrases_width_ver = []
            for tp_i, label_scores in enumerate(emerging_label_scores):
                if np.sum(label_scores) == 0:
                    emerge_phrases_width_ver.append([])
                    continue
                emerge_phrases_ver.append([
                    dictionary[label_ids[t_i][w_id]]
                    for w_id in np.argsort(label_scores)[:-4:-1]
                ])
                emerge_phrases_width_ver.append([
                    dictionary[label_ids[t_i][w_id]]
                    for w_id in np.argsort(label_scores)[:-4:-1]
                ])
            emerge_phrases.append(emerge_phrases_ver)
            # merge emerge to label
            label_emerge_ver = [
                set(l) | set(e)
                for l, e in zip(label_phrases_ver, emerge_phrases_width_ver)
            ]
            topic_width = count_width(dictionary, label_emerge_ver, count[t_i],
                                      sensi_label[t_i], label_ids[t_i])
            for tp_i, sent_scores in enumerate(emerging_sent_scores):
                if np.sum(sent_scores) == 0:
                    continue
                emerge_sents_ver.append([
                    rawinput_sent[sent_ids[t_i][s_id]]
                    for s_id in np.argsort(sent_scores)[:-4:-1]
                ])
            emerge_sents.append(emerge_sents_ver)
            # write topic width
            if save:
                for theta in topic_width:
                    fout_topic_width.write("%f\t" % theta)
                fout_topic_width.write("\n")

        ############################################
        if val_index:
            validation(validate_files[apk], label_phrases, label_sents,
                       emerge_phrases, emerge_sents)
        ############################################

        if save:
            fout_labels.close()
            fout_sents.close()
            fout_emerging.close()
            fout_emerging_sent.close()
            fout_topic_width.close()
        apk_jsds[apk] = jsds
    return apk_jsds


def topic_labeling_(count, total_count, label_ids, sensi, phi, mu, lam):
    topic_label_scores = rank_topic_label(count, total_count, phi, label_ids,
                                          mu)
    topic_label_scores += lam * sensi
    return topic_label_scores


# rank the label according to similarity of the topic dist and divergence of other topic dist
def rank_topic_label(count, total_count, phi, label_ids, mu=0.2):
    # matrix implementation for speed-up
    # construct topic matrix
    mu_div = mu / (len(phi) - 1)
    c_phi = phi * (1 + mu_div) - np.sum(phi, 0) * mu_div
    # construct label count matrix
    c_label_m = np.empty((len(label_ids), len(phi[0])), dtype=float)
    for ind, label_id in enumerate(label_ids):
        for w_id in range(len(phi[0])):
            c_label_m[ind, w_id] = count.get(
                (label_id, w_id)) * total_count / float(
                    (count.get(w_id) + 1) *
                    (count.get(label_id) + 1)) if (label_id,
                                                   w_id) in count else 1.0
    c_label_m = np.log(c_label_m)
    # compute score matrix
    topic_label_scores = np.dot(c_phi, np.transpose(c_label_m))
    return topic_label_scores


def topic_detect(rawinput_sents, dic, phi, last_phi, count, last_count,
                 total_count, last_total_count, label_ids, sent_ids,
                 sensi_label, sensi_sent, jsds, theta, mu, lam):
    # matrix implementation for speed-up
    # construct count label matrix
    c_label_m = np.empty((len(label_ids), len(phi[0])), dtype=float)
    c_last_label_m = np.empty((len(label_ids), len(phi[0])), dtype=float)
    for ind, label_id in enumerate(label_ids):
        for w_id in range(len(phi[0])):
            c_label_m[ind, w_id] = count.get(
                (label_id, w_id)) * total_count / float(
                    (count.get(w_id) + 1) *
                    (count.get(w_id) + 1)) if (label_id,
                                               w_id) in count else 1.0
            c_last_label_m[ind, w_id] = last_count.get(
                (label_id, w_id)) * last_total_count / float(
                    (last_count.get(w_id) + 1) *
                    (last_count.get(label_id) + 1)) if (
                        label_id, w_id) in last_count else 1.0

    c_label_m = np.log(c_label_m)
    c_last_label_m = np.log(c_last_label_m)
    # construct sentence count matrix
    sent_count = np.empty((len(sent_ids), len(phi[0])), dtype=float)
    for ind, s_id in enumerate(sent_ids):
        bow = dic.doc2bow(rawinput_sents[s_id])
        len_s = len(rawinput_sents[s_id])
        for w_id in range(len(phi[0])):
            sent_count[ind, w_id] = 0.00001
        for k, v in bow:
            sent_count[ind, k] = v / float(len_s)
    # # construct residuals
    # phi_logphi = np.log(phi) * phi
    # phi_logphi_last = np.log(last_phi) * last_phi

    # read topic distribution \phi
    emerging_label_scores_rst = np.zeros((len(phi), len(label_ids)))
    emerging_sent_scores_rst = np.zeros((len(phi), len(sent_ids)))
    js_d = []
    for t_i, phi_i in enumerate(phi):
        # labeling
        js_divergence = JSD(phi_i, last_phi[t_i])
        js_d.append(js_divergence)
        jsds.append(js_divergence)
        # logging.info("JSD for phi is %f"%js_divergence)
    # compute mean and variance of jsds
    js_mean = np.mean(jsds[:-3 * len(phi) - 1:-1])
    js_std = np.std(jsds[:-3 * len(phi) - 1:-1])
    # logging.info("JSD threshold is %f"%(js_mean+1.25*js_std))
    emerging_index = np.array(js_d) > js_mean + 1.25 * js_std
    # TOPIC DETECT
    phi_e = phi[emerging_index]
    phi_last_e = last_phi[emerging_index]
    E = float(np.sum(emerging_index))
    if E == 0:
        return emerging_label_scores_rst, emerging_sent_scores_rst
    # TOPIC DETECT: construct phi - last_phi
    phi_m = (1 + mu / E) * phi_e - theta * last_phi[
        emerging_index] - mu / E * np.sum(phi_e, 0)
    # TOPIC DETECT: construct residuals
    residuals_m = (1 + mu / E) * np.log(phi_e) * phi_e - theta * np.log(
        phi_last_e) * phi_last_e - mu / E * np.sum(np.log(phi_e) * phi_e, 0)
    # TOPIC DETECT: compute labels
    emerging_label_scores = np.dot(
        (1 + mu / E) * phi_e - mu / E * np.sum(phi_e, 0),
        np.transpose(c_label_m)) - theta * np.dot(
            last_phi[emerging_index],
            np.transpose(c_last_label_m)) + lam * sensi_label
    emerging_sent_scores = np.dot(phi_m, np.transpose(
        np.log(sent_count))) - np.sum(residuals_m, 1,
                                      keepdims=True) + lam * sensi_sent

    emerging_label_scores_rst[emerging_index] = emerging_label_scores
    emerging_sent_scores_rst[emerging_index] = emerging_sent_scores

    return emerging_label_scores_rst, emerging_sent_scores_rst


# rank sentence representation for topic
def topic_label_sent(dic, phi, rawinput_sents, sent_ids, sensi, mu, lam):
    # construct topic matrix
    mu_div = mu / (len(phi) - 1)
    c_phi = phi * (1 + mu_div) - np.sum(phi, 0) * mu_div
    # construct residual
    phi_logphi = phi * np.log(phi)
    residual_1 = mu_div * np.sum(phi_logphi)  # residual_1 is a value
    residual_2 = (1 + mu_div) * np.sum(
        phi_logphi, 1, keepdims=True)  # residual_2 is a n_topic*1
    # construct sentence count matrix
    sent_count = np.empty((len(sent_ids), len(phi[0])), dtype=float)
    for ind, s_id in enumerate(sent_ids):
        bow = dic.doc2bow(rawinput_sents[s_id])
        len_s = len(rawinput_sents[s_id])
        for w_id in range(len(phi[0])):
            sent_count[ind, w_id] = 0.00001
        for k, v in bow:
            sent_count[ind, k] = v / float(len_s)

    phi_sent = np.dot(c_phi, np.transpose(
        np.log(sent_count))) + residual_1 - residual_2 + lam * sensi
    return phi_sent


def count_width(dictionary, label_phrases_ver, counts, sensi_labels,
                label_ids):
    count_width_rst = []
    for phrases in label_phrases_ver:
        t_count = 0
        for phrase in phrases:
            pid = dictionary.token2id.get(phrase)
            t_count += np.log(counts.get(pid) +
                              1) * sensi_labels[label_ids.index(pid)]
        count_width_rst.append(t_count)
    return np.array(count_width_rst)


def validation(logfile, label_phrases, label_sents, emerge_phrases,
               emerge_sents):
    # read changelog
    clog = []
    with open(logfile) as fin:
        for line in fin.readlines():
            line = line.strip()
            issue_kw = map(lambda s: s.strip().split(), line.split(","))
            clog.append(issue_kw)
    # check alignment
    if len(clog) != len(label_phrases):
        logging.error("length not corrected: %d, %d" %
                      (len(clog), len(label_phrases)))
        exit(0)
    # compare topic label using keyword
    # load word2vec model
    wv_model = Word2Vec.load(os.path.join("model", "wv", "word2vec_app.model"))
    label_phrase_precisions = []
    label_phrase_recalls = []
    label_sent_precisions = []
    label_sent_recalls = []
    em_phrase_precisions = []
    em_phrase_recalls = []
    em_sent_precisions = []
    em_sent_recalls = []
    # two list: [['keyword1', 'keyword2', ...], ['keyword1', 'keyword2', ...]]
    #           [['label1', 'label2', ...], ['label1', 'label2', ...]]
    for id, ver in enumerate(clog):
        if ver == [[]]:  # skip the empty version changelog
            continue
        label_phrase_match_set = set()
        label_phrase_issue_match_set = set()
        label_sent_match_set = set()
        label_sent_issue_match_set = set()
        em_phrase_match_set = set()
        em_phrase_issue_match_set = set()
        em_sent_match_set = set()
        em_sent_issue_match_set = set()

        if id != len(clog) - 1 and clog[id + 1] != [
            []
        ]:  # merge changelog with next version
            m_ver = ver + clog[id + 1]
        else:
            m_ver = ver
        # phrase
        for issue in m_ver:
            for kw in issue:
                kw_match = False
                for w in label_phrases[id]:
                    label_match = False
                    for w_s in w.split("_"):
                        if sim_w(kw, w_s, wv_model) > 0.6:
                            # hit
                            #logging.info("hit: %s -> %s"%(w, kw))
                            label_match = True
                            kw_match = True
                            break
                    if label_match:  # if label match found, add label to match set
                        label_phrase_match_set.add(w)
                if kw_match:  # if kw match found, add issue to match set
                    label_phrase_issue_match_set.add("_".join(issue))

        # sentence
        for issue in m_ver:
            for kw in issue:
                kw_match = False
                for sent in label_sents[id]:
                    for w in sent:
                        label_match = False
                        for w_s in w.split("_"):
                            if sim_w(kw, w_s, wv_model) > 0.6:
                                # hit
                                #logging.info("hit: %s -> %s"%(w, kw))
                                label_match = True
                                kw_match = True
                                break
                        if label_match:
                            label_sent_match_set.add(
                                "_".join(sent)
                            )  # if label match found, skip to next sentence
                            break
                if kw_match:
                    label_sent_issue_match_set.add("_".join(issue))

        # check emerging issue label
        # merge current version and next version
        # if id != len(clog) - 1:
        #     m_ver = ver + clog[id+1]
        # else:
        #     m_ver = ver
        if id != 0:  # skip the first epoch
            for issue in m_ver:
                for kw in issue:
                    kw_match = False
                    for tws in emerge_phrases[id - 1]:
                        for w in tws:
                            label_match = False
                            for w_s in w.split("_"):
                                if sim_w(kw, w_s, wv_model) > 0.6:
                                    # hit
                                    #logging.info("hit: %s -> %s" % (w, kw))
                                    label_match = True
                                    kw_match = True
                                    break
                            if label_match:
                                em_phrase_match_set.add("_".join(tws))
                                break
                    if kw_match:
                        em_phrase_issue_match_set.add("_".join(issue))

            # sentence
            for issue in m_ver:
                for kw in issue:
                    kw_match = False
                    for tsents in emerge_sents[id - 1]:
                        sent = list(itertools.chain.from_iterable(tsents))
                        label_match = False
                        for w in sent:
                            for w_s in w.split("_"):
                                if sim_w(kw, w_s, wv_model) > 0.6:
                                    # hit
                                    #logging.info("hit: %s -> %s" % (w, kw))
                                    label_match = True
                                    kw_match = True
                                    break
                            if label_match:
                                em_sent_match_set.add(
                                    "_".join(sent)
                                )  # if label match found, skip to next sentence
                                break
                    if kw_match:
                        em_sent_issue_match_set.add("_".join(issue))

        # compute
        label_phrase_precision = len(label_phrase_match_set) / float(
            len(label_phrases[id]))
        label_phrase_recall = len(label_phrase_issue_match_set) / float(
            len(m_ver))
        label_sent_precision = len(label_sent_match_set) / float(
            len(label_sents[id]))
        label_sent_recall = len(label_sent_issue_match_set) / float(len(m_ver))
        label_phrase_precisions.append(label_phrase_precision)
        label_phrase_recalls.append(label_phrase_recall)
        label_sent_precisions.append(label_sent_precision)
        label_sent_recalls.append(label_sent_recall)

        if id != 0:
            if len(emerge_phrases[id - 1]) != 0:
                em_phrase_precision = len(em_phrase_match_set) / float(
                    len(emerge_phrases[id - 1]))
                em_phrase_precisions.append(em_phrase_precision)
            em_phrase_recall = len(em_phrase_issue_match_set) / float(len(ver))
            if len(emerge_sents[id - 1]) != 0:
                em_sent_precision = len(em_sent_match_set) / float(
                    len(emerge_sents[id - 1]))
                em_sent_precisions.append(em_sent_precision)
            em_sent_recall = len(em_sent_issue_match_set) / float(len(ver))
            em_phrase_recalls.append(em_phrase_recall)
            em_sent_recalls.append(em_sent_recall)
    label_phrase_fscore = 2 * np.mean(label_phrase_recalls) * np.mean(
        em_phrase_precisions) / (np.mean(label_phrase_recalls) +
                                 np.mean(em_phrase_precisions))
    label_sent_fscore = 2 * np.mean(label_sent_recalls) * np.mean(
        em_sent_precisions) / (np.mean(label_sent_recalls) +
                               np.mean(em_sent_precisions))
    logging.info(
        "Phrase label precision: %s\trecall: %f" %
        (np.mean(label_phrase_precisions), np.mean(label_phrase_recalls)))
    logging.info("Sentence label precision: %s\trecall: %f" %
                 (np.mean(label_sent_precisions), np.mean(label_sent_recalls)))
    logging.info("Emerging phrase precision: %s\trecall: %f" %
                 (np.mean(em_phrase_precisions), np.mean(em_phrase_recalls)))
    logging.info("Emerging sentence precision: %s\trecall: %f" %
                 (np.mean(em_sent_precisions), np.mean(em_sent_recalls)))
    logging.info("Phrase F1 score: %f" % label_phrase_fscore)
    logging.info("Sentence F1 score: %f" % label_sent_fscore)
    with open("result/statistics.txt", "a") as fout:
        fout.write("%s\t%f\t%f\t%f\t%f\t%f\t%f\n" %
                   (logfile, np.mean(label_phrase_recalls),
                    np.mean(label_sent_recalls), np.mean(em_phrase_precisions),
                    np.mean(em_sent_precisions), label_phrase_fscore,
                    label_sent_fscore))


def sim_w(w1, w2, wv_model):
    if w1 not in wv_model or w2 not in wv_model:
        return 0.0
    return wv_model.similarity(w1, w2)


#def meminfo(str):
#   print(str, resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1000)


def save_phrase(review_path, bigram_num, trigram):
    extract_phrases(app_files)


# def save_phrase(review_path, bigram_num, trigram):
#     for apk, app in app_files:
#         extract_phrases(app)


#if __name__ == '__main__':
def main_run(topicnum, winsize, bigrammin, trigrammin, fn):
    global topic_num, win_size, bigram_min, trigram_min, info_num, store_num, candidate_num, app_files
    topic_num = topicnum
    win_size = winsize
    bigram_min = bigrammin
    trigram_min = trigrammin
    print(topic_num, win_size, bigram_min, trigram_min, info_num, store_num,
          candidate_num)

    app_files = "file_upload/" + fn
    extract_phrases(app_files, bigram_min, trigram_min)
    load_phrase()

    timed_reviews = extract_review(app_files)
    #print(timed_reviews)

    OLDA_input = build_AOLDA_input_version(timed_reviews)
    start_t = time.time()
    apk_phis = OLDA_fit(OLDA_input, topic_num, win_size)
    phrases = generate_labeling_candidates(OLDA_input)
    topic_labeling(OLDA_input, apk_phis, phrases, 1.0, 0.75, 0.0, save=True)
    print("Totally takes %.2f seconds" % (time.time() - start_t))