文本分类器

新闻类别1 tab 新闻内容
新闻类别1 tab 新闻内容
新闻类别2 tab 新闻内容
。。。
新闻类别20 tab 新闻内容

# coding: utf8
import math
total_train_num = 0 # 总样本数
label_num_dict = {} # 每个类别样本数
word_label_num_dict = {} # 每个词在每个类别下样本数
label_word_count_dict = {} # 每个类别所有样本总词数
word_set = set() # 单词表，用于laplace平滑

train_data = '../data/20ng-train-no-stop.txt'
with open(train_data) as f:
    for line in f:
        line = line.strip()
        line_items = line.split('\t')
        if len(line_items) != 2 or line_items[0] == '' or line_items[1] == '':
            continue
        total_train_num += 1
        label = line_items[0]
        words = line_items[1].split()
        label_num_dict[label] = label_num_dict.get(label, 0) + 1
        for word in words:
            word_set.add(word)
            word_label_num_dict[word+'_'+label] = word_label_num_dict.get(word+'_'+label, 0) + 1
            label_word_count_dict[label] = label_word_count_dict.get(label, 0) + 1

# for laplace smoothing
for word in word_set:
    for label in label_num_dict:
        word_label_num_dict[word+'_'+label] = word_label_num_dict.get(word+'_'+label, 0) + 1
        label_word_count_dict[label] = label_word_count_dict.get(label, 0) + 1

def get_label_prob(label, words_list):
    # priority prob
    p = math.log(label_num_dict.get(label) / total_train_num)
    for word in words_list:
        if word not in word_set:
            continue
        p += math.log((word_label_num_dict.get(word+'_'+label) / label_word_count_dict[label]))
    return p

test_data = '../data/20ng-test-no-stop.txt'
test_total = 0
correct_num = 0
with open(test_data) as f:
    for line in f:
        line = line.strip()
        line_items = line.split('\t')
        if len(line_items) != 2 or line_items[0] == '' or line_items[1] == '':
            continue
        test_total += 1
        real_label = line_items[0]
        words = line_items[1].split()
        predict_list = []
        for label in label_num_dict:
            predict_list.append([label, get_label_prob(label, words)])
        predict_label = max(predict_list, key=lambda x:x[1])[0]
        if predict_label == real_label:
            correct_num += 1

print(correct_num / test_total)
# output：0.815

# coding: utf8

import tensorflow as tf
import collections
import numpy as np
import time

epoches = 10 # 训练数据有限，数据过10遍
words_length = 300 # 数据维度
n_words = 60000 # 词总数
batch_size = 30
embedding_size = 10
learning_rate = 0.01
train_data_path = '../data/20ng-train-no-stop.txt'

train_data_path = '../data/20ng-train-no-stop.txt'
raw_data_record_list = []
words_list = []
label_set = set()
with open(train_data_path) as f:
    for line in f:
        line = line.strip()
        line_items = line.split('\t')
        if len(line_items) != 2 or line_items[0] == '' or line_items[1] == '':
            continue
        label = line_items[0]
        words = line_items[1].split()
        if len(words) > words_length:
            words = words[:words_length]
        while len(words) < words_length:
            words.append('&')
        words_list.extend(words)
        label_set.add(label)
        raw_data_record_list.append([words, label])

# build dictionary
count = collections.Counter(words_list).most_common(n_words)
word_dict = {}
# every word with unique id
for word, _ in count:
    word_dict[word] = len(word_dict)

with open('./model/word_dict', 'w') as fw:
    for word, ids in word_dict.items():
        fw.write(word + '\t' + str(ids) + '\n')

label_dict = {}
for label in label_set:
    label_dict[label] = len(label_dict)

with open('./model/label_dict', 'w') as fw:
    for word, ids in label_dict.items():
        fw.write(word + '\t' + str(ids) + '\n')

data_record_list = []
for data in raw_data_record_list:
    words = [word_dict.get(k, 0) for k in data[0]]
    labels = label_dict[data[1]]
    data_record_list.append(([words, labels]))

# generate train data
def generate_batch(batch_size, data_record_list):
    for i in range(0, len(data_record_list), batch_size):
        batch = [k[0] for k in data_record_list[i:i+batch_size]]
        labels = [k[1] for k in data_record_list[i:i+batch_size]]
        yield batch, labels

# build graph
vocabulary_size = len(word_dict)
num_classes = len(label_dict)
graph = tf.Graph()
with graph.as_default():
    # input data
    with tf.name_scope('inputs'):
        train_inputs = tf.placeholder(tf.int64, shape=[None, words_length], name='inputs')
        train_labels = tf.placeholder(tf.int64, shape=[None], name='labels')
    with tf.device('/cpu:0'):
        # Look up embeddings for inputs.
        with tf.name_scope('embeddings'):
            embeddings = tf.Variable(
                tf.random_uniform([vocabulary_size, embedding_size], -1.0, 1.0))
            embed = tf.reduce_mean(tf.nn.embedding_lookup(embeddings, train_inputs), 1)
        input_layer = embed
        logits = tf.layers.dense(
            inputs=input_layer,
            units=num_classes,
            activation=None,
        )
        predictions = tf.argmax(logits, axis=-1, name='predictions')
        correct_predictions = tf.equal(predictions, train_labels)
        accuracy = tf.reduce_mean(tf.cast(correct_predictions, "float"), name="accuracy")
        mean_loss = tf.reduce_mean(
            tf.nn.sparse_softmax_cross_entropy_with_logits(
                labels=train_labels,
                logits=logits,
            ),
        )
        tf.summary.scalar('loss', mean_loss)
        tf.summary.scalar('accuracy', accuracy)
        train_step = tf.train.AdamOptimizer(learning_rate).minimize(mean_loss, global_step=tf.train.get_global_step())
        summary_op = tf.summary.merge_all()
    init = tf.global_variables_initializer()
    sess = tf.Session()
    sess.run(init)
    summary_writer = tf.summary.FileWriter('./events', sess.graph)
    num = 0
    start_time = time.time()
    for k in range(epoches):
        np.random.shuffle(data_record_list)
        batch_generator = generate_batch(batch_size, data_record_list)
        for batch, labels in batch_generator:
            num += 1
            loss, _, _summary = sess.run([mean_loss, train_step, summary_op], feed_dict={train_inputs: batch, train_labels: labels})
            summary_writer.add_summary(_summary, num)
    end_time = time.time()
    print('total time:{}'.format(end_time - start_time))
    saver = tf.train.Saver()
    saver.save(sess, './model/fast_text.model')

# coding: utf8

import tensorflow as tf
import collections
import numpy as np
import time

epoches = 20
words_length = 30
n_words = 60000
batch_size = 30
embedding_size = 100
learning_rate = 0.1

train_data_path = '../data/20ng-train-no-stop.txt'
raw_data_record_list = []
words_list = []
label_set = set()
with open(train_data_path) as f:
    for line in f:
        line = line.strip()
        line_items = line.split('\t')
        if len(line_items) != 2 or line_items[0] == '' or line_items[1] == '':
            continue
        label = line_items[0]
        words = line_items[1].split()
        if len(words) > words_length:
            words = words[:words_length]
        while len(words) < words_length:
            words.append('&')
        words_list.extend(words)
        label_set.add(label)
        raw_data_record_list.append([words, label])

# build dictionary
count = collections.Counter(words_list).most_common(n_words)
word_dict = {}
# every word with unique id
for word, _ in count:
    word_dict[word] = len(word_dict)

reverse_word_dict = dict((v,k) for k,v in word_dict.items())

with open('./model/word_dict', 'w') as fw:
    for word, ids in word_dict.items():
        fw.write(word + '\t' + str(ids) + '\n')

label_dict = {}
for label in label_set:
    label_dict[label] = len(label_dict)

reverse_label_dict = dict((v,k) for k,v in label_dict.items())

with open('./model/label_dict', 'w') as fw:
    for word, ids in label_dict.items():
        fw.write(word + '\t' + str(ids) + '\n')

data_record_list = []
for data in raw_data_record_list:
    words = [word_dict.get(k, 0) for k in data[0]]
    labels = label_dict[data[1]]
    data_record_list.append([words, labels])

# generate train data
def generate_batch(batch_size, data_record_list):
    for i in range(0, len(data_record_list), batch_size):
        batch = [k[0] for k in data_record_list[i:i+batch_size]]
        labels = [k[1] for k in data_record_list[i:i+batch_size]]
        yield batch, labels

# build graph
vocabulary_size = len(word_dict)
num_classes = len(label_dict)
graph = tf.Graph()
filter_sizes = list(range(3,6))
num_filters = 8
with graph.as_default():
    # input data
    input_x = tf.placeholder(tf.int64, [None, words_length], name="input_x")
    input_y = tf.placeholder(tf.int64, [None], name="input_y")
    dropout_keep_prob = tf.placeholder(tf.float32, name="dropout_keep_prob")
    with tf.device('/cpu:0'), tf.name_scope("embedding"):
        W = tf.Variable(
            tf.random_uniform([vocabulary_size, embedding_size], -1.0, 1.0),
            name="W")
        embedded_chars = tf.nn.embedding_lookup(W, input_x)
        embedded_chars_expanded = tf.expand_dims(embedded_chars, -1)
        def get_pool(filters, size):
            conv = tf.layers.conv2d(
                inputs=embedded_chars_expanded,
                filters=filters,
                kernel_size=[size, embedding_size],
                padding='valid',
                activation=tf.nn.relu,
                use_bias=True,
            )
            pool = tf.layers.max_pooling2d(
                inputs=conv,
                pool_size=[words_length-size+1, 1],
                strides=1,
            )
            return pool
        pool2 = get_pool(6, 2)
        pool3 = get_pool(6, 3)
        pool4 = get_pool(6, 4)
        pool5 = get_pool(5, 5)
        pool6 = get_pool(5, 6)
        pool7 = get_pool(5, 7)
        pool = tf.concat(
            values=[pool2, pool3, pool4, pool5, pool6, pool7],
            axis=3
        )
        pool = tf.reshape(pool, [-1, pool.shape[3]])
        logits = tf.layers.dense(
            inputs=pool,
            units=num_classes,
        )
        predictions = tf.argmax(logits, axis=-1, name='predictions')
        losses = tf.nn.sparse_softmax_cross_entropy_with_logits(
                labels = input_y,
                logits = logits
                )
        loss = tf.reduce_mean(losses)
        correct_predictions = tf.equal(predictions, input_y)
        accuracy = tf.reduce_mean(tf.cast(correct_predictions, "float"), name="accuracy")
        tf.summary.scalar('loss', loss)
        tf.summary.scalar('acccuray', accuracy)
        optimizer = tf.train.AdamOptimizer(1e-4)
        grads_and_vars = optimizer.compute_gradients(loss)
        train_op = optimizer.apply_gradients(grads_and_vars)
        summary_op = tf.summary.merge_all()
        init = tf.global_variables_initializer()
        sess = tf.Session()
        sess.run(init)
        summary_writer = tf.summary.FileWriter('./events', sess.graph)
        index = 0
        for k in range(epoches):
            np.random.shuffle(data_record_list)
            batch_generator = generate_batch(batch_size, data_record_list)
            for x_batch, y_batch in batch_generator:
                index += 1
                feed_dict = {
                    input_x: x_batch,
                    input_y: y_batch,
                    dropout_keep_prob: 0.9
                }
                _, _loss, _accuracy, _summary = sess.run(
                        [train_op, loss, accuracy, summary_op],
                    feed_dict)
                summary_writer.add_summary(_summary, index)