Summarizing text – Orbifold Consulting

    import numpy as np
    import tensorflow as tf


    # Data can be downloaded at https://github.com/harvardnlp/sent-summary


    article_filename = 'Data/sumdata/train/train.article.txt'
    title_filename = 'Data/sumdata/train/train.title.txt'

    with open(article_filename) as article_file:
        articles = article_file.readlines()
    with open(title_filename) as title_file:
        titles = title_file.readlines()


    def create_lookup_tables(text):
        vocab = set(text.split())
        vocab_to_int = {'&lt;S&gt;': 0, '&lt;E&gt;': 1, '&lt;UNK&gt;': 2, '&lt;PAD&gt;': 3 }

        for i, v in enumerate(vocab, len(vocab_to_int)):
            vocab_to_int[v] = i

        int_to_vocab = {i: v for v, i in vocab_to_int.items()}

        return vocab_to_int, int_to_vocab

    vocab_to_int_article, int_to_vocab_article = create_lookup_tables([x.lower() for x in articles])
    vocab_to_int_title, int_to_vocab_title = create_lookup_tables([x.lower() for x in titles])


    def text_to_ids(source_text, target_text, source_vocab_to_int, target_vocab_to_int):
        source_id_text = [[source_vocab_to_int[word] for word in sentence.split()] for sentence in source_text.split('\n')]
        target_id_text = [[target_vocab_to_int[word] for word in sentence.split()]+[target_vocab_to_int['&lt;E&gt;']] for sentence in target_text.split('\n')]

        return source_id_text, target_id_text

    X, y = text_to_ids(articles.lower(), titles.lower(), vocab_to_int_articles, vocab_to_int_titles)


    learning_rate = 0.001
    hidden_units = 400
    embedding_size = 200
    n_layers = 1
    dropout = 0.5
    n_iters = 40


    encoder_forward_cell = tf.contrib.rnn.GRUCell(state_size)
    encoder_backward_cell = tf.contrib.rnn.GRUCell(state_size)
    decoder_cell = tf.contrib.rnn.GRUCell(state_size)

    encoder_forward_cell = tf.contrib.rnn.DropoutWrapper(encoder_forward_cell, output_keep_prob = (1-dropout))
    encoder_backward_cell = tf.contrib.rnn.DropoutWrapper(encoder_backward_cell, output_keep_prob = (1-dropout))
    decoder_cell = tf.contrib.rnn.DropoutWrapper(decoder_cell, output_keep_prob = (1-dropout))


    with tf.variable_scope("seq2seq", dtype=dtype):
        with tf.variable_scope("encoder"):
            encoder_embedding = tf.get_variable("embedding", [source_vocab_size, embedding_size],initializer=embedding_init)
            encoder_inputs_embedding = tf.nn.embedding_lookup(encoder_embedding, self.encoder_inputs)
            encoder_outputs, encoder_states = tf.nn.bidirectional_dynamic_rnn(encoder_forward_cell, encoder_backward_cell, encoder_inputs_embedding, sequence_length=self.encoder_len, dtype=dtype)

        with tf.variable_scope("init_state"):
            init_state = fc_layer(
                tf.concat(encoder_states, 1), state_size)
            # the shape of bidirectional_dynamic_rnn is weird
            # None for batch_size
            self.init_state = init_state
            self.init_state.set_shape([self.batch_size, state_size])
            self.att_states = tf.concat(encoder_outputs, 2)
            self.att_states.set_shape([self.batch_size, None, state_size*2])

        with tf.variable_scope("attention"):
            attention = tf.contrib.seq2seq.BahdanauAttention(
                state_size, self.att_states, self.encoder_len)
            decoder_cell = tf.contrib.seq2seq.DynamicAttentionWrapper(
                decoder_cell, attention, state_size * 2)
            wrapper_state = tf.contrib.seq2seq.DynamicAttentionWrapperState(
                self.init_state, self.prev_att)

        with tf.variable_scope("decoder") as scope:
            decoder_emb = tf.get_variable("embedding", [target_vocab_size, embedding_size],initializer=emb_init)
            decoder_cell = tf.contrib.rnn.OutputProjectionWrapper(decoder_cell, target_vocab_size)
            decoder_inputs_emb = tf.nn.embedding_lookup(decoder_emb, self.decoder_inputs)

            helper = tf.contrib.seq2seq.TrainingHelper(decoder_inputs_emb, self.decoder_len)
            decoder = tf.contrib.seq2seq.BasicDecoder(decoder_cell, helper, wrapper_state)

            outputs, final_state = tf.contrib.seq2seq.dynamic_decode(decoder)

            outputs_logits = outputs[0]
            self.outputs = outputs_logits

            weights = tf.sequence_mask(self.decoder_len, dtype=tf.float32)

            loss_t = tf.contrib.seq2seq.sequence_loss(outputs_logits, self.decoder_targets, weights, average_across_timesteps=False, average_across_batch=False)
            self.loss = tf.reduce_sum(loss_t) / self.batch_size

            params = tf.trainable_variables()
            opt = tf.train.AdadeltaOptimizer(self.learning_rate, epsilon=1e-6)
            gradients = tf.gradients(self.loss, params)
            clipped_gradients, norm = tf.clip_by_global_norm(gradients, max_gradient)
            self.updates = opt.apply_gradients(zip(clipped_gradients, params), global_step=self.global_step)


    def get_batches(int_text, batch_size, seq_length):
        n_batches = int(len(int_text) / (batch_size * seq_length))
        inputs = np.array(int_text[: n_batches * batch_size * seq_length])
        outputs = np.array(int_text[1: n_batches * batch_size * seq_length + 1])

        x = np.split(inputs.reshape(batch_size, -1), n_batches, 1)
        y = np.split(outputs.reshape(batch_size, -1), n_batches, 1)

        return np.array(list(zip(x, y)))


    with tf.Session() as sess:
            model = create_model(sess, False)
            loss = 0.0
            current_step = sess.run(model.global_step)

            while current_step &lt;= n_iters:
                rand = np.random.random_sample()
                bucket_id = min([i for i in range(len(train_buckets_scale))
                                 if train_buckets_scale[i] &gt; rand])

                encoder_inputs, decoder_inputs, encoder_len, decoder_len = model.get_batches(train_set, bucket_id)
                step_loss, _ = model.step(sess, encoder_inputs, decoder_inputs, encoder_len, decoder_len, False, train_writer)
                loss += step_loss * batch_size / np.sum(decoder_len)
                current_step += 1

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

import numpy as np

import tensorflow as tf

# Data can be downloaded at https://github.com/harvardnlp/sent-summary

article_filename = 'Data/sumdata/train/train.article.txt'

title_filename = 'Data/sumdata/train/train.title.txt'

with open(article_filename) as article_file:

articles = article_file.readlines()

with open(title_filename) as title_file:

titles = title_file.readlines()

def create_lookup_tables(text):

vocab = set(text.split())

vocab_to_int = {'<S>': 0, '<E>': 1, '<UNK>': 2, '<PAD>': 3 }

for i, v in enumerate(vocab, len(vocab_to_int)):

vocab_to_int[v] = i

int_to_vocab = {i: v for v, i in vocab_to_int.items()}

return vocab_to_int, int_to_vocab

vocab_to_int_article, int_to_vocab_article = create_lookup_tables([x.lower() for x in articles])

vocab_to_int_title, int_to_vocab_title = create_lookup_tables([x.lower() for x in titles])

def text_to_ids(source_text, target_text, source_vocab_to_int, target_vocab_to_int):

source_id_text = [[source_vocab_to_int[word] for word in sentence.split()] for sentence in source_text.split('\n')]

target_id_text = [[target_vocab_to_int[word] for word in sentence.split()]+[target_vocab_to_int['<E>']] for sentence in target_text.split('\n')]

return source_id_text, target_id_text

X, y = text_to_ids(articles.lower(), titles.lower(), vocab_to_int_articles, vocab_to_int_titles)

learning_rate = 0.001

hidden_units = 400

embedding_size = 200

n_layers = 1

dropout = 0.5

n_iters = 40

encoder_forward_cell = tf.contrib.rnn.GRUCell(state_size)

encoder_backward_cell = tf.contrib.rnn.GRUCell(state_size)

decoder_cell = tf.contrib.rnn.GRUCell(state_size)

encoder_forward_cell = tf.contrib.rnn.DropoutWrapper(encoder_forward_cell, output_keep_prob = (1-dropout))

encoder_backward_cell = tf.contrib.rnn.DropoutWrapper(encoder_backward_cell, output_keep_prob = (1-dropout))

decoder_cell = tf.contrib.rnn.DropoutWrapper(decoder_cell, output_keep_prob = (1-dropout))

with tf.variable_scope("seq2seq", dtype=dtype):

with tf.variable_scope("encoder"):

encoder_embedding = tf.get_variable("embedding", [source_vocab_size, embedding_size],initializer=embedding_init)

encoder_inputs_embedding = tf.nn.embedding_lookup(encoder_embedding, self.encoder_inputs)

encoder_outputs, encoder_states = tf.nn.bidirectional_dynamic_rnn(encoder_forward_cell, encoder_backward_cell, encoder_inputs_embedding, sequence_length=self.encoder_len, dtype=dtype)

with tf.variable_scope("init_state"):

init_state = fc_layer(

tf.concat(encoder_states, 1), state_size)

# the shape of bidirectional_dynamic_rnn is weird

# None for batch_size

self.init_state = init_state

self.init_state.set_shape([self.batch_size, state_size])

self.att_states = tf.concat(encoder_outputs, 2)

self.att_states.set_shape([self.batch_size, None, state_size*2])

with tf.variable_scope("attention"):

attention = tf.contrib.seq2seq.BahdanauAttention(

state_size, self.att_states, self.encoder_len)

decoder_cell = tf.contrib.seq2seq.DynamicAttentionWrapper(

decoder_cell, attention, state_size * 2)

wrapper_state = tf.contrib.seq2seq.DynamicAttentionWrapperState(

self.init_state, self.prev_att)

with tf.variable_scope("decoder") as scope:

decoder_emb = tf.get_variable("embedding", [target_vocab_size, embedding_size],initializer=emb_init)

decoder_cell = tf.contrib.rnn.OutputProjectionWrapper(decoder_cell, target_vocab_size)

decoder_inputs_emb = tf.nn.embedding_lookup(decoder_emb, self.decoder_inputs)

helper = tf.contrib.seq2seq.TrainingHelper(decoder_inputs_emb, self.decoder_len)

decoder = tf.contrib.seq2seq.BasicDecoder(decoder_cell, helper, wrapper_state)

outputs, final_state = tf.contrib.seq2seq.dynamic_decode(decoder)

outputs_logits = outputs[0]

self.outputs = outputs_logits

weights = tf.sequence_mask(self.decoder_len, dtype=tf.float32)

loss_t = tf.contrib.seq2seq.sequence_loss(outputs_logits, self.decoder_targets, weights, average_across_timesteps=False, average_across_batch=False)

self.loss = tf.reduce_sum(loss_t) / self.batch_size

params = tf.trainable_variables()

opt = tf.train.AdadeltaOptimizer(self.learning_rate, epsilon=1e-6)

gradients = tf.gradients(self.loss, params)

clipped_gradients, norm = tf.clip_by_global_norm(gradients, max_gradient)

self.updates = opt.apply_gradients(zip(clipped_gradients, params), global_step=self.global_step)

def get_batches(int_text, batch_size, seq_length):

n_batches = int(len(int_text) / (batch_size * seq_length))

inputs = np.array(int_text[: n_batches * batch_size * seq_length])

outputs = np.array(int_text[1: n_batches * batch_size * seq_length + 1])

x = np.split(inputs.reshape(batch_size, -1), n_batches, 1)

y = np.split(outputs.reshape(batch_size, -1), n_batches, 1)

return np.array(list(zip(x, y)))

with tf.Session() as sess:

model = create_model(sess, False)

loss = 0.0

current_step = sess.run(model.global_step)

while current_step <= n_iters:

rand = np.random.random_sample()

bucket_id = min([i for i in range(len(train_buckets_scale))

if train_buckets_scale[i] > rand])

encoder_inputs, decoder_inputs, encoder_len, decoder_len = model.get_batches(train_set, bucket_id)

step_loss, _ = model.step(sess, encoder_inputs, decoder_inputs, encoder_len, decoder_len, False, train_writer)

loss += step_loss * batch_size / np.sum(decoder_len)

current_step += 1

You might also like