|
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 |
import numpy as np import tensorflow as tf # Data can be downloaded at https://github.com/harvardnlp/sent-summary article_filename = 'Data/sumdata/train/train.article.txt' title_filename = 'Data/sumdata/train/train.title.txt' with open(article_filename) as article_file: articles = article_file.readlines() with open(title_filename) as title_file: titles = title_file.readlines() def create_lookup_tables(text): vocab = set(text.split()) vocab_to_int = {'<S>': 0, '<E>': 1, '<UNK>': 2, '<PAD>': 3 } for i, v in enumerate(vocab, len(vocab_to_int)): vocab_to_int[v] = i int_to_vocab = {i: v for v, i in vocab_to_int.items()} return vocab_to_int, int_to_vocab vocab_to_int_article, int_to_vocab_article = create_lookup_tables([x.lower() for x in articles]) vocab_to_int_title, int_to_vocab_title = create_lookup_tables([x.lower() for x in titles]) def text_to_ids(source_text, target_text, source_vocab_to_int, target_vocab_to_int): source_id_text = [[source_vocab_to_int[word] for word in sentence.split()] for sentence in source_text.split('\n')] target_id_text = [[target_vocab_to_int[word] for word in sentence.split()]+[target_vocab_to_int['<E>']] for sentence in target_text.split('\n')] return source_id_text, target_id_text X, y = text_to_ids(articles.lower(), titles.lower(), vocab_to_int_articles, vocab_to_int_titles) learning_rate = 0.001 hidden_units = 400 embedding_size = 200 n_layers = 1 dropout = 0.5 n_iters = 40 encoder_forward_cell = tf.contrib.rnn.GRUCell(state_size) encoder_backward_cell = tf.contrib.rnn.GRUCell(state_size) decoder_cell = tf.contrib.rnn.GRUCell(state_size) encoder_forward_cell = tf.contrib.rnn.DropoutWrapper(encoder_forward_cell, output_keep_prob = (1-dropout)) encoder_backward_cell = tf.contrib.rnn.DropoutWrapper(encoder_backward_cell, output_keep_prob = (1-dropout)) decoder_cell = tf.contrib.rnn.DropoutWrapper(decoder_cell, output_keep_prob = (1-dropout)) with tf.variable_scope("seq2seq", dtype=dtype): with tf.variable_scope("encoder"): encoder_embedding = tf.get_variable("embedding", [source_vocab_size, embedding_size],initializer=embedding_init) encoder_inputs_embedding = tf.nn.embedding_lookup(encoder_embedding, self.encoder_inputs) encoder_outputs, encoder_states = tf.nn.bidirectional_dynamic_rnn(encoder_forward_cell, encoder_backward_cell, encoder_inputs_embedding, sequence_length=self.encoder_len, dtype=dtype) with tf.variable_scope("init_state"): init_state = fc_layer( tf.concat(encoder_states, 1), state_size) # the shape of bidirectional_dynamic_rnn is weird # None for batch_size self.init_state = init_state self.init_state.set_shape([self.batch_size, state_size]) self.att_states = tf.concat(encoder_outputs, 2) self.att_states.set_shape([self.batch_size, None, state_size*2]) with tf.variable_scope("attention"): attention = tf.contrib.seq2seq.BahdanauAttention( state_size, self.att_states, self.encoder_len) decoder_cell = tf.contrib.seq2seq.DynamicAttentionWrapper( decoder_cell, attention, state_size * 2) wrapper_state = tf.contrib.seq2seq.DynamicAttentionWrapperState( self.init_state, self.prev_att) with tf.variable_scope("decoder") as scope: decoder_emb = tf.get_variable("embedding", [target_vocab_size, embedding_size],initializer=emb_init) decoder_cell = tf.contrib.rnn.OutputProjectionWrapper(decoder_cell, target_vocab_size) decoder_inputs_emb = tf.nn.embedding_lookup(decoder_emb, self.decoder_inputs) helper = tf.contrib.seq2seq.TrainingHelper(decoder_inputs_emb, self.decoder_len) decoder = tf.contrib.seq2seq.BasicDecoder(decoder_cell, helper, wrapper_state) outputs, final_state = tf.contrib.seq2seq.dynamic_decode(decoder) outputs_logits = outputs[0] self.outputs = outputs_logits weights = tf.sequence_mask(self.decoder_len, dtype=tf.float32) loss_t = tf.contrib.seq2seq.sequence_loss(outputs_logits, self.decoder_targets, weights, average_across_timesteps=False, average_across_batch=False) self.loss = tf.reduce_sum(loss_t) / self.batch_size params = tf.trainable_variables() opt = tf.train.AdadeltaOptimizer(self.learning_rate, epsilon=1e-6) gradients = tf.gradients(self.loss, params) clipped_gradients, norm = tf.clip_by_global_norm(gradients, max_gradient) self.updates = opt.apply_gradients(zip(clipped_gradients, params), global_step=self.global_step) def get_batches(int_text, batch_size, seq_length): n_batches = int(len(int_text) / (batch_size * seq_length)) inputs = np.array(int_text[: n_batches * batch_size * seq_length]) outputs = np.array(int_text[1: n_batches * batch_size * seq_length + 1]) x = np.split(inputs.reshape(batch_size, -1), n_batches, 1) y = np.split(outputs.reshape(batch_size, -1), n_batches, 1) return np.array(list(zip(x, y))) with tf.Session() as sess: model = create_model(sess, False) loss = 0.0 current_step = sess.run(model.global_step) while current_step <= n_iters: rand = np.random.random_sample() bucket_id = min([i for i in range(len(train_buckets_scale)) if train_buckets_scale[i] > rand]) encoder_inputs, decoder_inputs, encoder_len, decoder_len = model.get_batches(train_set, bucket_id) step_loss, _ = model.step(sess, encoder_inputs, decoder_inputs, encoder_len, decoder_len, False, train_writer) loss += step_loss * batch_size / np.sum(decoder_len) current_step += 1 |
