Nsryjdtyk

I have a time series data looking something like this:

I am trying to model this with a sequence to sequence RNN in pytorch. It trains well and I can see the loss going down. But on testing it gives the same out put irrespective of the input.

My Model:

class RNNModel(nn.Module):



def __init__(self, predictor_size, hidden_size, num_layers, dropout = 0.3, output_size=83):

    super(RNNModel, self).__init__()

    self.drop = nn.Dropout(dropout)

    self.rnn = nn.GRU(predictor_size, hidden_size, num_layers=num_layers, dropout = dropout)

    self.decoder = nn.Linear(hidden_size, output_size)

    self.init_weights()

    self.hidden_size = hidden_size

    self.num_layers = num_layers



def init_weights(self):

    initrange = 0.1

    self.decoder.bias.data.fill_(0)

    self.decoder.weight.data.uniform_(-initrange, initrange)



def forward(self, input, hidden):

    output, hidden = self.rnn(input, hidden)

    output = self.drop(output)

    decoded = self.decoder(output.view(output.size(0) * output.size(1), output.size(2)))

    return decoded.view(output.size(0), output.size(1), decoded.size(1)), hidden



def init_hidden(self, batch_size):

    weight = next(self.parameters()).data

    return Variable(weight.new(self.num_layers, batch_size, self.hidden_size).zero_())

Train Method:

def train(data_source, lr):

    # turn on training mode that enables dropout



    model.train()

    total_loss = 0

    hidden = model.init_hidden(bs_train)

    optimizer = optim.Adam(model.parameters(), lr = lr)



    for batch, i in enumerate(range(0, data_source.size(0) - 1, bptt_size)):



        data, targets = get_batch(data_source, i)



        # Starting each batch, we detach the hidden state from how it was previously produced

        # so that model doesen't ry to backprop to all the way start of the dataset

        # unrolling of the graph will go from the last iteration to the first iteration

        hidden = Variable(hidden.data)

        if cuda.is_available():

            hidden = hidden.cuda()

        optimizer.zero_grad()



        output, hidden = model(data, hidden)

        loss = criterion(output, targets)

        loss.backward()



        # clip_grad_norm to prevent gradient explosion

        torch.nn.utils.clip_grad_norm(model.parameters(), clip)



        optimizer.step()

        total_loss += len(data) * loss.data

        # return accumulated loss for all the iterations

        return total_loss[0] / len(data_source)

Evaluation Method:

def evaluate(data_source):

    # turn on evaluation to disable dropout

    model.eval()

    model.train(False)

    total_loss = 0

    hidden = model.init_hidden(bs_valid)



    for i in range(0, data_source.size(0) - 1, bptt_size):

        data, targets = get_batch(data_source, i, evaluation = True)



        if cuda.is_available():

            hidden = hidden.cuda()



        output, hidden = model(data, hidden)

        total_loss += len(data) * criterion(output, targets).data

        hidden = Variable(hidden.data)



    return total_loss[0]/len(data_source)

Training Loop:

best_val_loss = None

best_epoch = 0

def run(epochs, lr):

    val_losses = 

    num_epochs = 

    global best_val_loss

    global best_epoch

    for epoch in range(0, epochs):

        train_loss = train(train_set, lr)

        val_loss = evaluate(test_set)

        num_epochs.append(epoch)

        val_losses.append(val_loss)

        print("Train Loss: ", train_loss, " Validation Loss: ", val_loss)



        if not best_val_loss or val_loss < best_val_loss:

            best_val_loss = val_loss

            torch.save(model.state_dict(), "./4.model.pth")

            best_epoch = epoch

    return num_epochs, val_losses

Loss with epochs:

Getting the output:

model = RNNModel(predictor_size, hidden_size, num_layers, dropout_pct, output_size)

model.load_state_dict(torch.load("./4.model.pth"))



if cuda.is_available():

    model.cuda()



model.eval()

model.train(False)

hidden = model.init_hidden(1)

inp = torch.Tensor(var[105])

input = Variable(inp.contiguous().view(1,1,predictor_size), volatile=True)

if cuda.is_available():

    input.data = input.data.cuda()

output, hidden = model(input, hidden)

op = output.squeeze().data.cpu()

print(op)

Here I always get the same output irrespective of datapoint I give as input. Can somebody please tell me what I am doing wrong.