Computing a Loss Landscape for an RNN¶

This notebook demonstrates how you can use PyLossLandscapes to compute a loss landscape and save the resulting data to be analyzed later. We will use an RNN to classify names by their ethnic origin (from https://pytorch.org/tutorials/intermediate/char_rnn_classification_tutorial.html).

Building and training the model¶

In [1]:

# first we have to download the dataset
import urllib.request
import subprocess
urllib.request.urlretrieve("https://download.pytorch.org/tutorial/data.zip", "data.zip")

# first we have to download the dataset import urllib.request import subprocess urllib.request.urlretrieve("https://download.pytorch.org/tutorial/data.zip", "data.zip")

Out[1]:

('data.zip', <http.client.HTTPMessage at 0x7ff234583280>)

In [2]:

# unzip the archive
subprocess.run(["unzip", "data.zip"])

# unzip the archive subprocess.run(["unzip", "data.zip"])

Archive:  data.zip

replace data/eng-fra.txt? [y]es, [n]o, [A]ll, [N]one, [r]ename:  NULL
(EOF or read error, treating as "[N]one" ...)

Out[2]:

CompletedProcess(args=['unzip', 'data.zip'], returncode=1)

In [3]:

import torch

# Check if CUDA is available
device = torch.device('cpu')
if torch.cuda.is_available():
    device = torch.device('cuda')

torch.set_default_device(device)
print(f"Using device = {torch.get_default_device()}")

import torch # Check if CUDA is available device = torch.device('cpu') if torch.cuda.is_available(): device = torch.device('cuda') torch.set_default_device(device) print(f"Using device = {torch.get_default_device()}")

Using device = cuda:0

In [4]:

import string
import unicodedata

# We can use "_" to represent an out-of-vocabulary character, that is, any character we are not handling in our model
allowed_characters = string.ascii_letters + " .,;'" + "_"
n_letters = len(allowed_characters)

# Turn a Unicode string to plain ASCII, thanks to https://stackoverflow.com/a/518232/2809427
def unicodeToAscii(s):
    return ''.join(
        c for c in unicodedata.normalize('NFD', s)
        if unicodedata.category(c) != 'Mn'
        and c in allowed_characters
    )

import string import unicodedata # We can use "_" to represent an out-of-vocabulary character, that is, any character we are not handling in our model allowed_characters = string.ascii_letters + " .,;'" + "_" n_letters = len(allowed_characters) # Turn a Unicode string to plain ASCII, thanks to https://stackoverflow.com/a/518232/2809427 def unicodeToAscii(s): return ''.join( c for c in unicodedata.normalize('NFD', s) if unicodedata.category(c) != 'Mn' and c in allowed_characters )

In [5]:

# Find letter index from all_letters, e.g. "a" = 0
def letterToIndex(letter):
    # return our out-of-vocabulary character if we encounter a letter unknown to our model
    if letter not in allowed_characters:
        return allowed_characters.find("_")
    else:
        return allowed_characters.find(letter)

# Turn a line into a <line_length x 1 x n_letters>,
# or an array of one-hot letter vectors
def lineToTensor(line):
    tensor = torch.zeros(len(line), 1, n_letters)
    for li, letter in enumerate(line):
        tensor[li][0][letterToIndex(letter)] = 1
    return tensor

# Find letter index from all_letters, e.g. "a" = 0 def letterToIndex(letter): # return our out-of-vocabulary character if we encounter a letter unknown to our model if letter not in allowed_characters: return allowed_characters.find("_") else: return allowed_characters.find(letter) # Turn a line into a , # or an array of one-hot letter vectors def lineToTensor(line): tensor = torch.zeros(len(line), 1, n_letters) for li, letter in enumerate(line): tensor[li][0][letterToIndex(letter)] = 1 return tensor

In [6]:

from io import open
import glob
import os
import time

import torch
from torch.utils.data import Dataset, DataLoader

class NamesDataset(Dataset):

    def __init__(self, data_dir):
        self.data_dir = data_dir #for provenance of the dataset
        self.load_time = time.localtime #for provenance of the dataset
        labels_set = set() #set of all classes

        self.data = []
        self.data_tensors = []
        self.labels = []
        self.labels_tensors = []

        #read all the ``.txt`` files in the specified directory
        text_files = glob.glob(os.path.join(data_dir, '*.txt'))
        for filename in text_files:
            label = os.path.splitext(os.path.basename(filename))[0]
            labels_set.add(label)
            lines = open(filename, encoding='utf-8').read().strip().split('\n')
            for name in lines:
                self.data.append(name)
                self.data_tensors.append(lineToTensor(name))
                self.labels.append(label)

        #Cache the tensor representation of the labels
        self.labels_uniq = list(labels_set)
        for idx in range(len(self.labels)):
            temp_tensor = torch.tensor([self.labels_uniq.index(self.labels[idx])], dtype=torch.long)
            self.labels_tensors.append(temp_tensor)

    def __len__(self):
        return len(self.data)

    def __getitem__(self, idx):
        data_item = self.data[idx]
        data_label = self.labels[idx]
        data_tensor = self.data_tensors[idx]
        label_tensor = self.labels_tensors[idx]

        return label_tensor, data_tensor, data_label, data_item

from io import open import glob import os import time import torch from torch.utils.data import Dataset, DataLoader class NamesDataset(Dataset): def __init__(self, data_dir): self.data_dir = data_dir #for provenance of the dataset self.load_time = time.localtime #for provenance of the dataset labels_set = set() #set of all classes self.data = [] self.data_tensors = [] self.labels = [] self.labels_tensors = [] #read all the ``.txt`` files in the specified directory text_files = glob.glob(os.path.join(data_dir, '*.txt')) for filename in text_files: label = os.path.splitext(os.path.basename(filename))[0] labels_set.add(label) lines = open(filename, encoding='utf-8').read().strip().split('\n') for name in lines: self.data.append(name) self.data_tensors.append(lineToTensor(name)) self.labels.append(label) #Cache the tensor representation of the labels self.labels_uniq = list(labels_set) for idx in range(len(self.labels)): temp_tensor = torch.tensor([self.labels_uniq.index(self.labels[idx])], dtype=torch.long) self.labels_tensors.append(temp_tensor) def __len__(self): return len(self.data) def __getitem__(self, idx): data_item = self.data[idx] data_label = self.labels[idx] data_tensor = self.data_tensors[idx] label_tensor = self.labels_tensors[idx] return label_tensor, data_tensor, data_label, data_item

In [7]:

alldata = NamesDataset("data/names")
#dataloader = DataLoader(alldata, batch_size=64, shuffle=False, num_workers=1, generator=torch.Generator(device='cuda'))

alldata = NamesDataset("data/names") #dataloader = DataLoader(alldata, batch_size=64, shuffle=False, num_workers=1, generator=torch.Generator(device='cuda'))

In [8]:

train_set, test_set = torch.utils.data.random_split(alldata, [.85, .15], generator=torch.Generator(device=device).manual_seed(2024))

train_set, test_set = torch.utils.data.random_split(alldata, [.85, .15], generator=torch.Generator(device=device).manual_seed(2024))

In [9]:

import torch.nn as nn
import torch.nn.functional as F

class CharRNN(nn.Module):
    def __init__(self, input_size, hidden_size, output_size):
        super(CharRNN, self).__init__()

        self.rnn = nn.RNN(input_size, hidden_size)
        self.h2o = nn.Linear(hidden_size, output_size)
        self.softmax = nn.LogSoftmax(dim=1)

    def forward(self, line_tensor):
        rnn_out, hidden = self.rnn(line_tensor)
        output = self.h2o(hidden[0])
        output = self.softmax(output)

        return output

import torch.nn as nn import torch.nn.functional as F class CharRNN(nn.Module): def __init__(self, input_size, hidden_size, output_size): super(CharRNN, self).__init__() self.rnn = nn.RNN(input_size, hidden_size) self.h2o = nn.Linear(hidden_size, output_size) self.softmax = nn.LogSoftmax(dim=1) def forward(self, line_tensor): rnn_out, hidden = self.rnn(line_tensor) output = self.h2o(hidden[0]) output = self.softmax(output) return output

In [10]:

n_hidden = 128
rnn = CharRNN(n_letters, n_hidden, len(alldata.labels_uniq))

n_hidden = 128 rnn = CharRNN(n_letters, n_hidden, len(alldata.labels_uniq))

In [11]:

def label_from_output(output, output_labels):
    top_n, top_i = output.topk(1)
    label_i = top_i[0].item()
    return output_labels[label_i], label_i

def label_from_output(output, output_labels): top_n, top_i = output.topk(1) label_i = top_i[0].item() return output_labels[label_i], label_i

In [12]:

def get_params_grad(model):
    """
    get model parameters and corresponding gradients
    """
    params = []
    grads = []
    for param in model.parameters():
        if not param.requires_grad:
            continue
        params.append(param)
        grads.append(torch.tensor(0.0) if param.grad is None else param.grad + 0.)
    return params, grads

def get_params_grad(model): """ get model parameters and corresponding gradients """ params = [] grads = [] for param in model.parameters(): if not param.requires_grad: continue params.append(param) grads.append(torch.tensor(0.0) if param.grad is None else param.grad + 0.) return params, grads

In [13]:

import random
import numpy as np

def train(rnn, training_data, n_epoch = 10, n_batch_size = 64, report_every = 50, learning_rate = 0.2, criterion = nn.NLLLoss()):
    """
    Learn on a batch of training_data for a specified number of iterations and reporting thresholds
    """
    # Keep track of losses for plotting
    current_loss = 0
    all_losses = []
    rnn.train()
    optimizer = torch.optim.SGD(rnn.parameters(), lr=learning_rate)

    start = time.time()
    print(f"training on data set with n = {len(training_data)}")

    for iter in range(1, n_epoch + 1):
        rnn.zero_grad() # clear the gradients

        # create some minibatches
        # we cannot use dataloaders because each of our names is a different length
        batches = list(range(len(training_data)))
        random.shuffle(batches)
        batches = np.array_split(batches, len(batches) //n_batch_size )

        for idx, batch in enumerate(batches):
            batch_loss = 0
            for i in batch: #for each example in this batch
                (label_tensor, text_tensor, label, text) = training_data[i]
                text_tensor = lineToTensor(text)
                output = rnn.forward(text_tensor)
                loss = criterion(output, label_tensor)
                batch_loss += loss

            # optimize parameters
            batch_loss.backward()
            nn.utils.clip_grad_norm_(rnn.parameters(), 3)
            optimizer.step()
            optimizer.zero_grad()

            current_loss += batch_loss.item() / len(batch)

        all_losses.append(current_loss / len(batches) )
        if iter % report_every == 0:
            print(f"{iter} ({iter / n_epoch:.0%}): \t average batch loss = {all_losses[-1]}")
        current_loss = 0

    return all_losses

import random import numpy as np def train(rnn, training_data, n_epoch = 10, n_batch_size = 64, report_every = 50, learning_rate = 0.2, criterion = nn.NLLLoss()): """ Learn on a batch of training_data for a specified number of iterations and reporting thresholds """ # Keep track of losses for plotting current_loss = 0 all_losses = [] rnn.train() optimizer = torch.optim.SGD(rnn.parameters(), lr=learning_rate) start = time.time() print(f"training on data set with n = {len(training_data)}") for iter in range(1, n_epoch + 1): rnn.zero_grad() # clear the gradients # create some minibatches # we cannot use dataloaders because each of our names is a different length batches = list(range(len(training_data))) random.shuffle(batches) batches = np.array_split(batches, len(batches) //n_batch_size ) for idx, batch in enumerate(batches): batch_loss = 0 for i in batch: #for each example in this batch (label_tensor, text_tensor, label, text) = training_data[i] text_tensor = lineToTensor(text) output = rnn.forward(text_tensor) loss = criterion(output, label_tensor) batch_loss += loss # optimize parameters batch_loss.backward() nn.utils.clip_grad_norm_(rnn.parameters(), 3) optimizer.step() optimizer.zero_grad() current_loss += batch_loss.item() / len(batch) all_losses.append(current_loss / len(batches) ) if iter % report_every == 0: print(f"{iter} ({iter / n_epoch:.0%}): \t average batch loss = {all_losses[-1]}") current_loss = 0 return all_losses

In [14]:

#all_losses = train(rnn, train_set, n_epoch=27, learning_rate=0.15, report_every=5)

#all_losses = train(rnn, train_set, n_epoch=27, learning_rate=0.15, report_every=5)

In [15]:

# save the model
#torch.save(rnn.state_dict(), 'model.pth')

# save the model #torch.save(rnn.state_dict(), 'model.pth')

In [16]:

rnn = CharRNN(n_letters, n_hidden, len(alldata.labels_uniq))
rnn.load_state_dict(torch.load('model.pth'))

rnn = CharRNN(n_letters, n_hidden, len(alldata.labels_uniq)) rnn.load_state_dict(torch.load('model.pth'))

Out[16]:

<All keys matched successfully>

Computing the landscape¶

In [17]:

from landscaper import LossLandscape, PyHessian

from landscaper import LossLandscape, PyHessian

In [18]:

data = [(dt, lbl_t) for (lbl_t, dt, dl, di) in train_set][:50]

data = [(dt, lbl_t) for (lbl_t, dt, dl, di) in train_set][:50]

In [19]:

# create a hessian calculator based on our model
with torch.backends.cudnn.flags(enabled=False):
    hessian_comp = PyHessian(rnn, nn.NLLLoss(), data, device, try_cache=True)

# create a hessian calculator based on our model with torch.backends.cudnn.flags(enabled=False): hessian_comp = PyHessian(rnn, nn.NLLLoss(), data, device, try_cache=True)

Setting model to eval mode. PyHessian will not work with models in training mode!

In [20]:

# for the loss landscape computation to work, we need to define how the loss should be calculated
lfn = nn.NLLLoss()
def loss_function(model, data):
    batch_loss = 0
    for d in data: 
        tt, lbl_t = d
        output = rnn.forward(tt)
        loss = lfn(output, lbl_t)
        batch_loss += loss
    return batch_loss / len(data)

# for the loss landscape computation to work, we need to define how the loss should be calculated lfn = nn.NLLLoss() def loss_function(model, data): batch_loss = 0 for d in data: tt, lbl_t = d output = rnn.forward(tt) loss = lfn(output, lbl_t) batch_loss += loss return batch_loss / len(data)

In [21]:

with torch.backends.cudnn.flags(enabled=False):
    evals, evecs = hessian_comp.eigenvalues(top_n=3, maxIter=100)

with torch.backends.cudnn.flags(enabled=False): evals, evecs = hessian_comp.eigenvalues(top_n=3, maxIter=100)

Eigenvectors computed:   0%|                                    | 0/3 [00:00<?, ?it/s]
Iteration:   0%|                                              | 0/100 [00:00<?, ?it/s]
Iteration:   1%|▍                                     | 1/100 [00:00<00:31,  3.09it/s]
Spectral gap: 1697.8530524356609:   1%|▏              | 1/100 [00:00<00:31,  3.09it/s]
Spectral gap: 1697.8530524356609:   2%|▎              | 2/100 [00:00<00:20,  4.75it/s]
Spectral gap: 4.0317867166203465:   2%|▎              | 2/100 [00:00<00:20,  4.75it/s]
Spectral gap: 4.0317867166203465:   3%|▍              | 3/100 [00:00<00:16,  5.71it/s]
Spectral gap: 0.24763802237362204:   3%|▍             | 3/100 [00:00<00:16,  5.71it/s]
Spectral gap: 0.24763802237362204:   4%|▌             | 4/100 [00:00<00:15,  6.34it/s]
Spectral gap: 0.03862144724807775:   4%|▌             | 4/100 [00:00<00:15,  6.34it/s]
Spectral gap: 0.03862144724807775:   5%|▋             | 5/100 [00:00<00:14,  6.73it/s]
Spectral gap: 0.009471561228870908:   5%|▋            | 5/100 [00:00<00:14,  6.73it/s]
Spectral gap: 0.009471561228870908:   6%|▊            | 6/100 [00:00<00:13,  6.97it/s]
Spectral gap: 0.002809408647258704:   6%|▊            | 6/100 [00:01<00:13,  6.97it/s]
Spectral gap: 0.002809408647258704:   7%|▉            | 7/100 [00:01<00:13,  7.12it/s]
Spectral gap: 0.000890555740272384:   7%|▉            | 7/100 [00:01<00:13,  7.12it/s]
Eigenvectors computed:  33%|█████████▎                  | 1/3 [00:01<00:02,  1.27s/it]
Iteration:   0%|                                              | 0/100 [00:00<?, ?it/s]
Iteration:   1%|▍                                     | 1/100 [00:00<00:13,  7.46it/s]
Spectral gap: 39.31103214689547:   1%|▏               | 1/100 [00:00<00:13,  7.46it/s]
Spectral gap: 39.31103214689547:   2%|▎               | 2/100 [00:00<00:13,  7.47it/s]
Spectral gap: 14.286212563922914:   2%|▎              | 2/100 [00:00<00:13,  7.47it/s]
Spectral gap: 14.286212563922914:   3%|▍              | 3/100 [00:00<00:12,  7.60it/s]
Spectral gap: 1.2850148610548655:   3%|▍              | 3/100 [00:00<00:12,  7.60it/s]
Spectral gap: 1.2850148610548655:   4%|▌              | 4/100 [00:00<00:12,  7.65it/s]
Spectral gap: 0.46345546155986367:   4%|▌             | 4/100 [00:00<00:12,  7.65it/s]
Spectral gap: 0.46345546155986367:   5%|▋             | 5/100 [00:00<00:12,  7.68it/s]
Spectral gap: 0.28869205862514735:   5%|▋             | 5/100 [00:00<00:12,  7.68it/s]
Spectral gap: 0.28869205862514735:   6%|▊             | 6/100 [00:00<00:12,  7.68it/s]
Spectral gap: 0.2058390318749211:   6%|▉              | 6/100 [00:00<00:12,  7.68it/s]
Spectral gap: 0.2058390318749211:   7%|█              | 7/100 [00:00<00:12,  7.70it/s]
Spectral gap: 0.13864459134254078:   7%|▉             | 7/100 [00:01<00:12,  7.70it/s]
Spectral gap: 0.13864459134254078:   8%|█             | 8/100 [00:01<00:13,  6.90it/s]
Spectral gap: 0.08120663668496621:   8%|█             | 8/100 [00:01<00:13,  6.90it/s]
Spectral gap: 0.08120663668496621:   9%|█▎            | 9/100 [00:01<00:16,  5.54it/s]
Spectral gap: 0.03516392243545905:   9%|█▎            | 9/100 [00:01<00:16,  5.54it/s]
Spectral gap: 0.03516392243545905:  10%|█▎           | 10/100 [00:01<00:18,  4.84it/s]
Spectral gap: 0.0010554006469288759:  10%|█          | 10/100 [00:01<00:18,  4.84it/s]
Spectral gap: 0.0010554006469288759:  11%|█▏         | 11/100 [00:01<00:19,  4.46it/s]
Spectral gap: 0.030771533773091123:  11%|█▎          | 11/100 [00:02<00:19,  4.46it/s]
Spectral gap: 0.030771533773091123:  12%|█▍          | 12/100 [00:02<00:20,  4.23it/s]
Spectral gap: 0.05737213734516458:  12%|█▌           | 12/100 [00:02<00:20,  4.23it/s]
Spectral gap: 0.05737213734516458:  13%|█▋           | 13/100 [00:02<00:21,  4.07it/s]
Spectral gap: 0.08388828328501464:  13%|█▋           | 13/100 [00:02<00:21,  4.07it/s]
Spectral gap: 0.08388828328501464:  14%|█▊           | 14/100 [00:02<00:21,  4.02it/s]
Spectral gap: 0.11328186591901049:  14%|█▊           | 14/100 [00:02<00:21,  4.02it/s]
Spectral gap: 0.11328186591901049:  15%|█▉           | 15/100 [00:02<00:21,  3.94it/s]
Spectral gap: 0.14913527084928568:  15%|█▉           | 15/100 [00:03<00:21,  3.94it/s]
Spectral gap: 0.14913527084928568:  16%|██           | 16/100 [00:03<00:21,  3.91it/s]
Spectral gap: 0.19706223034570425:  16%|██           | 16/100 [00:03<00:21,  3.91it/s]
Spectral gap: 0.19706223034570425:  17%|██▏          | 17/100 [00:03<00:21,  3.87it/s]
Spectral gap: 0.2682296728912812:  17%|██▍           | 17/100 [00:03<00:21,  3.87it/s]
Spectral gap: 0.2682296728912812:  18%|██▌           | 18/100 [00:03<00:21,  3.86it/s]
Spectral gap: 0.39110971077861834:  18%|██▎          | 18/100 [00:03<00:21,  3.86it/s]
Spectral gap: 0.39110971077861834:  19%|██▍          | 19/100 [00:03<00:21,  3.86it/s]
Spectral gap: 0.6704764190216516:  19%|██▋           | 19/100 [00:04<00:21,  3.86it/s]
Spectral gap: 0.6704764190216516:  20%|██▊           | 20/100 [00:04<00:20,  3.88it/s]
Spectral gap: 2.079697605957135:  20%|███            | 20/100 [00:04<00:20,  3.88it/s]
Spectral gap: 2.079697605957135:  21%|███▏           | 21/100 [00:04<00:20,  3.87it/s]
Spectral gap: 1.9290011460818919:  21%|██▉           | 21/100 [00:04<00:20,  3.87it/s]
Spectral gap: 1.9290011460818919:  22%|███           | 22/100 [00:04<00:20,  3.85it/s]
Spectral gap: 0.6465649029298314:  22%|███           | 22/100 [00:05<00:20,  3.85it/s]
Spectral gap: 0.6465649029298314:  23%|███▏          | 23/100 [00:05<00:19,  3.85it/s]
Spectral gap: 0.378170835351169:  23%|███▍           | 23/100 [00:05<00:19,  3.85it/s]
Spectral gap: 0.378170835351169:  24%|███▌           | 24/100 [00:05<00:18,  4.16it/s]
Spectral gap: 0.2594624447179541:  24%|███▎          | 24/100 [00:05<00:18,  4.16it/s]
Spectral gap: 0.2594624447179541:  25%|███▌          | 25/100 [00:05<00:15,  4.79it/s]
Spectral gap: 0.19146257883566434:  25%|███▎         | 25/100 [00:05<00:15,  4.79it/s]
Spectral gap: 0.19146257883566434:  26%|███▍         | 26/100 [00:05<00:13,  5.39it/s]
Spectral gap: 0.14697873261647626:  26%|███▍         | 26/100 [00:05<00:13,  5.39it/s]
Spectral gap: 0.14697873261647626:  27%|███▌         | 27/100 [00:05<00:12,  5.91it/s]
Spectral gap: 0.11550525932310667:  27%|███▌         | 27/100 [00:05<00:12,  5.91it/s]
Spectral gap: 0.11550525932310667:  28%|███▋         | 28/100 [00:05<00:11,  6.33it/s]
Spectral gap: 0.09211198479917945:  28%|███▋         | 28/100 [00:05<00:11,  6.33it/s]
Spectral gap: 0.09211198479917945:  29%|███▊         | 29/100 [00:05<00:10,  6.66it/s]
Spectral gap: 0.07415667185335098:  29%|███▊         | 29/100 [00:06<00:10,  6.66it/s]
Spectral gap: 0.07415667185335098:  30%|███▉         | 30/100 [00:06<00:10,  6.82it/s]
Spectral gap: 0.06007720972970835:  30%|███▉         | 30/100 [00:06<00:10,  6.82it/s]
Spectral gap: 0.06007720972970835:  31%|████         | 31/100 [00:06<00:10,  6.87it/s]
Spectral gap: 0.048877214022435445:  31%|███▋        | 31/100 [00:06<00:10,  6.87it/s]
Spectral gap: 0.048877214022435445:  32%|███▊        | 32/100 [00:06<00:09,  6.84it/s]
Spectral gap: 0.03988005652554547:  32%|████▏        | 32/100 [00:06<00:09,  6.84it/s]
Spectral gap: 0.03988005652554547:  33%|████▎        | 33/100 [00:06<00:09,  6.78it/s]
Spectral gap: 0.03260385406018266:  33%|████▎        | 33/100 [00:06<00:09,  6.78it/s]
Spectral gap: 0.03260385406018266:  34%|████▍        | 34/100 [00:06<00:09,  6.71it/s]
Spectral gap: 0.026692368652848544:  34%|████        | 34/100 [00:06<00:09,  6.71it/s]
Spectral gap: 0.026692368652848544:  35%|████▏       | 35/100 [00:06<00:09,  6.68it/s]
Spectral gap: 0.021874154013015543:  35%|████▏       | 35/100 [00:06<00:09,  6.68it/s]
Spectral gap: 0.021874154013015543:  36%|████▎       | 36/100 [00:06<00:09,  6.75it/s]
Spectral gap: 0.017937924059742566:  36%|████▎       | 36/100 [00:07<00:09,  6.75it/s]
Spectral gap: 0.017937924059742566:  37%|████▍       | 37/100 [00:07<00:09,  6.82it/s]
Spectral gap: 0.014717396976979493:  37%|████▍       | 37/100 [00:07<00:09,  6.82it/s]
Spectral gap: 0.014717396976979493:  38%|████▌       | 38/100 [00:07<00:09,  6.80it/s]
Spectral gap: 0.012079568450458307:  38%|████▌       | 38/100 [00:07<00:09,  6.80it/s]
Spectral gap: 0.012079568450458307:  39%|████▋       | 39/100 [00:07<00:08,  6.83it/s]
Spectral gap: 0.00991708428900409:  39%|█████        | 39/100 [00:07<00:08,  6.83it/s]
Spectral gap: 0.00991708428900409:  40%|█████▏       | 40/100 [00:07<00:08,  6.78it/s]
Spectral gap: 0.008143250497765318:  40%|████▊       | 40/100 [00:07<00:08,  6.78it/s]
Spectral gap: 0.008143250497765318:  41%|████▉       | 41/100 [00:07<00:08,  6.79it/s]
Spectral gap: 0.0066878499303776075:  41%|████▌      | 41/100 [00:07<00:08,  6.79it/s]
Spectral gap: 0.0066878499303776075:  42%|████▌      | 42/100 [00:07<00:08,  6.92it/s]
Spectral gap: 0.005492805863886888:  42%|█████       | 42/100 [00:07<00:08,  6.92it/s]
Spectral gap: 0.005492805863886888:  43%|█████▏      | 43/100 [00:07<00:08,  6.92it/s]
Spectral gap: 0.004511920780915883:  43%|█████▏      | 43/100 [00:08<00:08,  6.92it/s]
Spectral gap: 0.004511920780915883:  44%|█████▎      | 44/100 [00:08<00:08,  6.85it/s]
Spectral gap: 0.0037064357249065715:  44%|████▊      | 44/100 [00:08<00:08,  6.85it/s]
Spectral gap: 0.0037064357249065715:  45%|████▉      | 45/100 [00:08<00:07,  6.89it/s]
Spectral gap: 0.003044861506334771:  45%|█████▍      | 45/100 [00:08<00:07,  6.89it/s]
Spectral gap: 0.003044861506334771:  46%|█████▌      | 46/100 [00:08<00:07,  7.09it/s]
Spectral gap: 0.002501603022987549:  46%|█████▌      | 46/100 [00:08<00:07,  7.09it/s]
Spectral gap: 0.002501603022987549:  47%|█████▋      | 47/100 [00:08<00:07,  7.25it/s]
Spectral gap: 0.00205539177335859:  47%|██████       | 47/100 [00:08<00:07,  7.25it/s]
Spectral gap: 0.00205539177335859:  48%|██████▏      | 48/100 [00:08<00:07,  7.32it/s]
Spectral gap: 0.0016882919932496086:  48%|█████▎     | 48/100 [00:08<00:07,  7.32it/s]
Spectral gap: 0.0016882919932496086:  49%|█████▍     | 49/100 [00:08<00:06,  7.34it/s]
Spectral gap: 0.0013872037816450257:  49%|█████▍     | 49/100 [00:08<00:06,  7.34it/s]
Spectral gap: 0.0013872037816450257:  50%|█████▌     | 50/100 [00:08<00:06,  7.32it/s]
Spectral gap: 0.0011399207493945171:  50%|█████▌     | 50/100 [00:09<00:06,  7.32it/s]
Spectral gap: 0.0011399207493945171:  51%|█████▌     | 51/100 [00:09<00:06,  7.24it/s]
Spectral gap: 0.0009365524035984121:  51%|█████▌     | 51/100 [00:09<00:06,  7.24it/s]
Eigenvectors computed:  67%|██████████████████▋         | 2/3 [00:10<00:05,  5.92s/it]
Iteration:   0%|                                              | 0/100 [00:00<?, ?it/s]
Iteration:   1%|▍                                     | 1/100 [00:00<00:20,  4.85it/s]
Spectral gap: 241.18247703177144:   1%|▏              | 1/100 [00:00<00:20,  4.85it/s]
Spectral gap: 241.18247703177144:   2%|▎              | 2/100 [00:00<00:23,  4.14it/s]
Spectral gap: 1.1663354566621487:   2%|▎              | 2/100 [00:00<00:23,  4.14it/s]
Spectral gap: 1.1663354566621487:   3%|▍              | 3/100 [00:00<00:24,  3.97it/s]
Spectral gap: 0.6824635010861562:   3%|▍              | 3/100 [00:00<00:24,  3.97it/s]
Spectral gap: 0.6824635010861562:   4%|▌              | 4/100 [00:00<00:24,  3.91it/s]
Spectral gap: 5.013314000281176:   4%|▋               | 4/100 [00:01<00:24,  3.91it/s]
Spectral gap: 5.013314000281176:   5%|▊               | 5/100 [00:01<00:24,  3.88it/s]
Spectral gap: 1.3211244534095679:   5%|▊              | 5/100 [00:01<00:24,  3.88it/s]
Spectral gap: 1.3211244534095679:   6%|▉              | 6/100 [00:01<00:24,  3.86it/s]
Spectral gap: 0.4319394479434034:   6%|▉              | 6/100 [00:01<00:24,  3.86it/s]
Spectral gap: 0.4319394479434034:   7%|█              | 7/100 [00:01<00:24,  3.84it/s]
Spectral gap: 0.19227522361333102:   7%|▉             | 7/100 [00:02<00:24,  3.84it/s]
Spectral gap: 0.19227522361333102:   8%|█             | 8/100 [00:02<00:24,  3.83it/s]
Spectral gap: 0.09423674720709461:   8%|█             | 8/100 [00:02<00:24,  3.83it/s]
Spectral gap: 0.09423674720709461:   9%|█▎            | 9/100 [00:02<00:23,  3.81it/s]
Spectral gap: 0.04852638857367241:   9%|█▎            | 9/100 [00:02<00:23,  3.81it/s]
Spectral gap: 0.04852638857367241:  10%|█▎           | 10/100 [00:02<00:23,  3.81it/s]
Spectral gap: 0.025861557422916102:  10%|█▏          | 10/100 [00:02<00:23,  3.81it/s]
Spectral gap: 0.025861557422916102:  11%|█▎          | 11/100 [00:02<00:23,  3.80it/s]
Spectral gap: 0.014180306348438426:  11%|█▎          | 11/100 [00:03<00:23,  3.80it/s]
Spectral gap: 0.014180306348438426:  12%|█▍          | 12/100 [00:03<00:23,  3.79it/s]
Spectral gap: 0.007978698200502057:  12%|█▍          | 12/100 [00:03<00:23,  3.79it/s]
Spectral gap: 0.007978698200502057:  13%|█▌          | 13/100 [00:03<00:23,  3.77it/s]
Spectral gap: 0.004599146156083687:  13%|█▌          | 13/100 [00:03<00:23,  3.77it/s]
Spectral gap: 0.004599146156083687:  14%|█▋          | 14/100 [00:03<00:22,  3.78it/s]
Spectral gap: 0.0027128767883270095:  14%|█▌         | 14/100 [00:03<00:22,  3.78it/s]
Spectral gap: 0.0027128767883270095:  15%|█▋         | 15/100 [00:03<00:22,  3.76it/s]
Spectral gap: 0.0016353614633023042:  15%|█▋         | 15/100 [00:04<00:22,  3.76it/s]
Spectral gap: 0.0016353614633023042:  16%|█▊         | 16/100 [00:04<00:22,  3.80it/s]
Spectral gap: 0.0010055447923407108:  16%|█▊         | 16/100 [00:04<00:22,  3.80it/s]
Spectral gap: 0.0010055447923407108:  17%|█▊         | 17/100 [00:04<00:21,  3.83it/s]
Spectral gap: 0.0006301425099729107:  17%|█▊         | 17/100 [00:04<00:21,  3.83it/s]
Eigenvectors computed: 100%|████████████████████████████| 3/3 [00:15<00:00,  5.04s/it]

In [22]:

print(evals)

print(evals)

[-162.9615478515625, -99.86493682861328, 90.47868347167969]

In [23]:

landscape = LossLandscape.compute(
    rnn,
    data,
    evecs,
    loss_function, # loss function
    dim=2,
    distance=0.5,
    device=device,
)

landscape = LossLandscape.compute( rnn, data, evecs, loss_function, # loss function dim=2, distance=0.5, device=device, )

Computing 1681 points in 2D space...

Computing 2D landscape: 100%|█████████████████████| 1681/1681 [00:22<00:00, 73.71it/s]

Loss hypercube stats - min: 4.7444376945495605, max: 11.950156211853027, mean: 7.034827068687407

In [24]:

landscape.save("example.npz")

landscape.save("example.npz")

In [25]:

landscape  = LossLandscape.load_from_npz("example.npz")

landscape = LossLandscape.load_from_npz("example.npz")

In [26]:

landscape.show()

landscape.show()

Attempting log-scale surface plot...

In [38]:

landscape.show_profile()

landscape.show_profile()

Out[38]:

In [28]:

landscape.show_contour()

landscape.show_contour()

In [29]:

landscape.show_persistence_barcode()

landscape.show_persistence_barcode()

In [30]:

# we can use this function to convert the merge tree 
# into a networkx graph and visualize it!
from landscaper.tda import digraph_mt
g = digraph_mt(landscape.get_sublevel_tree())

# we can use this function to convert the merge tree # into a networkx graph and visualize it! from landscaper.tda import digraph_mt g = digraph_mt(landscape.get_sublevel_tree())

In [31]:

import networkx as nx
nx.draw_planar(g, with_labels=True)

import networkx as nx nx.draw_planar(g, with_labels=True)

In [32]:

landscape.smad()

landscape.smad()

Out[32]:

0.05093240737915039

In [ ]: