1
$\begingroup$

I'm a beginner.

If I am using a Convolutional Neural Networks with Triplet Loss as a loss function (also combined with GAN and a Classifier) for building a model that performs Speaker Verification, what labels should I use for the training process?

I used the speakers' numbers as labels for the training but the model actually doesn't learn anything (and embeddings are all similar) and I think that this is the cause of the problem. Are the labels should be 0 and 1? If any additional details are needed, I will add them.

First, in the pre-processing stage, the model has to read audio samples from the dataset (dataset is TIMIT) and perform features extraction (which has to be done with Wavelet), after this, the images are fed into the Encoder, a Convolutional Neural Network with 4 layers, kernel size is 5, this is the Encoder class:

class Encoder(nn.Module):
    def __init__(self, expansion, blocks, embedding_dim):
        super(Encoder, self).__init__()
        self.convnet = self._make_layers(expansion, blocks)
        self.pool = nn.AdaptiveAvgPool2d([2,2])
        self.fc = nn.Linear(expansion * blocks * 2 * 2 * 2, embedding_dim)
    
    def _make_layers(self, expansion, blocks):
        layers = []
        for i in range(blocks):
            if i == 0:
                layers.append(nn.Conv2d(1, expansion, kernel_size = 5, stride = 2, padding = 2))
            else:
                layers.append(nn.Conv2d(expansion, expansion * 2, kernel_size = 5, stride = 2, padding = 2))
                expansion = expansion * 2
            layers.append(nn.BatchNorm2d(expansion))
            layers.append(nn.ReLU())
        return nn.Sequential(*layers)

    def forward(self, x):
        x = self.convnet(x)
        x = self.pool(x)
        x = torch.flatten(x, start_dim = 1)
        x = self.fc(x)
        return F.normalize(x)

The number of speakers per batch is 20 and utterances per speaker is 10. It is a bit complicated to explain the whole model architecture so I'm adding an image: image

This is the train function:

def train(epoch, 
          encoder,
          generator,
          discriminator,
          classifier,
          triplet_criterion, 
          bce_criterion,
          softmax_criterion,
          generator_optimizer,
          discriminator_optimizer,
          train_loader,
          metric): # 
    encoder.train() # 
    generator.train()
    discriminator.train()
    classifier.train()

    triplet_sum_loss, generator_loss, discriminator_loss, classifier_loss, sum_samples = 0, 0, 0, 0, 0
    progress_bar = tqdm(enumerate(train_loader))
    for batch_idx, (data, label) in progress_bar:
        sum_samples += len(data)
        #data = data.to(device)
        #label = label.to(device)  

        # Adversarial ground truths
        #valid = torch.FloatTensor(utts_per_spk * spks_per_batch, 1).fill_(1.0).to(device)
        #fake = torch.FloatTensor(utts_per_spk * spks_per_batch, 1).fill_(0.0).to(device)
        valid = torch.FloatTensor(utts_per_spk * spks_per_batch, 1).fill_(1.0)
        fake = torch.FloatTensor(utts_per_spk * spks_per_batch, 1).fill_(0.0)
        # triplet loss
        generator_optimizer.zero_grad()
        #print(data)
        embedding = encoder(data)
        #print(embedding)
        triplet_loss, non_zero_triplets = triplet_criterion(embedding, label) # loss
        metric(non_zero_triplets)
        triplet_sum_loss += triplet_loss.item() * len(data)
        #logger.log_value('non_zero_triplets', metric.value())
        #logger.log_value('triplet_loss', triplet_sum_loss / sum_samples)

        # generator loss
        #z = torch.rand(spks_per_batch * utts_per_spk, latent_dim).to(device) # 
        z = torch.rand(spks_per_batch * utts_per_spk, latent_dim)
        #print("embedding is ", embedding)
        #print("z is ", z)
        fake_fbank = generator(embedding, z) # fake fbank
        #print("fake fbank is ", fake_fbank)
        validity = discriminator(fake_fbank) # 
        #print("validity is ", validity)
        #print("valid is ", valid)
        g_loss = bce_criterion(validity, valid) # 计算loss
        #logger.log_value('g_loss', g_loss)
        triplet_g_loss = 0.2 * g_loss + 0.1 * triplet_loss
        triplet_g_loss.backward() # bp
        generator_optimizer.step()

        generator_loss += g_loss.item() * len(data)

        # discriminator loss train
        #print(data)
        discriminator_optimizer.zero_grad()
        validity_real = discriminator(data) 
        d_real_loss = bce_criterion(validity_real, valid) # Loss for real images
        validity_fake = discriminator(fake_fbank.detach()) 
        d_fake_loss = bce_criterion(validity_fake, fake) # Loss for fake images
        d_loss = (d_real_loss + d_fake_loss) / 2 # Total discriminator loss
        #logger.log_value('d_loss', d_loss)
        discriminator_loss += d_loss.item() * len(data)
        
        # softmax loss train
        input = torch.cat((data, fake_fbank.detach()), 0)
        label = torch.cat((label, label), -1)
        output = classifier(input)
        softmax_loss = softmax_criterion(output, label)
        #logger.log_value('soft_loss', softmax_loss)
        classifier_loss += softmax_loss.item() * len(data)

        discriminator_classifier_loss = 0.2 * softmax_loss + 0.5 * d_loss
        discriminator_classifier_loss.backward()
        discriminator_optimizer.step()

        #logger.step()

        progress_bar.set_description(
            'Train Epoch: {:3d} [{:4d}/{:4d} ({:3.3f}%)] TriLoss: {:.4f} Nonzero: {:.4f} GenLoss: {:.4f} DisLoss: {:.4f} ClaLoss: {:.4f}'.format(
                epoch, batch_idx + 1, len(train_loader),
                100. * (batch_idx + 1) / len(train_loader),
                triplet_sum_loss / sum_samples,
                metric.value(),
                generator_loss / sum_samples,
                discriminator_loss / sum_samples,
                classifier_loss / sum_samples))

    torch.save({'epoch': epoch, 'encoder_state_dict': encoder.state_dict(),
                'generator_state_dict': generator.state_dict(),
                'discriminator_state_dict': discriminator.state_dict(),
                'classifier_state_dict': classifier.state_dict(),
                'generator_optimizer': generator_optimizer.state_dict(),
                'discriminator_optimizer': discriminator_optimizer.state_dict()},
                '{}/net_{}.pth'.format(model_dir, epoch)) # net_{}.pth
    torch.save({'epoch': epoch, 'encoder_state_dict': encoder.state_dict(),
                'generator_state_dict': generator.state_dict(),
                'discriminator_state_dict': discriminator.state_dict(),
                'classifier_state_dict': classifier.state_dict(),
                'generator_optimizer': generator_optimizer.state_dict(),
                'discriminator_optimizer': discriminator_optimizer.state_dict()},
                '{}/net.pth'.format(final_dir)) # net.pth
    return data.detach(), fake_fbank.detach()
```
$\endgroup$
4
  • $\begingroup$ not being able to learn can have very many reasons: bad initialization, network too small, too big, insufficiently diverse data, data too diverse for the sample size… It's also not clear what exactly you're learning, where your GAN comes into play etc. So: yes, you'll need to supply us with much much more detail of your overall thing! $\endgroup$ Commented May 11, 2022 at 16:09
  • $\begingroup$ Thanks for adding the architecture! $\endgroup$ Commented May 11, 2022 at 17:22
  • $\begingroup$ Welcome! If any additional details are needed, I will add them. $\endgroup$
    – yba
    Commented May 11, 2022 at 17:24
  • $\begingroup$ Would this question fit better over on ai.stackexchange.com? $\endgroup$
    – Gillespie
    Commented May 12, 2022 at 21:03

0

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service and acknowledge you have read our privacy policy.