I'm a beginner.
If I am using a Convolutional Neural Networks with Triplet Loss as a loss function (also combined with GAN and a Classifier) for building a model that performs Speaker Verification, what labels should I use for the training process?
I used the speakers' numbers as labels for the training but the model actually doesn't learn anything (and embeddings are all similar) and I think that this is the cause of the problem. Are the labels should be 0 and 1? If any additional details are needed, I will add them.
First, in the pre-processing stage, the model has to read audio samples from the dataset (dataset is TIMIT) and perform features extraction (which has to be done with Wavelet), after this, the images are fed into the Encoder, a Convolutional Neural Network with 4 layers, kernel size is 5, this is the Encoder class:
class Encoder(nn.Module):
def __init__(self, expansion, blocks, embedding_dim):
super(Encoder, self).__init__()
self.convnet = self._make_layers(expansion, blocks)
self.pool = nn.AdaptiveAvgPool2d([2,2])
self.fc = nn.Linear(expansion * blocks * 2 * 2 * 2, embedding_dim)
def _make_layers(self, expansion, blocks):
layers = []
for i in range(blocks):
if i == 0:
layers.append(nn.Conv2d(1, expansion, kernel_size = 5, stride = 2, padding = 2))
else:
layers.append(nn.Conv2d(expansion, expansion * 2, kernel_size = 5, stride = 2, padding = 2))
expansion = expansion * 2
layers.append(nn.BatchNorm2d(expansion))
layers.append(nn.ReLU())
return nn.Sequential(*layers)
def forward(self, x):
x = self.convnet(x)
x = self.pool(x)
x = torch.flatten(x, start_dim = 1)
x = self.fc(x)
return F.normalize(x)
The number of speakers per batch is 20 and utterances per speaker is 10.
It is a bit complicated to explain the whole model architecture so I'm adding an image: 
This is the train function:
def train(epoch,
encoder,
generator,
discriminator,
classifier,
triplet_criterion,
bce_criterion,
softmax_criterion,
generator_optimizer,
discriminator_optimizer,
train_loader,
metric): #
encoder.train() #
generator.train()
discriminator.train()
classifier.train()
triplet_sum_loss, generator_loss, discriminator_loss, classifier_loss, sum_samples = 0, 0, 0, 0, 0
progress_bar = tqdm(enumerate(train_loader))
for batch_idx, (data, label) in progress_bar:
sum_samples += len(data)
#data = data.to(device)
#label = label.to(device)
# Adversarial ground truths
#valid = torch.FloatTensor(utts_per_spk * spks_per_batch, 1).fill_(1.0).to(device)
#fake = torch.FloatTensor(utts_per_spk * spks_per_batch, 1).fill_(0.0).to(device)
valid = torch.FloatTensor(utts_per_spk * spks_per_batch, 1).fill_(1.0)
fake = torch.FloatTensor(utts_per_spk * spks_per_batch, 1).fill_(0.0)
# triplet loss
generator_optimizer.zero_grad()
#print(data)
embedding = encoder(data)
#print(embedding)
triplet_loss, non_zero_triplets = triplet_criterion(embedding, label) # loss
metric(non_zero_triplets)
triplet_sum_loss += triplet_loss.item() * len(data)
#logger.log_value('non_zero_triplets', metric.value())
#logger.log_value('triplet_loss', triplet_sum_loss / sum_samples)
# generator loss
#z = torch.rand(spks_per_batch * utts_per_spk, latent_dim).to(device) #
z = torch.rand(spks_per_batch * utts_per_spk, latent_dim)
#print("embedding is ", embedding)
#print("z is ", z)
fake_fbank = generator(embedding, z) # fake fbank
#print("fake fbank is ", fake_fbank)
validity = discriminator(fake_fbank) #
#print("validity is ", validity)
#print("valid is ", valid)
g_loss = bce_criterion(validity, valid) # 计算loss
#logger.log_value('g_loss', g_loss)
triplet_g_loss = 0.2 * g_loss + 0.1 * triplet_loss
triplet_g_loss.backward() # bp
generator_optimizer.step()
generator_loss += g_loss.item() * len(data)
# discriminator loss train
#print(data)
discriminator_optimizer.zero_grad()
validity_real = discriminator(data)
d_real_loss = bce_criterion(validity_real, valid) # Loss for real images
validity_fake = discriminator(fake_fbank.detach())
d_fake_loss = bce_criterion(validity_fake, fake) # Loss for fake images
d_loss = (d_real_loss + d_fake_loss) / 2 # Total discriminator loss
#logger.log_value('d_loss', d_loss)
discriminator_loss += d_loss.item() * len(data)
# softmax loss train
input = torch.cat((data, fake_fbank.detach()), 0)
label = torch.cat((label, label), -1)
output = classifier(input)
softmax_loss = softmax_criterion(output, label)
#logger.log_value('soft_loss', softmax_loss)
classifier_loss += softmax_loss.item() * len(data)
discriminator_classifier_loss = 0.2 * softmax_loss + 0.5 * d_loss
discriminator_classifier_loss.backward()
discriminator_optimizer.step()
#logger.step()
progress_bar.set_description(
'Train Epoch: {:3d} [{:4d}/{:4d} ({:3.3f}%)] TriLoss: {:.4f} Nonzero: {:.4f} GenLoss: {:.4f} DisLoss: {:.4f} ClaLoss: {:.4f}'.format(
epoch, batch_idx + 1, len(train_loader),
100. * (batch_idx + 1) / len(train_loader),
triplet_sum_loss / sum_samples,
metric.value(),
generator_loss / sum_samples,
discriminator_loss / sum_samples,
classifier_loss / sum_samples))
torch.save({'epoch': epoch, 'encoder_state_dict': encoder.state_dict(),
'generator_state_dict': generator.state_dict(),
'discriminator_state_dict': discriminator.state_dict(),
'classifier_state_dict': classifier.state_dict(),
'generator_optimizer': generator_optimizer.state_dict(),
'discriminator_optimizer': discriminator_optimizer.state_dict()},
'{}/net_{}.pth'.format(model_dir, epoch)) # net_{}.pth
torch.save({'epoch': epoch, 'encoder_state_dict': encoder.state_dict(),
'generator_state_dict': generator.state_dict(),
'discriminator_state_dict': discriminator.state_dict(),
'classifier_state_dict': classifier.state_dict(),
'generator_optimizer': generator_optimizer.state_dict(),
'discriminator_optimizer': discriminator_optimizer.state_dict()},
'{}/net.pth'.format(final_dir)) # net.pth
return data.detach(), fake_fbank.detach()
```
