FastText

Label (space separated): natural language processing

FastText

FastText paper link

Review

FastText is not a special kind of institution, but an idea that is to get results faster.

FastText(pytorch) for text categorization

class FastText(BasicModule):
    def __init__(self, config, vectors=None):
        super(FastText, self).__init__()
        self.config = config

        self.embedding = nn.Embedding(config.vocab_size, config.embedding_dim)
        if vectors is not None:
            self.embedding.weight.data.copy_(vectors) ## Embed the word vector into self.embedding

        self.pre = nn.Sequential(
            ## This is equivalent to converting the original 300-dimensional vector into 600-dimensional
            nn.Linear(config.embedding_dim, config.embedding_dim * 2),
            nn.BatchNorm1d(config.embedding_dim * 2),
            nn.ReLU(True)
        )
        self.fc = nn.Sequential(
            ## Here we will convert the vector of 600 dimensions into 19 dimensions, that is, the dimension transformed into the desired y (target value) by two linear layers. The following line is the first linear layer.
            nn.Linear(config.embedding_dim * 2, config.linear_hidden_size),
            nn.BatchNorm1d(config.linear_hidden_size),
            nn.ReLU(inplace=True),
            ## Here is the second linear layer
            nn.Linear(config.linear_hidden_size, config.label_size)
        )

    def forward(self, sentence):
        embed = self.embedding(sentence)  # seq * batch * emb 2000 * 64 * 300 ## Every sentence here is 64 sentences, 2000 words per sentence, so the dimension of the sentence is 2000*64
        embed_size = embed.size() ## 2000 * 64 * 300
        embed.contiguous().view(-1, self.config.embedding_dim) # 128000 * 300 This is equivalent to splicing every word in every sentence in the batch, so 12800 is like this.
        out = self.pre(embed.contiguous().view(-1, self.config.embedding_dim)).view(embed_size[0], embed_size[1], -1)
        mean_out = torch.mean(out, dim=0).squeeze()  # batch * 2emb 64 ​​* 600 ## This is equivalent to averaging each sentence of 2000 length

        logit = self.fc(mean_out)
        return logit