
import numpy as np
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers
from tensorflow.keras.utils import plot_model

# define model
vocab_size = 1000
embed_dim = 50  # tamanho do Embedding de cada token
maxlen = 200  # Tamanho máximo da sentença


# Seu código aqui
tokenizer_layer = tf.keras.layers.experimental.preprocessing.TextVectorization(
    standardize='lower_and_strip_punctuation',
    split='whitespace', ngrams=None, output_mode="int",
    output_sequence_length=maxlen, pad_to_max_tokens=False)
tokenizer_layer.adapt(np.array(["O rato roeu a roupa do rei de roma"]))

inputs = layers.Input(shape=(1,), dtype=tf.string, name='input_text')
embedding_layer = layers.Embedding(input_dim=vocab_size, output_dim=embed_dim)


x = tokenizer_layer(inputs)    
x = embedding_layer(x)
x = layers.LSTM(units=1)(x)
x = layers.Dropout(0.1)(x)
x = layers.Dense(20, activation="relu")(x)
x = layers.Dropout(0.1)(x)
outputs = layers.Dense(2, activation="softmax")(x)
model = keras.Model(inputs=inputs, outputs=outputs)


model.compile(loss="categorical_crossentropy", metrics=["accuracy"])


from IPython.display import Video

Video("images/TransformerEncoderExample.mp4")


￼

class MultiHeadSelfAttention(layers.Layer):
    def __init__(self, embed_dim, num_heads ):
        super(MultiHeadSelfAttention, self).__init__()
        if embed_dim % num_heads != 0:
            raise ValueError(
                f"dimensão do embedding = {embed_dim} deve ser "  +
                "divisível pelo n. de cabeças = {num_heads}"
            )
        self.embed_dim = embed_dim
        self.num_heads = num_heads
        self.projection_dim = embed_dim // num_heads
        self.query_dense = layers.Dense(embed_dim)
        self.key_dense = layers.Dense(embed_dim)
        self.value_dense = layers.Dense(embed_dim)
        self.combine_heads = layers.Dense(embed_dim)

    def attention(self, query, key, value):
        score = tf.matmul(query, key, transpose_b=True)
        dim_key = tf.cast(tf.shape(key)[-1], tf.float32)
        scaled_score = score / tf.math.sqrt(dim_key)
        weights = tf.nn.softmax(scaled_score, axis=-1)
        output = tf.matmul(weights, value)
        return output, weights

    def separate_heads(self, x, batch_size):
        # divide o embedding_dim em n_cabeças de projection_dim (batch_size e seq_len são mantidos)
        x = tf.reshape(x, (batch_size, -1, self.num_heads, self.projection_dim))
        # reordena o n_cabeças com a seq_len
        return tf.transpose(x, perm=[0, 2, 1, 3])

    def call(self, inputs):
        # x.shape = [batch_size, seq_len, embedding_dim]
        batch_size = tf.shape(inputs)[0]
        query = self.query_dense(inputs)  # (batch_size, seq_len, embed_dim)
        key = self.key_dense(inputs)      # (batch_size, seq_len, embed_dim)
        value = self.value_dense(inputs)  # (batch_size, seq_len, embed_dim)
        query = self.separate_heads(
            query, batch_size
        )  # (batch_size, num_heads, seq_len, projection_dim)
        key = self.separate_heads(
            key, batch_size
        )  # (batch_size, num_heads, seq_len, projection_dim)
        value = self.separate_heads(
            value, batch_size
        )  # (batch_size, num_heads, seq_len, projection_dim)
        attention, weights = self.attention(query, key, value)
        attention = tf.transpose(
            attention, perm=[0, 2, 1, 3]
        )  # (batch_size, seq_len, num_heads, projection_dim)
        concat_attention = tf.reshape(
            attention, (batch_size, -1, self.embed_dim)
        )  # (batch_size, seq_len, embed_dim)
        output = self.combine_heads(
            concat_attention
        )  # (batch_size, seq_len, embed_dim)
        return output


class TokenAndPositionEmbedding(layers.Layer):
    def __init__(self, maxlen, vocab_size, embed_dim):
        super(TokenAndPositionEmbedding, self).__init__()
        positions = np.array(range(0, maxlen, 1)).reshape(-1, 1)
        even_i = np.array(range(0, embed_dim, 2))
        odd_i = np.array(range(1, embed_dim, 2))

        self.word_embedding = layers.Embedding(
            input_dim=vocab_size, output_dim=embed_dim,
        )
        positional_encoding = np.zeros((maxlen, embed_dim))
        positional_encoding[:, 0::2] = np.sin(
            positions / (10000 ** (even_i / embed_dim))
        )
        positional_encoding[:, 1::2] = np.cos(
            positions / (10000 ** ((odd_i - 1) / embed_dim))
        )
        self.positional_encoding = layers.Embedding(
            input_dim=maxlen, output_dim=embed_dim, weights=[positional_encoding], trainable=False
        )
        #trainable é false pq é uma fórmula só, entao nao precisa aprender

    def call(self, x):  # o que  é x aqui?
        # x é um indice
        positions = tf.range(start=0, limit=maxlen, delta=1)
        positions = self.positional_encoding(positions)
        x = self.word_embedding(x)
        return x + positions


class TransformerBlock(layers.Layer):
    def __init__(self, embed_dim, num_heads, ff_dim, rate=0.1):
        super(TransformerBlock, self).__init__()
        self.att = MultiHeadSelfAttention(embed_dim, num_heads)
        self.ffn = keras.Sequential(
            [layers.Dense(ff_dim, activation="relu"), 
             layers.Dense(embed_dim),
            ]
        )
        self.layernorm1 = layers.LayerNormalization(epsilon=1e-6)
        self.layernorm2 = layers.LayerNormalization(epsilon=1e-6)
        self.dropout1 = layers.Dropout(rate)
        self.dropout2 = layers.Dropout(rate)

    def call(self, inputs, training):
        attn_output = self.att(inputs)
        attn_output = self.dropout1(attn_output, training=training)
        out1 = self.layernorm1(inputs + attn_output)
        ffn_output = self.ffn(out1)
        ffn_output = self.dropout2(ffn_output, training=training)
        return self.layernorm2(out1 + ffn_output)


vocab_size = 30000
embed_dim = 768  # tamanho do Embedding de cada token
num_heads = 12  # N. de cabeças de atenção
ff_dim = 12  # tamanho da camada oculta nas redes feed forward dentro do transformer
maxlen = 512 #ja incluindo CLS e SEP

tokenizer_layer = tf.keras.layers.experimental.preprocessing.TextVectorization(
    standardize="lower_and_strip_punctuation",
    split="whitespace",
    ngrams=None,
    output_mode="int",
    output_sequence_length=maxlen,
    pad_to_max_tokens=False,
)
inputs = layers.Input(shape=(1,), dtype=tf.string, name="input_text")
embedding_layer = TokenAndPositionEmbedding(maxlen, vocab_size, embed_dim)
transformer_block = TransformerBlock(embed_dim, num_heads, ff_dim)


x = tokenizer_layer(inputs)    
x = embedding_layer(x)
for i in range(12):
    x = transformer_block(x)
x = x[:,0,:] #cls token
x = layers.Dropout(0.1)(x)
outputs = layers.Dense(2, activation="softmax")(x)
model = keras.Model(inputs=inputs, outputs=outputs)


plot_model(model, show_shapes=True, show_layer_names=True)


from transformers import BertTokenizer, pipeline, TFBertForPreTraining


REF_MODEL = 'neuralmind/bert-base-portuguese-cased'


model = TFBertForPreTraining.from_pretrained(REF_MODEL, from_pt=True)

Some weights of the PyTorch model were not used when initializing the TF 2.0 model TFBertForPreTraining: ['cls.predictions.decoder.weight']
- This IS expected if you are initializing TFBertForPreTraining from a PyTorch model trained on another task or with another architecture (e.g. initializing a TFBertForSequenceClassification model from a BertForPretraining model).
- This IS NOT expected if you are initializing TFBertForPreTraining from a PyTorch model that you expect to be exactly identical (e.g. initializing a TFBertForSequenceClassification model from a BertForSequenceClassification model).
All the weights of TFBertForPreTraining were initialized from the PyTorch model.
If your task is similar to the task the model of the ckeckpoint was trained on, you can already use TFBertForPreTraining for predictions without further training.


unmasker = pipeline('fill-mask', model=REF_MODEL)
unmasker("Eu bebi muita [MASK] no carnaval")[:3]

Some weights of the model checkpoint at neuralmind/bert-base-portuguese-cased were not used when initializing BertForMaskedLM: ['cls.seq_relationship.weight', 'cls.seq_relationship.bias']
- This IS expected if you are initializing BertForMaskedLM from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPretraining model).
- This IS NOT expected if you are initializing BertForMaskedLM from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).

[{'sequence': '[CLS] Eu bebi muita água no carnaval [SEP]',
  'score': 0.3878271281719208,
  'token': 1991,
  'token_str': 'água'},
 {'sequence': '[CLS] Eu bebi muita cerveja no carnaval [SEP]',
  'score': 0.3672230541706085,
  'token': 18099,
  'token_str': 'cerveja'},
 {'sequence': '[CLS] Eu bebi muita coisa no carnaval [SEP]',
  'score': 0.18478259444236755,
  'token': 5664,
  'token_str': 'coisa'}]


from transformers import BertTokenizer, TFBertModel

tokenizer = BertTokenizer.from_pretrained(REF_MODEL)
model = TFBertModel.from_pretrained(REF_MODEL, from_pt=True)
text = "O rato roeu a roupa do Rei de Roma"
encoded_input = tokenizer(text, return_tensors='tf')
output = model(encoded_input)
output[0].shape, output[1].shape

Some weights of the PyTorch model were not used when initializing the TF 2.0 model TFBertModel: ['cls.predictions.transform.dense.bias', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.seq_relationship.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.bias', 'cls.seq_relationship.weight', 'cls.predictions.decoder.weight']
- This IS expected if you are initializing TFBertModel from a PyTorch model trained on another task or with another architecture (e.g. initializing a TFBertForSequenceClassification model from a BertForPretraining model).
- This IS NOT expected if you are initializing TFBertModel from a PyTorch model that you expect to be exactly identical (e.g. initializing a TFBertForSequenceClassification model from a BertForSequenceClassification model).
All the weights of TFBertModel were initialized from the PyTorch model.
If your task is similar to the task the model of the ckeckpoint was trained on, you can already use TFBertModel for predictions without further training.

(TensorShape([1, 13, 768]), TensorShape([1, 768]))


encoded_input['input_ids'].shape

TensorShape([1, 13])


from transformers import (
    TFBertForSequenceClassification,
    TFBertForMultipleChoice,
    TFBertForTokenClassification,
    TFBertForQuestionAnswering,
)


model = TFBertForSequenceClassification.from_pretrained(REF_MODEL, from_pt=True)
model.summary()

Some weights of the PyTorch model were not used when initializing the TF 2.0 model TFBertForSequenceClassification: ['cls.predictions.transform.dense.bias', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.seq_relationship.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.bias', 'cls.seq_relationship.weight', 'cls.predictions.decoder.weight']
- This IS expected if you are initializing TFBertForSequenceClassification from a PyTorch model trained on another task or with another architecture (e.g. initializing a TFBertForSequenceClassification model from a BertForPretraining model).
- This IS NOT expected if you are initializing TFBertForSequenceClassification from a PyTorch model that you expect to be exactly identical (e.g. initializing a TFBertForSequenceClassification model from a BertForSequenceClassification model).
Some weights or buffers of the TF 2.0 model TFBertForSequenceClassification were not initialized from the PyTorch model and are newly initialized: ['classifier.weight', 'classifier.bias']
You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.

Model: "tf_bert_for_sequence_classification_22"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
bert (TFBertMainLayer)       multiple                  108923136 
_________________________________________________________________
dropout_1400 (Dropout)       multiple                  0         
_________________________________________________________________
classifier (Dense)           multiple                  1538      
=================================================================
Total params: 108,924,674
Trainable params: 108,924,674
Non-trainable params: 0
_________________________________________________________________


model = TFBertForMultipleChoice.from_pretrained(REF_MODEL, from_pt=True)
model.summary()

Some weights of the PyTorch model were not used when initializing the TF 2.0 model TFBertForMultipleChoice: ['cls.predictions.transform.dense.bias', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.seq_relationship.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.bias', 'cls.seq_relationship.weight', 'cls.predictions.decoder.weight']
- This IS expected if you are initializing TFBertForMultipleChoice from a PyTorch model trained on another task or with another architecture (e.g. initializing a TFBertForSequenceClassification model from a BertForPretraining model).
- This IS NOT expected if you are initializing TFBertForMultipleChoice from a PyTorch model that you expect to be exactly identical (e.g. initializing a TFBertForSequenceClassification model from a BertForSequenceClassification model).
Some weights or buffers of the TF 2.0 model TFBertForMultipleChoice were not initialized from the PyTorch model and are newly initialized: ['classifier.weight', 'classifier.bias']
You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.

Model: "tf_bert_for_multiple_choice_4"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
bert (TFBertMainLayer)       multiple                  108923136 
_________________________________________________________________
dropout_1438 (Dropout)       multiple                  0         
_________________________________________________________________
classifier (Dense)           multiple                  769       
=================================================================
Total params: 108,923,905
Trainable params: 108,923,905
Non-trainable params: 0
_________________________________________________________________


model = TFBertForTokenClassification.from_pretrained(REF_MODEL, from_pt=True)
model.summary()

Some weights of the PyTorch model were not used when initializing the TF 2.0 model TFBertForTokenClassification: ['cls.predictions.transform.dense.bias', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.seq_relationship.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.bias', 'cls.seq_relationship.weight', 'cls.predictions.decoder.weight']
- This IS expected if you are initializing TFBertForTokenClassification from a PyTorch model trained on another task or with another architecture (e.g. initializing a TFBertForSequenceClassification model from a BertForPretraining model).
- This IS NOT expected if you are initializing TFBertForTokenClassification from a PyTorch model that you expect to be exactly identical (e.g. initializing a TFBertForSequenceClassification model from a BertForSequenceClassification model).
Some weights or buffers of the TF 2.0 model TFBertForTokenClassification were not initialized from the PyTorch model and are newly initialized: ['classifier.weight', 'classifier.bias']
You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.

Model: "tf_bert_for_token_classification_2"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
bert (TFBertMainLayer)       multiple                  108923136 
_________________________________________________________________
dropout_1516 (Dropout)       multiple                  0         
_________________________________________________________________
classifier (Dense)           multiple                  1538      
=================================================================
Total params: 108,924,674
Trainable params: 108,924,674
Non-trainable params: 0
_________________________________________________________________


model = TFBertForQuestionAnswering.from_pretrained(REF_MODEL, from_pt=True)
model.summary()

Some weights of the PyTorch model were not used when initializing the TF 2.0 model TFBertForQuestionAnswering: ['cls.predictions.transform.dense.bias', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.seq_relationship.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.bias', 'cls.seq_relationship.weight', 'cls.predictions.decoder.weight']
- This IS expected if you are initializing TFBertForQuestionAnswering from a PyTorch model trained on another task or with another architecture (e.g. initializing a TFBertForSequenceClassification model from a BertForPretraining model).
- This IS NOT expected if you are initializing TFBertForQuestionAnswering from a PyTorch model that you expect to be exactly identical (e.g. initializing a TFBertForSequenceClassification model from a BertForSequenceClassification model).
Some weights or buffers of the TF 2.0 model TFBertForQuestionAnswering were not initialized from the PyTorch model and are newly initialized: ['qa_outputs.weight', 'qa_outputs.bias']
You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.

Model: "tf_bert_for_question_answering_4"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
bert (TFBertMainLayer)       multiple                  108923136 
_________________________________________________________________
qa_outputs (Dense)           multiple                  1538      
=================================================================
Total params: 108,924,674
Trainable params: 108,924,674
Non-trainable params: 0
_________________________________________________________________


from transformers import (
    TFRobertaForSequenceClassification,
    TFDistilBertForQuestionAnswering,
)

Agenda¶

Momento Computer Vision em NLP¶

Transfer Learning¶

Mas como isso é possível em NLP?¶

Modelo de Linguagem¶

Regra de Probabilidade Condicional¶

ULMFit¶

Mas e a Arquitetura?¶

Mas atenção não é tudo que a gente precisa?¶

BERT¶

Encoder Layer¶

A Loss no BERT¶

Tarefa 1 Masked Language Model¶

Tarefa 2 Next Sentence Prediction (NSP)¶

Fine Tuning¶

Classificando uma única sentença: Usando a transformação do [CLS]¶

Rotulando uma sentença: Uma FF para cada palavra¶

Arquiteturas de Refinamento¶

BERT e seus irmãos¶

RoBERTa¶

Albert¶

E muitos outros¶

XLNet¶

SpanBERT¶

ELECTRA¶

DistilBERT¶

Miscelânea¶

Gradient Accumulation¶

FP16¶

Obrigado! :)¶