Python não pode aplicar fit_generator ao modelo keras com entrada múltipla

Oct 25 2020

Eu tenho o seguinte modelo - este é LSTM + CNN com 3 entradas.

E eu construí esta função de gerador para treinar o modelo usando fit_generator (com base nisso: https://stanford.edu/~shervine/blog/keras-how-to-generate-data-on-the-fly):

class MultiInputDataGenerator(keras.utils.Sequence):
    'Generates data for Keras'

    def __init__(self, list_IDs, labels, shuffle=True):
        'Initialization'
        self.batch_size = 8
        self.labels = labels
        self.list_IDs = list_IDs
        self.n_classes = 5
        self.shuffle = shuffle
        self.on_epoch_end()

def __len__(self):
    'Denotes the number of batches per epoch'
    return int(np.floor(len(self.list_IDs) / self.batch_size))

def __getitem__(self, index):
    'Generate one batch of data'
    # Generate indexes of the batch
    indexes = self.indexes[index*self.batch_size:(index+1)*self.batch_size]

    # Find list of IDs
    list_IDs_temp = [self.list_IDs[k] for k in indexes]

    # Generate data
    X, y = self.__data_generation(list_IDs_temp)

    return X, y

def on_epoch_end(self):
    'Updates indexes after each epoch'
    self.indexes = np.arange(len(self.list_IDs))
    if self.shuffle == True:
        np.random.shuffle(self.indexes)

def __data_generation(self, list_IDs_temp):
    'Generates data containing batch_size samples' # X : (n_samples, *dim, n_channels)
    # Initialization
    
    X = np.empty((self.batch_size, 1, 3), dtype=object)
    y = np.empty((self.batch_size), dtype=object)

    # Generate data
    for i, ID in enumerate(list_IDs_temp):
        X_id = []
        x_features = df.iloc[id][et_cols].values #ET_COLS are 14 columns so I get 1X14 here
        x_text = df.iloc[id].text_col #x_text is 1X768
        x_vid = df.iloc[id].frame_col #x_vid is (3,244,244)
        
        X_id.append(x_features)
        X_id.append(x_text) 
        X_id.append(x_vid)
           
        X[i,] = X_id
        y[i] = self.labels[ID]

    y_mat = tf.convert_to_tensor(pd.get_dummies(y))
    return X, y_mat


training_generator = MultiModelDataGenerator(generator_partition['train'], generator_labels)
validation_generator = MultiModelDataGenerator(generator_partition['val'], generator_labels)
net = build_LSTMCNN_net()
net.compile(keras.optimizers.Adam(0.001),'categorical_crossentropy',metrics=['acc'])
net.fit_generator(generator=training_generator,
                    validation_data=validation_generator,)
                    use_multiprocessing=True)#,    workers=6)

E recebo o erro:

---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-38-669153f703e6> in <module>()
      
      net.fit_generator(generator=training_generator,
--->                      validation_data=validation_generator,)
                          #use_multiprocessing=True)#,    workers=6)

/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/constant_op.py in convert_to_eager_tensor(value, ctx, dtype)
     96       dtype = dtypes.as_dtype(dtype).as_datatype_enum
     97   ctx.ensure_initialized()
---> 98   return ops.EagerTensor(value, ctx.device_name, dtype)
     99 
    100 

ValueError: Failed to convert a NumPy array to a Tensor (Unsupported object type numpy.ndarray).

Também tentei várias variações, como adicionar:

x_features = np.asarray(x_features).astype(object)
x_text = np.asarray(x_text).astype(object)
x_vid = np.asarray(x_text).astype(object)

Ou em X[i,] = [X_id]vez de X[i,] = X_idMas nada funcionou Alguma ideia de como resolver o problema?

Editado: Ao adicionar:

astype(np.float32) 

e
tf.convert_to_tensor(X)

Recebo o erro: ValueError Traceback (chamada mais recente, última) em ()

      net.fit_generator(generator=training_generator,
--->                      validation_data=validation_generator,
                          use_multiprocessing=True,    workers=6)


/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/constant_op.py in convert_to_eager_tensor(value, ctx, dtype)
            dtype = dtypes.as_dtype(dtype).as_datatype_enum
        ctx.ensure_initialized()
--->    return ops.EagerTensor(value, ctx.device_name, dtype)
  

Respostas

meTchaikovsky Oct 30 2020 at 05:57

Antes de resolver o problema, vamos primeiro resumir o conjunto de dados com o qual você está trabalhando. Com base na sua descrição, criei um exemplo DataFrameque pode se parecer com o seu

import pandas as pd

dataset_size = 500
train_idx,val_idx = train_test_split(range(dataset_size),test_size=0.2,) 

# create an example DataFrame that I assume will be resemble yours 
example_df = pd.DataFrame({'vids':np.random.randint(0,10000,dataset_size)})
# create feature columns 
for ind in range(14): example_df['feature_%i' % ind] = np.random.rand(dataset_size)
# each cell contains a list 
example_df['text'] = np.random.randint(dataset_size)
example_df['text'] = example_df['text'].astype('object')
for ind in range(dataset_size):example_df.at[ind,'text'] = np.random.rand(768).tolist()
# create the label column
example_df['label'] = np.random.randint(low=0,high=5,size=dataset_size)

# extract information from the dataframe, and create data generators 
all_vids = example_df['vids'].values
feature_columns = ['feature_%i' % ind for ind in range(14)]
all_features = example_df[feature_columns].values
all_text = example_df['text'].values
all_labels = example_df['label'].values

Como você pode ver, a coluna texté uma coluna de listas, em que cada lista contém 768 itens. A coluna labelscontém os rótulos dos exemplos, não importa se você usa codificação one-hot ou outros tipos de codificação, desde que sua forma corresponda à forma da camada de saída do modelo geral de rede neural. A coluna vidsé uma coluna de seeds para gerar imagens aleatórias em tempo real.


Resolvendo o problema (com base no conjunto de dados acima)

Você pode usar esta sintaxe return {'feature':features,'text':text,'vid':vid},ypara o método __getitem__, em vez de empilhar as três matrizes de entrada.

Para explicar isso, vamos primeiro construir um modelo de brinquedo parecido com o seu

from tensorflow.keras.models import Model
from tensorflow.keras.layers import Input,Dense,Flatten,Add


def features_part(x):
    y = Dense(14)(x)
    y = Dense(10,activation='linear')(y)
    return y

def text_part(x):
    y = Dense(768)(x)
    y = Dense(10,activation='linear')(y)
    return y

def vid_part(x):
    y = Flatten()(x)
    y = Dense(10,activation='linear')(y)
    return y

input_features = Input(shape=(14,),name='feature')
input_text = Input(shape=(768,),name='text')
input_vid = Input(shape=(3,244,244,),name='vid')

feature_block = features_part(input_features)
text_block = text_part(input_text)
vid_block = vid_part(input_vid)
added = Add()([feature_block,text_block,vid_block])
# you have five classes at the end of the day 
pred = Dense(1)(added)
# build model
model = Model(inputs=[input_features,input_text,input_vid],outputs=pred)
model.compile(loss='mae',optimizer='adam',metrics=['mae'])

A coisa mais importante sobre este modelo é que eu especifiquei os nomes das três camadas de entrada

input_features = Input(shape=(14,),name='feature')
input_text = Input(shape=(768,),name='text')
input_vid = Input(shape=(3,244,244,),name='vid')

Para este modelo, você pode construir um gerador como

# provide a seed for generating a random image 
def fn2img(seed):
    np.random.seed(seed)
    # fake an image with three channels 
    return np.random.randint(low=0,high=255,size=(3,244,244))


class MultiInputDataGenerator(keras.utils.Sequence):

    def __init__(self, 
                 all_inds,labels, 
                 features,text,vid, 
                 shuffle=True):
        self.batch_size = 8
        self.labels = labels
        self.all_inds = all_inds
        self.shuffle = shuffle
        self.on_epoch_end()
        
        self.features = features
        self.text = text
        self.vid = vid

    def __len__(self): 
        return int(np.floor(len(self.all_inds) / self.batch_size))


    def __getitem__(self,index):
        indexes = self.indexes[index*self.batch_size:(index+1)*self.batch_size]
        batch_indices = [self.all_inds[k] for k in indexes]
        features,text,vid,y = self.__data_generation(batch_indices)

        return {'feature':features,'text':text,'vid':vid},y

    def on_epoch_end(self):
        self.indexes = np.arange(len(self.all_inds))
        if self.shuffle == True:
            np.random.shuffle(self.indexes)

    def __data_generation(self,batch_indices):
        # Generate data
        features = self.features[batch_indices,:]
        # note that you need to stack the slice in order to reshape it to (num_samples,768)
        text = np.stack(self.text[batch_indices])
        # since batch_size is not a super large number, you can stack here
        vid = np.stack([fn2img(seed) for seed in self.vid[batch_indices]])
        y = self.labels[batch_indices]

        return features,text,vid,y

como você pode ver, o __getitem__método retorna um dicionário {'feature':features,'text':text,'vid':vid},y. As chaves do dicionário correspondem aos nomes das três camadas de entrada. Além disso, as imagens aleatórias são geradas instantaneamente.

Para ter certeza de que tudo funciona, você pode executar o script abaixo,

import numpy as np
import pandas as pd
from tensorflow import keras 
from sklearn.model_selection import train_test_split

from tensorflow.keras.models import Model
from tensorflow.keras.layers import Input,Dense,Flatten,Add


# provide a seed for generating a random image
def fn2img(seed):
    np.random.seed(seed)
    # fake an image with three channels
    return np.random.randint(low=0,high=255,size=(3,244,244))


class MultiInputDataGenerator(keras.utils.Sequence):

    def __init__(self,
                 all_inds,labels,
                 features,text,vid,
                 shuffle=True):
        self.batch_size = 8
        self.labels = labels
        self.all_inds = all_inds
        self.shuffle = shuffle
        self.on_epoch_end()
        
        self.features = features
        self.text = text
        self.vid = vid

    def __len__(self):
        return int(np.floor(len(self.all_inds) / self.batch_size))


    def __getitem__(self,index):
        indexes = self.indexes[index*self.batch_size:(index+1)*self.batch_size]
        batch_indices = [self.all_inds[k] for k in indexes]
        features,text,vid,y = self.__data_generation(batch_indices)

        return {'feature':features,'text':text,'vid':vid},y

    def on_epoch_end(self):
        self.indexes = np.arange(len(self.all_inds))
        if self.shuffle == True:
            np.random.shuffle(self.indexes)

    def __data_generation(self,batch_indices):
        # Generate data
        features = self.features[batch_indices,:]
        # note that you need to stack the slice in order to reshape it to (num_samples,768)
        text = np.stack(self.text[batch_indices])
        # since batch_size is not a super large number, you can stack here
        vid = np.stack([fn2img(seed) for seed in self.vid[batch_indices]])
        y = self.labels[batch_indices]

        return features,text,vid,y


# fake a dataset
dataset_size = 500
train_idx,val_idx = train_test_split(range(dataset_size),test_size=0.2,)

# create an example DataFrame that I assume will be resemble yours
example_df = pd.DataFrame({'vids':np.random.randint(0,10000,dataset_size)})
# create feature columns
for ind in range(14): example_df['feature_%i' % ind] = np.random.rand(dataset_size)
# each cell contains a list
example_df['text'] = np.random.randint(dataset_size)
example_df['text'] = example_df['text'].astype('object')
for ind in range(dataset_size):example_df.at[ind,'text'] = np.random.rand(768).tolist()
# create the label column
example_df['label'] = np.random.randint(low=0,high=5,size=dataset_size)

# extract information from the dataframe, and create data generators
all_vids = example_df['vids'].values
feature_columns = ['feature_%i' % ind for ind in range(14)]
all_features = example_df[feature_columns].values
all_text = example_df['text'].values
all_labels = example_df['label'].values

training_generator = MultiInputDataGenerator(train_idx,all_labels,all_features,all_text,all_vids)

# create model
def features_part(x):
    y = Dense(14)(x)
    y = Dense(10,activation='linear')(y)
    return y

def text_part(x):
    y = Dense(768)(x)
    y = Dense(10,activation='linear')(y)
    return y

def vid_part(x):
    y = Flatten()(x)
    y = Dense(10,activation='linear')(y)
    return y

input_features = Input(shape=(14,),name='feature')
input_text = Input(shape=(768,),name='text')
input_vid = Input(shape=(3,244,244,),name='vid')

feature_block = features_part(input_features)
text_block = text_part(input_text)
vid_block = vid_part(input_vid)
added = Add()([feature_block,text_block,vid_block])
# you have five classes at the end of the day 
pred = Dense(1)(added)
# build model
model = Model(inputs=[input_features,input_text,input_vid],outputs=pred)
model.compile(loss='mae',optimizer='adam',metrics=['mae'])

model.fit_generator(generator=training_generator,epochs=10)

print(model.history.history)