Python non può applicare fit_generator al modello keras con più input

Question

Ho il seguente modello: questo è LSTM + CNN con 3 ingressi.

E ho costruito questa funzione generatore per addestrare il modello utilizzando fit_generator (basato su questo: https://stanford.edu/~shervine/blog/keras-how-to-generate-data-on-the-fly):

class MultiInputDataGenerator(keras.utils.Sequence):
    'Generates data for Keras'

    def __init__(self, list_IDs, labels, shuffle=True):
        'Initialization'
        self.batch_size = 8
        self.labels = labels
        self.list_IDs = list_IDs
        self.n_classes = 5
        self.shuffle = shuffle
        self.on_epoch_end()

def __len__(self):
    'Denotes the number of batches per epoch'
    return int(np.floor(len(self.list_IDs) / self.batch_size))

def __getitem__(self, index):
    'Generate one batch of data'
    # Generate indexes of the batch
    indexes = self.indexes[index*self.batch_size:(index+1)*self.batch_size]

    # Find list of IDs
    list_IDs_temp = [self.list_IDs[k] for k in indexes]

    # Generate data
    X, y = self.__data_generation(list_IDs_temp)

    return X, y

def on_epoch_end(self):
    'Updates indexes after each epoch'
    self.indexes = np.arange(len(self.list_IDs))
    if self.shuffle == True:
        np.random.shuffle(self.indexes)

def __data_generation(self, list_IDs_temp):
    'Generates data containing batch_size samples' # X : (n_samples, *dim, n_channels)
    # Initialization
    
    X = np.empty((self.batch_size, 1, 3), dtype=object)
    y = np.empty((self.batch_size), dtype=object)

    # Generate data
    for i, ID in enumerate(list_IDs_temp):
        X_id = []
        x_features = df.iloc[id][et_cols].values #ET_COLS are 14 columns so I get 1X14 here
        x_text = df.iloc[id].text_col #x_text is 1X768
        x_vid = df.iloc[id].frame_col #x_vid is (3,244,244)
        
        X_id.append(x_features)
        X_id.append(x_text) 
        X_id.append(x_vid)
           
        X[i,] = X_id
        y[i] = self.labels[ID]

    y_mat = tf.convert_to_tensor(pd.get_dummies(y))
    return X, y_mat


training_generator = MultiModelDataGenerator(generator_partition['train'], generator_labels)
validation_generator = MultiModelDataGenerator(generator_partition['val'], generator_labels)
net = build_LSTMCNN_net()
net.compile(keras.optimizers.Adam(0.001),'categorical_crossentropy',metrics=['acc'])
net.fit_generator(generator=training_generator,
                    validation_data=validation_generator,)
                    use_multiprocessing=True)#,    workers=6)

E ottengo l'errore:

---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-38-669153f703e6> in <module>()
      
      net.fit_generator(generator=training_generator,
--->                      validation_data=validation_generator,)
                          #use_multiprocessing=True)#,    workers=6)

/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/constant_op.py in convert_to_eager_tensor(value, ctx, dtype)
     96       dtype = dtypes.as_dtype(dtype).as_datatype_enum
     97   ctx.ensure_initialized()
---> 98   return ops.EagerTensor(value, ctx.device_name, dtype)
     99 
    100 

ValueError: Failed to convert a NumPy array to a Tensor (Unsupported object type numpy.ndarray).

Ho anche provato diverse varianti come l'aggiunta di:

x_features = np.asarray(x_features).astype(object)
x_text = np.asarray(x_text).astype(object)
x_vid = np.asarray(x_text).astype(object)

O X[i,] = [X_id]invece di X[i,] = X_idMa nessuno ha funzionato Qualche idea su come risolvere il problema?

Modificato: quando si aggiunge:

astype(np.float32) 

e
tf.convert_to_tensor(X)

Ottengo l'errore: ValueError Traceback (la chiamata più recente per ultima) in ()

      net.fit_generator(generator=training_generator,
--->                      validation_data=validation_generator,
                          use_multiprocessing=True,    workers=6)


/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/constant_op.py in convert_to_eager_tensor(value, ctx, dtype)
            dtype = dtypes.as_dtype(dtype).as_datatype_enum
        ctx.ensure_initialized()
--->    return ops.EagerTensor(value, ctx.device_name, dtype)
  

Answer 1

Prima di risolvere il problema, riassumiamo prima il set di dati con cui stai lavorando. Sulla base della tua descrizione, ho creato un esempio DataFrameche potrebbe assomigliare al tuo

import pandas as pd

dataset_size = 500
train_idx,val_idx = train_test_split(range(dataset_size),test_size=0.2,) 

# create an example DataFrame that I assume will be resemble yours 
example_df = pd.DataFrame({'vids':np.random.randint(0,10000,dataset_size)})
# create feature columns 
for ind in range(14): example_df['feature_%i' % ind] = np.random.rand(dataset_size)
# each cell contains a list 
example_df['text'] = np.random.randint(dataset_size)
example_df['text'] = example_df['text'].astype('object')
for ind in range(dataset_size):example_df.at[ind,'text'] = np.random.rand(768).tolist()
# create the label column
example_df['label'] = np.random.randint(low=0,high=5,size=dataset_size)

# extract information from the dataframe, and create data generators 
all_vids = example_df['vids'].values
feature_columns = ['feature_%i' % ind for ind in range(14)]
all_features = example_df[feature_columns].values
all_text = example_df['text'].values
all_labels = example_df['label'].values

Come puoi vedere, la colonna textè una colonna di elenchi, in cui ogni elenco contiene 768 elementi. La colonna labelscontiene le etichette degli esempi, non importa se si utilizza la codifica one-hot o altri tipi di codifica, purché la sua forma corrisponda alla forma del livello di output del modello di rete neurale complessivo. La colonna vidsè una colonna di seeds per la generazione di immagini casuali al volo.

---

Risoluzione del problema (in base al set di dati sopra)

È possibile utilizzare questa sintassi return {'feature':features,'text':text,'vid':vid},yper il metodo __getitem__, invece di impilare i tre array di input.

Per spiegare questo, costruiamo prima un modello di giocattolo simile al tuo

from tensorflow.keras.models import Model
from tensorflow.keras.layers import Input,Dense,Flatten,Add


def features_part(x):
    y = Dense(14)(x)
    y = Dense(10,activation='linear')(y)
    return y

def text_part(x):
    y = Dense(768)(x)
    y = Dense(10,activation='linear')(y)
    return y

def vid_part(x):
    y = Flatten()(x)
    y = Dense(10,activation='linear')(y)
    return y

input_features = Input(shape=(14,),name='feature')
input_text = Input(shape=(768,),name='text')
input_vid = Input(shape=(3,244,244,),name='vid')

feature_block = features_part(input_features)
text_block = text_part(input_text)
vid_block = vid_part(input_vid)
added = Add()([feature_block,text_block,vid_block])
# you have five classes at the end of the day 
pred = Dense(1)(added)
# build model
model = Model(inputs=[input_features,input_text,input_vid],outputs=pred)
model.compile(loss='mae',optimizer='adam',metrics=['mae'])

La cosa più importante di questo modello è che ho specificato i nomi dei tre livelli di input

input_features = Input(shape=(14,),name='feature')
input_text = Input(shape=(768,),name='text')
input_vid = Input(shape=(3,244,244,),name='vid')

Per questo modello, puoi costruire un generatore come

# provide a seed for generating a random image 
def fn2img(seed):
    np.random.seed(seed)
    # fake an image with three channels 
    return np.random.randint(low=0,high=255,size=(3,244,244))


class MultiInputDataGenerator(keras.utils.Sequence):

    def __init__(self, 
                 all_inds,labels, 
                 features,text,vid, 
                 shuffle=True):
        self.batch_size = 8
        self.labels = labels
        self.all_inds = all_inds
        self.shuffle = shuffle
        self.on_epoch_end()
        
        self.features = features
        self.text = text
        self.vid = vid

    def __len__(self): 
        return int(np.floor(len(self.all_inds) / self.batch_size))


    def __getitem__(self,index):
        indexes = self.indexes[index*self.batch_size:(index+1)*self.batch_size]
        batch_indices = [self.all_inds[k] for k in indexes]
        features,text,vid,y = self.__data_generation(batch_indices)

        return {'feature':features,'text':text,'vid':vid},y

    def on_epoch_end(self):
        self.indexes = np.arange(len(self.all_inds))
        if self.shuffle == True:
            np.random.shuffle(self.indexes)

    def __data_generation(self,batch_indices):
        # Generate data
        features = self.features[batch_indices,:]
        # note that you need to stack the slice in order to reshape it to (num_samples,768)
        text = np.stack(self.text[batch_indices])
        # since batch_size is not a super large number, you can stack here
        vid = np.stack([fn2img(seed) for seed in self.vid[batch_indices]])
        y = self.labels[batch_indices]

        return features,text,vid,y

come puoi vedere, il __getitem__metodo restituisce un dizionario {'feature':features,'text':text,'vid':vid},y. Le chiavi del dizionario corrispondono ai nomi dei tre livelli di input. Inoltre, le immagini casuali vengono generate al volo.

Per assicurarti che tutto funzioni, puoi eseguire lo script seguente,

import numpy as np
import pandas as pd
from tensorflow import keras 
from sklearn.model_selection import train_test_split

from tensorflow.keras.models import Model
from tensorflow.keras.layers import Input,Dense,Flatten,Add


# provide a seed for generating a random image
def fn2img(seed):
    np.random.seed(seed)
    # fake an image with three channels
    return np.random.randint(low=0,high=255,size=(3,244,244))


class MultiInputDataGenerator(keras.utils.Sequence):

    def __init__(self,
                 all_inds,labels,
                 features,text,vid,
                 shuffle=True):
        self.batch_size = 8
        self.labels = labels
        self.all_inds = all_inds
        self.shuffle = shuffle
        self.on_epoch_end()
        
        self.features = features
        self.text = text
        self.vid = vid

    def __len__(self):
        return int(np.floor(len(self.all_inds) / self.batch_size))


    def __getitem__(self,index):
        indexes = self.indexes[index*self.batch_size:(index+1)*self.batch_size]
        batch_indices = [self.all_inds[k] for k in indexes]
        features,text,vid,y = self.__data_generation(batch_indices)

        return {'feature':features,'text':text,'vid':vid},y

    def on_epoch_end(self):
        self.indexes = np.arange(len(self.all_inds))
        if self.shuffle == True:
            np.random.shuffle(self.indexes)

    def __data_generation(self,batch_indices):
        # Generate data
        features = self.features[batch_indices,:]
        # note that you need to stack the slice in order to reshape it to (num_samples,768)
        text = np.stack(self.text[batch_indices])
        # since batch_size is not a super large number, you can stack here
        vid = np.stack([fn2img(seed) for seed in self.vid[batch_indices]])
        y = self.labels[batch_indices]

        return features,text,vid,y


# fake a dataset
dataset_size = 500
train_idx,val_idx = train_test_split(range(dataset_size),test_size=0.2,)

# create an example DataFrame that I assume will be resemble yours
example_df = pd.DataFrame({'vids':np.random.randint(0,10000,dataset_size)})
# create feature columns
for ind in range(14): example_df['feature_%i' % ind] = np.random.rand(dataset_size)
# each cell contains a list
example_df['text'] = np.random.randint(dataset_size)
example_df['text'] = example_df['text'].astype('object')
for ind in range(dataset_size):example_df.at[ind,'text'] = np.random.rand(768).tolist()
# create the label column
example_df['label'] = np.random.randint(low=0,high=5,size=dataset_size)

# extract information from the dataframe, and create data generators
all_vids = example_df['vids'].values
feature_columns = ['feature_%i' % ind for ind in range(14)]
all_features = example_df[feature_columns].values
all_text = example_df['text'].values
all_labels = example_df['label'].values

training_generator = MultiInputDataGenerator(train_idx,all_labels,all_features,all_text,all_vids)

# create model
def features_part(x):
    y = Dense(14)(x)
    y = Dense(10,activation='linear')(y)
    return y

def text_part(x):
    y = Dense(768)(x)
    y = Dense(10,activation='linear')(y)
    return y

def vid_part(x):
    y = Flatten()(x)
    y = Dense(10,activation='linear')(y)
    return y

input_features = Input(shape=(14,),name='feature')
input_text = Input(shape=(768,),name='text')
input_vid = Input(shape=(3,244,244,),name='vid')

feature_block = features_part(input_features)
text_block = text_part(input_text)
vid_block = vid_part(input_vid)
added = Add()([feature_block,text_block,vid_block])
# you have five classes at the end of the day 
pred = Dense(1)(added)
# build model
model = Model(inputs=[input_features,input_text,input_vid],outputs=pred)
model.compile(loss='mae',optimizer='adam',metrics=['mae'])

model.fit_generator(generator=training_generator,epochs=10)

print(model.history.history)