VAE编程报错无法解决

Question

python代码，VAE架构，加入了CNN，处理物联网2023数据集，decoder维度有问题，一直输出报错。

import numpy as np
import tensorflow as tf
from tensorflow.keras.layers import Input, Dense, Lambda,Conv1D,Flatten,Activation, SpatialDropout1D,Reshape
from tensorflow.keras.models import Model,Sequential
from tensorflow.keras import backend as K
from tensorflow.keras.datasets import mnist
from sklearn.preprocessing import MinMaxScaler, StandardScaler

import numpy as np
import pandas as pd
import time

# Load the dataset
# csv文件路径
csv_path_train = 'CICIoT2023/CICIoT2023/benign.csv'
# 读取数据
X_train = pd.read_csv(csv_path_train)
X_train = X_train.values
X_train = np.nan_to_num(MinMaxScaler().fit_transform(StandardScaler().fit_transform(X_train)))

X_train = np.reshape(X_train, (-1, 100, 46))
print(f"train:{X_train.shape}")
idx = np.random.randint(0, X_train.shape[0], 16)
imgs = X_train[idx]
# print(imgs.shape)
print(f"imgs:{imgs.shape}")
# noise = np.random.normal(0, 1, (16, 100, 1))
# # print(noise.shape)
# print(f"noise:{noise.shape}")

# csv文件路径
csv_path_test = 'CICIoT2023/CICIoT2023/ceshi.csv'
Y_test = pd.read_csv(csv_path_test)
Y_test_normal = Y_test[Y_test.label == 'BenignTraffic'].drop(labels='label', axis=1).values
Y_test_normal = np.nan_to_num(MinMaxScaler().fit_transform(StandardScaler().fit_transform(Y_test_normal)))
Y_test_abnormal = Y_test[Y_test.label != 'BenignTraffic'].drop(labels='label', axis=1).values
Y_test_abnormal = np.nan_to_num(MinMaxScaler().fit_transform(StandardScaler().fit_transform(Y_test_abnormal)))
Y_test_normal = np.reshape(Y_test_normal, (-1, 100, 46))
Y_test_abnormal = np.reshape(Y_test_abnormal, (-1, 100, 46))

# Define VAE architecture
#original_dim = X_train.shape[1]
#latent_dim = 2
batch_size=100
original_dim=784
latent_dim=2
intermediate_dim = 256
epochs=50

def sampling(args):
    z_mean, z_log_var = args
    print(f"z_mean:{z_mean.shape}")
    batch = K.shape(z_mean)[0]
    dim = K.int_shape(z_mean)[1]
    epsilon = K.random_normal(shape=(batch, dim, 1))
    return z_mean + K.exp(0.5 * z_log_var) * epsilon
    #epsilon_reshaped = tf.reshape(epsilon, [-1, 392, 512])
    #return z_mean + K.exp(0.5 * z_log_var) * epsilon_reshaped


inputs = Input(shape=(original_dim,100))# 这里应该是原始数据的维度
#h_inputs= Input(shape=(batch_size,original_dim))
h = Conv1D(1024, kernel_size=3, strides=2, padding='same', activation='relu')(inputs)#卷积层定义
# 均值输出层，输出的维度应与潜在空间的维度匹配
z_mean=Dense(512, activation='relu')(h)
# 对数方差输出层，同上
z_log_var=Dense(512, activation='relu')(h)
# 采样层，生成潜在空间的样本
z = Lambda(sampling)([z_mean, z_log_var])

#def tcn_layer(x1, dilation_rate):
x1 = Conv1D(filters=64, kernel_size=2, dilation_rate=2, padding='causal')(inputs)
    #x1 = Activation('relu')(x1)
    #x1 = SpatialDropout1D(0.2)(x1)
    #return x1

#encoder = Model(inputs, [z_mean, z_log_var,z], name='encoder')
def build_encoder():
    model=Sequential()
    model.add(inputs)
    model.add(Conv1D(filters=64, kernel_size=2, activation='relu'))
    model.add(SpatialDropout1D(0.2))
    model.add(Flatten())
    model.add(Dense(2, activation='softmax'))
    #model.outputs=outputs
    model.summary()
    return model

encoder=build_encoder()

latent_inputs = Input(shape=(latent_dim,100))
x = Dense(intermediate_dim, activation='relu')(latent_inputs)
outputs = Dense(original_dim, activation='sigmoid')(x)

#decoder = Model(latent_inputs, outputs, name='decoder')
def build_decoder():
    model = Sequential()
    model.add(latent_inputs)
    model.add(Conv1D(filters=64, kernel_size=2, activation='relu'))
    model.add(SpatialDropout1D(0.2))
    #model.add(Dense(784, activation='sigmoid'))  # 添加一个全连接层，输出维度为 784
    model.add(Dense(units=2 * 784, activation='sigmoid'))
    #model.add(outputs)
    #model.add(Flatten())
    #model.add(Reshape((1,1)))
    model.add(Dense(2, activation='softmax'))
    model.summary()
    return model

decoder=build_decoder()
print(f"encoder(inputs):{encoder(inputs)}")

#outputs = decoder(encoder(inputs))
#outputs = decoder(tf.expand_dims(z_mean, axis=-1))
# 首先，获取编码器的输出
encoded_output = encoder(inputs)

outputs = decoder(encoded_output)

vae = Model(inputs, outputs, name='vae')

# Define VAE loss
reconstruction_loss = tf.keras.losses.binary_crossentropy(inputs, outputs)
reconstruction_loss *= original_dim
kl_loss = 1 + z_log_var - K.square(z_mean) - K.exp(z_log_var)
kl_loss = K.sum(kl_loss, axis=-1)
kl_loss *= -0.5
vae_loss = K.mean(reconstruction_loss + kl_loss)
vae.add_loss(vae_loss)
vae.compile(optimizer='adam')

# Train VAE
vae.fit(X_train, epochs=10, batch_size=32, validation_data=(Y_test))

打印模型信息如下：

报错如下：

Traceback (most recent call last):
  File "E:/PythonProject/test02/VAE-TCN.py", line 118, in <module>
    outputs = decoder(encoded_output)
  File "D:\Software\Anaconda\envs\test02\lib\site-packages\tensorflow_core\python\keras\engine\base_layer.py", line 737, in __call__
    self.name)
  File "D:\Software\Anaconda\envs\test02\lib\site-packages\tensorflow_core\python\keras\engine\input_spec.py", line 177, in assert_input_compatibility
    str(x.shape.as_list()))
ValueError: Input 0 of layer sequential_1 is incompatible with the layer: expected ndim=3, found ndim=2. Full shape received: [None, 2]

进程已结束，退出代码 1

Answer 1

以下回复参考：皆我百晓生、券券喵儿等免费微信小程序作答:

根据你提供的报错信息，问题出在解码器的输入与期望的维度不匹配上。具体来说，你的解码器期望的输入维度是3（即形状为(batch_size, height, width)），但实际上你提供的输入只有2个维度（形状为[None, 2]）。这种不匹配通常发生在数据预处理或模型构建阶段。让我们一步步解决这个问题。

首先，检查你的数据集形状是否正确。你需要确保输入数据的形状与你想要输入到VAE模型中的形状相匹配。在你的代码中，你尝试将编码器的输出直接作为解码器的输入，但编码器的输出形状可能不是你预期的那样。你需要确保经过编码器处理后的数据形状是(batch_size, latent_dim)，然后再传递给解码器。你可能需要调整编码器的结构或使用适当的预处理步骤来确保这一点。

其次，检查你的解码器结构是否正确。根据你的代码，解码器似乎期望一个三维的输入（可能是(batch_size, height, width)），但你需要确保解码器的输入层接受正确的输入形状。你可能需要调整解码器的输入层以接受二维或一维的输入，或者确保在传递给解码器之前对数据进行适当的扩展或重塑。

假设你想保持你的编码器输出是(batch_size, latent_dim)，并且你想让这个输出作为解码器的输入，你需要确保解码器的第一个层可以接受这种形状。你可以尝试调整解码器的结构，使其接受二维输入，如下所示：

def build_decoder():
    model = Sequential()
    model.add(Input(shape=(latent_dim,)))  # 修改这里的形状以匹配编码器的输出
    model.add(Dense(units=784, activation='sigmoid'))  # 假设你的原始数据形状是(batch_size, 784)
    # ... 其他层 ...
    model.summary()
    return model

请注意，这只是一个示例，你需要根据你的具体情况进行调整。确保你的解码器结构适应你的数据和编码器的输出。希望这些建议能帮助你解决问题！如果还有其他问题或需要进一步的澄清，请告诉我。