tensorflow:从TFRecord读取时间序列数据

tensorflow:从TFRecord读取时间序列数据,tensorflow,tensorflow-datasets,tfrecord,Tensorflow,Tensorflow Datasets,Tfrecord,我使用SequenceExample protobuf将时间序列数据读/写到TFRecord文件中 我序列化了一对np数组,如下所示: writer = tf.python_io.TFRecordWriter(file_name) context = tf.train.Features( ... Feature( ... ) ... ) feature_data = tf.train.FeatureList(feature=[ tf.train.Featu

我使用SequenceExample protobuf将时间序列数据读/写到TFRecord文件中

我序列化了一对np数组,如下所示:

writer = tf.python_io.TFRecordWriter(file_name)

context = tf.train.Features( ... Feature( ... ) ... )

feature_data = tf.train.FeatureList(feature=[
                  tf.train.Feature(float_list=tf.train.FloatList(value=
                                   np.random.normal(size=([4065000,]))])
labels = tf.train.FeatureList(feature=[
                  tf.train.Feature(int64_list=tf.train.Int64List(value=
                           np.random.random_integers(0,10,size=([1084,]))])

##feature_data and labels are of similar, but varying lengths

feature_list = {"feature_data": feature_data,
                "labels": labels}

feature_lists = tf.train.FeatureLists(feature_list=feature_list)
example = tf.train.SequenceExample(context=context,
                                   feature_lists=feature_lists)

        ## serialize and close
dataset = tf.data.TFRecordDataset("data/tf_record.tfrecords")
dataset = dataset.map(decode)
dataset = dataset.make_one_shot_iterator().get_next()

### reshape tensors and feed to estimator###
def decode(serialized_proto):
    context_features = {...}
    sequence_features = {"feature_data": tf.FixedLenSequenceFeature((None,), 
                                                                tf.float32),
                         "labels": tf.FixedLenSequenceFeature(((None,), 
                                                                 tf.int64)}

    context, sequence = tf.parse_single_sequence_example(serialized_proto,
                                        context_features=context_features,
                                        sequence_features=sequence_features)

    return context, sequence
在尝试读取.tfrecords文件时,我遇到了很多错误,主要是因为SequenceExample protobuf将时间序列数据写入一系列值(例如值:-12.2549,值:-18.1372,…值:13.1234)。我读取.tfrecords文件的代码如下:

writer = tf.python_io.TFRecordWriter(file_name)

context = tf.train.Features( ... Feature( ... ) ... )

feature_data = tf.train.FeatureList(feature=[
                  tf.train.Feature(float_list=tf.train.FloatList(value=
                                   np.random.normal(size=([4065000,]))])
labels = tf.train.FeatureList(feature=[
                  tf.train.Feature(int64_list=tf.train.Int64List(value=
                           np.random.random_integers(0,10,size=([1084,]))])

##feature_data and labels are of similar, but varying lengths

feature_list = {"feature_data": feature_data,
                "labels": labels}

feature_lists = tf.train.FeatureLists(feature_list=feature_list)
example = tf.train.SequenceExample(context=context,
                                   feature_lists=feature_lists)

        ## serialize and close
dataset = tf.data.TFRecordDataset("data/tf_record.tfrecords")
dataset = dataset.map(decode)
dataset = dataset.make_one_shot_iterator().get_next()

### reshape tensors and feed to estimator###
def decode(serialized_proto):
    context_features = {...}
    sequence_features = {"feature_data": tf.FixedLenSequenceFeature((None,), 
                                                                tf.float32),
                         "labels": tf.FixedLenSequenceFeature(((None,), 
                                                                 tf.int64)}

    context, sequence = tf.parse_single_sequence_example(serialized_proto,
                                        context_features=context_features,
                                        sequence_features=sequence_features)

    return context, sequence
My decode()函数的定义如下:

writer = tf.python_io.TFRecordWriter(file_name)

context = tf.train.Features( ... Feature( ... ) ... )

feature_data = tf.train.FeatureList(feature=[
                  tf.train.Feature(float_list=tf.train.FloatList(value=
                                   np.random.normal(size=([4065000,]))])
labels = tf.train.FeatureList(feature=[
                  tf.train.Feature(int64_list=tf.train.Int64List(value=
                           np.random.random_integers(0,10,size=([1084,]))])

##feature_data and labels are of similar, but varying lengths

feature_list = {"feature_data": feature_data,
                "labels": labels}

feature_lists = tf.train.FeatureLists(feature_list=feature_list)
example = tf.train.SequenceExample(context=context,
                                   feature_lists=feature_lists)

        ## serialize and close
dataset = tf.data.TFRecordDataset("data/tf_record.tfrecords")
dataset = dataset.map(decode)
dataset = dataset.make_one_shot_iterator().get_next()

### reshape tensors and feed to estimator###
def decode(serialized_proto):
    context_features = {...}
    sequence_features = {"feature_data": tf.FixedLenSequenceFeature((None,), 
                                                                tf.float32),
                         "labels": tf.FixedLenSequenceFeature(((None,), 
                                                                 tf.int64)}

    context, sequence = tf.parse_single_sequence_example(serialized_proto,
                                        context_features=context_features,
                                        sequence_features=sequence_features)

    return context, sequence
其中一个错误如下:

writer = tf.python_io.TFRecordWriter(file_name)

context = tf.train.Features( ... Feature( ... ) ... )

feature_data = tf.train.FeatureList(feature=[
                  tf.train.Feature(float_list=tf.train.FloatList(value=
                                   np.random.normal(size=([4065000,]))])
labels = tf.train.FeatureList(feature=[
                  tf.train.Feature(int64_list=tf.train.Int64List(value=
                           np.random.random_integers(0,10,size=([1084,]))])

##feature_data and labels are of similar, but varying lengths

feature_list = {"feature_data": feature_data,
                "labels": labels}

feature_lists = tf.train.FeatureLists(feature_list=feature_list)
example = tf.train.SequenceExample(context=context,
                                   feature_lists=feature_lists)

        ## serialize and close
dataset = tf.data.TFRecordDataset("data/tf_record.tfrecords")
dataset = dataset.map(decode)
dataset = dataset.make_one_shot_iterator().get_next()

### reshape tensors and feed to estimator###
def decode(serialized_proto):
    context_features = {...}
    sequence_features = {"feature_data": tf.FixedLenSequenceFeature((None,), 
                                                                tf.float32),
                         "labels": tf.FixedLenSequenceFeature(((None,), 
                                                                 tf.int64)}

    context, sequence = tf.parse_single_sequence_example(serialized_proto,
                                        context_features=context_features,
                                        sequence_features=sequence_features)

    return context, sequence
对于输入形状为[]、[0]、[]、[]、[]、[]、[]、[]、[]、[]、[]、[]、[]、[]、[]、[]、[]、[]的“ParseSingleSequenceExample/ParseSingleSequenceExample”(op:“ParseSingleSequenceExample”)未完全定义形状[?]


我的主要问题是如何考虑数据集的结构。我不确定我是否真正理解返回数据的结构。我很难遍历这个数据集并返回可变大小的张量。提前谢谢

只有当特征形状已知时,才能使用
tf.FixedLenSequenceFeature
。否则,请改用
tf.VarLenFeature