Python Tensorflow:UnicodeDecodeError:&x27；charmap'；编解码器可以'；t解码3289位置的字节0x8f：字符映射到<；未定义>；

我正在尝试运行此模块：

import sys
import time
import os
import tensorflow as tf
import numpy as np
from collections import namedtuple
from data import Vocab
from batcher import Batcher
from model import SummarizationModel
from decode import BeamSearchDecoder
import util

FLAGS = tf.app.flags.FLAGS

# Where to find data
tf.app.flags.DEFINE_string('data_path', '', 'Path expression to tf.Example datafiles. Can include wildcards to access multiple datafiles.')
tf.app.flags.DEFINE_string('vocab_path', '', 'Path expression to text vocabulary file.')

# Important settings
tf.app.flags.DEFINE_string('mode', 'train', 'must be one of train/eval/decode')
tf.app.flags.DEFINE_boolean('single_pass', False, 'For decode mode only. If True, run eval on the full dataset using a fixed checkpoint, i.e. take the current checkpoint, and use it to produce one summary for each example in the dataset, write the summaries to file and then get ROUGE scores for the whole dataset. If False (default), run concurrent decoding, i.e. repeatedly load latest checkpoint, use it to produce summaries for randomly-chosen examples and log the results to screen, indefinitely.')

# Where to save output
tf.app.flags.DEFINE_string('log_root', '', 'Root directory for all logging.')
tf.app.flags.DEFINE_string('exp_name', '', 'Name for experiment. Logs will be saved in a directory with this name, under log_root.')

# Hyperparameters
tf.app.flags.DEFINE_integer('hidden_dim', 256, 'dimension of RNN hidden states')
tf.app.flags.DEFINE_integer('emb_dim', 128, 'dimension of word embeddings')
tf.app.flags.DEFINE_integer('batch_size', 16, 'minibatch size')
tf.app.flags.DEFINE_integer('max_enc_steps', 400, 'max timesteps of encoder (max source text tokens)')
tf.app.flags.DEFINE_integer('max_dec_steps', 100, 'max timesteps of decoder (max summary tokens)')
tf.app.flags.DEFINE_integer('beam_size', 4, 'beam size for beam search decoding.')
tf.app.flags.DEFINE_integer('min_dec_steps', 35, 'Minimum sequence length of generated summary. Applies only for beam search decoding mode')
tf.app.flags.DEFINE_integer('vocab_size', 50000, 'Size of vocabulary. These will be read from the vocabulary file in order. If the vocabulary file contains fewer words than this number, or if this number is set to 0, will take all words in the vocabulary file.')
tf.app.flags.DEFINE_float('lr', 0.15, 'learning rate')
tf.app.flags.DEFINE_float('adagrad_init_acc', 0.1, 'initial accumulator value for Adagrad')
tf.app.flags.DEFINE_float('rand_unif_init_mag', 0.02, 'magnitude for lstm cells random uniform inititalization')
tf.app.flags.DEFINE_float('trunc_norm_init_std', 1e-4, 'std of trunc norm init, used for initializing everything else')
tf.app.flags.DEFINE_float('max_grad_norm', 2.0, 'for gradient clipping')

# Pointer-generator or baseline model
tf.app.flags.DEFINE_boolean('pointer_gen', True, 'If True, use pointer-generator model. If False, use baseline model.')

# Coverage hyperparameters
tf.app.flags.DEFINE_boolean('coverage', False, 'Use coverage mechanism. Note, the experiments reported in the ACL paper train WITHOUT coverage until converged, and then train for a short phase WITH coverage afterwards. i.e. to reproduce the results in the ACL paper, turn this off for most of training then turn on for a short phase at the end.')
tf.app.flags.DEFINE_float('cov_loss_wt', 1.0, 'Weight of coverage loss (lambda in the paper). If zero, then no incentive to minimize coverage loss.')
tf.app.flags.DEFINE_boolean('convert_to_coverage_model', False, 'Convert a non-coverage model to a coverage model. Turn this on and run in train mode. Your current model will be copied to a new version (same name with _cov_init appended) that will be ready to run with coverage flag turned on, for the coverage training stage.')


def calc_running_avg_loss(loss, running_avg_loss, summary_writer, step, decay=0.99):
  """Calculate the running average loss via exponential decay.
  This is used to implement early stopping w.r.t. a more smooth loss curve than the raw loss curve.

  Args:
    loss: loss on the most recent eval step
    running_avg_loss: running_avg_loss so far
    summary_writer: FileWriter object to write for tensorboard
    step: training iteration step
    decay: rate of exponential decay, a float between 0 and 1. Larger is smoother.

  Returns:
    running_avg_loss: new running average loss
  """
  if running_avg_loss == 0:  # on the first iteration just take the loss
    running_avg_loss = loss
  else:
    running_avg_loss = running_avg_loss * decay + (1 - decay) * loss
  running_avg_loss = min(running_avg_loss, 12)  # clip
  loss_sum = tf.Summary()
  tag_name = 'running_avg_loss/decay=%f' % (decay)
  loss_sum.value.add(tag=tag_name, simple_value=running_avg_loss)
  summary_writer.add_summary(loss_sum, step)
  tf.logging.info('running_avg_loss: %f', running_avg_loss)
  return running_avg_loss


def convert_to_coverage_model():
  """Load non-coverage checkpoint, add initialized extra variables for coverage, and save as new checkpoint"""
  tf.logging.info("converting non-coverage model to coverage model..")

  # initialize an entire coverage model from scratch
  sess = tf.Session(config=util.get_config())
  print("initializing everything...")
  sess.run(tf.global_variables_initializer())

  # load all non-coverage weights from checkpoint
  saver = tf.train.Saver([v for v in tf.global_variables() if "coverage" not in v.name and "Adagrad" not in v.name])
  print("restoring non-coverage variables...")
  curr_ckpt = util.load_ckpt(saver, sess)
  print("restored.")

  # save this model and quit
  new_fname = curr_ckpt + '_cov_init'
  print("saving model to %s..." % (new_fname))
  new_saver = tf.train.Saver() # this one will save all variables that now exist
  new_saver.save(sess, new_fname)
  print("saved.")
  exit()


def setup_training(model, batcher):
  """Does setup before starting training (run_training)"""
  train_dir = os.path.join(FLAGS.log_root, "train")
  if not os.path.exists(train_dir): os.makedirs(train_dir)

  default_device = tf.device('/cpu:0')
  with default_device:
    model.build_graph() # build the graph
    if FLAGS.convert_to_coverage_model:
      assert FLAGS.coverage, "To convert your non-coverage model to a coverage model, run with convert_to_coverage_model=True and coverage=True"
      convert_to_coverage_model()
    saver = tf.train.Saver(max_to_keep=1) # only keep 1 checkpoint at a time

  sv = tf.train.Supervisor(logdir=train_dir,
                     is_chief=True,
                     saver=saver,
                     summary_op=None,
                     save_summaries_secs=60, # save summaries for tensorboard every 60 secs
                     save_model_secs=60, # checkpoint every 60 secs
                     global_step=model.global_step)
  summary_writer = sv.summary_writer
  tf.logging.info("Preparing or waiting for session...")
  sess_context_manager = sv.prepare_or_wait_for_session(config=util.get_config())
  tf.logging.info("Created session.")
  try:
    run_training(model, batcher, sess_context_manager, sv, summary_writer) # this is an infinite loop until interrupted
  except KeyboardInterrupt:
    tf.logging.info("Caught keyboard interrupt on worker. Stopping supervisor...")
    sv.stop()


def run_training(model, batcher, sess_context_manager, sv, summary_writer):
  """Repeatedly runs training iterations, logging loss to screen and writing summaries"""
  tf.logging.info("starting run_training")
  with sess_context_manager as sess:
    while True: # repeats until interrupted
      batch = batcher.next_batch()

      tf.logging.info('running training step...')
      t0=time.time()
      results = model.run_train_step(sess, batch)
      t1=time.time()
      tf.logging.info('seconds for training step: %.3f', t1-t0)

      loss = results['loss']
      tf.logging.info('loss: %f', loss) # print the loss to screen
      if FLAGS.coverage:
        coverage_loss = results['coverage_loss']
        tf.logging.info("coverage_loss: %f", coverage_loss) # print the coverage loss to screen

      # get the summaries and iteration number so we can write summaries to tensorboard
      summaries = results['summaries'] # we will write these summaries to tensorboard using summary_writer
      train_step = results['global_step'] # we need this to update our running average loss

      summary_writer.add_summary(summaries, train_step) # write the summaries
      if train_step % 100 == 0: # flush the summary writer every so often
        summary_writer.flush()


def run_eval(model, batcher, vocab):
  """Repeatedly runs eval iterations, logging to screen and writing summaries. Saves the model with the best loss seen so far."""
  model.build_graph() # build the graph
  saver = tf.train.Saver(max_to_keep=3) # we will keep 3 best checkpoints at a time
  sess = tf.Session(config=util.get_config())
  eval_dir = os.path.join(FLAGS.log_root, "eval") # make a subdir of the root dir for eval data
  bestmodel_save_path = os.path.join(eval_dir, 'bestmodel') # this is where checkpoints of best models are saved
  summary_writer = tf.summary.FileWriter(eval_dir)
  running_avg_loss = 0 # the eval job keeps a smoother, running average loss to tell it when to implement early stopping
  best_loss = None  # will hold the best loss achieved so far

  while True:
    _ = util.load_ckpt(saver, sess) # load a new checkpoint
    batch = batcher.next_batch() # get the next batch

    # run eval on the batch
    t0=time.time()
    results = model.run_eval_step(sess, batch)
    t1=time.time()
    tf.logging.info('seconds for batch: %.2f', t1-t0)

    # print the loss and coverage loss to screen
    loss = results['loss']
    tf.logging.info('loss: %f', loss)
    if FLAGS.coverage:
      coverage_loss = results['coverage_loss']
      tf.logging.info("coverage_loss: %f", coverage_loss)

    # add summaries
    summaries = results['summaries']
    train_step = results['global_step']
    summary_writer.add_summary(summaries, train_step)

    # calculate running avg loss
    running_avg_loss = calc_running_avg_loss(np.asscalar(loss), running_avg_loss, summary_writer, train_step)

    # If running_avg_loss is best so far, save this checkpoint (early stopping).
    # These checkpoints will appear as bestmodel-<iteration_number> in the eval dir
    if best_loss is None or running_avg_loss < best_loss:
      tf.logging.info('Found new best model with %.3f running_avg_loss. Saving to %s', running_avg_loss, bestmodel_save_path)
      saver.save(sess, bestmodel_save_path, global_step=train_step, latest_filename='checkpoint_best')
      best_loss = running_avg_loss

    # flush the summary writer every so often
    if train_step % 100 == 0:
      summary_writer.flush()


def main(unused_argv):
  if len(unused_argv) != 1: # prints a message if you've entered flags incorrectly
    raise Exception("Problem with flags: %s" % unused_argv)

  tf.logging.set_verbosity(tf.logging.INFO) # choose what level of logging you want
  tf.logging.info('Starting seq2seq_attention in %s mode...', (FLAGS.mode))

  # Change log_root to FLAGS.log_root/FLAGS.exp_name and create the dir if necessary
  FLAGS.log_root = os.path.join(FLAGS.log_root, FLAGS.exp_name)
  if not os.path.exists(FLAGS.log_root):
    if FLAGS.mode=="train":
      os.makedirs(FLAGS.log_root)
    else:
      raise Exception("Logdir %s doesn't exist. Run in train mode to create it." % (FLAGS.log_root))

  vocab = Vocab(FLAGS.vocab_path, FLAGS.vocab_size) # create a vocabulary

  # If in decode mode, set batch_size = beam_size
  # Reason: in decode mode, we decode one example at a time.
  # On each step, we have beam_size-many hypotheses in the beam, so we need to make a batch of these hypotheses.
  if FLAGS.mode == 'decode':
    FLAGS.batch_size = FLAGS.beam_size

  # If single_pass=True, check we're in decode mode
  if FLAGS.single_pass and FLAGS.mode!='decode':
    raise Exception("The single_pass flag should only be True in decode mode")

  # Make a namedtuple hps, containing the values of the hyperparameters that the model needs
  hparam_list = ['mode', 'lr', 'adagrad_init_acc', 'rand_unif_init_mag', 'trunc_norm_init_std', 'max_grad_norm', 'hidden_dim', 'emb_dim', 'batch_size', 'max_dec_steps', 'max_enc_steps', 'coverage', 'cov_loss_wt', 'pointer_gen']
  hps_dict = {}
  for key,val in FLAGS.__flags.items(): # for each flag
    if key in hparam_list: # if it's in the list
      hps_dict[key] = val # add it to the dict
  hps = namedtuple("HParams", list(hps_dict.keys()))(**hps_dict)

  # Create a batcher object that will create minibatches of data
  batcher = Batcher(FLAGS.data_path, vocab, hps, single_pass=FLAGS.single_pass)

  tf.set_random_seed(111) # a seed value for randomness

  if hps.mode == 'train':
    print("creating model...")
    model = SummarizationModel(hps, vocab)
    setup_training(model, batcher)
  elif hps.mode == 'eval':
    model = SummarizationModel(hps, vocab)
    run_eval(model, batcher, vocab)
  elif hps.mode == 'decode':
    decode_model_hps = hps  # This will be the hyperparameters for the decoder model
    decode_model_hps = hps._replace(max_dec_steps=1) # The model is configured with max_dec_steps=1 because we only ever run one step of the decoder at a time (to do beam search). Note that the batcher is initialized with max_dec_steps equal to e.g. 100 because the batches need to contain the full summaries
    model = SummarizationModel(decode_model_hps, vocab)
    decoder = BeamSearchDecoder(model, batcher, vocab)
    decoder.decode() # decode indefinitely (unless single_pass=True, in which case deocde the dataset exactly once)
  else:
    raise ValueError("The 'mode' flag must be one of train/eval/decode")

if __name__ == '__main__':
  tf.app.run()

导入系统 导入时间 导入操作系统 导入tensorflow作为tf 将numpy作为np导入 从集合导入namedtuple 从数据导入Vocab 从批处理程序导入批处理程序 从模型导入摘要模型 从解码导入BeamSearchDecoder 导入util FLAGS=tf.app.FLAGS.FLAGS #哪里可以找到数据 tf.app.flags.DEFINE_string（'data_path'，''，tf的路径表达式。示例数据文件。可以包含通配符以访问多个数据文件。“） tf.app.flags.DEFINE_string（'vocab_path'，''，''，文本词汇表文件的路径表达式'） #重要设置 tf.app.flags.DEFINE_字符串（'mode'、'train'、'必须是train/eval/decode'之一） tf.app.flags.DEFINE_boolean（'single_pass'，False'，仅用于解码模式。如果为True，则使用固定检查点对完整数据集运行eval，即使用当前检查点，并使用它为数据集中的每个示例生成一个摘要，将摘要写入文件，然后获取整个数据集的胭脂分数。如果为False（默认值），运行并发解码，即重复加载最新检查点，使用它为随机选择的示例生成摘要，并将结果无限期地记录到屏幕上。“） #在哪里保存输出 tf.app.flags.DEFINE_string（'log_root'，''，''，所有日志记录的根目录'） tf.app.flags.DEFINE_string（'exp_name'，''，'name for experience.log将保存在日志根目录下的同名目录中。'）） #超参数 tf.app.flags.DEFINE_integer（'hidden_dim'，256，'RNN hidden states的维度'） tf.app.flags.DEFINE_integer（'emb_dim'，128，'单词嵌入的维度'） tf.app.flags.DEFINE_integer（'batch_size'，16，'minibatch size'）） tf.app.flags.DEFINE_integer（'max_enc_steps'，400，'max timesteps of encoder（max source text tokens））） tf.app.flags.DEFINE_integer（'max_dec_steps'，100，'max timesteps of decoder（max summary tokens）'） tf.app.flags.DEFINE_integer（'波束大小'，4，'波束搜索解码的波束大小'） tf.app.flags.DEFINE_integer（'min_dec_steps'，35，'生成摘要的最小序列长度。仅适用于波束搜索解码模式'） tf.app.flags.DEFINE_integer（'vocab_size'，50000，'词汇大小）。这些将按顺序从词汇文件中读取。如果词汇文件包含的单词少于此数字，或者如果此数字设置为0，则将使用词汇文件中的所有单词。“） tf.app.flags.DEFINE_float（'lr'，0.15，'学习率'） tf.app.flags.DEFINE_float（'adagrad_init_acc'，0.1，'adagrad的初始累加器值'） tf.app.flags.DEFINE_float（'rand_unif_init_mag'，0.02，'lstm单元随机均匀初始化的幅度'） tf.app.flags.DEFINE_float（'trunc_norm_init_std'，1e-4，'trunc norm init std，用于初始化其他所有内容'） tf.app.flags.DEFINE_float（'max_grad_norm'，2.0，'用于渐变剪裁'） #指针生成器或基线模型 tf.app.flags.DEFINE_boolean（'pointer_gen'，True'，如果True，则使用指针生成器模型。如果False，则使用基线模型。“） #覆盖超参数 tf.app.flags.DEFINE_boolean（'coverage'，False'，Use coverage mechanism.注意，ACL文件中报告的实验在聚合前不覆盖，然后在聚合后覆盖一个短阶段进行训练。也就是说，要在ACL文件中重现结果，在大多数训练中关闭此选项，然后在结束时打开一个短阶段。'）） tf.app.flags.DEFINE_float（'cov_loss_wt'，1.0，'覆盖损失的权重（论文中的lambda）。如果为零，则不鼓励最小化覆盖损失。'）） tf.app.flags.DEFINE_boolean（'convert_to_coverage_model'，False，'convert a non-coverage model to a coverage model.。启用此选项并在训练模式下运行。您当前的模型将复制到新版本（与附加了_cov_init的名称相同），该版本将在覆盖率训练阶段启用覆盖率标志后准备运行。“） def calc_running_avg_loss（损耗，running_avg_loss，summary_writer，阶跃，衰减=0.99）： “”“通过指数衰减计算运行平均损耗。 这用于实现提前停止w.r.t.比原始损失曲线更平滑的损失曲线。 Args： 丢失：在最近的评估步骤中丢失 运行平均损耗：目前运行平均损耗 summary_writer:要为tensorboard编写的FileWriter对象 步骤：训练迭代步骤 衰减：指数衰减的速率，介于0和1之间的浮动。越大越平滑。 返回： 运行平均损耗：新的运行平均损耗 """ 如果在第一次迭代中运行_avg_loss==0:#只需接受损失 运行平均损耗=损耗 其他： 运行平均损耗=运行平均损耗*衰减+（1-衰减）*损耗 运行平均损耗=最小值（运行平均损耗，12）#剪辑 损失总额=tf.汇总（） 标记名称='运行平均损耗/衰减=%f'（衰减） 损失总和值添加（标记=标记名称，简单值=运行平均损失） 摘要作者。添加摘要（损失总额，步骤） tf.logging.info（'running_avg_loss:%f'，running_avg_loss） 返回运行平均损耗 def convert_to_coverage_model（）： “”“加载非覆盖检查点，为覆盖添加初始化的额外变量，并另存为新检查点”“” tf.logging.info（“将非覆盖模型转换为覆盖模型…”） #从头开始初始化整个覆盖模型 sess=tf.Session（config=util.get\u config（）） 打印（“初始化所有内容…”） sess.run（tf.global\u variables\u initializer（）） #从检查点加载所有非覆盖权重 saver=tf.train.saver（[v代表tf.global_变量中的v（），如果v.name中没有“coverage”，v.name中没有“Adagrad”）） 打印（“还原非覆盖范围变量…”） curr\u ckpt=util.load\u ckpt（saver，sess） 打印（“已还原”） #保存此模型并退出 new_fname=curr_ckpt+“_cov_init” 打印（“将模型保存到%s…”%（新名称）） new_saver=tf.train.saver（） 新建\u saver.save（sess，新建\u fname） 打印（“已保存”） 退出（） def设置_培训（型号、bat

INFO:tensorflow:Starting seq2seq_attention in train mode...
Warning: incorrectly formatted line in vocabulary file: b'0800A 555A 111 356\n'

Warning: incorrectly formatted line in vocabulary file: b'1800A 333A 000 139\n'

Warning: incorrectly formatted line in vocabulary file: b'2A 1/2 124\n'

Traceback (most recent call last):
  File "run_summarization.py", line 284, in <module>
    tf.app.run()
  File "c:\python35\lib\site-packages\tensorflow\python\platform\app.py", line 48, in run
    _sys.exit(main(_sys.argv[:1] + flags_passthrough))
  File "run_summarization.py", line 242, in main
    vocab = Vocab(FLAGS.vocab_path, FLAGS.vocab_size) # create a vocabulary
  File "C:\Users\lenovo-pc\Desktop\pointer-generator-master\data.py", line 58, in __init__
    for line in vocab_f:
  File "c:\python35\lib\encodings\cp1252.py", line 23, in decode
    return codecs.charmap_decode(input,self.errors,decoding_table)[0]
UnicodeDecodeError: 'charmap' codec can't decode byte 0x8f in position 3289: character maps to <undefined>