Python 运行图像分割时在第一次迭代中获得NAN

我正在GCP上运行以下代码。我有一个8cpu和一个特斯拉K80 gpu。有时，当我运行它时，第一次迭代会得到一个NAN值。有时我会运行它，第一个历元运行得非常好，它似乎在学习，损耗在减少，精度在0.9以上，然后第二个历元，损耗将直接进行。有什么原因吗

#!/usr/bin/env python
# coding: utf-8

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import tensorflow as tf
from sklearn.model_selection import train_test_split
import keras
import cv2
from keras.utils import Sequence


# In[3]:


from keras.layers.convolutional import Conv2D, Conv2DTranspose
from keras.layers.pooling import MaxPooling2D
from keras.layers.merge import concatenate
from keras.losses import binary_crossentropy
from keras.callbacks import EarlyStopping




import numpy as np 
import os
import skimage.io as io
import skimage.transform as trans
import numpy as np
from keras.models import *
from keras.layers import *
from keras.optimizers import *


path = '/home/umfarooq0/RooftopSolar/'

long_lat_file = 'polygonVertices_LatitudeLongitude.csv'

pol_long_lat = pd.read_csv(path + long_lat_file)

pol_long_lat.head()

pixel_coord_file = 'polygonVertices_PixelCoordinates.csv'

pol_coord = pd.read_csv(path + pixel_coord_file)

pol_coord.shape

pol_coord.iloc[0,:]

pol_coord['join'] = pol_coord[pol_coord.columns[2:]].apply(
    lambda x: ','.join(x.dropna().astype(str)),
    axis=1)

except_vert_file = 'polygonDataExceptVertices.csv'

except_vert = pd.read_csv(path + except_vert_file)

# we need to create a variable which has image name and
# we assign a class to each image name, based on how many polygons they have
def create_class(x):
    if x['polygon_id'] < 20:
        return int(0)
    elif x['polygon_id'] == 20:
        return int(1)
    elif 20 < x['polygon_id'] < 50:
        return int(2)
    elif x['polygon_id'] > 50:
        return int(3)

df2_vals = except_vert.groupby(['image_name']).count()['polygon_id']
df2_vals = pd.DataFrame(df2_vals)
df2_vals['class'] = df2_vals.apply(create_class,axis = 1)

df_coord = pd.merge(except_vert,pol_coord,on = 'polygon_id')

def rle_to_mask(rle_string,height,width):
    '''
    convert RLE(run length encoding) string to numpy array

    Parameters: 
    rleString (str): Description of arg1 
    height (int): height of the mask
    width (int): width of the mask 

    Returns: 
    numpy.array: numpy array of the mask
    '''
    rows, cols = height, width
    if rle_string == -1:
        return np.zeros((height, width))
    else:
        rleNumbers = [int(float(numstring)) for numstring in rle_string.split(' ')]
        #rleNumbers = rle_string
        rlePairs = np.array(rleNumbers).reshape(-1,2)

        img = np.zeros(rows*cols,dtype=np.uint8)
        for index,length in rlePairs:
            index -= 1
            img[index:index+length] = 255
        img = img.reshape(cols,rows)
        img = img.T
        return img

def mask_to_rle(mask):
    '''
    Convert a mask into RLE

    Parameters: 
    mask (numpy.array): binary mask of numpy array where 1 - mask, 0 - background

    Returns: 
    sring: run length encoding 
    '''
    pixels= mask.T.flatten()
    pixels = np.concatenate([[0], pixels, [0]])
    runs = np.where(pixels[1:] != pixels[:-1])[0] + 1
    runs[1::2] -= runs[::2]
    return ' '.join(str(x) for x in runs)


# In[13]:


class DataGenerator(Sequence):
    def __init__(self, list_ids, labels, image_dir, batch_size=2,
                 img_h=512, img_w=512, shuffle= False):
        #self.steps_per_epoch = steps_per_epoch
        self.list_ids = list_ids
        self.labels = labels
        self.image_dir = image_dir
        self.batch_size = batch_size
        self.img_h = img_h
        self.img_w = img_w
        self.shuffle = shuffle
        self.on_epoch_end()

    def __len__(self):
        'denotes the number of batches per epoch'
        return int(np.floor(len(self.list_ids)) / self.batch_size)

    def __getitem__(self, index):
        'generate one batch of data'
        indexes = self.indexes[index*self.batch_size:(index+1)*self.batch_size]
        # get list of IDs
        list_ids_temp = [self.list_ids[k] for k in indexes]
        # generate data
        X, y = self.__data_generation(list_ids_temp)
        # return data 
        return X, y

    def on_epoch_end(self):
        'update ended after each epoch'
        self.indexes = np.arange(len(self.list_ids))
        if self.shuffle:
            np.random.shuffle(self.indexes)

    def __data_generation(self, list_ids_temp):
        'generate data containing batch_size samples'
        X = np.empty((self.batch_size, self.img_h, self.img_w, 1))
        y = np.empty((self.batch_size, self.img_h, self.img_w, 4)) #  this was originally 4, but changed to 1

        cls_ = []
        imn = []

        for idx, id in enumerate(list_ids_temp):
            file_path =  os.path.join(self.image_dir, id+'.tif')
            lc = os.path.exists(file_path)
            if lc is True:

                image = cv2.imread(file_path, 0)

               #print(id + '__load_image')


                im_sz = image.size

                if im_sz > 0:
                    #print('check_size')

                    image_resized = cv2.resize(image, (self.img_w, self.img_h))

                    image_resized = np.array(image_resized, dtype=np.float64)
                    # standardization of the image
                    image_resized -= image_resized.mean()
                    image_resized /= image_resized.std()

                    mask = np.empty((img_h, img_w, 4))


                    rle = self.labels.get(id)
                    total_classes = [0,1,2,3]

                    # we need to get what class each id is
                    class_ =int(df2_vals[df2_vals.index == id ]['class'][0])

                   # cls_.append(class_)
                   # imn.append(id)

                    if rle is None:
                        class_mask = np.zeros((5000, 5000))
                    else:
                        class_mask = rle_to_mask(rle, width=5000, height=5000)

                    class_mask_resized = cv2.resize(class_mask, (self.img_w, self.img_h))
                    mask[...,class_] = class_mask_resized
                    total_classes.remove(class_)
                    for ix in total_classes:
                        class_mask = np.zeros((5000, 5000))
                        class_mask_resized = cv2.resize(class_mask, (self.img_w, self.img_h))
                        mask[...,ix] = class_mask_resized

                    # if there is no mask create empty mask



                    X[idx,] = np.expand_dims(image_resized, axis=2)
                    y[idx,] = mask


        # normalize Y
        #y = (y > 0).astype(int)

        return X,y

in_un = except_vert.image_name.unique()

len(except_vert.image_name.unique())


in_un = pd.DataFrame(in_un,columns = ['image_name'])

sample_data = in_un.merge(df_coord,how = 'inner', on='image_name')

sample_data['join'] = sample_data['join'].apply(lambda x: x.replace(","," "))

train_image_ids = in_un
val_size = 20
train_image_ids = train_image_ids[train_image_ids.image_name != '11ska505815']
train_image_ids = train_image_ids[train_image_ids.image_name != '10sfh465105']



X_train, X_val = train_test_split(train_image_ids, test_size=val_size, random_state=42)


# In[199]:


masks = {}
for index, row in  sample_data[ sample_data['join']!=-1].iterrows():
    masks[row['image_name']] = row['join']

img_h = 512
img_w = 512
train_image_dir = path + 'train_data'
batch_size = 4

params = {'img_h': img_h,
          'img_w': img_w,
          'image_dir': train_image_dir,
          'batch_size': batch_size,

          'shuffle': True}

X_train = np.array(X_train)
X_train = X_train.reshape(X_train.shape[0])

X_val = np.array(X_val)
X_val = X_val.reshape(X_val.shape[0])

training_generator = DataGenerator(X_train, masks, **params)
validation_generator = DataGenerator(X_val, masks, **params)



def unet(pretrained_weights = None,input_size = (512,512,1)):
    inputs = Input(input_size)
    conv1 = Conv2D(64, 3, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(inputs)
    conv1 = Conv2D(64, 3, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(conv1)
    pool1 = MaxPooling2D(pool_size=(2, 2))(conv1)
    conv2 = Conv2D(128, 3, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(pool1)
    conv2 = Conv2D(128, 3, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(conv2)
    pool2 = MaxPooling2D(pool_size=(2, 2))(conv2)
    conv3 = Conv2D(256, 3, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(pool2)
    conv3 = Conv2D(256, 3, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(conv3)
    pool3 = MaxPooling2D(pool_size=(2, 2))(conv3)
    conv4 = Conv2D(512, 3, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(pool3)
    conv4 = Conv2D(512, 3, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(conv4)
    drop4 = Dropout(0.5)(conv4)
    pool4 = MaxPooling2D(pool_size=(2, 2))(drop4)

    conv5 = Conv2D(1024, 3, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(pool4)
    conv5 = Conv2D(1024, 3, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(conv5)
    drop5 = Dropout(0.5)(conv5)

    up6 = Conv2D(512, 2, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(UpSampling2D(size = (2,2))(drop5))
    merge6 = concatenate([drop4,up6], axis = 3)
    conv6 = Conv2D(512, 3, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(merge6)
    conv6 = Conv2D(512, 3, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(conv6)

    up7 = Conv2D(256, 2, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(UpSampling2D(size = (2,2))(conv6))
    merge7 = concatenate([conv3,up7], axis = 3)
    conv7 = Conv2D(256, 3, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(merge7)
    conv7 = Conv2D(256, 3, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(conv7)

    up8 = Conv2D(128, 2, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(UpSampling2D(size = (2,2))(conv7))
    merge8 = concatenate([conv2,up8], axis = 3)
    conv8 = Conv2D(128, 3, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(merge8)
    conv8 = Conv2D(128, 3, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(conv8)

    up9 = Conv2D(64, 2, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(UpSampling2D(size = (2,2))(conv8))
    merge9 = concatenate([conv1,up9], axis = 3)
    conv9 = Conv2D(64, 3, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(merge9)
    conv9 = Conv2D(64, 3, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(conv9)
    conv9 = Conv2D(2, 3, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')(conv9)
    conv10 = Conv2D(4, 1, activation = 'sigmoid')(conv9)

    model = Model(input = inputs, output = conv10)

    model.compile(optimizer = Adam(lr = 1e-6), loss = 'binary_crossentropy', metrics = ['accuracy'])


    #model.summary()

    if(pretrained_weights):
        model.load_weights(pretrained_weights)

    return model


model = unet()
epochs = 10


history = model.fit_generator(generator=training_generator, validation_data=validation_generator, epochs=epochs, verbose=1)

model.save('RooftopSolar_1.h5')

#/usr/bin/env python
#编码：utf-8
作为pd进口熊猫
将numpy作为np导入
将matplotlib.pyplot作为plt导入
导入tensorflow作为tf
从sklearn.model\u选择导入列车\u测试\u拆分
进口干酪
进口cv2
从keras.utils导入序列
#在[3]中：
从keras.layers.Conv2D卷积导入，Conv2DTranspose
从keras.layers.pooling导入MaxPoolig2D
从keras.layers.merge导入连接
从keras.com导入二进制交叉熵
从keras.callbacks导入早期终止
将numpy作为np导入
导入操作系统
将skimage.io导入为io
导入skimage.transform作为trans
将numpy作为np导入
从keras.models导入*
从keras.layers导入*
从keras.optimizers导入*
path='/home/umfarooq0/ROOTTOPSOLAR/'
long_lat_file='polygonVertices_latudelongitude.csv'
pol_long_lat=pd.read_csv（路径+long_lat_文件）
pol_long_lat.head（）
pixel_coord_文件='PolygonVertexts_PixelCoordinates.csv'
pol_coord=pd.read_csv（路径+像素_coord_文件）
pol_coord.形状
pol_coord.iloc[0，：]
pol_coord['join']=pol_coord[pol_coord.columns[2:][2]。应用(
lambda x:'，'.join（x.dropna（）.astype（str）），
轴=1）
除了_vert_file='PolygondaExceptVertices.csv'
Exception\u vert=pd.read\u csv（路径+Exception\u vert\u文件）
#我们需要创建一个具有图像名称和
#我们根据每个图像的多边形数量为它们指定一个类
def创建_类（x）：
如果x['polygon_id']<20：
返回整数（0）
elif x['polygon_id']==20：
返回整数（1）
elif 2050：
返回整数（3）
df2\u vals=除垂直groupby（['image\u name']）.count（）['polygon\u id']
df2\u VAL=pd.数据帧（df2\u VAL）
df2_vals['class']=df2_vals.apply（创建_类，轴=1）
df_-coord=pd.merge（除了_-vert，pol_-coord，on='polygon_-id'）
定义rle_到_掩码（rle_字符串、高度、宽度）：
'''
将RLE（运行长度编码）字符串转换为numpy数组
参数：
rleString（str）：arg1的说明
高度（int）：遮罩的高度
宽度（int）：遮罩的宽度
返回：
数组：掩码的numpy数组
'''
行，列=高度，宽度
如果rle_字符串==-1：
返回np.零（（高度、宽度））
其他：
rleNumbers=[int（float（numstring））表示rle_string.split（“”）中的numstring
#rleNumbers=rle_字符串
rlePairs=np.数组（RLEDumbers）.重塑（-1,2）
img=np.zero（行*cols，数据类型=np.uint8）
对于索引，成对长度：
索引-=1
img[索引：索引+长度]=255
img=img.重塑（列、行）
img=img.T
返回img
def遮罩至遮罩（遮罩）：
'''
将掩码转换为RLE
参数：
掩码（numpy.array）：numpy数组的二进制掩码，其中1-掩码，0-背景
返回：
sring：运行长度编码
'''
像素=mask.T.展平（）
像素=np。连接（[[0]，像素，[0]]）
运行=np。其中（像素[1:::！=像素[：-1]）[0]+1
运行[1:：2]-=运行[：：2]
返回“”。加入（str（x）用于运行中的x）
#在[13]中：
类数据生成器（序列）：
def uuu init uuuu（self，list_id，labels，image_dir，batch_size=2，
img_h=512，img_w=512，shuffle=False）：
#self.steps\u per\u epoch=steps\u per\u epoch
self.list\u id=list\u id
self.labels=标签
self.image\u dir=image\u dir
self.batch\u size=批次大小
self.img\u h=img\u h
self.img\u w=img\u w
self.shuffle=洗牌
self.on_epoch_end（）
定义（自我）：
'表示每个历元的批次数'
返回整数（np.floor（len（self.list\u id））/self.batch\u size）
定义uu获取项目uu（自身，索引）：
“生成一批数据”
索引=自索引[索引*自批大小：（索引+1）*自批大小]
#获取ID列表
list_id_temp=[索引中k的self.list_id[k]
#生成数据
十、 y=自身数据生成（列表ID临时）
#返回数据
返回X，y
终端上的def（自身）：
“更新在每个历元后结束”
self.index=np.arange（len（self.list_id））
如果self.shuffle：
np.random.shuffle（自索引）
定义数据生成（自身、列表ID和临时）：
“生成包含批量大小样本的数据”
X=np.空（（self.batch\u size，self.img\u h，self.img\u w，1））
y=np.empty（（self.batch_size，self.img_h，self.img_w，4））#这原来是4，但改为1
cls_U8;=[]
imn=[]
对于idx，枚举中的id（列表id临时）：
file_path=os.path.join（self.image_dir，id+'.tif'）
lc=os.path.exists（文件路径）
如果lc为真：
image=cv2.imread（文件路径，0）
#打印（id+“\加载\图像”）
im_sz=image.size
如果im_sz>0：
#打印（“检查大小”）
image\u resized=cv2.resize（image，（self.img\u w，self.img\u h））
image\u resized=np.array（image\u resized，dtype=np.float64）
#图像的标准化
image_resized-=image_resized.mean（）
image\u resized/=image\u resized.std（）
掩码=np.空（（img_h，img_w，4））
rle=self.labels.get（id）
课程总数=[0,1,2,3]
#我们需要知道每个id是什么类
类=int（df2\u vals[df2\u vals.index==id]['class'][0]）
#cls_uu0.append（类_u0）
#imn.append（id）
如果rle是