Docker Rancher、Kubernetes和StorageOS：持久存储、卷装载问题？_Docker_Kubernetes_Rancheros

Docker Rancher、Kubernetes和StorageOS：持久存储、卷装载问题？

docker kubernetes

Docker Rancher、Kubernetes和StorageOS：持久存储、卷装载问题？,docker,kubernetes,rancheros,Docker,Kubernetes,Rancheros,我需要帮助解决一个问题。我已经在kubernetes上安装了storagesos群集。我能够创建卷。但当我创建PVC并将它们安装到我的K8吊舱时，吊舱被卡在容器创建状态中。从日志中我可以收集到：从卷映射到主机/var/lib/storageos/volumes/mypvcid的卷目录/文件夹不存在 storageos群集容器无法在AWS上的主机VM RHEL上创建这些目录/文件夹。请帮帮我，我已经困了好几天了。谢谢以下是日志： StorageOS日志- > kubectl lo

我需要帮助解决一个问题。我已经在kubernetes上安装了storagesos群集。我能够创建卷。但当我创建PVC并将它们安装到我的K8吊舱时，吊舱被卡在容器创建状态中。从日志中我可以收集到：

从卷映射到主机/var/lib/storageos/volumes/mypvcid的卷目录/文件夹不存在 storageos群集容器无法在AWS上的主机VM RHEL上创建这些目录/文件夹。请帮帮我，我已经困了好几天了。谢谢以下是日志：

StorageOS日志-

    > kubectl logs storageos-dpj6q
time="2018-07-11T18:38:30Z" level=info msg="starting server" address=172.31.14.99 cluster= hostname=ip-172-31-14-99 id=4dd922fe-d998-c67a-122e-1ea755ec61e9 join=d732967d-93c4-4018-8e0e-a9170065e8e2 labels="map[]" module=command version="StorageOS 1.0.0 (f8915fa), built: 2018-05-25T190132Z"
time="2018-07-11T18:38:30Z" level=info msg="starting api server" action=create category=server endpoint="0.0.0.0:5705" module=cp
time="2018-07-11T18:38:30Z" level=info msg="starting embedded etcd server" action=create category=etcd cluster_id= initialised=true members= module=cp
time="2018-07-11T18:38:30Z" level=info msg="by using this product, you are agreeing to the terms of the StorageOS Ltd. End User Subscription Agreement (EUSA) found at: https://eusa.storageos.com" module=command
time="2018-07-11T18:38:30Z" level=info msg="started temporary docker volume plugin api while control plane starts"
time="2018-07-11T18:38:38Z" level=info msg="embedded etcd server started successfuly" action=create category=etcd module=cp
time="2018-07-11T18:38:38Z" level=info msg="new messaging routes found, [re]starting nats" action=update category=nats module=cp
time="2018-07-11T18:38:39Z" level=info msg="connected to store" action=wait address="http://127.0.0.1:5706" backend=embedded category=etcd module=cp
time="2018-07-11T18:38:39Z" level=info msg="temporary docker volume plugin api shutdown"
time="2018-07-11T18:38:39Z" level=info msg="leader election process started" action=election category=leader module=ha
time="2018-07-11T18:38:39Z" level=info msg="docker plugin api started, exporting filesystems at /var/lib/storageos/filesystems"
time="2018-07-11T18:38:39Z" level=error msg="startup force unmount failed" device_dir=/var/lib/storageos/volumes error="exit status 32" module=supervisor output="umount: /var/lib/storageos/volumes: not mounted"
time="2018-07-11T18:38:39Z" level=info msg="reaper started"
time="2018-07-11T18:38:39Z" level=info msg="dataplane notifications server starting..." path="unix:///var/run/storageos/dataplane-notifications.sock"
time="2018-07-11T18:38:39Z" level=info msg="syncer started"
time="2018-07-11T18:38:39Z" level=info msg="module=\"storageos-stats\", release=\"1.0.0\", buildId=\"release/1.0.0-rc1_5\", scmId=\"15e59658ac5bccb957b74cb96b2cf44dc175f3b2\" level=info" module=supervisor
time="2018-07-11T18:38:40Z" level=info msg="StorageOS Volume Presentation level=info" module=supervisor
time="2018-07-11T18:38:40Z" level=info msg="module=\"storageos-fs-director\", release=\"1.0.0\", buildId=\"release/1.0.0-rc1_5\", scmId=\"15e59658ac5bccb957b74cb96b2cf44dc175f3b2\" level=info" module=supervisor
time="2018-07-11T18:38:40Z" level=info msg="fuse: mountpoint is not empty" module=supervisor
time="2018-07-11T18:38:40Z" level=info msg="fuse: if you are sure this is safe, use the 'nonempty' mount option" module=supervisor
time="2018-07-11T18:38:40Z" level=info msg="Error creating FUSE channel! level=fatal" module=supervisor
time="2018-07-11T18:38:40Z" level=info msg="module=\"server\", release=\"1.0.0\", buildId=\"release/1.0.0-rc1_5\", scmId=\"15e59658ac5bccb957b74cb96b2cf44dc175f3b2\" level=info" module=supervisor
time="2018-07-11T18:38:40Z" level=info msg="module=\"client\", release=\"1.0.0\", buildId=\"release/1.0.0-rc1_5\", scmId=\"15e59658ac5bccb957b74cb96b2cf44dc175f3b2\" level=info" module=supervisor
time="2018-07-11T18:38:40Z" level=info msg="StorageOS DirectFS v1 server (server v0.1 protocol v1.3) start level=info" module=supervisor
time="2018-07-11T18:38:40Z" level=info msg="StorageOS DIRECTOR category=director level=info" category=director module=supervisor
time="2018-07-11T18:38:40Z" level=info msg="module=\"storageos-director\", release=\"1.0.0\", buildId=\"release/1.0.0-rc1_5\", scmId=\"15e59658ac5bccb957b74cb96b2cf44dc175f3b2\" level=info" category=director module=supervisor
time="2018-07-11T18:38:40Z" level=info msg="CACHE: configure ram of 256MB exceeds maximum recommended of 66 MiB category=corecache level=warn" category=corecache module=supervisor
time="2018-07-11T18:38:40Z" level=info msg="CACHE: configured cache ram value is 256MiB, using 66MiB category=corecache level=warn" category=corecache module=supervisor
time="2018-07-11T18:38:41Z" level=info msg="StorageOS DirectFS v1 client (server v0.1 protocol v1.3) start category=clinit level=info" category=clinit module=supervisor
time="2018-07-11T18:38:41Z" level=info msg="StorageOS RDB plugin category=rdbplginit level=info" category=rdbplginit module=supervisor
time="2018-07-11T18:38:41Z" level=info msg="=> dir: /var/lib/storageos/data category=rdbplginit level=info" category=rdbplginit module=supervisor
time="2018-07-11T18:38:41Z" level=info msg="=> databases: 1 category=rdbplginit level=info" category=rdbplginit module=supervisor
time="2018-07-11T18:38:41Z" level=info msg="=> nodes: 1 category=rdbplginit level=info" category=rdbplginit module=supervisor
time="2018-07-11T18:38:41Z" level=info msg="module=\"storageos-rdbplugin\", release=\"1.0.0\", buildId=\"release/1.0.0-rc1_5\", scmId=\"15e59658ac5bccb957b74cb96b2cf44dc175f3b2\" level=info" category=rdbplginit module=supervisor
time="2018-07-11T18:38:41Z" level=info msg="Initialising rdb databases category=rdbplginit level=info" category=rdbplginit module=supervisor
time="2018-07-11T18:38:42Z" level=info msg="rdb peristing db at /var/lib/storageos/data/db1 category=rdbplginit level=info" category=rdbplginit module=supervisor
time="2018-07-11T18:38:42Z" level=info msg="Ready category=rdbplginit level=info" category=rdbplginit module=supervisor
time="2018-07-11T18:38:44Z" level=info msg="started watching volumes" module=watcher
time="2018-07-11T18:38:44Z" level=info msg="started watching nodes" module=watcher
time="2018-07-11T18:38:44Z" level=info msg="starting dataplane state controller worker '3'" module=statesync
time="2018-07-11T18:38:44Z" level=info msg="starting dataplane state controller worker '0'" module=statesync
time="2018-07-11T18:38:44Z" level=info msg="starting dataplane state controller worker '1'" module=statesync
time="2018-07-11T18:38:44Z" level=info msg="starting dataplane state controller worker '2'" module=statesync
time="2018-07-11T18:38:44Z" level=info msg=started action=watch category=client key=storageos/nodes module=store
time="2018-07-11T18:38:44Z" level=info msg=started action=watch category=client key=storageos/volumes module=store
time="2018-07-11T18:38:44Z" level=info msg="starting api servers" action=start category=http module=cp
time="2018-07-11T18:38:44Z" level=info msg="server running" module=command
time="2018-07-11T18:38:44Z" level=error msg="watch cancelled" action=watch category=client error="etcdserver: mvcc: required revision has been compacted" key=storageos/volumes module=store
time="2018-07-11T18:38:44Z" level=warning msg="volume watcher: got received error 'watch stopped'" module=watcher
time="2018-07-11T18:38:46Z" level=info msg="config change: will add server map entry [72389@172.31.29.31:5703] level=info" category=rdbplginit module=supervisor
time="2018-07-11T18:38:46Z" level=info msg="configuration changes applied level=info" category=rdbplginit module=supervisor
time="2018-07-11T18:38:46Z" level=info msg="[72389@172.31.29.31:5703]:: attempt to establish channel category=clconmon level=info" category=clconmon module=supervisor
time="2018-07-11T18:38:46Z" level=info msg="[72389@172.31.29.31:5703]: connection established (fd=13) category=clconn level=info" category=clconn module=supervisor
time="2018-07-11T18:38:46Z" level=info msg="void FsConfig::presentation_lun_notify_receiver(FsConfig::FsConfigStore *, ConfigStore::Event<uint32_t, FsConfigPresentation>): Failed to stat backing store file '/var/lib/storageos/volumes/bst-197928' category=config level=warn" category=config module=supervisor
time="2018-07-11T18:38:46Z" level=info msg="void FsConfig::presentation_lun_notify_receiver(FsConfig::FsConfigStore *, ConfigStore::Event<uint32_t, FsConfigPresentation>): Failed to stat backing store file '/var/lib/storageos/volumes/bst-212792' category=config level=warn" category=config module=supervisor
time="2018-07-11T18:38:46Z" level=info msg="virtual bool FsConfig::PresentationEventSemantics::Validate(event_type): Not adding pr_filename '2560c04f-995b-2ad1-bc03-edd9df29b57c' for volume 56980 - already exists forvolume 56980 category=fscfg level=warn" category=fscfg module=supervisor
time="2018-07-11T18:38:46Z" level=info msg="validator 'device_validator' rejected Event{type CREATE} category=libcfg level=warn" category=libcfg module=supervisor
time="2018-07-11T18:38:46Z" level=error msg="filesystem client: presentation create failed" action=create error="<nil>" module=statesync reason="Create refused by validator" volume_uuid=2560c04f-995b-2ad1-bc03-edd9df29b57c
time="2018-07-11T18:38:46Z" level=info msg="virtual bool FsConfig::PresentationEventSemantics::Validate(event_type): Not adding pr_filename 'f536c52c-daf3-4d3f-ccba-2f4c1db4aea1' for volume 72389 - already exists forvolume 72389 category=fscfg level=warn" category=fscfg module=supervisor
time="2018-07-11T18:38:46Z" level=info msg="validator 'device_validator' rejected Event{type CREATE} category=libcfg level=warn" category=libcfg module=supervisor
time="2018-07-11T18:38:46Z" level=error msg="filesystem client: presentation create failed" action=create error="<nil>" module=statesync reason="Create refused by validator" volume_uuid=f536c52c-daf3-4d3f-ccba-2f4c1db4aea1
time="2018-07-11T18:38:47Z" level=info msg="Accepted connection from 172.31.29.31:65228 (fd=8) level=info" category=libcfg module=supervisor
time="2018-07-11T18:38:47Z" level=info msg="0x0000DE94@172.31.29.31:65228 connected level=info" category=libcfg module=supervisor
time="2018-07-11T19:04:36Z" level=error msg="got err while waiting for volume creation" action=create category=volume error="no available nodes for replica found, error: no node matched filtering constraints, filters: anti_affinity_filter" module=cp namespace=default volume=vol1
time="2018-07-11T19:04:58Z" level=error msg="got err while waiting for volume creation" action=create category=volume error="no node matched filtering constraints, filters: node_selector_filter" module=cp namespace=default volume=vol2
time="2018-07-11T19:05:33Z" level=error msg="got err while waiting for volume creation" action=create category=volume error="no node matched filtering constraints, filters: " module=cp namespace=default volume=vol3

更新1：在按照下面的答案为k8设置标志之后

我仍然得到相同的错误，而豆荚创建。未在节点/主机VM RHEL上创建pvc目录。日志与上述“kubectl描述吊舱”的日志相同

Storageos pod正在运行的节点：

[root@ip-172-31-24-214 storageos]# pwd
/var/lib/storageos

[root@ip-172-31-24-214 storageos]# ls -l
total 0
drwxr-xr-x. 3 root root 17 Jul 13 10:07 data
drwxr-xr-x. 2 root root  6 Jul 13 10:07 filesystems
drwxr-xr-x. 3 root root 22 Jul 13 10:07 kv
drwxr--r--. 2 root root 27 Jul 13 10:06 logs
drwxr-x---. 2 root root 16 Jul 13 10:06 state
drwxr-xr-x. 2 root root  6 Jul 13 10:07 volumes

[root@ip-172-31-24-214 storageos]# cd volumes/
[root@ip-172-31-24-214 volumes]# ls -l
total 0

您的安装中似乎未启用MountPropagation。docker和Kubernetes k8s 1.10都支持默认情况下需要进行相应设置

您是否可以发布所有节点的/var/lib/storageos/volumes中是否有任何文件和设备

为k8s功能门的docker位、点2和点3签出以下文档链接。

您必须更改api控制器清单以添加功能门标志

Kubernetes 1.10在默认情况下启用了该特性门。然而，我不知道Rancher是如何部署配置的。如果kubelet正在系统中运行，则必须在systemd kubelet.service文件中添加功能门。如果它在容器中运行，则必须在其config-volume=/var/lib/storageos:/var/lib/storageos:rshared中添加具有共享装载标志的卷。helm图表具有变量cluster.sharedDir，您可以为容器化kubelet定义该变量，它将使用kubelet volume dir托管StorageOS的设备文件

Rancher，尤其是在使用rke时，通过tha cluster.yml文件部署其配置。有关更多详细信息，请参阅rke文档

要添加该特性门，您必须以以下方式调整cluster.yml并重新运行rke up-config cluster.yml：

希望，这有帮助

干杯，

达米安看起来遥不可及。你可以在这里或其他地方发布日志吗？我可以访问imgur链接，但我还是在上面添加了日志@KonstantinVustinI在所有节点上都有dir/var/lib/storageos/volumes，这些节点上的storageos吊舱运行时仍然面临问题。我确保设置了“-feature gates MountPropagation=true”。我添加了/var/lib/storageos:/var/lib/storageos:rshared，但现在当容器尝试装载卷时，我收到以下错误：MountVolume.pvc-a8b3bec5-01eb-11e9-97c1-00505691ceae卷的安装失败：API错误未经身份验证的安全端点访问，请在身份验证后重试：找不到仍然存在相同问题的帐户。正如上面pod日志中所述，我已使用此文档通过helm time=2018-07-15T11:03:14Z level=error msg=filesystem client:presentation create failed action=create error=module=statesync reason=create被验证器拒绝volume_uuid=fe5e6baf-7378-7822-333f-d98633bb5cdc安装StorageOS

> kubectl get sc
NAME      PROVISIONER
fast      kubernetes.io/storageos
>  kubectl get pv
NAME                                       CAPACITY   ACCESS MODES   RECLAIM POLICY   STATUS     CLAIM                STORAGECLASS      REASON    AGE
pvc-4ac4fc7c-8471-11e8-90d5-0ad368ad109e   10Gi       RWO            Delete           Released   default/volx         nfs-provisioner             1d
pvc-7a4ee536-851c-11e8-90d5-0ad368ad109e   20Gi       RWO            Delete           Bound      default/pgsql-data   fast                        15h
pvc-bc643fc6-8519-11e8-90d5-0ad368ad109e   10Gi       RWO,RWX        Delete           Bound      default/sos3         fast                        15h
> kubectl get pvc
NAME         STATUS    VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS   AGE
pgsql-data   Bound     pvc-7a4ee536-851c-11e8-90d5-0ad368ad109e   20Gi       RWO            fast           15h
sos3         Bound     pvc-bc643fc6-8519-11e8-90d5-0ad368ad109e   10Gi       RWO,RWX        fast           15h

[root@ip-172-31-24-214 storageos]# pwd
/var/lib/storageos

[root@ip-172-31-24-214 storageos]# ls -l
total 0
drwxr-xr-x. 3 root root 17 Jul 13 10:07 data
drwxr-xr-x. 2 root root  6 Jul 13 10:07 filesystems
drwxr-xr-x. 3 root root 22 Jul 13 10:07 kv
drwxr--r--. 2 root root 27 Jul 13 10:06 logs
drwxr-x---. 2 root root 16 Jul 13 10:06 state
drwxr-xr-x. 2 root root  6 Jul 13 10:07 volumes

[root@ip-172-31-24-214 storageos]# cd volumes/
[root@ip-172-31-24-214 volumes]# ls -l
total 0

# Necessary for password protected SSH Keys:
ssh_agent_auth: true

# Nodes Definitions
nodes:
  - address: 10.70.60.3  # hostname or IP to access nodes
    user: docker # root user (usually 'root')
    role: [controlplane,etcd,worker] # K8s roles for node
    ssh_key_path: ~/.ssh/id_rsa # path to PEM file
...

services:
  kube-apiserver:
    extra_args:
      feature-gates: "PersistentLocalVolumes=true,VolumeScheduling=true"
  kubelet:
    extra_args:
      feature-gates: "PersistentLocalVolumes=true,VolumeScheduling=true"

# Default versions
system_images:
    kubernetes: rancher/hyperkube:v1.10.5-rancher1
...