使用k8s版本 1.16,部署controller报错后降级了sidecar csi-provisioner => v1.6.1
未使用carina-scheduler,使用default scheduler绑定节点执行carina测试demo
问题:deployment部署成功后进入容器,/var/lib/www/html 没有mount
deploy文件
apiVersion: apps/v1
kind: Deployment
metadata:
name: carina-deployment
namespace: carina
labels:
app: web-server
spec:
replicas: 1
selector:
matchLabels:
app: web-server
template:
metadata:
annotations:
cni: macvlan
labels:
app: web-server
spec:
nodeSelector:
kubernetes.io/hostname: 172.23.36.5
containers:
- name: web-server
image: nginx:latest
imagePullPolicy: "IfNotPresent"
volumeMounts:
- name: mypvc
mountPath: /var/lib/www/html
volumes:
- name: mypvc
persistentVolumeClaim:
claimName: csi-carina-pvc
readOnly: false
集群内pv,lv,sc
kubelet 日志
kubelet.log
carina-node 日志
carina-node.log
进入carina-node容器执行命令
root@shylf-t-k8s-node-18:~ # docker exec -it 0e8b504934d8 bash
[root@shylf-t-k8s-node-18 /]# df -h
Filesystem Size Used Avail Use% Mounted on
overlay 893G 26G 867G 3% /
udev 63G 0 63G 0% /dev
shm 64M 0 64M 0% /dev/shm
/dev/sdb 893G 26G 867G 3% /csi
tmpfs 13G 1.4G 12G 11% /run/mount
/dev/sda1 47G 7.5G 40G 16% /var/log/carina
tmpfs 63G 0 63G 0% /sys/fs/cgroup
tmpfs 63G 12K 63G 1% /var/lib/kubelet/pods/c6a9e39c-8f82-4738-9629-f4a662fd88bc/volumes/kubernetes.io~secret/default-token-zstfk
tmpfs 63G 12K 63G 1% /var/lib/kubelet/pods/dfde4bea-cd81-4ec0-ae1b-3d0960fb6cc1/volumes/kubernetes.io~secret/default-token-zstfk
tmpfs 63G 12K 63G 1% /var/lib/kubelet/pods/86944153-3c9b-11ec-8561-049fca30d189/volumes/kubernetes.io~secret/default-token-gfsdw
tmpfs 63G 12K 63G 1% /var/lib/kubelet/pods/4f66c584-3e17-11ec-8561-049fca30d189/volumes/kubernetes.io~secret/default-token-dj45d
tmpfs 63G 12K 63G 1% /var/lib/kubelet/pods/89e4ef0e-58b0-4d09-81aa-21019b58f8b4/volumes/kubernetes.io~secret/volcano-controllers-token-5k4kn
tmpfs 63G 12K 63G 1% /var/lib/kubelet/pods/4aed7639-44a3-4db2-9d43-153fba35028a/volumes/kubernetes.io~secret/kruise-daemon-token-khwgc
tmpfs 63G 12K 63G 1% /var/lib/kubelet/pods/4d638e6a-9fa8-4ec7-a010-69798c89bcc4/volumes/kubernetes.io~secret/kubecost-kube-state-metrics-token-czcv2
tmpfs 63G 12K 63G 1% /var/lib/kubelet/pods/98b87ba2-cf4e-45b0-8962-65a4707a5791/volumes/kubernetes.io~secret/kubecost-prometheus-node-exporter-token-4xgxv
tmpfs 63G 12K 63G 1% /var/lib/kubelet/pods/ddc94974-7a4e-4d0d-ae7f-9d6af689dfcd/volumes/kubernetes.io~secret/default-token-zstfk
tmpfs 63G 12K 63G 1% /var/lib/kubelet/pods/8f78b4fb-4092-4630-8b93-ca815561607f/volumes/kubernetes.io~secret/default-token-p54ss
tmpfs 63G 0 63G 0% /var/lib/kubelet/pods/72f19bce-76f6-400e-8f3e-48fe56c28c78/volumes/kubernetes.io~empty-dir/socket-dir
tmpfs 63G 8.0K 63G 1% /var/lib/kubelet/pods/72f19bce-76f6-400e-8f3e-48fe56c28c78/volumes/kubernetes.io~secret/certs
tmpfs 63G 12K 63G 1% /var/lib/kubelet/pods/72f19bce-76f6-400e-8f3e-48fe56c28c78/volumes/kubernetes.io~secret/carina-csi-controller-token-lcrp9
tmpfs 63G 12K 63G 1% /run/secrets/kubernetes.io/serviceaccount
tmpfs 63G 12K 63G 1% /var/lib/kubelet/pods/9c9efec6-fdcc-4a18-aff5-dfb8979564be/volumes/kubernetes.io~secret/default-token-6lz94
/dev/carina/volume-pvc-c01289bc-5188-436b-90fc-6d32cab20fc1 6.0G 33M 6.0G 1% /data/docker/kubelet/pods/9c9efec6-fdcc-4a18-aff5-dfb8979564be/volumes/kubernetes.io~csi/pvc-c01289bc-5188-436b-90fc-6d32cab20fc1/mount
tmpfs 63G 12K 63G 1% /var/lib/kubelet/pods/d5ddd88d-8e11-4211-8d0f-c1982787d1d9/volumes/kubernetes.io~secret/default-token-zstfk
[root@shylf-t-k8s-node-18 /]# lvs
LV VG Attr LSize Pool Origin Data% Meta% Move Log Cpy%Sync Convert
mylv carina-vg-hdd -wi-a----- 10.00g
thin-pvc-c01289bc-5188-436b-90fc-6d32cab20fc1 carina-vg-hdd twi-aotz-- 6.00g 99.47 47.85
volume-pvc-c01289bc-5188-436b-90fc-6d32cab20fc1 carina-vg-hdd Vwi-aotz-- 6.00g thin-pvc-c01289bc-5188-436b-90fc-6d32cab20fc1 99.47
[root@shylf-t-k8s-node-18 /]# pvs
PV VG Fmt Attr PSize PFree
/dev/mapper/loop0p1 carina-vg-hdd lvm2 a-- <19.53g <19.52g
/dev/mapper/loop1p1 carina-vg-hdd lvm2 a-- 24.41g 8.40g
/dev/mapper/loop2p1 carina-vg-hdd lvm2 a-- 29.29g 29.29g
[root@shylf-t-k8s-node-18 /]# vgs
VG #PV #LV #SN Attr VSize VFree
carina-vg-hdd 3 3 0 wz--n- 73.23g 57.21g
进入nginx容器
root@shylf-t-k8s-node-18:~ # docker exec -it b3a711b9349c bash
root@carina-deployment-bc8959776-vln75:/# df -h
Filesystem Size Used Avail Use% Mounted on
overlay 893G 26G 867G 3% /
tmpfs 64M 0 64M 0% /dev
tmpfs 63G 0 63G 0% /sys/fs/cgroup
/dev/sdb 893G 26G 867G 3% /etc/hosts
shm 64M 0 64M 0% /dev/shm
tmpfs 63G 12K 63G 1% /run/secrets/kubernetes.io/serviceaccount
tmpfs 63G 0 63G 0% /proc/acpi
tmpfs 63G 0 63G 0% /sys/firmware
root@carina-deployment-bc8959776-vln75:/#
宿主机
df -h
root@shylf-t-k8s-node-18:~ # df -h
Filesystem Size Used Avail Use% Mounted on
udev 63G 0 63G 0% /dev
tmpfs 13G 1.4G 12G 11% /run
/dev/sda1 47G 7.5G 40G 16% /
tmpfs 63G 460K 63G 1% /dev/shm
tmpfs 5.0M 0 5.0M 0% /run/lock
tmpfs 63G 0 63G 0% /sys/fs/cgroup
/dev/sda3 165G 32G 133G 20% /data
/dev/sdb 893G 26G 867G 3% /data/docker
tmpfs 63G 12K 63G 1% /data/docker/kubelet/pods/c6a9e39c-8f82-4738-9629-f4a662fd88bc/volumes/kubernetes.io~secret/default-token-zstfk
tmpfs 63G 12K 63G 1% /data/docker/kubelet/pods/dfde4bea-cd81-4ec0-ae1b-3d0960fb6cc1/volumes/kubernetes.io~secret/default-token-zstfk
tmpfs 63G 12K 63G 1% /data/docker/kubelet/pods/86944153-3c9b-11ec-8561-049fca30d189/volumes/kubernetes.io~secret/default-token-gfsdw
tmpfs 63G 12K 63G 1% /data/docker/kubelet/pods/4f66c584-3e17-11ec-8561-049fca30d189/volumes/kubernetes.io~secret/default-token-dj45d
tmpfs 63G 12K 63G 1% /data/docker/kubelet/pods/89e4ef0e-58b0-4d09-81aa-21019b58f8b4/volumes/kubernetes.io~secret/volcano-controllers-token-5k4kn
tmpfs 63G 12K 63G 1% /data/docker/kubelet/pods/4aed7639-44a3-4db2-9d43-153fba35028a/volumes/kubernetes.io~secret/kruise-daemon-token-khwgc
tmpfs 63G 12K 63G 1% /data/docker/kubelet/pods/4d638e6a-9fa8-4ec7-a010-69798c89bcc4/volumes/kubernetes.io~secret/kubecost-kube-state-metrics-token-czcv2
tmpfs 63G 12K 63G 1% /data/docker/kubelet/pods/98b87ba2-cf4e-45b0-8962-65a4707a5791/volumes/kubernetes.io~secret/kubecost-prometheus-node-exporter-token-4xgxv
tmpfs 63G 12K 63G 1% /data/docker/kubelet/pods/ddc94974-7a4e-4d0d-ae7f-9d6af689dfcd/volumes/kubernetes.io~secret/default-token-zstfk
tmpfs 63G 12K 63G 1% /data/docker/kubelet/pods/8f78b4fb-4092-4630-8b93-ca815561607f/volumes/kubernetes.io~secret/default-token-p54ss
tmpfs 63G 0 63G 0% /data/docker/kubelet/pods/72f19bce-76f6-400e-8f3e-48fe56c28c78/volumes/kubernetes.io~empty-dir/socket-dir
tmpfs 63G 8.0K 63G 1% /data/docker/kubelet/pods/72f19bce-76f6-400e-8f3e-48fe56c28c78/volumes/kubernetes.io~secret/certs
tmpfs 63G 12K 63G 1% /data/docker/kubelet/pods/72f19bce-76f6-400e-8f3e-48fe56c28c78/volumes/kubernetes.io~secret/carina-csi-controller-token-lcrp9
overlay 893G 26G 867G 3% /data/docker/overlay2/a216d4228c9c3c045c6e4855906444b74ff39a9ee23ec0fb9dd1882aacf2ebf0/merged
overlay 893G 26G 867G 3% /data/docker/overlay2/526b8df3824b74887b7450dfab234ebf109c1bb7e500f866987ce9039269d3d0/merged
shm 64M 0 64M 0% /data/docker/containers/caa29edd3a5b8e138e37bcf91a95adead757dbf460deb3f7b745a7dfc0c93de7/mounts/shm
shm 64M 0 64M 0% /data/docker/containers/064c3b334433a68749cd477d1db14cbeb6104dbade3286ebcfc3ea633701233c/mounts/shm
overlay 893G 26G 867G 3% /data/docker/overlay2/71e132c2b079223cbc62f4a88de131809c4b7f1611fbcc7719abc4bd46654c87/merged
shm 64M 0 64M 0% /data/docker/containers/531d44a6048b2ce7e1d2f6a61604ecdecdb906f38ef90e47594665029a3583a7/mounts/shm
overlay 893G 26G 867G 3% /data/docker/overlay2/19653af8958402eefff0a01b1f8a8c676bfefc9617207e6fe71eba3bda5d1d46/merged
shm 64M 0 64M 0% /data/docker/containers/a86dd2b0d1e169680ec3cef8ba3a357b1ea2766d39a920290e9fdc3a6fca865e/mounts/shm
overlay 893G 26G 867G 3% /data/docker/overlay2/544bda05f222b55d1126e5f51c1c7559f8db819ab02ea4eb633635d408353e84/merged
overlay 893G 26G 867G 3% /data/docker/overlay2/8cc47db16916c53d3324ad4b8fd251036808602fbe837353c5f70e71efa4d2f4/merged
shm 64M 0 64M 0% /data/docker/containers/1542e464b9ffa9488478962415ec61589aef02d02f7ceee381837c943772a4ef/mounts/shm
overlay 893G 26G 867G 3% /data/docker/overlay2/667083a79276e053ab38db18d459596ebe89aea07bf72897e8cd3d9154f2cb0d/merged
overlay 893G 26G 867G 3% /data/docker/overlay2/541d373f498fd0aae9732569dc9ceb3d5edbf395da34153ce31daca5a6637814/merged
shm 64M 0 64M 0% /data/docker/containers/617dfa1afde0d1ca3a0dfe17ea96a27ec0ab8ee2536be344a0f31d5d17a76ae3/mounts/shm
overlay 893G 26G 867G 3% /data/docker/overlay2/16fae5669dcb9f44aee19b42a340acacede6fdb41f610f178f71785a0bab1d6d/merged
shm 64M 0 64M 0% /data/docker/containers/4d7b8c7cb079752cd1c2cfcf5ac3d55997696273fc957e286481b923add98b69/mounts/shm
overlay 893G 26G 867G 3% /data/docker/overlay2/6e4bd0e7003ffc089171f35c347c7e35f5b39e3c81c48740e09caf2f838f6e0b/merged
shm 64M 0 64M 0% /data/docker/containers/385987b7f5071e0119c4e1cd67cff21a48898be2252e2fe063102ec10cee42fc/mounts/shm
overlay 893G 26G 867G 3% /data/docker/overlay2/703de9e6fa8465ab8cc79c7aac019c9e8cb5bf031352b483d92c4061f6afe64b/merged
shm 64M 0 64M 0% /data/docker/containers/78c97664414f17c3d2a4b3b3192681793da4fb47e45f4e192761d30a710ac78d/mounts/shm
overlay 893G 26G 867G 3% /data/docker/overlay2/7a2f7ac15692e724c2da318427ddacc11badd6dee13bc58eac51aa349ac0c1da/merged
overlay 893G 26G 867G 3% /data/docker/overlay2/69d4eaf83bc1c0fde95f0fbfdaaf6293b8166ffec78a86b0f287ef3bd9793b47/merged
shm 64M 0 64M 0% /data/docker/containers/c328c93dfcedad046375a6d5c7ae61c159b4a1ccbfabd6cf84ede72fc3af5b80/mounts/shm
overlay 893G 26G 867G 3% /data/docker/overlay2/2b66361e05449c67b58666579f7bc763012ed1722599cfcc853adeb91b6eeffe/merged
overlay 893G 26G 867G 3% /data/docker/overlay2/98a9216bf26b3f6fb4e01205e69c6a61fa3946c0e0d4a2ee3cd0166e66921bb5/merged
overlay 893G 26G 867G 3% /data/docker/overlay2/55848bcb70e3f7d69e033ff0279848a1dde960a64e37d738d9dbe7899d6c34e2/merged
overlay 893G 26G 867G 3% /data/docker/overlay2/42e90451be6a7ec4dc9345764e8079d3beee8b205402e90d7db09fa02a260f34/merged
overlay 893G 26G 867G 3% /data/docker/overlay2/fb7a968851ca0f2832fbc7d515c5676ffeb52ba8b63d874c46ef29d43f763d82/merged
overlay 893G 26G 867G 3% /data/docker/overlay2/f404fdff1634045c92e58ea95536fbd7427b295881e4a969c94af608e734aa15/merged
overlay 893G 26G 867G 3% /data/docker/overlay2/3e80823a44e5d21f1138528f30a5e7df33af63e8f6b35706a7ae392fecc59db6/merged
tmpfs 13G 0 13G 0% /run/user/0
tmpfs 63G 12K 63G 1% /data/docker/kubelet/pods/5cd34f3e-8f5d-402e-ac45-5129ccc89dea/volumes/kubernetes.io~secret/carina-csi-node-token-mr2fk
overlay 893G 26G 867G 3% /data/docker/overlay2/d7e31342404d08d5fd4676d41ec7aaaf3d9ee5d8f98c1376cad972613c93a0ac/merged
shm 64M 0 64M 0% /data/docker/containers/939cdcc03f8e7b986dbe981eaa895de4d25adc0021a5e81cd144b9438adb85f3/mounts/shm
overlay 893G 26G 867G 3% /data/docker/overlay2/506e0d518ad983e10a29a2aed73707bdea0f40f70c85408fe5a326ed1e87220b/merged
overlay 893G 26G 867G 3% /data/docker/overlay2/3b5df922c0ce360e132b56c70407fe3c49b991c6bf277af05a06a3533ee985a5/merged
overlay 893G 26G 867G 3% /data/docker/overlay2/5125665eab4d1ed3b046f766588d83576c20e36dd32984520b5a0f852e407d3f/merged
tmpfs 63G 12K 63G 1% /data/docker/kubelet/pods/9c9efec6-fdcc-4a18-aff5-dfb8979564be/volumes/kubernetes.io~secret/default-token-6lz94
overlay 893G 26G 867G 3% /data/docker/overlay2/02245acd4ae110d14e805b69ce6fb589d391f9faee669a7659224a6c74c9b30d/merged
shm 64M 0 64M 0% /data/docker/containers/d6299df3d906b1495e81dc09ba54ea05cac467e4b5f87ae2f8edc8e09b31fe65/mounts/shm
overlay 893G 26G 867G 3% /data/docker/overlay2/467f745acd8f320de388690fa330bebf9601570cc199326bde64ba2dd16f0b52/merged
tmpfs 63G 12K 63G 1% /data/docker/kubelet/pods/5525ffff-228f-403c-8eb3-9fa3764f6779/volumes/kubernetes.io~secret/default-token-zstfk
overlay 893G 26G 867G 3% /data/docker/overlay2/fce3315104b4a463a8eeba2c57d418e59d82425bdf935dc44c7af9fd4dc7a017/merged
shm 64M 0 64M 0% /data/docker/containers/ab3ed3e62ad99a7bb4b62312757e7c527f3385e30bb270c80048d164c205a967/mounts/shm
root@shylf-t-k8s-node-18:~ # fdisk -l
Disk /dev/sdb: 893.1 GiB, 958999298048 bytes, 1873045504 sectors
Units: sectors of 1 * 512 = 512 bytes
Sector size (logical/physical): 512 bytes / 4096 bytes
I/O size (minimum/optimal): 4096 bytes / 4096 bytes
Disk /dev/sda: 222.6 GiB, 238999830528 bytes, 466796544 sectors
Units: sectors of 1 * 512 = 512 bytes
Sector size (logical/physical): 512 bytes / 4096 bytes
I/O size (minimum/optimal): 4096 bytes / 4096 bytes
Disklabel type: dos
Disk identifier: 0x0f794366
Device Boot Start End Sectors Size Id Type
/dev/sda1 * 2048 97656831 97654784 46.6G 83 Linux
/dev/sda2 97656832 121094143 23437312 11.2G 82 Linux swap / Solaris
/dev/sda3 121094144 466794495 345700352 164.9G 83 Linux
Disk /dev/loop0: 19.5 GiB, 20971520000 bytes, 40960000 sectors
Units: sectors of 1 * 512 = 512 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 512 bytes / 512 bytes
Disk /dev/loop1: 24.4 GiB, 26214400000 bytes, 51200000 sectors
Units: sectors of 1 * 512 = 512 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 512 bytes / 512 bytes
Disk /dev/loop2: 29.3 GiB, 31457280000 bytes, 61440000 sectors
Units: sectors of 1 * 512 = 512 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 512 bytes / 512 bytes
Disk /dev/mapper/carina--vg--hdd-volume--pvc--c01289bc--5188--436b--90fc--6d32cab20fc1: 6 GiB, 6442450944 bytes, 12582912 sectors
Units: sectors of 1 * 512 = 512 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 65536 bytes / 65536 bytes
Disk /dev/mapper/carina--vg--hdd-mylv: 10 GiB, 10737418240 bytes, 20971520 sectors
Units: sectors of 1 * 512 = 512 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 512 bytes / 512 bytes