diff --git a/bin/clusterConfig.sh b/bin/clusterConfig.sh deleted file mode 100644 index 08912d9..0000000 --- a/bin/clusterConfig.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -for i in "$@" -do - case $i in - -c=*|--config=*) - CONFIG_FILE="${i#*=}" - shift - ;; - -d=*|--domain=*) - SUB_DOMAIN="${i#*=}" - shift - ;; - *) - # put usage here: - ;; - esac -done \ No newline at end of file diff --git a/bin/createEnvScript.sh b/bin/createEnvScript.sh deleted file mode 100755 index 3a9308c..0000000 --- a/bin/createEnvScript.sh +++ /dev/null @@ -1,116 +0,0 @@ -#!/bin/bash - -EDGE=false -SUB_DOMAIN="" -INDEX="" -CONFIG_FILE=${LAB_CONFIG_FILE} - -for i in "$@" -do - case ${i} in - -e|--edge) - EDGE=true - shift - ;; - -c=*|--config=*) - CONFIG_FILE="${i#*=}" - shift - ;; - -d=*|--domain=*) - SUB_DOMAIN="${i#*=}" - shift - ;; - *) - echo "USAGE: createEnvScript -e -c=path/to/config/file -d=sub-domain-name" - ;; - esac -done - -EDGE_NETWORK=$(yq e ".network" ${CONFIG_FILE}) -LAB_DOMAIN=$(yq e ".domain" ${CONFIG_FILE}) -BASTION_HOST=$(yq e ".bastion-ip" ${CONFIG_FILE}) -EDGE_ROUTER=$(yq e ".router" ${CONFIG_FILE}) -EDGE_NETMASK=$(yq e ".netmask" ${CONFIG_FILE}) - -mkdir -p ${OKD_LAB_PATH}/work-dir - -if [[ ${SUB_DOMAIN} != "" ]] -then - DONE=false - DOMAIN_COUNT=$(yq e ".sub-domain-configs" ${CONFIG_FILE} | yq e 'length' -) - let i=0 - while [[ i -lt ${DOMAIN_COUNT} ]] - do - domain_name=$(yq e ".sub-domain-configs.[${i}].name" ${CONFIG_FILE}) - if [[ ${domain_name} == ${SUB_DOMAIN} ]] - then - INDEX=${i} - DONE=true - break - fi - i=$(( ${i} + 1 )) - done - if [[ ${DONE} == "false" ]] - then - echo "Domain Entry Not Found In Config File." - exit 1 - fi - SUB_DOMAIN=$(yq e ".sub-domain-configs.[${INDEX}].name" ${CONFIG_FILE}) - ROUTER=$(yq e ".sub-domain-configs.[${INDEX}].router-ip" ${CONFIG_FILE}) - NETWORK=$(yq e ".sub-domain-configs.[${INDEX}].network" ${CONFIG_FILE}) - EDGE_IP=$(yq e ".sub-domain-configs.[${INDEX}].router-edge-ip" ${CONFIG_FILE}) - NETMASK=$(yq e ".sub-domain-configs.[${INDEX}].netmask" ${CONFIG_FILE}) - -IFS=. read -r i1 i2 i3 i4 << EOI -${ROUTER} -EOI -LB_IP=${i1}.${i2}.${i3}.$(( ${i4} + 1 )) - -cat << EOF > ${OKD_LAB_PATH}/work-dir/internal-router -export EDGE_NETWORK=${EDGE_NETWORK} -export NETWORK=${NETWORK} -export NETMASK=${NETMASK} -export DOMAIN=${SUB_DOMAIN}.${LAB_DOMAIN} -export BASTION_HOST=${BASTION_HOST} -export EDGE_ROUTER=${EDGE_ROUTER} -export EDGE_IP=${EDGE_IP} -export ROUTER=${ROUTER} -export LB_IP=${LB_IP} -EOF - -A=$( echo ${SUB_DOMAIN} | tr "[:lower:]" "[:upper:]" ) -cat << EOF > ${OKD_LAB_PATH}/work-dir/edge-router-sub -export ${A}_ROUTER=${EDGE_IP} -export ${A}_NETWORK=${NETWORK} -EOF - -cat << EOF > ${OKD_LAB_PATH}/work-dir/edge-zone -zone "${SUB_DOMAIN}.${LAB_DOMAIN}" { - type stub; - masters { ${ROUTER}; }; - file "stub.${SUB_DOMAIN}.${LAB_DOMAIN}"; -}; - -EOF - -fi - - - -if [[ ${EDGE} == true ]] -then - -cat << EOF > ${OKD_LAB_PATH}/work-dir/edge-router -export NETWORK=${EDGE_NETWORK} -export DOMAIN=${LAB_DOMAIN} -export BASTION_HOST=${BASTION_HOST} -export ROUTER=${EDGE_ROUTER} -export NETMASK=${EDGE_NETMASK} -EOF -fi - - - - - - diff --git a/bin/createRootFs.sh b/bin/createRootFs.sh deleted file mode 100644 index e49cd38..0000000 --- a/bin/createRootFs.sh +++ /dev/null @@ -1,111 +0,0 @@ -#!/bin/bash - -SSH="ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" -SCP="scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" -CONFIG_FILE=${LAB_CONFIG_FILE} - -for i in "$@" -do - case $i in - -c=*|--config=*) - CONFIG_FILE="${i#*=}" - shift # past argument=value - ;; - -d=*|--domain=*) - SUB_DOMAIN="${i#*=}" - shift - ;; - *) - # put usage here: - ;; - esac -done - -DONE=false -DOMAIN_COUNT=$(yq e ".sub-domain-configs" ${CONFIG_FILE} | yq e 'length' -) -let i=0 -while [[ i -lt ${DOMAIN_COUNT} ]] -do - domain_name=$(yq e ".sub-domain-configs.[${i}].name" ${CONFIG_FILE}) - if [[ ${domain_name} == ${SUB_DOMAIN} ]] - then - INDEX=${i} - DONE=true - break - fi - i=$(( ${i} + 1 )) -done -if [[ ${DONE} == "false" ]] -then - echo "Domain Entry Not Found In Config File." - exit 1 -fi - -SUB_DOMAIN=$(yq e ".sub-domain-configs.[${INDEX}].name" ${CONFIG_FILE}) -CLUSTER_CONFIG=$(yq e ".sub-domain-configs.[${INDEX}].cluster-config-file" ${CONFIG_FILE}) -CLUSTER_NAME=$(yq e ".cluster-name" ${CLUSTER_CONFIG}) -OKD_RELEAE_IMAGE=$(openshift-install version | grep image | cut -d" " -f3) - -podman machine init fcos -podman machine start fcos - -FCOS_SSH_PORT=$(cat ~/.config/containers/podman/machine/qemu/fcos.json | jq -r '.Port') - -${SCP} -i ~/.ssh/fcos -P ${FCOS_SSH_PORT} ${OKD_LAB_PATH}/ipxe-work-dir/fcos/${CLUSTER_NAME}-${SUB_DOMAIN}/rootfs.img core@localhost:/tmp/rootfs.img - -cat << EOF > /tmp/createRootFs.sh -#!/bin/bash - -set -x - -podman pull -q ${OKD_RELEAE_IMAGE} -OS_CONTENT_IMAGE=\$(podman run --quiet --rm --net=none ${OKD_RELEAE_IMAGE} image machine-os-content) -podman pull -q \${OS_CONTENT_IMAGE} -CONTAINER_ID=\$(podman create --net=none --name ostree-container \${OS_CONTENT_IMAGE}) -mkdir -p /usr/local/fcos-image/os-content -podman cp \${CONTAINER_ID}:/ /usr/local/fcos-image/os-content -mkdir -p /usr/local/fcos-image/rootfs -cpio --extract -D /usr/local/fcos-image/rootfs < /tmp/rootfs.img -unsquashfs -d /usr/local/fcos-image/new-fs /usr/local/fcos-image/rootfs/root.squashfs -rm -rf /usr/local/fcos-image/new-fs/ostree/repo -mv /usr/local/fcos-image/os-content/srv/repo /usr/local/fcos-image/new-fs/ostree/repo -rm -f /usr/local/fcos-image/rootfs/root.squashfs -mksquashfs /usr/local/fcos-image/new-fs/ /usr/local/fcos-image/rootfs/root.squashfs -comp zstd -rm -rf /usr/local/fcos-image/new-fs/ /usr/local/fcos-image/os-content -ls /usr/local/fcos-image/rootfs > /usr/local/fcos-image/cpio.list -cpio -D /usr/local/fcos-image/rootfs --create < /usr/local/fcos-image/cpio.list > /usr/local/fcos-image/rootfs.img -EOF - -${SCP} -i ~/.ssh/fcos -P ${FCOS_SSH_PORT} /tmp/createRootFs.sh core@localhost:/tmp/createRootFs.sh - -${SSH} -i ~/.ssh/fcos -p ${FCOS_SSH_PORT} core@localhost "sudo chmod 755 /tmp/createRootFs.sh" -${SSH} -i ~/.ssh/fcos -p ${FCOS_SSH_PORT} core@localhost "sudo /tmp/createRootFs.sh" - -${SCP} -i ~/.ssh/fcos -P ${FCOS_SSH_PORT} core@localhost:/usr/local/fcos-image/rootfs.img ${OKD_LAB_PATH}/ipxe-work-dir/fcos/okd4-sno-${SUB_DOMAIN}/bootstrap-rootfs.img - -# ${SSH} -i ~/.ssh/fcos -p ${FCOS_SSH_PORT} core@localhost "sudo podman pull -q ${OKD_RELEAE_IMAGE}" -# OS_CONTENT_IMAGE=$(${SSH} -i ~/.ssh/fcos -p ${FCOS_SSH_PORT} core@localhost "sudo podman run --quiet --rm --net=none ${OKD_RELEAE_IMAGE} image machine-os-content") -# ${SSH} -i ~/.ssh/fcos -p ${FCOS_SSH_PORT} core@localhost "sudo podman pull -q ${OS_CONTENT_IMAGE}" -# CONTAINER_ID=$(${SSH} -i ~/.ssh/fcos -p ${FCOS_SSH_PORT} core@localhost "sudo podman create --net=none --name ostree-container ${OS_CONTENT_IMAGE}") - -# ${SSH} -i ~/.ssh/fcos -p ${FCOS_SSH_PORT} core@localhost "sudo mkdir -p /usr/local/fcos-image/os-content && sudo podman cp ${CONTAINER_ID}:/ /usr/local/fcos-image/os-content" - -# ${SCP} -i ~/.ssh/fcos -P ${FCOS_SSH_PORT} ${OKD_LAB_PATH}/ipxe-work-dir/fcos/okd4-sno-${SUB_DOMAIN}/rootfs.img core@localhost:/tmp/rootfs.img - -# ${SSH} -i ~/.ssh/fcos -p ${FCOS_SSH_PORT} core@localhost "sudo mkdir -p /usr/local/fcos-image/rootfs && sudo cpio --extract -D /usr/local/fcos-image/rootfs < /tmp/rootfs.img" - -# ${SSH} -i ~/.ssh/fcos -p ${FCOS_SSH_PORT} core@localhost "sudo unsquashfs -d /usr/local/fcos-image/new-fs /usr/local/fcos-image/rootfs/root.squashfs" - -# ${SSH} -i ~/.ssh/fcos -p ${FCOS_SSH_PORT} core@localhost "sudo rm -rf /usr/local/fcos-image/new-fs/ostree/repo && sudo mv /usr/local/fcos-image/os-content/srv/repo /usr/local/fcos-image/new-fs/ostree/repo" - -# ${SSH} -i ~/.ssh/fcos -p ${FCOS_SSH_PORT} core@localhost "sudo rm -f /usr/local/fcos-image/rootfs/root.squashfs && sudo mksquashfs /usr/local/fcos-image/new-fs/ /usr/local/fcos-image/rootfs/root.squashfs" - -# ${SSH} -i ~/.ssh/fcos -p ${FCOS_SSH_PORT} core@localhost "sudo rm -rf /usr/local/fcos-image/new-fs/ /usr/local/fcos-image/os-content" - -# ${SSH} -i ~/.ssh/fcos -p ${FCOS_SSH_PORT} core@localhost "sudo ls /usr/local/fcos-image/rootfs > /usr/local/fcos-image/cpio.list && sudo cpio -D /usr/local/fcos-image/rootfs --create < /usr/local/fcos-image/cpio.list > /usr/local/fcos-image/rootfs.img" - -# ${SCP} -i ~/.ssh/fcos -P ${FCOS_SSH_PORT} core@localhost:/usr/local/fcos-image/rootfs.img ${OKD_LAB_PATH}/ipxe-work-dir/fcos/okd4-sno-${SUB_DOMAIN}/bootstrap-rootfs.img - -# podman machine stop fcos -# sleep 10 -# podman machine rm --force fcos diff --git a/bin/deployKvmHosts.sh b/bin/deployKvmHosts.sh deleted file mode 100755 index 16a1a67..0000000 --- a/bin/deployKvmHosts.sh +++ /dev/null @@ -1,279 +0,0 @@ -#!/bin/bash - -SSH="ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" -SCP="scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" -CONFIG_FILE=${LAB_CONFIG_FILE} - -CREATE_DNS="false" -HOST_NAME="" -SUB_DOMAIN="" -INDEX="" - -for i in "$@" -do -case $i in - -c=*|--config=*) - CONFIG_FILE="${i#*=}" - shift - ;; - -h=*|--host=*) - HOST_NAME="${i#*=}" - shift - ;; - -d=*|--domain=*) - SUB_DOMAIN="${i#*=}" - shift - ;; - *) - # Put usage here: - ;; -esac -done - -function createPartInfo() { - -local disk1=${1} -local disk2=${2} - -if [[ ${disk2} == "NA" ]] -then -cat < ${OKD_LAB_PATH}/boot-work-dir/${mac_addr//:/-}.ipxe -#!ipxe - -kernel ${INSTALL_URL}/repos/BaseOS/x86_64/os/isolinux/vmlinuz net.ifnames=1 ifname=nic0:${mac_addr} ip=${ip_addr}::${ROUTER}:${NETMASK}:${hostname}.${DOMAIN}:nic0:none nameserver=${ROUTER} inst.ks=${INSTALL_URL}/kickstart/${mac_addr//:/-}.ks inst.repo=${INSTALL_URL}/repos/BaseOS/x86_64/os initrd=initrd.img -initrd ${INSTALL_URL}/repos/BaseOS/x86_64/os/isolinux/initrd.img - -boot -EOF -} - -function createKickStartFile() { - -local hostname=${1} -local mac_addr=${2} -local ip_addr=${3} -local disk1=${4} -local disk2=${5} - -PART_INFO=$(createPartInfo ${disk1} ${disk2} ) -DISK_LIST=${disk1} -if [[ ${disk2} != "NA" ]] -then - DISK_LIST="${disk1},${disk2}" -fi - -cat << EOF > ${OKD_LAB_PATH}/boot-work-dir/${mac_addr//:/-}.ks -#version=RHEL8 -cmdline -keyboard --vckeymap=us --xlayouts='us' -lang en_US.UTF-8 -repo --name="install" --baseurl=${INSTALL_URL}/repos/BaseOS/x86_64/os/ -url --url="${INSTALL_URL}/repos/BaseOS/x86_64/os" -rootpw --iscrypted ${LAB_PWD} -firstboot --disable -skipx -services --enabled="chronyd" -timezone America/New_York --isUtc - -# Disk partitioning information -ignoredisk --only-use=${DISK_LIST} -bootloader --append=" crashkernel=auto" --location=mbr --boot-drive=${disk1} -clearpart --drives=${DISK_LIST} --all --initlabel -zerombr -part /boot --fstype="xfs" --ondisk=${disk1} --size=1024 -part /boot/efi --fstype="efi" --ondisk=${disk1} --size=600 --fsoptions="umask=0077,shortname=winnt" -${PART_INFO} -logvol swap --fstype="swap" --size=16064 --name=swap --vgname=centos -logvol / --fstype="xfs" --grow --maxsize=2000000 --size=1024 --name=root --vgname=centos - -# Network Config -network --hostname=${hostname} -network --device=nic0 --noipv4 --noipv6 --no-activate --onboot=no -network --bootproto=static --device=br0 --bridgeslaves=nic0 --gateway=${ROUTER} --ip=${ip_addr} --nameserver=${ROUTER} --netmask=${NETMASK} --noipv6 --activate --bridgeopts="stp=false" --onboot=yes - -eula --agreed - -%packages -@^minimal-environment -kexec-tools -%end - -%addon com_redhat_kdump --enable --reserve-mb='auto' - -%end - -%anaconda -pwpolicy root --minlen=6 --minquality=1 --notstrict --nochanges --notempty -pwpolicy user --minlen=6 --minquality=1 --notstrict --nochanges --emptyok -pwpolicy luks --minlen=6 --minquality=1 --notstrict --nochanges --notempty -%end - -%post -dnf config-manager --add-repo ${INSTALL_URL}/postinstall/local-repos.repo -dnf config-manager --disable appstream -dnf config-manager --disable baseos -dnf config-manager --disable extras - -mkdir -p /root/.ssh -chmod 700 /root/.ssh -curl -o /root/.ssh/authorized_keys ${INSTALL_URL}/postinstall/authorized_keys -chmod 600 /root/.ssh/authorized_keys -dnf -y module install virt -dnf -y install wget git net-tools bind-utils bash-completion nfs-utils rsync libguestfs-tools virt-install iscsi-initiator-utils -dnf -y update -echo "InitiatorName=iqn.$(hostname)" > /etc/iscsi/initiatorname.iscsi -echo "options kvm_intel nested=1" >> /etc/modprobe.d/kvm.conf -systemctl enable libvirtd -mkdir /VirtualMachines -mkdir -p /root/bin -curl -o /root/bin/rebuildhost.sh ${INSTALL_URL}/postinstall/rebuildhost.sh -chmod 700 /root/bin/rebuildhost.sh -curl -o /etc/chrony.conf ${INSTALL_URL}/postinstall/chrony.conf -echo '@reboot root nmcli con mod "br0 slave 1" ethtool.feature-tso off' >> /etc/crontab -%end - -reboot - -EOF - -} - -function createDnsRecords() { - - local hostname=${1} - local ip_octet=${2} - - echo "${hostname}.${DOMAIN}. IN A ${NET_PREFIX}.${ip_octet} ; ${hostname}-${DOMAIN}-kvm" >> ${OKD_LAB_PATH}/boot-work-dir/forward.zone - echo "${ip_octet}.${NET_PREFIX_ARPA} IN PTR ${hostname}.${DOMAIN}. ; ${hostname}-${DOMAIN}-kvm" >> ${OKD_LAB_PATH}/boot-work-dir/reverse.zone -} - -function buildHostConfig() { - local index=${1} - - hostname=$(yq e ".kvm-hosts.[${index}].host-name" ${CLUSTER_CONFIG}) - mac_addr=$(yq e ".kvm-hosts.[${index}].mac-addr" ${CLUSTER_CONFIG}) - ip_octet=$(yq e ".kvm-hosts.[${index}].ip-octet" ${CLUSTER_CONFIG}) - ip_addr=${NET_PREFIX}.${ip_octet} - disk1=$(yq e ".kvm-hosts.[${index}].disks.disk1" ${CLUSTER_CONFIG}) - disk2=$(yq e ".kvm-hosts.[${index}].disks.disk2" ${CLUSTER_CONFIG}) - - TEST=$(dig ${hostname}.${DOMAIN} +short) - if [[ ${TEST} == "${ip_addr}" ]] - then - echo "DNS Record exists, skipping DNS record creation" - else - createDnsRecords ${hostname} ${ip_octet} - CREATE_DNS="true" - fi - createBootFile ${hostname} ${mac_addr} ${ip_addr} - createKickStartFile ${hostname} ${mac_addr} ${ip_addr} ${disk1} ${disk2} -} - -if [[ ${SUB_DOMAIN} != "" ]] -then - DONE=false - DOMAIN_COUNT=$(yq e ".sub-domain-configs" ${CONFIG_FILE} | yq e 'length' -) - let i=0 - while [[ i -lt ${DOMAIN_COUNT} ]] - do - domain_name=$(yq e ".sub-domain-configs.[${i}].name" ${CONFIG_FILE}) - if [[ ${domain_name} == ${SUB_DOMAIN} ]] - then - INDEX=${i} - DONE=true - break - fi - i=$(( ${i} + 1 )) - done - if [[ ${DONE} == "false" ]] - then - echo "Domain Entry Not Found In Config File." - exit 1 - fi - SUB_DOMAIN=$(yq e ".sub-domain-configs.[${INDEX}].name" ${CONFIG_FILE}) - ROUTER=$(yq e ".sub-domain-configs.[${INDEX}].router-ip" ${CONFIG_FILE}) - NETWORK=$(yq e ".sub-domain-configs.[${INDEX}].network" ${CONFIG_FILE}) - NETMASK=$(yq e ".sub-domain-configs.[${INDEX}].netmask" ${CONFIG_FILE}) - CLUSTER_CONFIG=$(yq e ".sub-domain-configs.[${INDEX}].cluster-config-file" ${CONFIG_FILE}) - DOMAIN="${SUB_DOMAIN}.${LAB_DOMAIN}" -else - DOMAIN=$(yq e ".domain" ${CONFIG_FILE}) - ROUTER=$(yq e ".router" ${CONFIG_FILE}) - NETWORK=$(yq e ".network" ${CONFIG_FILE}) - NETMASK=$(yq e ".netmask" ${CONFIG_FILE}) - CLUSTER_CONFIG=${CONFIG_FILE} -fi - -BASTION_HOST=$(yq e ".bastion-ip" ${CONFIG_FILE}) -INSTALL_URL="http://${BASTION_HOST}/install" - -LAB_PWD=$(cat ${OKD_LAB_PATH}/lab_host_pw) -IFS=. read -r i1 i2 i3 i4 << EOF -${NETWORK} -EOF -NET_PREFIX=${i1}.${i2}.${i3} -NET_PREFIX_ARPA=${i3}.${i2}.${i1} - -# Create temporary work directory -mkdir -p ${OKD_LAB_PATH}/boot-work-dir - -HOST_COUNT=$(yq e ".kvm-hosts" ${CLUSTER_CONFIG} | yq e 'length' -) - -if [[ ${HOST_NAME} == "" ]] -then - let i=0 - while [[ i -lt ${HOST_COUNT} ]] - do - buildHostConfig ${i} - i=$(( ${i} + 1 )) - done -else - DONE=false - let i=0 - while [[ i -lt ${HOST_COUNT} ]] - do - host_name=$(yq e ".kvm-hosts.[${i}].host-name" ${CLUSTER_CONFIG}) - if [[ ${host_name} == ${HOST_NAME} ]] - then - buildHostConfig ${i} - DONE=true - break - fi - i=$(( ${i} + 1 )) - done - if [[ ${DONE} == "false" ]] - then - echo "KVM HOST Entry Not Found In Config File." - exit 1 - fi -fi - -${SCP} -r ${OKD_LAB_PATH}/boot-work-dir/*.ks root@${BASTION_HOST}:/usr/local/www/install/kickstart -${SCP} -r ${OKD_LAB_PATH}/boot-work-dir/*.ipxe root@${ROUTER}:/data/tftpboot/ipxe -if [[ ${CREATE_DNS} == "true" ]] -then - cat ${OKD_LAB_PATH}/boot-work-dir/forward.zone | ${SSH} root@${ROUTER} "cat >> /etc/bind/db.${DOMAIN}" - cat ${OKD_LAB_PATH}/boot-work-dir/reverse.zone | ${SSH} root@${ROUTER} "cat >> /etc/bind/db.${NET_PREFIX_ARPA}" - ${SSH} root@${ROUTER} "/etc/init.d/named stop && /etc/init.d/named start" -fi - -rm -rf ${OKD_LAB_PATH}/boot-work-dir diff --git a/bin/deployOkdNodes.sh b/bin/deployOkdNodes.sh deleted file mode 100755 index e097d6f..0000000 --- a/bin/deployOkdNodes.sh +++ /dev/null @@ -1,516 +0,0 @@ -#!/bin/bash - -INIT_CLUSTER=false -ADD_WORKER=false -SSH="ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" -SCP="scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" -CONFIG_FILE=${LAB_CONFIG_FILE} -CP_REPLICAS="3" -SNO_BIP="" -SNO="false" -BIP="false" -# This script will set up the infrastructure to deploy an OKD 4.X cluster -# Follow the documentation at https://upstreamwithoutapaddle.com/home-lab/lab-intro/ - -for i in "$@" -do - case $i in - -c=*|--config=*) - CONFIG_FILE="${i#*=}" - shift # past argument=value - ;; - -i|--init) - INIT_CLUSTER=true - shift - ;; - -w|--worker) - ADD_WORKER=true - shift - ;; - -d=*|--domain=*) - SUB_DOMAIN="${i#*=}" - shift - ;; - *) - # put usage here: - ;; - esac -done - -function createControlPlaneDNS() { -cat << EOF > ${OKD_LAB_PATH}/dns-work-dir/forward.zone -${CLUSTER_NAME}-bootstrap.${DOMAIN}. IN A ${NET_PREFIX}.49 ; ${CLUSTER_NAME}-${DOMAIN}-bs -${CLUSTER_NAME}-lb01.${DOMAIN}. IN A ${NET_PREFIX}.2 ; ${CLUSTER_NAME}-${DOMAIN}-cp -*.apps.${CLUSTER_NAME}.${DOMAIN}. IN A ${NET_PREFIX}.2 ; ${CLUSTER_NAME}-${DOMAIN}-cp -api.${CLUSTER_NAME}.${DOMAIN}. IN A ${NET_PREFIX}.2 ; ${CLUSTER_NAME}-${DOMAIN}-cp -api-int.${CLUSTER_NAME}.${DOMAIN}. IN A ${NET_PREFIX}.2 ; ${CLUSTER_NAME}-${DOMAIN}-cp -${CLUSTER_NAME}-master-0.${DOMAIN}. IN A ${NET_PREFIX}.60 ; ${CLUSTER_NAME}-${DOMAIN}-cp -etcd-0.${DOMAIN}. IN A ${NET_PREFIX}.60 ; ${CLUSTER_NAME}-${DOMAIN}-cp -${CLUSTER_NAME}-master-1.${DOMAIN}. IN A ${NET_PREFIX}.61 ; ${CLUSTER_NAME}-${DOMAIN}-cp -etcd-1.${DOMAIN}. IN A ${NET_PREFIX}.61 ; ${CLUSTER_NAME}-${DOMAIN}-cp -${CLUSTER_NAME}-master-2.${DOMAIN}. IN A ${NET_PREFIX}.62 ; ${CLUSTER_NAME}-${DOMAIN}-cp -etcd-2.${DOMAIN}. IN A ${NET_PREFIX}.62 ; ${CLUSTER_NAME}-${DOMAIN}-cp -_etcd-server-ssl._tcp.${CLUSTER_NAME}.${DOMAIN} 86400 IN SRV 0 10 2380 etcd-0.${CLUSTER_NAME}.${DOMAIN}. ; ${CLUSTER_NAME}-${DOMAIN}-cp -_etcd-server-ssl._tcp.${CLUSTER_NAME}.${DOMAIN} 86400 IN SRV 0 10 2380 etcd-1.${CLUSTER_NAME}.${DOMAIN}. ; ${CLUSTER_NAME}-${DOMAIN}-cp -_etcd-server-ssl._tcp.${CLUSTER_NAME}.${DOMAIN} 86400 IN SRV 0 10 2380 etcd-2.${CLUSTER_NAME}.${DOMAIN}. ; ${CLUSTER_NAME}-${DOMAIN}-cp -EOF - -cat << EOF > ${OKD_LAB_PATH}/dns-work-dir/reverse.zone -2 IN PTR ${CLUSTER_NAME}-lb01.${DOMAIN}. ; ${CLUSTER_NAME}-${DOMAIN}-cp -49 IN PTR ${CLUSTER_NAME}-bootstrap.${DOMAIN}. ; ${CLUSTER_NAME}-${DOMAIN}-bs -60 IN PTR ${CLUSTER_NAME}-master-0.${DOMAIN}. ; ${CLUSTER_NAME}-${DOMAIN}-cp -61 IN PTR ${CLUSTER_NAME}-master-1.${DOMAIN}. ; ${CLUSTER_NAME}-${DOMAIN}-cp -62 IN PTR ${CLUSTER_NAME}-master-2.${DOMAIN}. ; ${CLUSTER_NAME}-${DOMAIN}-cp -EOF - -} - -function createSnoDNS() { -cat << EOF > ${OKD_LAB_PATH}/dns-work-dir/forward.zone -*.apps.${CLUSTER_NAME}.${DOMAIN}. IN A ${NET_PREFIX}.${NODE_IP} ; ${CLUSTER_NAME}-${DOMAIN}-cp -api.${CLUSTER_NAME}.${DOMAIN}. IN A ${NET_PREFIX}.${NODE_IP} ; ${CLUSTER_NAME}-${DOMAIN}-cp -api-int.${CLUSTER_NAME}.${DOMAIN}. IN A ${NET_PREFIX}.${NODE_IP} ; ${CLUSTER_NAME}-${DOMAIN}-cp -${CLUSTER_NAME}-sno-0.${DOMAIN}. IN A ${NET_PREFIX}.${NODE_IP} ; ${CLUSTER_NAME}-${DOMAIN}-cp -etcd-0.${DOMAIN}. IN A ${NET_PREFIX}.${NODE_IP} ; ${CLUSTER_NAME}-${DOMAIN}-cp -_etcd-server-ssl._tcp.${CLUSTER_NAME}.${DOMAIN} 86400 IN SRV 0 10 2380 etcd-0.${CLUSTER_NAME}.${DOMAIN}. ; ${CLUSTER_NAME}-${DOMAIN}-cp -EOF - -cat << EOF > ${OKD_LAB_PATH}/dns-work-dir/reverse.zone -${NODE_IP} IN PTR ${CLUSTER_NAME}-sno-0.${DOMAIN}. ; ${CLUSTER_NAME}-${DOMAIN}-cp -EOF - -} - -function createInstallConfig() { - - local install_dev=${1} - -if [[ ${SNO} == "true" ]] -then -read -r -d '' SNO_BIP << EOF -BootstrapInPlace: - InstallationDisk: /dev/${install_dev} -EOF -CP_REPLICAS="1" -fi - -cat << EOF > ${OKD_LAB_PATH}/install-config-upi.yaml -apiVersion: v1 -baseDomain: ${DOMAIN} -metadata: - name: ${CLUSTER_NAME} -networking: - networkType: OVNKubernetes - clusterNetwork: - - cidr: 10.100.0.0/14 - hostPrefix: 23 - serviceNetwork: - - 172.30.0.0/16 -compute: -- name: worker - replicas: 0 -controlPlane: - name: master - replicas: ${CP_REPLICAS} -platform: - none: {} -pullSecret: '${PULL_SECRET}' -sshKey: ${SSH_KEY} -additionalTrustBundle: | -${NEXUS_CERT} -imageContentSources: -- mirrors: - - ${REGISTRY}/okd - source: quay.io/openshift/okd -- mirrors: - - ${REGISTRY}/okd - source: quay.io/openshift/okd-content -${SNO_BIP} -EOF -} - -function configOkdNode() { - - local ip_addr=${1} - local host_name=${2} - local mac=${3} - local role=${4} - -cat << EOF > ${OKD_LAB_PATH}/ipxe-work-dir/ignition/${mac//:/-}.yml -variant: fcos -version: ${BUTANE_SPEC_VERSION} -ignition: - config: - merge: - - local: ${role}.ign -kernel_arguments: - should_exist: - - mitigations=auto - should_not_exist: - - mitigations=auto,nosmt - should_not_exist: - - mitigations=off -storage: - files: - - path: /etc/zincati/config.d/90-disable-feature.toml - mode: 0644 - contents: - inline: | - [updates] - enabled = false - - path: /etc/systemd/network/25-nic0.link - mode: 0644 - contents: - inline: | - [Match] - MACAddress=${mac} - [Link] - Name=nic0 - - path: /etc/NetworkManager/system-connections/nic0.nmconnection - mode: 0600 - overwrite: true - contents: - inline: | - [connection] - type=ethernet - interface-name=nic0 - - [ethernet] - mac-address=${mac} - - [ipv4] - method=manual - addresses=${ip_addr}/${NETMASK} - gateway=${ROUTER} - dns=${ROUTER} - dns-search=${DOMAIN} - - path: /etc/hostname - mode: 0420 - overwrite: true - contents: - inline: | - ${host_name} - - path: /etc/chrony.conf - mode: 0644 - overwrite: true - contents: - inline: | - pool ${BASTION_HOST} iburst - driftfile /var/lib/chrony/drift - makestep 1.0 3 - rtcsync - logdir /var/log/chrony -EOF - -cat ${OKD_LAB_PATH}/ipxe-work-dir/ignition/${mac//:/-}.yml | butane -d ${OKD_LAB_PATH}/ipxe-work-dir/ -o ${OKD_LAB_PATH}/ipxe-work-dir/ignition/${mac//:/-}.ign - -} - -function createPxeFile() { - local mac=${1} - local platform=${2} - local boot_dev=${3} - -if [[ ${platform} == "qemu" ]] -then - CONSOLE_OPT="console=ttyS0" -fi - -if [[ ${BIP} == "true" ]] -then -cat << EOF > ${OKD_LAB_PATH}/ipxe-work-dir/${mac//:/-}.ipxe -#!ipxe - -kernel http://${BASTION_HOST}/install/fcos/${CLUSTER_NAME}-${SUB_DOMAIN}/vmlinuz edd=off net.ifnames=1 rd.neednet=1 ignition.firstboot ignition.config.url=http://${BASTION_HOST}/install/fcos/ignition/${CLUSTER_NAME}-${SUB_DOMAIN}/${mac//:/-}.ign ignition.platform.id=${platform} initrd=initrd initrd=rootfs.img ${CONSOLE_OPT} -initrd http://${BASTION_HOST}/install/fcos/${CLUSTER_NAME}-${SUB_DOMAIN}/initrd -initrd http://${BASTION_HOST}/install/fcos/${CLUSTER_NAME}-${SUB_DOMAIN}/rootfs.img - -boot -EOF -else -cat << EOF > ${OKD_LAB_PATH}/ipxe-work-dir/${mac//:/-}.ipxe -#!ipxe - -kernel http://${BASTION_HOST}/install/fcos/${CLUSTER_NAME}-${SUB_DOMAIN}/vmlinuz edd=off net.ifnames=1 rd.neednet=1 coreos.inst.install_dev=/dev/${boot_dev} coreos.inst.ignition_url=http://${BASTION_HOST}/install/fcos/ignition/${CLUSTER_NAME}-${SUB_DOMAIN}/${mac//:/-}.ign coreos.inst.platform_id=${platform} initrd=initrd initrd=rootfs.img ${CONSOLE_OPT} -initrd http://${BASTION_HOST}/install/fcos/${CLUSTER_NAME}-${SUB_DOMAIN}/initrd -initrd http://${BASTION_HOST}/install/fcos/${CLUSTER_NAME}-${SUB_DOMAIN}/rootfs.img - -boot -EOF -fi - -} - - -function createOkdVmNode() { - - local ip_addr=${1} - local host_name=${2} - local kvm_host=${3} - local role=${4} - local memory=${5} - local cpu=${6} - local root_vol=${7} - local ceph_vol=${8} - - # Create the VM - DISK_CONFIG="--disk size=${root_vol},path=/VirtualMachines/${host_name}/rootvol,bus=sata" - if [ ${ceph_vol} != "0" ] - then - DISK_CONFIG="${DISK_CONFIG} --disk size=${ceph_vol},path=/VirtualMachines/${host_name}/datavol,bus=sata" - fi - ${SSH} root@${kvm_host}.${DOMAIN} "mkdir -p /VirtualMachines/${host_name}" - ${SSH} root@${kvm_host}.${DOMAIN} "virt-install --print-xml 1 --name ${host_name} --memory ${memory} --vcpus ${cpu} --boot=hd,network,menu=on,useserial=on ${DISK_CONFIG} --network bridge=br0 --graphics none --noautoconsole --os-variant centos7.0 --cpu host-passthrough,match=exact > /VirtualMachines/${host_name}.xml" - ${SSH} root@${kvm_host}.${DOMAIN} "virsh define /VirtualMachines/${host_name}.xml" -} - -DONE=false -DOMAIN_COUNT=$(yq e ".sub-domain-configs" ${CONFIG_FILE} | yq e 'length' -) -let i=0 -while [[ i -lt ${DOMAIN_COUNT} ]] -do - domain_name=$(yq e ".sub-domain-configs.[${i}].name" ${CONFIG_FILE}) - if [[ ${domain_name} == ${SUB_DOMAIN} ]] - then - INDEX=${i} - DONE=true - break - fi - i=$(( ${i} + 1 )) -done -if [[ ${DONE} == "false" ]] -then - echo "Domain Entry Not Found In Config File." - exit 1 -fi - -LAB_DOMAIN=$(yq e ".domain" ${CONFIG_FILE}) -BASTION_HOST=$(yq e ".bastion-ip" ${CONFIG_FILE}) -SUB_DOMAIN=$(yq e ".sub-domain-configs.[${INDEX}].name" ${CONFIG_FILE}) -ROUTER=$(yq e ".sub-domain-configs.[${INDEX}].router-ip" ${CONFIG_FILE}) -NETWORK=$(yq e ".sub-domain-configs.[${INDEX}].network" ${CONFIG_FILE}) -NETMASK=$(yq e ".sub-domain-configs.[${INDEX}].netmask" ${CONFIG_FILE}) -CLUSTER_CONFIG=$(yq e ".sub-domain-configs.[${INDEX}].cluster-config-file" ${CONFIG_FILE}) -DOMAIN="${SUB_DOMAIN}.${LAB_DOMAIN}" -REGISTRY=$(yq e ".proxy-registry" ${CLUSTER_CONFIG}) -CLUSTER_NAME=$(yq e ".cluster-name" ${CLUSTER_CONFIG}) -PULL_SECRET=$(yq e ".secret-file" ${CLUSTER_CONFIG}) -BUTANE_SPEC_VERSION=$(yq e ".butane-spec-version" ${CLUSTER_CONFIG}) -INSTALL_URL="http://${BASTION_HOST}/install" - -IFS=. read -r i1 i2 i3 i4 << EOF -${NETWORK} -EOF -NET_PREFIX=${i1}.${i2}.${i3} -NET_PREFIX_ARPA=${i3}.${i2}.${i1} - -rm -rf ${OKD_LAB_PATH}/ipxe-work-dir -rm -rf ${OKD_LAB_PATH}/dns-work-dir -mkdir -p ${OKD_LAB_PATH}/ipxe-work-dir/ignition -mkdir -p ${OKD_LAB_PATH}/ipxe-work-dir/fcos/${CLUSTER_NAME}-${SUB_DOMAIN} -mkdir -p ${OKD_LAB_PATH}/dns-work-dir - -if [[ ${INIT_CLUSTER} == "true" ]] -then - SSH_KEY=$(cat ${OKD_LAB_PATH}/id_rsa.pub) - PULL_SECRET=$(cat ${OKD_LAB_PATH}/pull_secret.json) - NEXUS_CERT=$(openssl s_client -showcerts -connect ${REGISTRY} /dev/null|openssl x509 -outform PEM | while read line; do echo " ${line}"; done) - CP_COUNT=$(yq e ".control-plane.okd-hosts" ${CLUSTER_CONFIG} | yq e 'length' -) - if [[ ${CP_COUNT} == "1" ]] - then - SNO="true" - elif [[ ${CP_COUNT} != "3" ]] - then - echo "There must be 3 host entries for the control plane for a full cluster, or 1 entry for a Single Node cluster." - exit 1 - fi - - # Create and deploy ignition files single-node-ignition-config - rm -rf ${OKD_LAB_PATH}/okd-install-dir - mkdir ${OKD_LAB_PATH}/okd-install-dir - - if [[ ${SNO} == "false" ]] # Create Bootstrap Node - then - # Create ignition files - createInstallConfig "null" - cp ${OKD_LAB_PATH}/install-config-upi.yaml ${OKD_LAB_PATH}/okd-install-dir/install-config.yaml - openshift-install --dir=${OKD_LAB_PATH}/okd-install-dir create ignition-configs - cp ${OKD_LAB_PATH}/okd-install-dir/*.ign ${OKD_LAB_PATH}/ipxe-work-dir/ - # Create Bootstrap Node: - host_name=${CLUSTER_NAME}-bootstrap - ip_addr=${NET_PREFIX}.49 - boot_dev=sda - platform=qemu - if [[ $(yq e ".bootstrap.metal" ${CLUSTER_CONFIG}) == "true" ]] - then - mac_addr=$(yq e ".bootstrap.mac-addr" ${CLUSTER_CONFIG}) - mkdir -p ${OKD_LAB_PATH}/bootstrap - qemu-img create -f qcow2 ${OKD_LAB_PATH}/bootstrap/bootstrap-node.qcow2 50G - else - kvm_host=$(yq e ".bootstrap.kvm-host" ${CLUSTER_CONFIG}) - memory=$(yq e ".bootstrap.node-spec.memory" ${CLUSTER_CONFIG}) - cpu=$(yq e ".bootstrap.node-spec.cpu" ${CLUSTER_CONFIG}) - root_vol=$(yq e ".bootstrap.node-spec.root_vol" ${CLUSTER_CONFIG}) - createOkdVmNode ${ip_addr} ${host_name} ${kvm_host} bootstrap ${memory} ${cpu} ${root_vol} 0 - # Get the MAC address for eth0 in the new VM - var=$(${SSH} root@${kvm_host}.${DOMAIN} "virsh -q domiflist ${host_name} | grep br0") - mac_addr=$(echo ${var} | cut -d" " -f5) - yq e ".bootstrap.mac-addr = \"${mac_addr}\"" -i ${CLUSTER_CONFIG} - fi - # Create the ignition and iPXE boot files - configOkdNode ${ip_addr} ${host_name}.${DOMAIN} ${mac_addr} bootstrap - createPxeFile ${mac_addr} ${platform} ${boot_dev} - fi - - #Create Control Plane Nodes: - metal=$(yq e ".control-plane.metal" ${CLUSTER_CONFIG}) - if [[ ${metal} == "true" ]] - then - platform=metal - else - platform=qemu - fi - - if [[ ${SNO} == "true" ]] - then - NODE_IP=$(yq e ".control-plane.okd-hosts.[0].ip-octet" ${CLUSTER_CONFIG}) - ip_addr=${NET_PREFIX}.${NODE_IP} - host_name=${CLUSTER_NAME}-sno-0 - if [[ ${metal} == "true" ]] - then - mac_addr=$(yq e ".control-plane.okd-hosts.[0].mac-addr" ${CLUSTER_CONFIG}) - else - memory=$(yq e ".control-plane.node-spec.memory" ${CLUSTER_CONFIG}) - cpu=$(yq e ".control-plane.node-spec.cpu" ${CLUSTER_CONFIG}) - root_vol=$(yq e ".control-plane.node-spec.root_vol" ${CLUSTER_CONFIG}) - kvm_host=$(yq e ".control-plane.okd-hosts.[0].kvm-host" ${CLUSTER_CONFIG}) - # Create the VM - createOkdVmNode ${ip_addr} ${host_name} ${kvm_host} sno ${memory} ${cpu} ${root_vol} 0 - # Get the MAC address for eth0 in the new VM - var=$(${SSH} root@${kvm_host}.${DOMAIN} "virsh -q domiflist ${host_name} | grep br0") - mac_addr=$(echo ${var} | cut -d" " -f5) - yq e ".control-plane.okd-hosts.[0].mac-addr = \"${mac_addr}\"" -i ${CLUSTER_CONFIG} - fi - # Create the ignition and iPXE boot files - install_dev=$(yq e ".control-plane.okd-hosts.[0].sno-install-dev" ${CLUSTER_CONFIG}) - boot_dev=$(yq e ".control-plane.okd-hosts.[0].boot-dev" ${CLUSTER_CONFIG}) - if [[ "${install_dev}" == "${boot_dev}" ]] - then - BIP="true" - fi - createInstallConfig ${install_dev} - cp ${OKD_LAB_PATH}/install-config-upi.yaml ${OKD_LAB_PATH}/okd-install-dir/install-config.yaml - openshift-install --dir=${OKD_LAB_PATH}/okd-install-dir create single-node-ignition-config - cp ${OKD_LAB_PATH}/okd-install-dir/bootstrap-in-place-for-live-iso.ign ${OKD_LAB_PATH}/ipxe-work-dir/sno.ign - configOkdNode ${ip_addr} ${host_name}.${DOMAIN} ${mac_addr} sno - createPxeFile ${mac_addr} ${platform} ${boot_dev} - # Set the node values in the lab domain configuration file - yq e ".control-plane.okd-hosts.[0].name = \"${host_name}\"" -i ${CLUSTER_CONFIG} - yq e ".control-plane.okd-hosts.[0].ip-addr = \"${ip_addr}\"" -i ${CLUSTER_CONFIG} - createSnoDNS - else - for i in 0 1 2 - do - ip_addr=${NET_PREFIX}.6${i} - host_name=${CLUSTER_NAME}-master-${i} - if [[ ${metal} == "true" ]] - then - mac_addr=$(yq e ".control-plane.okd-hosts.[${i}].mac-addr" ${CLUSTER_CONFIG}) - boot_dev=$(yq e ".control-plane.okd-hosts.[${i}].boot-dev" ${CLUSTER_CONFIG}) - else - memory=$(yq e ".control-plane.node-spec.memory" ${CLUSTER_CONFIG}) - cpu=$(yq e ".control-plane.node-spec.cpu" ${CLUSTER_CONFIG}) - root_vol=$(yq e ".control-plane.node-spec.root_vol" ${CLUSTER_CONFIG}) - kvm_host=$(yq e ".control-plane.okd-hosts.[${i}].kvm-host" ${CLUSTER_CONFIG}) - boot_dev="sda" - # Create the VM - createOkdVmNode ${ip_addr} ${host_name} ${kvm_host} master ${memory} ${cpu} ${root_vol} 0 - # Get the MAC address for eth0 in the new VM - var=$(${SSH} root@${kvm_host}.${DOMAIN} "virsh -q domiflist ${host_name} | grep br0") - mac_addr=$(echo ${var} | cut -d" " -f5) - yq e ".control-plane.okd-hosts.[${i}].mac-addr = \"${mac_addr}\"" -i ${CLUSTER_CONFIG} - fi - # Create the ignition and iPXE boot files - configOkdNode ${ip_addr} ${host_name}.${DOMAIN} ${mac_addr} master - createPxeFile ${mac_addr} ${platform} ${boot_dev} - # Set the node values in the lab domain configuration file - yq e ".control-plane.okd-hosts.[${i}].name = \"${host_name}\"" -i ${CLUSTER_CONFIG} - yq e ".control-plane.okd-hosts.[${i}].ip-addr = \"${ip_addr}\"" -i ${CLUSTER_CONFIG} - done - # Create DNS Records: - createControlPlaneDNS - fi - - KERNEL_URL=$(openshift-install coreos print-stream-json | jq -r '.architectures.x86_64.artifacts.metal.formats.pxe.kernel.location') - INITRD_URL=$(openshift-install coreos print-stream-json | jq -r '.architectures.x86_64.artifacts.metal.formats.pxe.initramfs.location') - ROOTFS_URL=$(openshift-install coreos print-stream-json | jq -r '.architectures.x86_64.artifacts.metal.formats.pxe.rootfs.location') - - curl -o ${OKD_LAB_PATH}/ipxe-work-dir/fcos/${CLUSTER_NAME}-${SUB_DOMAIN}/vmlinuz ${KERNEL_URL} - curl -o ${OKD_LAB_PATH}/ipxe-work-dir/fcos/${CLUSTER_NAME}-${SUB_DOMAIN}/initrd ${INITRD_URL} - curl -o ${OKD_LAB_PATH}/ipxe-work-dir/fcos/${CLUSTER_NAME}-${SUB_DOMAIN}/rootfs.img ${ROOTFS_URL} - - ${SCP} -r ${OKD_LAB_PATH}/ipxe-work-dir/fcos/${CLUSTER_NAME}-${SUB_DOMAIN} root@${BASTION_HOST}:/usr/local/www/install/fcos/ -fi - -if [[ ${ADD_WORKER} == "true" ]] -then - if [[ ${INIT_CLUSTER} != "true" ]] - then - export KUBECONFIG="${OKD_LAB_PATH}/lab-config/${CLUSTER_NAME}.${DOMAIN}/kubeconfig" - ID=$(oc whoami) - if [[ ${ID} != "system:admin" ]] - then - echo "ERROR: Invalid kube_config: ${KUBECONFIG}" - exit 1 - fi - oc extract -n openshift-machine-api secret/worker-user-data --keys=userData --to=- > ${OKD_LAB_PATH}/ipxe-work-dir/worker.ign - fi - let NODE_COUNT=$(yq e ".compute-nodes" ${CLUSTER_CONFIG} | yq e 'length' -) - let i=0 - let j=70 - while [[ i -lt ${NODE_COUNT} ]] - do - host_name=${CLUSTER_NAME}-worker-${i} - ip_addr=${NET_PREFIX}.${j} - if [[ $(yq e ".compute-nodes.[${i}].metal" ${CLUSTER_CONFIG}) == "true" ]] - then - platform=metal - mac_addr=$(yq e ".compute-nodes.[${i}].mac-addr" ${CLUSTER_CONFIG}) - boot_dev=$(yq e ".compute-nodes.[${i}].boot-dev" ${CLUSTER_CONFIG}) - else - platform=qemu - boot_dev="sda" - memory=$(yq e ".compute-nodes.[${i}].node-spec.memory" ${CLUSTER_CONFIG}) - cpu=$(yq e ".compute-nodes.[${i}].node-spec.cpu" ${CLUSTER_CONFIG}) - root_vol=$(yq e ".compute-nodes.[${i}].node-spec.root_vol" ${CLUSTER_CONFIG}) - ceph_vol=$(yq e ".compute-nodes.[${i}].node-spec.ceph_vol" ${CLUSTER_CONFIG}) - kvm_host=$(yq e ".compute-nodes.[${i}].kvm-host" ${CLUSTER_CONFIG}) - # Create the VM - createOkdVmNode ${ip_addr} ${host_name} ${kvm_host} worker ${memory} ${cpu} ${root_vol} ${ceph_vol} - # Get the MAC address for eth0 in the new VM - var=$(${SSH} root@${kvm_host}.${DOMAIN} "virsh -q domiflist ${host_name} | grep br0") - mac_addr=$(echo ${var} | cut -d" " -f5) - yq e ".compute-nodes.[${i}].mac-addr = \"${mac_addr}\"" -i ${CLUSTER_CONFIG} - fi - # Create the ignition and iPXE boot files - configOkdNode ${ip_addr} ${host_name}.${DOMAIN} ${mac_addr} worker - createPxeFile ${mac_addr} ${platform} ${boot_dev} - # Set the node values in the lab domain configuration file - yq e ".compute-nodes.[${i}].name = \"${host_name}\"" -i ${CLUSTER_CONFIG} - yq e ".compute-nodes.[${i}].ip-addr = \"${ip_addr}\"" -i ${CLUSTER_CONFIG} - # Create DNS entries - echo "${host_name}.${DOMAIN}. IN A ${NET_PREFIX}.${j} ; ${host_name}-${DOMAIN}-wk" >> ${OKD_LAB_PATH}/dns-work-dir/forward.zone - echo "${j} IN PTR ${host_name}.${DOMAIN}. ; ${host_name}-${DOMAIN}-wk" >> ${OKD_LAB_PATH}/dns-work-dir/reverse.zone - - i=$(( ${i} + 1 )) - j=$(( ${j} + 1 )) - done -fi - -cat ${OKD_LAB_PATH}/dns-work-dir/forward.zone | ${SSH} root@${ROUTER} "cat >> /etc/bind/db.${DOMAIN}" -cat ${OKD_LAB_PATH}/dns-work-dir/reverse.zone | ${SSH} root@${ROUTER} "cat >> /etc/bind/db.${NET_PREFIX_ARPA}" -${SSH} root@${ROUTER} "/etc/init.d/named stop && /etc/init.d/named start" -${SSH} root@${BASTION_HOST} "mkdir -p /usr/local/www/install/fcos/ignition/${CLUSTER_NAME}-${SUB_DOMAIN}" -${SCP} -r ${OKD_LAB_PATH}/ipxe-work-dir/ignition/*.ign root@${BASTION_HOST}:/usr/local/www/install/fcos/ignition/${CLUSTER_NAME}-${SUB_DOMAIN}/ -${SSH} root@${BASTION_HOST} "chmod 644 /usr/local/www/install/fcos/ignition/${CLUSTER_NAME}-${SUB_DOMAIN}/*" -${SCP} -r ${OKD_LAB_PATH}/ipxe-work-dir/*.ipxe root@${ROUTER}:/data/tftpboot/ipxe/ diff --git a/bin/destroyNodes.sh b/bin/destroyNodes.sh deleted file mode 100755 index 3e4f411..0000000 --- a/bin/destroyNodes.sh +++ /dev/null @@ -1,281 +0,0 @@ -#!/bin/bash - -SSH="ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" -RESET_LB=false -DELETE_BOOTSTRAP=false -DELETE_CLUSTER=false -DELETE_WORKER=false -DELETE_KVM_HOST=false -W_HOST_NAME="" -W_HOST_INDEX="" -K_HOST_INDEX="" -M_HOST_INDEX="" -let NODE_COUNT=0 -SNO="false" - -CONFIG_FILE=${LAB_CONFIG_FILE} - -for i in "$@" -do - case $i in - -c=*|--config=*) - CONFIG_FILE="${i#*=}" - shift - ;; - -d=*|--domain=*) - SUB_DOMAIN="${i#*=}" - shift - ;; - -b|--bootstrap) - DELETE_BOOTSTRAP=true - shift - ;; - -w=*|--worker=*) - DELETE_WORKER=true - W_HOST_NAME="${i#*=}" - shift - ;; - -r|--reset) - RESET_LB=true - DELETE_CLUSTER=true - DELETE_WORKER=true - W_HOST_NAME="all" - shift - ;; - -k=*|--kvm-host=*) - DELETE_KVM_HOST=true - K_HOST_NAME="${i#*=}" - shift - ;; - -m=*|--master=*) - M_HOST_NAME="${i#*=}" - shift - ;; - *) - # put usage here: - ;; - esac -done - -# Destroy the VM -function deleteNodeVm() { - local host_name=${1} - local kvm_host=${2} - - ${SSH} root@${kvm_host}.${DOMAIN} "virsh destroy ${host_name}" - ${SSH} root@${kvm_host}.${DOMAIN} "virsh undefine ${host_name}" - ${SSH} root@${kvm_host}.${DOMAIN} "virsh pool-destroy ${host_name}" - ${SSH} root@${kvm_host}.${DOMAIN} "virsh pool-undefine ${host_name}" - ${SSH} root@${kvm_host}.${DOMAIN} "rm -rf /VirtualMachines/${host_name}" -} - -# Destroy a physical host: -function destroyMetal() { - local user=${1} - local hostname=${2} - local boot_dev=${3} - local ceph_dev=${4} - - if [[ ${ceph_dev} != "na" ]] && [[ ${ceph_dev} != "" ]] - then - ${SSH} -o ConnectTimeout=5 ${user}@${hostname}.${DOMAIN} "sudo dd if=/dev/zero of=/dev/${ceph_dev} bs=4096 count=1" - fi - ${SSH} -o ConnectTimeout=5 ${user}@${hostname}.${DOMAIN} "sudo dd if=/dev/zero of=/dev/${boot_dev} bs=4096 count=1 && sudo poweroff" -} - -# Remove the iPXE boot files -function deletePxeConfig() { - local mac_addr=${1} - - ${SSH} root@${ROUTER} "rm -f /data/tftpboot/ipxe/${mac_addr//:/-}.ipxe" - ${SSH} root@${BASTION_HOST} "rm -f /usr/local/www/install/fcos/ignition/${CLUSTER_NAME}/${mac_addr//:/-}.ign" -} - -# Remove DNS Records -function deleteDns() { - local key=${1} - ${SSH} root@${ROUTER} "cat /etc/bind/db.${DOMAIN} | grep -v ${key} > /tmp/db.${DOMAIN} && cp /tmp/db.${DOMAIN} /etc/bind/db.${DOMAIN}" - ${SSH} root@${ROUTER} "cat /etc/bind/db.${NET_PREFIX_ARPA} | grep -v ${key} > /tmp/db.${NET_PREFIX_ARPA} && cp /tmp/db.${NET_PREFIX_ARPA} /etc/bind/db.${NET_PREFIX_ARPA}" -} - -# Validate options and set vars -function validateAndSetVars() { - DONE=false - DOMAIN_COUNT=$(yq e ".sub-domain-configs" ${CONFIG_FILE} | yq e 'length' -) - let i=0 - while [[ i -lt ${DOMAIN_COUNT} ]] - do - domain_name=$(yq e ".sub-domain-configs.[${i}].name" ${CONFIG_FILE}) - if [[ ${domain_name} == ${SUB_DOMAIN} ]] - then - D_INDEX=${i} - DONE=true - break - fi - i=$(( ${i} + 1 )) - done - if [[ ${DONE} == "false" ]] - then - echo "Domain Entry Not Found In Config File." - exit 1 - fi - - LAB_DOMAIN=$(yq e ".domain" ${CONFIG_FILE}) - BASTION_HOST=$(yq e ".bastion-ip" ${CONFIG_FILE}) - SUB_DOMAIN=$(yq e ".sub-domain-configs.[${D_INDEX}].name" ${CONFIG_FILE}) - ROUTER=$(yq e ".sub-domain-configs.[${D_INDEX}].router-ip" ${CONFIG_FILE}) - NETWORK=$(yq e ".sub-domain-configs.[${D_INDEX}].network" ${CONFIG_FILE}) - CLUSTER_CONFIG=$(yq e ".sub-domain-configs.[${D_INDEX}].cluster-config-file" ${CONFIG_FILE}) - DOMAIN="${SUB_DOMAIN}.${LAB_DOMAIN}" - CLUSTER_NAME=$(yq e ".cluster-name" ${CLUSTER_CONFIG}) - - CP_COUNT=$(yq e ".control-plane.okd-hosts" ${CLUSTER_CONFIG} | yq e 'length' -) - if [[ ${CP_COUNT} == "1" ]] - then - SNO="true" - RESET_LB="false" - fi - - if [[ ${DELETE_WORKER} == "true" ]] && [[ ${W_HOST_NAME} != "all" ]] - then - if [[ ${W_HOST_NAME} == "" ]] - then - echo "-w | --worker must have a value" - exit 1 - fi - let i=0 - DONE=false - let NODE_COUNT=$(yq e .compute-nodes ${CLUSTER_CONFIG} | yq e 'length' -) - while [[ i -lt ${NODE_COUNT} ]] - do - host_name=$(yq e ".compute-nodes.[${i}].name" ${CLUSTER_CONFIG}) - if [[ ${host_name} == ${W_HOST_NAME} ]] - then - W_HOST_INDEX=${i} - DONE=true - break; - fi - i=$(( ${i} + 1 )) - done - fi - - if [[ ${DELETE_KVM_HOST} == "true" ]] - then - echo "This method is not implemented yet: -k|--kvm-host" - exit 1 - fi -} - -function deleteBootstrap() { - #Delete Bootstrap - if [[ $(yq e ".bootstrap.metal" ${CLUSTER_CONFIG}) == "true" ]] - then - kill $(ps -ef | grep qemu | grep bootstrap | awk '{print $2}') - rm -rf ${OKD_LAB_PATH}/bootstrap - else - host_name="${CLUSTER_NAME}-bootstrap" - kvm_host=$(yq e .bootstrap.kvm-host ${CLUSTER_CONFIG}) - deleteNodeVm ${host_name} ${kvm_host} - fi - deletePxeConfig $(yq e ".bootstrap.mac-addr" ${CLUSTER_CONFIG}) - deleteDns ${CLUSTER_NAME}-${DOMAIN}-bs - ${SSH} root@${ROUTER} "cp /etc/haproxy.no-bootstrap /etc/haproxy.cfg && /etc/init.d/haproxy stop && /etc/init.d/haproxy start" -} - -function deleteWorker() { - local index=${1} - - host_name=$(yq e ".compute-nodes.[${index}].name" ${CLUSTER_CONFIG}) - mac_addr=$(yq e ".control-plane.okd-hosts.[${index}].mac-addr" ${CLUSTER_CONFIG}) - if [[ $(yq e ".compute-nodes.[${index}].metal" ${CLUSTER_CONFIG}) == "true" ]] - then - boot_dev=$(yq e ".compute-nodes.[${index}].boot-dev" ${CLUSTER_CONFIG}) - ceph_dev=$(yq e ".compute-nodes.[${index}].ceph.ceph-dev" ${CLUSTER_CONFIG}) - destroyMetal core ${host_name} ${boot_dev} ${ceph_dev} - else - kvm_host=$(yq e .compute-nodes.[${index}].kvm-host ${CLUSTER_CONFIG}) - deleteNodeVm ${host_name} ${kvm_host} - fi - deleteDns ${host_name}-${DOMAIN} - deletePxeConfig ${mac_addr} -} - -function deleteCluster() { - #Delete Control Plane Nodes: - metal=$(yq e ".control-plane.metal" ${CLUSTER_CONFIG}) - if [[ ${SNO} == "true" ]] - then - mac_addr=$(yq e ".control-plane.okd-hosts.[${i}].mac-addr" ${CLUSTER_CONFIG}) - host_name=$(yq e ".control-plane.okd-hosts.[0].name" ${CLUSTER_CONFIG}) - if [[ ${metal} == "true" ]] - then - install_dev=$(yq e ".control-plane.okd-hosts.[0].sno-install-dev" ${CLUSTER_CONFIG}) - destroyMetal core ${host_name} ${install_dev} - else - kvm_host=$(yq e .control-plane.okd-hosts.[0].kvm-host ${CLUSTER_CONFIG}) - deleteNodeVm ${host_name} ${kvm_host} - fi - deletePxeConfig ${mac_addr} - else - for i in 0 1 2 - do - mac_addr=$(yq e ".control-plane.okd-hosts.[${i}].mac-addr" ${CLUSTER_CONFIG}) - host_name=$(yq e ".control-plane.okd-hosts.[${i}].name" ${CLUSTER_CONFIG}) - if [[ ${metal} == "true" ]] - then - boot_dev=$(yq e ".control-plane.okd-hosts.[${i}].boot-dev" ${CLUSTER_CONFIG}) - destroyMetal core ${host_name} ${boot_dev} - else - kvm_host=$(yq e .control-plane.okd-hosts.[${i}].kvm-host ${CLUSTER_CONFIG}) - deleteNodeVm ${host_name} ${kvm_host} - fi - deletePxeConfig ${mac_addr} - done - fi - deleteDns ${CLUSTER_NAME}-${DOMAIN}-cp -} - -validateAndSetVars - -IFS=. read -r i1 i2 i3 i4 << EOF -${NETWORK} -EOF -NET_PREFIX_ARPA=${i3}.${i2}.${i1} - -if [[ ${DELETE_BOOTSTRAP} == "true" ]] -then - deleteBootstrap -fi - -if [[ ${DELETE_WORKER} == "true" ]] -then - if [[ ${W_HOST_NAME} == "all" ]] # Delete all Nodes - then - let j=$(yq e .compute-nodes ${CLUSTER_CONFIG} | yq e 'length' -) - let i=0 - while [[ i -lt ${j} ]] - do - deleteWorker ${i} - i=$(( ${i} + 1 )) - done - else - deleteWorker ${W_HOST_INDEX} - fi -fi - -if [[ ${DELETE_CLUSTER} == "true" ]] -then - deleteCluster -fi - -if [[ ${RESET_LB} == "true" ]] -then - ${SSH} root@${ROUTER} "cp /etc/haproxy.bootstrap /etc/haproxy.cfg && /etc/init.d/haproxy stop && /etc/init.d/haproxy start" -fi - -if [[ ${DELETE_KVM_HOST} == "true" ]] -then - echo "-k not implemented" -fi - -${SSH} root@${ROUTER} "/etc/init.d/named stop && /etc/init.d/named start" diff --git a/bin/getOkdCmds.sh b/bin/getOkdCmds.sh deleted file mode 100755 index 6a7c990..0000000 --- a/bin/getOkdCmds.sh +++ /dev/null @@ -1,74 +0,0 @@ -#!/bin/bash - -CONFIG_FILE=${LAB_CONFIG_FILE} - -for i in "$@" -do -case $i in - -c=*|--config=*) - CONFIG_FILE="${i#*=}" - shift - ;; - -d=*|--domain=*) - SUB_DOMAIN="${i#*=}" - shift - ;; - -m|--mac) - OS_VER=mac - BUTANE_DLD=apple-darwin - shift - ;; - -l|--linux) - OS_VER=linux - BUTANE_DLD=unknown-linux-gnu - shift - ;; - *) - # Put usage here: - ;; -esac -done - -DONE=false -DOMAIN_COUNT=$(yq e ".sub-domain-configs" ${CONFIG_FILE} | yq e 'length' -) -let i=0 -while [[ i -lt ${DOMAIN_COUNT} ]] -do - domain_name=$(yq e ".sub-domain-configs.[${i}].name" ${CONFIG_FILE}) - if [[ ${domain_name} == ${SUB_DOMAIN} ]] - then - INDEX=${i} - DONE=true - break - fi - i=$(( ${i} + 1 )) -done -if [[ ${DONE} == "false" ]] -then - echo "Domain Entry Not Found In Config File." - exit 1 -fi - -CLUSTER_CONFIG=$(yq e ".sub-domain-configs.[${INDEX}].cluster-config-file" ${CONFIG_FILE}) -OKD_VERSION=$(yq e ".okd-version" ${CLUSTER_CONFIG}) -BUTANE_VERSION=$(yq e ".butane-version" ${CLUSTER_CONFIG}) - -mkdir -p ${OKD_LAB_PATH}/okd-cmds/${OKD_VERSION} -mkdir -p ${OKD_LAB_PATH}/tmp - -wget -O ${OKD_LAB_PATH}/tmp/oc.tar.gz https://github.com/openshift/okd/releases/download/${OKD_VERSION}/openshift-client-${OS_VER}-${OKD_VERSION}.tar.gz -wget -O ${OKD_LAB_PATH}/tmp/oc-install.tar.gz https://github.com/openshift/okd/releases/download/${OKD_VERSION}/openshift-install-${OS_VER}-${OKD_VERSION}.tar.gz -wget -O ${OKD_LAB_PATH}/okd-cmds/${OKD_VERSION}/butane https://github.com/coreos/butane/releases/download/${BUTANE_VERSION}/butane-x86_64-${BUTANE_DLD} - -tar -xzf ${OKD_LAB_PATH}/tmp/oc.tar.gz -C ${OKD_LAB_PATH}/okd-cmds/${OKD_VERSION} -tar -xzf ${OKD_LAB_PATH}/tmp/oc-install.tar.gz -C ${OKD_LAB_PATH}/okd-cmds/${OKD_VERSION} - -chmod 700 ${OKD_LAB_PATH}/okd-cmds/${OKD_VERSION}/* - -rm -rf ${OKD_LAB_PATH}/tmp - -for i in $(ls ${OKD_LAB_PATH}/okd-cmds/${OKD_VERSION}) -do - rm -f ${OKD_LAB_PATH}/bin/${i} - ln -s ${OKD_LAB_PATH}/okd-cmds/${OKD_VERSION}/${i} ${OKD_LAB_PATH}/bin/${i} -done diff --git a/bin/mirrorOkdRelease.sh b/bin/mirrorOkdRelease.sh deleted file mode 100755 index 896ba0e..0000000 --- a/bin/mirrorOkdRelease.sh +++ /dev/null @@ -1,59 +0,0 @@ -#!/bin/bash - -CONFIG_FILE=${LAB_CONFIG_FILE} - -for i in "$@" -do -case $i in - -c=*|--config=*) - CONFIG_FILE="${i#*=}" - HOST_CONFIG=${CONFIG_FILE} - shift - ;; - -d=*|--domain=*) - SUB_DOMAIN="${i#*=}" - shift - ;; - *) - # Put usage here: - ;; -esac -done - -DONE=false -DOMAIN_COUNT=$(yq e ".sub-domain-configs" ${CONFIG_FILE} | yq e 'length' -) -let i=0 -while [[ i -lt ${DOMAIN_COUNT} ]] -do - domain_name=$(yq e ".sub-domain-configs.[${i}].name" ${CONFIG_FILE}) - if [[ ${domain_name} == ${SUB_DOMAIN} ]] - then - INDEX=${i} - DONE=true - break - fi - i=$(( ${i} + 1 )) -done -if [[ ${DONE} == "false" ]] -then - echo "Domain Entry Not Found In Config File." - exit 1 -fi - -LAB_DOMAIN=$(yq e ".domain" ${CONFIG_FILE}) -SUB_DOMAIN=$(yq e ".sub-domain-configs.[${INDEX}].name" ${CONFIG_FILE}) -CLUSTER_CONFIG=$(yq e ".sub-domain-configs.[${INDEX}].cluster-config-file" ${CONFIG_FILE}) -DOMAIN="${SUB_DOMAIN}.${LAB_DOMAIN}" -LOCAL_REGISTRY=$(yq e ".local-registry" ${CLUSTER_CONFIG}) -OKD_REGISTRY=$(yq e ".remote-registry" ${CLUSTER_CONFIG}) -PULL_SECRET=$(yq e ".secret-file" ${CLUSTER_CONFIG}) -OKD_RELEASE=$(yq e ".okd-version" ${CLUSTER_CONFIG}) - -rm -rf ${OKD_LAB_PATH}/lab-config/work-dir -mkdir -p ${OKD_LAB_PATH}/lab-config/work-dir -mkdir -p ${OKD_LAB_PATH}/lab-config/release-sigs -oc adm -a ${PULL_SECRET} release mirror --from=${OKD_REGISTRY}:${OKD_RELEASE} --to=${LOCAL_REGISTRY}/okd --to-release-image=${LOCAL_REGISTRY}/okd:${OKD_RELEASE} --release-image-signature-to-dir=${OKD_LAB_PATH}/lab-config/work-dir - -SIG_FILE=$(ls ${OKD_LAB_PATH}/lab-config/work-dir) -mv ${OKD_LAB_PATH}/lab-config/work-dir/${SIG_FILE} ${OKD_LAB_PATH}/lab-config/release-sigs/${OKD_RELEASE}-sig.yaml -rm -rf ${OKD_LAB_PATH}/lab-config/work-dir diff --git a/bin/setlab.sh b/bin/setlab.sh deleted file mode 100644 index 25bc2ed..0000000 --- a/bin/setlab.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/bin/bash - -# export SET_LAB="true" - -SUB_DOMAIN="" -INDEX="" -CONFIG_FILE=${LAB_CONFIG_FILE} - -for i in "$@" -do - case ${i} in - -c=*|--config=*) - CONFIG_FILE="${i#*=}" - shift - ;; - *) - echo "USAGE: setlab.sh -c=path/to/config/file -d=sub-domain-name" - ;; - esac -done - -DONE=false -DOMAIN_COUNT=$(yq e ".sub-domain-configs" ${CONFIG_FILE} | yq e 'length' -) -let array_index=0 -while [[ array_index -lt ${DOMAIN_COUNT} ]] -do - domain_name=$(yq e ".sub-domain-configs.[${array_index}].name" ${CONFIG_FILE}) - echo "$(( ${array_index} + 1 )) - ${domain_name}" - array_index=$(( ${array_index} + 1 )) -done -echo "Enter the index of the domain that you want to work with:" -read ENTRY -INDEX=$(( ${ENTRY} - 1 )) - -export CLUSTER_CONFIG=$(yq e ".sub-domain-configs.[${INDEX}].cluster-config-file" ${CONFIG_FILE}) -export SUB_DOMAIN=$(yq e ".sub-domain-configs.[${INDEX}].name" ${CONFIG_FILE}) -export DOMAIN_ROUTER=$(yq e ".sub-domain-configs.[${INDEX}].router-ip" ${CONFIG_FILE}) -export DOMAIN_NETWORK=$(yq e ".sub-domain-configs.[${INDEX}].network" ${CONFIG_FILE}) -export LOCAL_REGISTRY=$(yq e ".local-registry" ${OKD_LAB_PATH}/lab-config/${SUB_DOMAIN}-cluster.yaml) -OKD_VERSION=$(yq e ".okd-version" ${CLUSTER_CONFIG}) -for i in $(ls ${OKD_LAB_PATH}/okd-cmds/${OKD_VERSION}) -do - rm -f ${OKD_LAB_PATH}/bin/${i} - ln -s ${OKD_LAB_PATH}/okd-cmds/${OKD_VERSION}/${i} ${OKD_LAB_PATH}/bin/${i} -done - -unset array_index - -# if [[ ${SHELL} == "" ]] || [[ ${SHELL} == "/bin/bash" ]] -# then -# SHELL=/bin/bash -# SH_EX="/bin/bash --norc" -# else -# SH_EX="/bin/zsh --no-rcs" -# fi - -# ${SH_EX} diff --git a/bin/shutdownCluster.sh b/bin/shutdownCluster.sh deleted file mode 100644 index 64ff5f1..0000000 --- a/bin/shutdownCluster.sh +++ /dev/null @@ -1,95 +0,0 @@ -#!/bin/bash - -SSH="ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" - -CONFIG_FILE=${LAB_CONFIG_FILE} -PAUSE=60 - -for i in "$@" -do - case $i in - -c=*|--config=*) - CONFIG_FILE="${i#*=}" - shift # past argument=value - ;; - -d=*|--domain=*) - SUB_DOMAIN="${i#*=}" - shift - ;; - *) - # put usage here: - ;; - esac -done - -DONE=false -DOMAIN_COUNT=$(yq e ".sub-domain-configs" ${CONFIG_FILE} | yq e 'length' -) -let i=0 -while [[ i -lt ${DOMAIN_COUNT} ]] -do - domain_name=$(yq e ".sub-domain-configs.[${i}].name" ${CONFIG_FILE}) - if [[ ${domain_name} == ${SUB_DOMAIN} ]] - then - INDEX=${i} - DONE=true - break - fi - i=$(( ${i} + 1 )) -done -if [[ ${DONE} == "false" ]] -then - echo "Domain Entry Not Found In Config File." - exit 1 -fi - -LAB_DOMAIN=$(yq e ".domain" ${CONFIG_FILE}) -SUB_DOMAIN=$(yq e ".sub-domain-configs.[${INDEX}].name" ${CONFIG_FILE}) -CLUSTER_CONFIG=$(yq e ".sub-domain-configs.[${INDEX}].cluster-config-file" ${CONFIG_FILE}) -CLUSTER_NAME=$(yq e ".cluster-name" ${CLUSTER_CONFIG}) -DOMAIN="${SUB_DOMAIN}.${LAB_DOMAIN}" -export KUBECONFIG="${OKD_LAB_PATH}/lab-config/${CLUSTER_NAME}.${DOMAIN}/kubeconfig" - -ID=$(oc whoami) -if [[ ${ID} != "system:admin" ]] -then - echo "ERROR: Invalid kube_config: ${KUBECONFIG}" - exit 1 -fi - -if [[ $(yq e ".compute-nodes.metal" ${CLUSTER_CONFIG}) == "true" ]] - then - let NODE_COUNT=$(yq e .compute-nodes.okd-hosts ${CLUSTER_CONFIG} | yq e 'length' -) - else - let NODE_COUNT=$(yq e .compute-nodes.kvm-hosts ${CLUSTER_CONFIG} | yq e 'length' -) -fi - -# Cordon Compute Nodes -let i=0 -while [[ i -lt ${NODE_COUNT} ]] -do - oc adm cordon okd4-worker-${i}.${DOMAIN} - i=$(( ${i} + 1 )) -done - -# Drain & Shutdown Compute Nodes -let i=0 -while [[ i -lt ${NODE_COUNT} ]] -do - oc adm drain okd4-worker-${i}.${DOMAIN} --ignore-daemonsets --force --grace-period=60 --delete-emptydir-data - ${SSH} -o ConnectTimeout=5 core@okd4-worker-${i}.${DOMAIN} "sudo systemctl poweroff" - i=$(( ${i} + 1 )) -done - -# Shutdown Control Plane Nodes -echo "Sleep ${PAUSE} seconds, then shutdown Control Plane" -let pause=${PAUSE} -while [ ${pause} -gt 0 ]; do - echo -ne "Giving Compute Nodes Time to Shutdown: ${pause}\033[0K\r" - sleep 1 - : $((pause--)) -done - -for i in 0 1 2 -do - ${SSH} -o ConnectTimeout=5 core@okd4-master-${i}.${DOMAIN} "sudo systemctl poweroff" -done \ No newline at end of file diff --git a/bin/startNodes.sh b/bin/startNodes.sh deleted file mode 100755 index dfe475d..0000000 --- a/bin/startNodes.sh +++ /dev/null @@ -1,123 +0,0 @@ -#!/bin/bash -SSH="ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" -BOOTSTRAP=false -MASTER=false -WORKER=false -CONFIG_FILE=${LAB_CONFIG_FILE} - -for i in "$@" -do - case $i in - -c=*|--config=*) - CONFIG_FILE="${i#*=}" - shift - ;; - -d=*|--domain=*) - SUB_DOMAIN="${i#*=}" - shift - ;; - -b|--bootstrap) - BOOTSTRAP=true - shift - ;; - -m|--master) - MASTER=true - shift - ;; - -w|--worker) - WORKER=true - shift - ;; - *) - # put usage here: - ;; - esac -done - -function startNode() { - local kvm_host=${1} - local host_name=${2} - ${SSH} root@${kvm_host}.${DOMAIN} "virsh start ${host_name}" -} - -DONE=false -DOMAIN_COUNT=$(yq e ".sub-domain-configs" ${CONFIG_FILE} | yq e 'length' -) -let i=0 -while [[ i -lt ${DOMAIN_COUNT} ]] -do - domain_name=$(yq e ".sub-domain-configs.[${i}].name" ${CONFIG_FILE}) - if [[ ${domain_name} == ${SUB_DOMAIN} ]] - then - INDEX=${i} - DONE=true - break - fi - i=$(( ${i} + 1 )) -done -if [[ ${DONE} == "false" ]] -then - echo "Domain Entry Not Found In Config File." - exit 1 -fi - -SUB_DOMAIN=$(yq e ".sub-domain-configs.[${INDEX}].name" ${CONFIG_FILE}) -CLUSTER_CONFIG=$(yq e ".sub-domain-configs.[${INDEX}].cluster-config-file" ${CONFIG_FILE}) -CLUSTER_NAME=$(yq e ".cluster-name" ${CLUSTER_CONFIG}) -DOMAIN="${SUB_DOMAIN}.${LAB_DOMAIN}" - -if [[ ${BOOTSTRAP} == "true" ]] -then - host_name="$(yq e .cluster-name ${CLUSTER_CONFIG})-bootstrap" - - if [[ $(yq e ".bootstrap.metal" ${CLUSTER_CONFIG}) == "true" ]] - then - memory=$(yq e ".bootstrap.node-spec.memory" ${CLUSTER_CONFIG}) - cpu=$(yq e ".bootstrap.node-spec.cpu" ${CLUSTER_CONFIG}) - root_vol=$(yq e ".bootstrap.node-spec.root_vol" ${CLUSTER_CONFIG}) - bridge_dev=$(yq e ".bootstrap.bridge-dev" ${CLUSTER_CONFIG}) - - mkdir -p ${OKD_LAB_PATH}/bootstrap - qemu-img create -f qcow2 ${OKD_LAB_PATH}/bootstrap/bootstrap-node.qcow2 ${root_vol}G - qemu-system-x86_64 -accel accel=hvf -m ${memory}M -smp ${cpu} -display none -nographic -drive file=${OKD_LAB_PATH}/bootstrap/bootstrap-node.qcow2,if=none,id=disk1 -device ide-hd,bus=ide.0,drive=disk1,id=sata0-0-0,bootindex=1 -boot n -netdev vde,id=nic0,sock=/var/run/vde.bridged.${bridge_dev}.ctl -device virtio-net-pci,netdev=nic0,mac=52:54:00:a1:b2:c3 - else - kvm_host=$(yq e .bootstrap.kvm-host ${CLUSTER_CONFIG}) - startNode ${kvm_host} ${host_name} - fi -fi - -if [[ ${MASTER} == "true" ]] -then - if [[ $(yq e ".control-plane.metal" ${CLUSTER_CONFIG}) == "true" ]] - then - echo "This script will not auto start bare-metal nodes. Please power them on manually." - else - for i in 0 1 2 - do - kvm_host=$(yq e .control-plane.okd-hosts.[${i}].kvm-host ${CLUSTER_CONFIG}) - host_name=$(yq e .control-plane.okd-hosts.[${i}].name ${CLUSTER_CONFIG}) - startNode ${kvm_host} ${host_name} - echo "Pause for 15 seconds to stagger node start up." - sleep 15 - done - fi -fi - -if [[ ${WORKER} == "true" ]] -then - let NODE_COUNT=$(yq e .compute-nodes ${CLUSTER_CONFIG} | yq e 'length' -) - let i=0 - while [[ i -lt ${NODE_COUNT} ]] - do - host_name=$(yq e .compute-nodes.[${i}].name ${CLUSTER_CONFIG}) - if [[ $(yq e .compute-nodes.[${i}].metal ${CLUSTER_CONFIG}) == "true" ]] - then - echo "This script will not auto start a bare-metal node. Please power on ${host_name} manually." - else - kvm_host=$(yq e .compute-nodes.[${i}].kvm-host ${CLUSTER_CONFIG}) - startNode ${kvm_host} ${host_name} - echo "Pause for 15 seconds to stagger node start up." - sleep 15 - fi - i=$(( ${i} + 1 )) - done -fi diff --git a/pipelines/gitea-stateful-set/gitea.Dockerfile b/pipelines/gitea-stateful-set/gitea.Dockerfile new file mode 100644 index 0000000..e69de29 diff --git a/pipelines/images/java-11-app-runner.Dockerfile b/pipelines/images/java-11-app-runner.Dockerfile index f9d577b..cb43b1a 100644 --- a/pipelines/images/java-11-app-runner.Dockerfile +++ b/pipelines/images/java-11-app-runner.Dockerfile @@ -1,4 +1,4 @@ -FROM registry.access.redhat.com/ubi8/ubi-minimal:8.4 +FROM registry.access.redhat.com/ubi8/ubi-minimal:8.5 ARG JAVA_PACKAGE=java-11-openjdk-headless ARG RUN_JAVA_VERSION=1.3.8 ENV LANG='en_US.UTF-8' LANGUAGE='en_US:en' diff --git a/pipelines/images/java-11-builder.Dockerfile b/pipelines/images/java-11-builder.Dockerfile index 05467e0..a1cc6f8 100644 --- a/pipelines/images/java-11-builder.Dockerfile +++ b/pipelines/images/java-11-builder.Dockerfile @@ -1,13 +1,13 @@ -FROM registry.access.redhat.com/ubi8/ubi-minimal:8.4 +FROM registry.access.redhat.com/ubi8/ubi-minimal:8.5 ARG MAVEN_VERSION=3.8.4 ARG BASE_URL=https://apache.osuosl.org/maven/maven-3/${MAVEN_VERSION}/binaries -ARG JAVA_PACKAGE=java-11-openjdk-headless +ARG JAVA_PACKAGE=java-11-openjdk-devel ARG MANDREL_VERSION=21.3.0.0-Final ARG USER_HOME_DIR="/maven" ARG WORK_DIR="/workspace" ARG GRAALVM_DIR=/opt/mandral ENV LANG='en_US.UTF-8' LANGUAGE='en_US:en' LC_ALL='en_US.UTF-8' -RUN microdnf install glibc-devel zlib-devel gcc libffi-devel libstdc++-devel gcc-c++ glibc-langpack-en openssl curl ca-certificates git tar which ${JAVA_PACKAGE} shadow-utils \ +RUN microdnf install glibc-devel zlib-devel gcc libffi-devel libstdc++-devel gcc-c++ glibc-langpack-en openssl curl ca-certificates git tar which ${JAVA_PACKAGE} shadow-utils\ && microdnf update \ && microdnf clean all \ && mkdir -p ${USER_HOME_DIR} \ @@ -30,5 +30,6 @@ RUN microdnf install glibc-devel zlib-devel gcc libffi-devel libstdc++-devel gcc ENV MAVEN_HOME=/usr/share/maven ENV MAVEN_CONFIG="${USER_HOME_DIR}/.m2" ENV GRAALVM_HOME=${GRAALVM_DIR} +ENV JAVA_HOME=/etc/alternatives/jre_11_openjdk -VOLUME ${WORK_DIR} \ No newline at end of file +VOLUME ${WORK_DIR} diff --git a/pipelines/images/java-11-dev-tools.Dockerfile b/pipelines/images/java-11-dev-tools.Dockerfile new file mode 100644 index 0000000..ba89c45 --- /dev/null +++ b/pipelines/images/java-11-dev-tools.Dockerfile @@ -0,0 +1,45 @@ +FROM registry.access.redhat.com/ubi8/ubi-minimal:8.5 +ARG MAVEN_VERSION=3.8.4 +ARG BASE_URL=https://apache.osuosl.org/maven/maven-3/${MAVEN_VERSION}/binaries +ARG JAVA_PACKAGE=java-11-openjdk-devel +ARG MANDREL_VERSION=21.3.0.0-Final +ARG USER_HOME_DIR="/workspace" +ARG WORK_DIR="/workspace" +ARG GRAALVM_DIR=/opt/mandral +ENV LANG='en_US.UTF-8' LANGUAGE='en_US:en' LC_ALL='en_US.UTF-8' +RUN microdnf install glibc-devel zlib-devel gcc libffi-devel libstdc++-devel gcc-c++ glibc-langpack-en openssl curl ca-certificates git tar which ${JAVA_PACKAGE} shadow-utils unzip vim bash ps\ + && microdnf update \ + && microdnf clean all \ + && rpm -ivh https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm \ + && microdnf install jq \ + && microdnf clean all \ + && rm -rf /var/cache/yum \ + && mkdir -p ${USER_HOME_DIR} \ + && chown 1001 ${USER_HOME_DIR} \ + && chmod "g+rwX" ${USER_HOME_DIR} \ + && chown 1001:root ${USER_HOME_DIR} \ + && mkdir -p ${WORK_DIR} \ + && chown 1001 ${WORK_DIR} \ + && chmod "g+rwX" ${WORK_DIR} \ + && chown 1001:root ${WORK_DIR} \ + && echo "securerandom.source=file:/dev/urandom" >> /etc/alternatives/jre/lib/security/java.security \ + && mkdir -p /usr/share/maven /usr/share/maven/ref \ + && curl -fsSL -o /tmp/apache-maven.tar.gz ${BASE_URL}/apache-maven-${MAVEN_VERSION}-bin.tar.gz \ + && tar -xzf /tmp/apache-maven.tar.gz -C /usr/share/maven --strip-components=1 \ + && rm -f /tmp/apache-maven.tar.gz \ + && ln -s /usr/share/maven/bin/mvn /usr/bin/mvn \ + && mkdir -p ${GRAALVM_DIR} \ + && curl -fsSL -o /tmp/mandrel-java11-linux-amd64-${MANDREL_VERSION}.tar.gz https://github.com/graalvm/mandrel/releases/download/mandrel-${MANDREL_VERSION}/mandrel-java11-linux-amd64-${MANDREL_VERSION}.tar.gz \ + && tar xzf /tmp/mandrel-java11-linux-amd64-${MANDREL_VERSION}.tar.gz -C ${GRAALVM_DIR} --strip-components=1 \ + && curl -fsSL -o /tmp/awscliv2.zip https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip \ + && unzip /tmp/awscliv2.zip -d /tmp \ + && /tmp/aws/install \ + && rm -rf /tmp/aws +ENV MAVEN_HOME=/usr/share/maven +ENV MAVEN_CONFIG="${USER_HOME_DIR}/.m2" +ENV GRAALVM_HOME=${GRAALVM_DIR} +ENV JAVA_HOME=/etc/alternatives/jre_11_openjdk +ENV HOME=${USER_HOME_DIR} +ENV SHELL=/bin/bash + +VOLUME ${WORK_DIR} diff --git a/pipelines/manifests/create-trigger-task.yaml b/pipelines/manifests/create-trigger-task.yaml index b62a4ba..2479544 100644 --- a/pipelines/manifests/create-trigger-task.yaml +++ b/pipelines/manifests/create-trigger-task.yaml @@ -78,5 +78,5 @@ spec: GIT_CREDS="$(cat /tekton/creds-secrets/gitea-secret/username):$(cat /tekton/creds-secrets/gitea-secret/password)" API_PATH="/api/v1/repos/${PROJECT_PATH}/hooks" API_URL="${GIT_URL}${API_PATH}" - curl --location --request POST ${API_URL} -u ${GIT_CREDS} --header 'Content-Type: application/json' --header 'Content-Type: application/json' --data-raw "{\"active\":true,\"branch_filter\":\"$(params.git-branch)\",\"config\":{\"content_type\":\"json\",\"url\":\"${HOOK_URL}\",\"http_method\": \"post\",\"secret\":\"${SECRET}\"},\"events\": [\"push\"],\"type\": \"gitea\"}" + curl --location --request POST ${API_URL} -u ${GIT_CREDS} --header 'Content-Type: application/json' --data-raw "{\"active\":true,\"branch_filter\":\"$(params.git-branch)\",\"config\":{\"content_type\":\"json\",\"url\":\"${HOOK_URL}\",\"http_method\": \"post\",\"secret\":\"${SECRET}\"},\"events\": [\"push\"],\"type\": \"gitea\"}" echo ----------------------------------- diff --git a/rook-ceph/install/cluster-reference.yaml b/rook-ceph/install/cluster-reference.yaml index cb2ac1e..619c8aa 100644 --- a/rook-ceph/install/cluster-reference.yaml +++ b/rook-ceph/install/cluster-reference.yaml @@ -16,13 +16,13 @@ metadata: spec: cephVersion: # The container image used to launch the Ceph daemon pods (mon, mgr, osd, mds, rgw). - # v14 is nautilus, v15 is octopus, and v16 is pacific. + # v15 is octopus, and v16 is pacific. # RECOMMENDATION: In production, use a specific version tag instead of the general v14 flag, which pulls the latest release and could result in different # versions running within the cluster. See tags available at https://hub.docker.com/r/ceph/ceph/tags/. - # If you want to be more precise, you can always use a timestamp tag such quay.io/ceph/ceph:v16.2.5-20210708 + # If you want to be more precise, you can always use a timestamp tag such quay.io/ceph/ceph:v16.2.7-20211208 # This tag might not contain a new Ceph version, just security fixes from the underlying operating system, which will reduce vulnerabilities - image: quay.io/ceph/ceph:v16.2.5 - # Whether to allow unsupported versions of Ceph. Currently `nautilus`, `octopus`, and `pacific` are supported. + image: quay.io/ceph/ceph:v16.2.7 + # Whether to allow unsupported versions of Ceph. Currently `octopus` and `pacific` are supported. # Future versions such as `pacific` would require this to be set to `true`. # Do not set to true in production. allowUnsupported: false @@ -33,7 +33,7 @@ spec: # Whether or not upgrade should continue even if a check fails # This means Ceph's status could be degraded and we don't recommend upgrading but you might decide otherwise # Use at your OWN risk - # To understand Rook's upgrade process of Ceph, read https://rook.io/docs/rook/master/ceph-upgrade.html#ceph-version-upgrades + # To understand Rook's upgrade process of Ceph, read https://rook.io/docs/rook/latest/ceph-upgrade.html#ceph-version-upgrades skipUpgradeChecks: false # Whether or not continue if PGs are not clean during an upgrade continueUpgradeAfterChecksEvenIfNotHealthy: false @@ -44,7 +44,8 @@ spec: # The default wait timeout is 10 minutes. waitTimeoutForHealthyOSDInMinutes: 10 mon: - # Set the number of mons to be started. Must be an odd number, and is generally recommended to be 3. + # Set the number of mons to be started. Generally recommended to be 3. + # For highest availability, an odd number of mons should be specified. count: 3 # The mons should be on unique nodes. For production, at least 3 nodes are recommended for this reason. # Mons should only be allowed on the same node for test environments where data loss is acceptable. @@ -165,6 +166,10 @@ spec: # osd: # cleanup: # prepareosd: +# clusterMetadata annotations will be applied to only `rook-ceph-mon-endpoints` configmap and the `rook-ceph-mon` and `rook-ceph-admin-keyring` secrets. +# And clusterMetadata annotations will not be merged with `all` annotations. +# clusterMetadata: +# kubed.appscode.com/sync: "true" # If no mgr annotations are set, prometheus scrape annotations will be set by default. # mgr: labels: @@ -177,6 +182,7 @@ spec: # monitoring is a list of key-value pairs. It is injected into all the monitoring resources created by operator. # These labels can be passed as LabelSelector to Prometheus # monitoring: +# crashcollector: resources: # The requests and limits set here, allow the mgr pod to use half of one CPU core and 1 gigabyte of memory # mgr: @@ -205,6 +211,7 @@ spec: # mon: rook-ceph-mon-priority-class # osd: rook-ceph-osd-priority-class # mgr: rook-ceph-mgr-priority-class +# crashcollector: rook-ceph-crashcollector-priority-class storage: # cluster level storage configuration and selection useAllNodes: true useAllDevices: true @@ -218,17 +225,17 @@ spec: # encryptedDevice: "true" # the default value for this option is "false" # Individual nodes and their config can be specified as well, but 'useAllNodes' above must be set to false. Then, only the named # nodes below will be used as storage resources. Each node's 'name' field should match their 'kubernetes.io/hostname' label. -# nodes: -# - name: "172.17.4.201" -# devices: # specific devices to use for storage can be specified for each node -# - name: "sdb" -# - name: "nvme01" # multiple osds can be created on high performance devices -# config: -# osdsPerDevice: "5" -# - name: "/dev/disk/by-id/ata-ST4000DM004-XXXX" # devices can be specified using full udev paths -# config: # configuration can be specified at the node level which overrides the cluster level config -# - name: "172.17.4.301" -# deviceFilter: "^sd." + # nodes: + # - name: "172.17.4.201" + # devices: # specific devices to use for storage can be specified for each node + # - name: "sdb" + # - name: "nvme01" # multiple osds can be created on high performance devices + # config: + # osdsPerDevice: "5" + # - name: "/dev/disk/by-id/ata-ST4000DM004-XXXX" # devices can be specified using full udev paths + # config: # configuration can be specified at the node level which overrides the cluster level config + # - name: "172.17.4.301" + # deviceFilter: "^sd." # when onlyApplyOSDPlacement is false, will merge both placement.All() and placement.osd onlyApplyOSDPlacement: false # The section for configuring management of daemon disruptions during upgrade or fencing. @@ -263,7 +270,7 @@ spec: status: disabled: false interval: 60s - # Change pod liveness probe, it works for all mon,mgr,osd daemons + # Change pod liveness probe timing or threshold values. Works for all mon,mgr,osd daemons. livenessProbe: mon: disabled: false @@ -271,3 +278,11 @@ spec: disabled: false osd: disabled: false + # Change pod startup probe timing or threshold values. Works for all mon,mgr,osd daemons. + startupProbe: + mon: + disabled: false + mgr: + disabled: false + osd: + disabled: false diff --git a/rook-ceph/install/cluster.yaml b/rook-ceph/install/cluster.yaml index 5f1cde3..6cb5548 100644 --- a/rook-ceph/install/cluster.yaml +++ b/rook-ceph/install/cluster.yaml @@ -5,7 +5,7 @@ metadata: namespace: rook-ceph spec: cephVersion: - image: ${PROXY_REGISTRY}/ceph/ceph:v16.2.6 + image: ${PROXY_REGISTRY}/ceph/ceph:v16.2.7 allowUnsupported: false dataDirHostPath: /var/lib/rook skipUpgradeChecks: false diff --git a/rook-ceph/install/common.yaml b/rook-ceph/install/common.yaml index d174d3f..b560c9a 100644 --- a/rook-ceph/install/common.yaml +++ b/rook-ceph/install/common.yaml @@ -1,168 +1,242 @@ -################################################################################################################### +#################################################################################################### # Create the common resources that are necessary to start the operator and the ceph cluster. # These resources *must* be created before the operator.yaml and cluster.yaml or their variants. -# The samples all assume that a single operator will manage a single cluster crd in the same "rook-ceph" namespace. -# -# If the operator needs to manage multiple clusters (in different namespaces), see the section below -# for "cluster-specific resources". The resources below that section will need to be created for each namespace -# where the operator needs to manage the cluster. The resources above that section do not be created again. -# -# Most of the sections are prefixed with a 'OLM' keyword which is used to build our CSV for an OLM (Operator Life Cycle manager) -################################################################################################################### +# The samples all assume that a single operator will manage a single cluster crd in the same +# "rook-ceph" namespace. +#################################################################################################### # Namespace where the operator and other rook resources are created apiVersion: v1 kind: Namespace metadata: name: rook-ceph # namespace:cluster -# OLM: BEGIN OBJECTBUCKET ROLEBINDING ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-object-bucket -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: rook-ceph-object-bucket -subjects: - - kind: ServiceAccount - name: rook-ceph-system - namespace: rook-ceph # namespace:operator -# OLM: END OBJECTBUCKET ROLEBINDING -# OLM: BEGIN OPERATOR ROLE ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: rook-ceph-admission-controller - namespace: rook-ceph # namespace:operator --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: - name: rook-ceph-admission-controller-role + name: cephfs-csi-nodeplugin rules: - - apiGroups: ["ceph.rook.io"] - resources: ["*"] - verbs: ["get", "watch", "list"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "update"] + - apiGroups: [""] + resources: ["namespaces"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list"] --- -kind: ClusterRoleBinding +kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: - name: rook-ceph-admission-controller-rolebinding -subjects: - - kind: ServiceAccount - name: rook-ceph-admission-controller - apiGroup: "" - namespace: rook-ceph # namespace:operator -roleRef: - kind: ClusterRole - name: rook-ceph-admission-controller-role - apiGroup: rbac.authorization.k8s.io + name: cephfs-external-provisioner-runner +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "delete", "update", "patch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments/status"] + verbs: ["patch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims/status"] + verbs: ["update", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["create", "get", "list", "watch", "update", "delete", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents/status"] + verbs: ["update", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots/status"] + verbs: ["update", "patch"] --- -# The cluster role for managing all the cluster-specific resources in a namespace apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - name: rook-ceph-cluster-mgmt + name: 'psp:rook' labels: operator: rook storage-backend: ceph + app.kubernetes.io/part-of: rook-ceph-operator rules: - apiGroups: - - "" - - apps - - extensions + - policy resources: - - secrets - - pods - - pods/log - - services - - configmaps - - deployments - - daemonsets + - podsecuritypolicies + resourceNames: + - 00-rook-privileged verbs: - - get - - list - - watch - - patch - - create - - update - - delete + - use --- -apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 metadata: - name: rook-ceph-system + name: rbd-csi-nodeplugin labels: operator: rook storage-backend: ceph + app.kubernetes.io/part-of: rook-ceph-operator rules: - # Most resources are represented by a string representation of their name, such as “pods”, just as it appears in the URL for the relevant API endpoint. - # However, some Kubernetes APIs involve a “subresource”, such as the logs for a pod. [...] - # To represent this in an RBAC role, use a slash to delimit the resource and subresource. - # https://kubernetes.io/docs/reference/access-authn-authz/rbac/#referring-to-resources - apiGroups: [""] - resources: ["pods", "pods/log"] + resources: ["secrets"] verbs: ["get", "list"] - apiGroups: [""] - resources: ["pods/exec"] - verbs: ["create"] + resources: ["nodes"] + verbs: ["get", "list", "update"] + - apiGroups: [""] + resources: ["namespaces"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["serviceaccounts"] + verbs: ["get"] --- -# The role for the operator to manage resources in its own namespace +kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 -kind: Role metadata: - name: rook-ceph-system - namespace: rook-ceph # namespace:operator + name: rbd-external-provisioner-runner +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "delete", "update", "patch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments/status"] + verbs: ["patch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["create", "get", "list", "watch", "update", "delete", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents/status"] + verbs: ["update", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots/status"] + verbs: ["update", "patch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims/status"] + verbs: ["update", "patch"] + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get"] + - apiGroups: ["replication.storage.openshift.io"] + resources: ["volumereplications", "volumereplicationclasses"] + verbs: ["create", "delete", "get", "list", "patch", "update", "watch"] + - apiGroups: ["replication.storage.openshift.io"] + resources: ["volumereplications/finalizers"] + verbs: ["update"] + - apiGroups: ["replication.storage.openshift.io"] + resources: ["volumereplications/status"] + verbs: ["get", "patch", "update"] + - apiGroups: ["replication.storage.openshift.io"] + resources: ["volumereplicationclasses/status"] + verbs: ["get"] + - apiGroups: [""] + resources: ["serviceaccounts"] + verbs: ["get"] +--- +# The cluster role for managing all the cluster-specific resources in a namespace +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: rook-ceph-cluster-mgmt labels: operator: rook storage-backend: ceph + app.kubernetes.io/part-of: rook-ceph-operator rules: - apiGroups: - "" - resources: - - pods - - configmaps - - services - verbs: - - get - - list - - watch - - patch - - create - - update - - delete - - apiGroups: - apps - extensions resources: - - daemonsets - - statefulsets + - secrets + - pods + - pods/log + - services + - configmaps - deployments + - daemonsets verbs: - get - list - watch + - patch - create - update - delete - - apiGroups: - - batch - resources: - - cronjobs - verbs: - - delete --- # The cluster role for managing the Rook CRDs apiVersion: rbac.authorization.k8s.io/v1 +# Rook watches for its CRDs in all namespaces, so this should be a cluster-scoped role unless the +# operator config `ROOK_CURRENT_NAMESPACE_ONLY=true`. kind: ClusterRole metadata: name: rook-ceph-global labels: operator: rook storage-backend: ceph + app.kubernetes.io/part-of: rook-ceph-operator rules: - apiGroups: - "" @@ -173,6 +247,11 @@ rules: - nodes - nodes/proxy - services + # Rook watches secrets which it uses to configure access to external resources. + # e.g., external Ceph cluster; TLS certificates for the admission controller or object store + - secrets + # Rook watches for changes to the rook-operator-config configmap + - configmaps verbs: - get - list @@ -180,10 +259,12 @@ rules: - apiGroups: - "" resources: + # Rook creates events for its custom resources - events - # PVs and PVCs are managed by the Rook provisioner + # Rook creates PVs and PVCs for OSDs managed by the Rook provisioner - persistentvolumes - persistentvolumeclaims + # Rook creates endpoints for mgr and object store access - endpoints verbs: - get @@ -213,18 +294,71 @@ rules: - create - update - delete - - apiGroups: - - ceph.rook.io + # The Rook operator must be able to watch all ceph.rook.io resources to reconcile them. + - apiGroups: ["ceph.rook.io"] resources: - - "*" + - cephclients + - cephclusters + - cephblockpools + - cephfilesystems + - cephnfses + - cephobjectstores + - cephobjectstoreusers + - cephobjectrealms + - cephobjectzonegroups + - cephobjectzones + - cephbuckettopics + - cephbucketnotifications + - cephrbdmirrors + - cephfilesystemmirrors + - cephfilesystemsubvolumegroups verbs: - - "*" - - apiGroups: - - rook.io + - get + - list + - watch + # Ideally the update permission is not required, but Rook needs it to add finalizers to resources. + - update + # Rook must have update access to status subresources for its custom resources. + - apiGroups: ["ceph.rook.io"] resources: - - "*" - verbs: - - "*" + - cephclients/status + - cephclusters/status + - cephblockpools/status + - cephfilesystems/status + - cephnfses/status + - cephobjectstores/status + - cephobjectstoreusers/status + - cephobjectrealms/status + - cephobjectzonegroups/status + - cephobjectzones/status + - cephbuckettopics/status + - cephbucketnotifications/status + - cephrbdmirrors/status + - cephfilesystemmirrors/status + - cephfilesystemsubvolumegroups/status + verbs: ["update"] + # The "*/finalizers" permission may need to be strictly given for K8s clusters where + # OwnerReferencesPermissionEnforcement is enabled so that Rook can set blockOwnerDeletion on + # resources owned by Rook CRs (e.g., a Secret owned by an OSD Deployment). See more: + # https://kubernetes.io/docs/reference/access-authn-authz/_print/#ownerreferencespermissionenforcement + - apiGroups: ["ceph.rook.io"] + resources: + - cephclients/finalizers + - cephclusters/finalizers + - cephblockpools/finalizers + - cephfilesystems/finalizers + - cephnfses/finalizers + - cephobjectstores/finalizers + - cephobjectstoreusers/finalizers + - cephobjectrealms/finalizers + - cephobjectzonegroups/finalizers + - cephobjectzones/finalizers + - cephbuckettopics/finalizers + - cephbucketnotifications/finalizers + - cephrbdmirrors/finalizers + - cephfilesystemmirrors/finalizers + - cephfilesystemsubvolumegroups/finalizers + verbs: ["update"] - apiGroups: - policy - apps @@ -236,7 +370,13 @@ rules: - deployments - replicasets verbs: - - "*" + - get + - list + - watch + - create + - update + - delete + - deletecollection - apiGroups: - healthchecking.openshift.io resources: @@ -283,6 +423,7 @@ metadata: labels: operator: rook storage-backend: ceph + app.kubernetes.io/part-of: rook-ceph-operator rules: - apiGroups: - "" @@ -314,86 +455,168 @@ rules: - list - watch --- +# Aspects of ceph-mgr that require access to the system namespace kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: - name: rook-ceph-object-bucket - labels: - operator: rook - storage-backend: ceph + name: rook-ceph-mgr-system rules: - apiGroups: - "" - verbs: - - "*" resources: - - secrets - configmaps - - apiGroups: - - storage.k8s.io - resources: - - storageclasses verbs: - get - list - watch - - apiGroups: - - "objectbucket.io" - verbs: - - "*" - resources: - - "*" -# OLM: END OPERATOR ROLE -# OLM: BEGIN SERVICE ACCOUNT SYSTEM --- -# The rook system service account used by the operator, agent, and discovery pods -apiVersion: v1 -kind: ServiceAccount +# Used for provisioning ObjectBuckets (OBs) in response to ObjectBucketClaims (OBCs). +# Note: Rook runs a copy of the lib-bucket-provisioner's OBC controller. +# OBCs can be created in any Kubernetes namespace, so this must be a cluster-scoped role. +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 metadata: - name: rook-ceph-system - namespace: rook-ceph # namespace:operator + name: rook-ceph-object-bucket labels: operator: rook storage-backend: ceph -# imagePullSecrets: -# - name: my-registry-secret - -# OLM: END SERVICE ACCOUNT SYSTEM -# OLM: BEGIN OPERATOR ROLEBINDING + app.kubernetes.io/part-of: rook-ceph-operator +rules: + - apiGroups: [""] + resources: ["secrets", "configmaps"] + verbs: + # OBC controller creates secrets and configmaps containing information for users about how to + # connect to object buckets. It deletes them when an OBC is deleted. + - get + - create + - update + - delete + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: + # OBC controller gets parameters from the OBC's storageclass + # Rook gets additional parameters from the OBC's storageclass + - get + - apiGroups: ["objectbucket.io"] + resources: ["objectbucketclaims"] + verbs: + # OBC controller needs to list/watch OBCs and get latest version of a reconciled OBC + - list + - watch + - get + # Ideally, update should not be needed, but the OBC controller updates the OBC with bucket + # information outside of the status subresource + - update + # OBC controller does not delete OBCs; users do this + - apiGroups: ["objectbucket.io"] + resources: ["objectbuckets"] + verbs: + # OBC controller needs to list/watch OBs and get latest version of a reconciled OB + - list + - watch + - get + # OBC controller creates an OB when an OBC's bucket has been provisioned by Ceph, updates them + # when an OBC is updated, and deletes them when the OBC is de-provisioned. + - create + - update + - delete + - apiGroups: ["objectbucket.io"] + resources: ["objectbucketclaims/status", "objectbuckets/status"] + verbs: + # OBC controller updates OBC and OB statuses + - update + - apiGroups: ["objectbucket.io"] + # This does not strictly allow the OBC/OB controllers to update finalizers. That is handled by + # the direct "update" permissions above. Instead, this allows Rook's controller to create + # resources which are owned by OBs/OBCs and where blockOwnerDeletion is set. + resources: ["objectbucketclaims/finalizers", "objectbuckets/finalizers"] + verbs: + - update --- -# Grant the operator, agent, and discovery agents access to resources in the namespace -kind: RoleBinding +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rook-ceph-osd +rules: + - apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list +--- +kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: rook-ceph-system - namespace: rook-ceph # namespace:operator labels: operator: rook storage-backend: ceph + app.kubernetes.io/part-of: rook-ceph-operator +rules: + # Most resources are represented by a string representation of their name, such as "pods", just as it appears in the URL for the relevant API endpoint. + # However, some Kubernetes APIs involve a "subresource", such as the logs for a pod. [...] + # To represent this in an RBAC role, use a slash to delimit the resource and subresource. + # https://kubernetes.io/docs/reference/access-authn-authz/rbac/#referring-to-resources + - apiGroups: [""] + resources: ["pods", "pods/log"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["pods/exec"] + verbs: ["create"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: cephfs-csi-nodeplugin +subjects: + - kind: ServiceAccount + name: rook-csi-cephfs-plugin-sa + namespace: rook-ceph # namespace:operator roleRef: + kind: ClusterRole + name: cephfs-csi-nodeplugin apiGroup: rbac.authorization.k8s.io - kind: Role - name: rook-ceph-system +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: cephfs-csi-provisioner-role subjects: - kind: ServiceAccount - name: rook-ceph-system + name: rook-csi-cephfs-provisioner-sa namespace: rook-ceph # namespace:operator +roleRef: + kind: ClusterRole + name: cephfs-external-provisioner-runner + apiGroup: rbac.authorization.k8s.io --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: - name: rook-ceph-system - labels: - operator: rook - storage-backend: ceph + name: rbd-csi-nodeplugin +subjects: + - kind: ServiceAccount + name: rook-csi-rbd-plugin-sa + namespace: rook-ceph # namespace:operator roleRef: - apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: rook-ceph-system + name: rbd-csi-nodeplugin + apiGroup: rbac.authorization.k8s.io +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rbd-csi-provisioner-role subjects: - kind: ServiceAccount - name: rook-ceph-system + name: rook-csi-rbd-provisioner-sa namespace: rook-ceph # namespace:operator +roleRef: + kind: ClusterRole + name: rbd-external-provisioner-runner + apiGroup: rbac.authorization.k8s.io --- # Grant the rook system daemons cluster-wide access to manage the Rook CRDs, PVCs, and storage classes kind: ClusterRoleBinding @@ -403,6 +626,7 @@ metadata: labels: operator: rook storage-backend: ceph + app.kubernetes.io/part-of: rook-ceph-operator roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole @@ -411,276 +635,158 @@ subjects: - kind: ServiceAccount name: rook-ceph-system namespace: rook-ceph # namespace:operator -# OLM: END OPERATOR ROLEBINDING -################################################################################################################# -# Beginning of cluster-specific resources. The example will assume the cluster will be created in the "rook-ceph" -# namespace. If you want to create the cluster in a different namespace, you will need to modify these roles -# and bindings accordingly. -################################################################################################################# -# Service account for the Ceph OSDs. Must exist and cannot be renamed. -# OLM: BEGIN SERVICE ACCOUNT OSD --- -apiVersion: v1 -kind: ServiceAccount +# Allow the ceph mgr to access cluster-wide resources necessary for the mgr modules +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 metadata: - name: rook-ceph-osd - namespace: rook-ceph # namespace:cluster -# imagePullSecrets: -# - name: my-registry-secret - -# OLM: END SERVICE ACCOUNT OSD -# OLM: BEGIN SERVICE ACCOUNT MGR + name: rook-ceph-mgr-cluster +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: rook-ceph-mgr-cluster +subjects: + - kind: ServiceAccount + name: rook-ceph-mgr + namespace: rook-ceph # namespace:cluster --- -# Service account for the Ceph Mgr. Must exist and cannot be renamed. -apiVersion: v1 -kind: ServiceAccount +kind: ClusterRoleBinding +# Give Rook-Ceph Operator permissions to provision ObjectBuckets in response to ObjectBucketClaims. +apiVersion: rbac.authorization.k8s.io/v1 metadata: - name: rook-ceph-mgr - namespace: rook-ceph # namespace:cluster -# imagePullSecrets: -# - name: my-registry-secret - -# OLM: END SERVICE ACCOUNT MGR -# OLM: BEGIN CMD REPORTER SERVICE ACCOUNT + name: rook-ceph-object-bucket +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: rook-ceph-object-bucket +subjects: + - kind: ServiceAccount + name: rook-ceph-system + namespace: rook-ceph # namespace:operator --- -apiVersion: v1 -kind: ServiceAccount +# Allow the ceph osd to access cluster-wide resources necessary for determining their topology location +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 metadata: - name: rook-ceph-cmd-reporter - namespace: rook-ceph # namespace:cluster -# OLM: END CMD REPORTER SERVICE ACCOUNT -# OLM: BEGIN CLUSTER ROLE + name: rook-ceph-osd +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: rook-ceph-osd +subjects: + - kind: ServiceAccount + name: rook-ceph-osd + namespace: rook-ceph # namespace:cluster --- -kind: Role +kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: - name: rook-ceph-osd - namespace: rook-ceph # namespace:cluster -rules: - - apiGroups: [""] - resources: ["configmaps"] - verbs: ["get", "list", "watch", "create", "update", "delete"] - - apiGroups: ["ceph.rook.io"] - resources: ["cephclusters", "cephclusters/finalizers"] - verbs: ["get", "list", "create", "update", "delete"] + name: rook-ceph-system + labels: + operator: rook + storage-backend: ceph + app.kubernetes.io/part-of: rook-ceph-operator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: rook-ceph-system +subjects: + - kind: ServiceAccount + name: rook-ceph-system + namespace: rook-ceph # namespace:operator --- -kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding metadata: - name: rook-ceph-osd -rules: - - apiGroups: - - "" - resources: - - nodes - verbs: - - get - - list + name: rook-ceph-system-psp + labels: + operator: rook + storage-backend: ceph + app.kubernetes.io/part-of: rook-ceph-operator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: 'psp:rook' +subjects: + - kind: ServiceAccount + name: rook-ceph-system + namespace: rook-ceph # namespace:operator --- -# Aspects of ceph-mgr that require access to the system namespace -kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding metadata: - name: rook-ceph-mgr-system -rules: - - apiGroups: - - "" - resources: - - configmaps - verbs: - - get - - list - - watch + name: rook-csi-cephfs-plugin-sa-psp +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: 'psp:rook' +subjects: + - kind: ServiceAccount + name: rook-csi-cephfs-plugin-sa + namespace: rook-ceph # namespace:operator --- -# Aspects of ceph-mgr that operate within the cluster's namespace -kind: Role apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding metadata: - name: rook-ceph-mgr - namespace: rook-ceph # namespace:cluster -rules: - - apiGroups: - - "" - resources: - - pods - - services - - pods/log - verbs: - - get - - list - - watch - - create - - update - - delete - - apiGroups: - - batch - resources: - - jobs - verbs: - - get - - list - - watch - - create - - update - - delete - - apiGroups: - - ceph.rook.io - resources: - - "*" - verbs: - - "*" -# OLM: END CLUSTER ROLE -# OLM: BEGIN CMD REPORTER ROLE ---- -kind: Role -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-cmd-reporter - namespace: rook-ceph # namespace:cluster -rules: - - apiGroups: - - "" - resources: - - pods - - configmaps - verbs: - - get - - list - - watch - - create - - update - - delete -# OLM: END CMD REPORTER ROLE -# OLM: BEGIN CLUSTER ROLEBINDING ---- -# Allow the operator to create resources in this cluster's namespace -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-cluster-mgmt - namespace: rook-ceph # namespace:cluster + name: rook-csi-cephfs-provisioner-sa-psp roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: rook-ceph-cluster-mgmt + name: 'psp:rook' subjects: - kind: ServiceAccount - name: rook-ceph-system + name: rook-csi-cephfs-provisioner-sa namespace: rook-ceph # namespace:operator --- -# Allow the osd pods in this namespace to work with configmaps -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-osd - namespace: rook-ceph # namespace:cluster -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: rook-ceph-osd -subjects: - - kind: ServiceAccount - name: rook-ceph-osd - namespace: rook-ceph # namespace:cluster ---- -# Allow the ceph mgr to access the cluster-specific resources necessary for the mgr modules -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-mgr - namespace: rook-ceph # namespace:cluster -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: rook-ceph-mgr -subjects: - - kind: ServiceAccount - name: rook-ceph-mgr - namespace: rook-ceph # namespace:cluster ---- -# Allow the ceph mgr to access the rook system resources necessary for the mgr modules -kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-mgr-system - namespace: rook-ceph # namespace:operator -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: rook-ceph-mgr-system -subjects: - - kind: ServiceAccount - name: rook-ceph-mgr - namespace: rook-ceph # namespace:cluster ---- -# Allow the ceph mgr to access cluster-wide resources necessary for the mgr modules kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 metadata: - name: rook-ceph-mgr-cluster + name: rook-csi-rbd-plugin-sa-psp roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: rook-ceph-mgr-cluster + name: 'psp:rook' subjects: - kind: ServiceAccount - name: rook-ceph-mgr - namespace: rook-ceph # namespace:cluster - + name: rook-csi-rbd-plugin-sa + namespace: rook-ceph # namespace:operator --- -# Allow the ceph osd to access cluster-wide resources necessary for determining their topology location -kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding metadata: - name: rook-ceph-osd + name: rook-csi-rbd-provisioner-sa-psp roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: rook-ceph-osd -subjects: - - kind: ServiceAccount - name: rook-ceph-osd - namespace: rook-ceph # namespace:cluster - -# OLM: END CLUSTER ROLEBINDING -# OLM: BEGIN CMD REPORTER ROLEBINDING ---- -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-cmd-reporter - namespace: rook-ceph # namespace:cluster -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: rook-ceph-cmd-reporter + name: 'psp:rook' subjects: - kind: ServiceAccount - name: rook-ceph-cmd-reporter - namespace: rook-ceph # namespace:cluster -# OLM: END CMD REPORTER ROLEBINDING -################################################################################################################# -# Beginning of pod security policy resources. The example will assume the cluster will be created in the -# "rook-ceph" namespace. If you want to create the cluster in a different namespace, you will need to modify -# the roles and bindings accordingly. -################################################################################################################# -# OLM: BEGIN CLUSTER POD SECURITY POLICY + name: rook-csi-rbd-provisioner-sa + namespace: rook-ceph # namespace:operator --- +# We expect most Kubernetes teams to follow the Kubernetes docs and have these PSPs. +# * privileged (for kube-system namespace) +# * restricted (for all logged in users) +# +# PSPs are applied based on the first match alphabetically. `rook-ceph-operator` comes after +# `restricted` alphabetically, so we name this `00-rook-privileged`, so it stays somewhere +# close to the top and so `rook-system` gets the intended PSP. This may need to be renamed in +# environments with other `00`-prefixed PSPs. +# +# More on PSP ordering: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#policy-order apiVersion: policy/v1beta1 kind: PodSecurityPolicy metadata: - # Note: Kubernetes matches PSPs to deployments alphabetically. In some environments, this PSP may - # need to be renamed with a value that will match before others. name: 00-rook-privileged annotations: - seccomp.security.alpha.kubernetes.io/allowedProfileNames: "runtime/default" - seccomp.security.alpha.kubernetes.io/defaultProfileName: "runtime/default" + seccomp.security.alpha.kubernetes.io/allowedProfileNames: 'runtime/default' + seccomp.security.alpha.kubernetes.io/defaultProfileName: 'runtime/default' spec: privileged: true allowedCapabilities: # required by CSI - SYS_ADMIN - # fsGroup - the flexVolume agent has fsGroup capabilities and could potentially be any group + - MKNOD fsGroup: rule: RunAsAny # runAsUser, supplementalGroups - Rook needs to run some pods as root @@ -702,7 +808,6 @@ spec: - projected # required for Rook - hostPath - - flexVolume # allowedHostPaths can be set to Rook's known host volume mount points when they are fully-known # allowedHostPaths: # - pathPrefix: "/run/udev" # for OSD prep @@ -736,107 +841,9 @@ spec: # Ceph mgr Prometheus Metrics - min: 9283 max: 9283 -# OLM: END CLUSTER POD SECURITY POLICY -# OLM: BEGIN POD SECURITY POLICY BINDINGS ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: "psp:rook" -rules: - - apiGroups: - - policy - resources: - - podsecuritypolicies - resourceNames: - - 00-rook-privileged - verbs: - - use ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: rook-ceph-system-psp -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: "psp:rook" -subjects: - - kind: ServiceAccount - name: rook-ceph-system - namespace: rook-ceph # namespace:operator ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: rook-ceph-default-psp - namespace: rook-ceph # namespace:cluster -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: psp:rook -subjects: - - kind: ServiceAccount - name: default - namespace: rook-ceph # namespace:cluster ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: rook-ceph-osd-psp - namespace: rook-ceph # namespace:cluster -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: psp:rook -subjects: - - kind: ServiceAccount - name: rook-ceph-osd - namespace: rook-ceph # namespace:cluster ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: rook-ceph-mgr-psp - namespace: rook-ceph # namespace:cluster -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: psp:rook -subjects: - - kind: ServiceAccount - name: rook-ceph-mgr - namespace: rook-ceph # namespace:cluster ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: rook-ceph-cmd-reporter-psp - namespace: rook-ceph # namespace:cluster -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: psp:rook -subjects: - - kind: ServiceAccount - name: rook-ceph-cmd-reporter - namespace: rook-ceph # namespace:cluster -# OLM: END CLUSTER POD SECURITY POLICY BINDINGS -# OLM: BEGIN CSI CEPHFS SERVICE ACCOUNT ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: rook-csi-cephfs-plugin-sa - namespace: rook-ceph # namespace:operator ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: rook-csi-cephfs-provisioner-sa - namespace: rook-ceph # namespace:operator -# OLM: END CSI CEPHFS SERVICE ACCOUNT -# OLM: BEGIN CSI CEPHFS ROLE + # port for CSIAddons + - min: 9070 + max: 9070 --- kind: Role apiVersion: rbac.authorization.k8s.io/v1 @@ -853,379 +860,373 @@ rules: - apiGroups: ["coordination.k8s.io"] resources: ["leases"] verbs: ["get", "watch", "list", "delete", "update", "create"] -# OLM: END CSI CEPHFS ROLE -# OLM: BEGIN CSI CEPHFS ROLEBINDING --- -kind: RoleBinding +kind: Role apiVersion: rbac.authorization.k8s.io/v1 metadata: - name: cephfs-csi-provisioner-role-cfg + name: rbd-csi-nodeplugin namespace: rook-ceph # namespace:operator -subjects: - - kind: ServiceAccount - name: rook-csi-cephfs-provisioner-sa - namespace: rook-ceph # namespace:operator -roleRef: - kind: Role - name: cephfs-external-provisioner-cfg - apiGroup: rbac.authorization.k8s.io -# OLM: END CSI CEPHFS ROLEBINDING -# OLM: BEGIN CSI CEPHFS CLUSTER ROLE +rules: + - apiGroups: ["csiaddons.openshift.io"] + resources: ["csiaddonsnodes"] + verbs: ["create"] --- -kind: ClusterRole +kind: Role apiVersion: rbac.authorization.k8s.io/v1 metadata: - name: cephfs-csi-nodeplugin + name: rbd-external-provisioner-cfg + namespace: rook-ceph # namespace:operator rules: - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "update"] - - apiGroups: [""] - resources: ["namespaces"] - verbs: ["get", "list"] - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments"] - verbs: ["get", "list", "watch", "update"] + resources: ["endpoints"] + verbs: ["get", "watch", "list", "delete", "update", "create"] - apiGroups: [""] resources: ["configmaps"] - verbs: ["get", "list"] + verbs: ["get", "list", "watch", "create", "delete", "update"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] + - apiGroups: ["csiaddons.openshift.io"] + resources: ["csiaddonsnodes"] + verbs: ["create"] --- -kind: ClusterRole +kind: Role apiVersion: rbac.authorization.k8s.io/v1 metadata: - name: cephfs-external-provisioner-runner + name: rook-ceph-cmd-reporter + namespace: rook-ceph # namespace:cluster rules: - - apiGroups: [""] - resources: ["secrets"] - verbs: ["get", "list"] - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "create", "delete", "update", "patch"] - - apiGroups: [""] - resources: ["persistentvolumeclaims"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["events"] - verbs: ["list", "watch", "create", "update", "patch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents"] - verbs: ["create", "get", "list", "watch", "update", "delete"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents/status"] - verbs: ["update"] - - apiGroups: ["apiextensions.k8s.io"] - resources: ["customresourcedefinitions"] - verbs: ["create", "list", "watch", "delete", "get", "update"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots/status"] - verbs: ["update"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments"] - verbs: ["get", "list", "watch", "update", "patch"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments/status"] - verbs: ["patch"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["persistentvolumeclaims/status"] - verbs: ["update", "patch"] -# OLM: END CSI CEPHFS CLUSTER ROLE -# OLM: BEGIN CSI CEPHFS CLUSTER ROLEBINDING ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: rook-csi-cephfs-plugin-sa-psp -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: "psp:rook" -subjects: - - kind: ServiceAccount - name: rook-csi-cephfs-plugin-sa - namespace: rook-ceph # namespace:operator ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: rook-csi-cephfs-provisioner-sa-psp -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: "psp:rook" -subjects: - - kind: ServiceAccount - name: rook-csi-cephfs-provisioner-sa - namespace: rook-ceph # namespace:operator ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: cephfs-csi-nodeplugin -subjects: - - kind: ServiceAccount - name: rook-csi-cephfs-plugin-sa - namespace: rook-ceph # namespace:operator -roleRef: - kind: ClusterRole - name: cephfs-csi-nodeplugin - apiGroup: rbac.authorization.k8s.io - + - apiGroups: + - "" + resources: + - pods + - configmaps + verbs: + - get + - list + - watch + - create + - update + - delete --- -kind: ClusterRoleBinding +# Aspects of ceph-mgr that operate within the cluster's namespace +kind: Role apiVersion: rbac.authorization.k8s.io/v1 metadata: - name: cephfs-csi-provisioner-role -subjects: - - kind: ServiceAccount - name: rook-csi-cephfs-provisioner-sa - namespace: rook-ceph # namespace:operator -roleRef: - kind: ClusterRole - name: cephfs-external-provisioner-runner - apiGroup: rbac.authorization.k8s.io -# OLM: END CSI CEPHFS CLUSTER ROLEBINDING -# OLM: BEGIN CSI RBD SERVICE ACCOUNT ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: rook-csi-rbd-plugin-sa - namespace: rook-ceph # namespace:operator ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: rook-csi-rbd-provisioner-sa - namespace: rook-ceph # namespace:operator -# OLM: END CSI RBD SERVICE ACCOUNT -# OLM: BEGIN CSI RBD ROLE + name: rook-ceph-mgr + namespace: rook-ceph # namespace:cluster +rules: + - apiGroups: + - "" + resources: + - pods + - services + - pods/log + verbs: + - get + - list + - watch + - create + - update + - delete + - apiGroups: + - batch + resources: + - jobs + verbs: + - get + - list + - watch + - create + - update + - delete + - apiGroups: + - ceph.rook.io + resources: + - "*" + verbs: + - "*" + - apiGroups: + - apps + resources: + - deployments/scale + - deployments + verbs: + - patch + - delete + - apiGroups: + - '' + resources: + - persistentvolumeclaims + verbs: + - delete --- kind: Role apiVersion: rbac.authorization.k8s.io/v1 metadata: - name: rbd-external-provisioner-cfg - namespace: rook-ceph # namespace:operator + name: rook-ceph-osd + namespace: rook-ceph # namespace:cluster rules: + # this is needed for rook's "key-management" CLI to fetch the vault token from the secret when + # validating the connection details - apiGroups: [""] - resources: ["endpoints"] - verbs: ["get", "watch", "list", "delete", "update", "create"] + resources: ["secrets"] + verbs: ["get"] - apiGroups: [""] resources: ["configmaps"] - verbs: ["get", "list", "watch", "create", "delete", "update"] - - apiGroups: ["coordination.k8s.io"] - resources: ["leases"] - verbs: ["get", "watch", "list", "delete", "update", "create"] -# OLM: END CSI RBD ROLE -# OLM: BEGIN CSI RBD ROLEBINDING ---- -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rbd-csi-provisioner-role-cfg - namespace: rook-ceph # namespace:operator -subjects: - - kind: ServiceAccount - name: rook-csi-rbd-provisioner-sa - namespace: rook-ceph # namespace:operator -roleRef: - kind: Role - name: rbd-external-provisioner-cfg - apiGroup: rbac.authorization.k8s.io -# OLM: END CSI RBD ROLEBINDING -# OLM: BEGIN CSI RBD CLUSTER ROLE + verbs: ["get", "list", "watch", "create", "update", "delete"] + - apiGroups: ["ceph.rook.io"] + resources: ["cephclusters", "cephclusters/finalizers"] + verbs: ["get", "list", "create", "update", "delete"] --- -kind: ClusterRole +# Aspects of ceph osd purge job that require access to the cluster namespace +kind: Role apiVersion: rbac.authorization.k8s.io/v1 metadata: - name: rbd-csi-nodeplugin + name: rook-ceph-purge-osd + namespace: rook-ceph # namespace:cluster rules: - - apiGroups: [""] - resources: ["secrets"] - verbs: ["get", "list"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "update"] - - apiGroups: [""] - resources: ["namespaces"] - verbs: ["get", "list"] - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments"] - verbs: ["get", "list", "watch", "update"] - apiGroups: [""] resources: ["configmaps"] - verbs: ["get", "list"] - - apiGroups: [""] - resources: ["serviceaccounts"] verbs: ["get"] + - apiGroups: ["apps"] + resources: ["deployments"] + verbs: ["get", "delete"] + - apiGroups: ["batch"] + resources: ["jobs"] + verbs: ["get", "list", "delete"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "update", "delete"] --- -kind: ClusterRole +# Allow the operator to manage resources in its own namespace apiVersion: rbac.authorization.k8s.io/v1 +kind: Role metadata: - name: rbd-external-provisioner-runner + name: rook-ceph-system + namespace: rook-ceph # namespace:operator + labels: + operator: rook + storage-backend: ceph + app.kubernetes.io/part-of: rook-ceph-operator rules: - - apiGroups: [""] - resources: ["secrets"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "create", "delete", "update", "patch"] - - apiGroups: [""] - resources: ["persistentvolumeclaims"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments"] - verbs: ["get", "list", "watch", "update", "patch"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments/status"] - verbs: ["patch"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "watch"] - - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["events"] - verbs: ["list", "watch", "create", "update", "patch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents"] - verbs: ["create", "get", "list", "watch", "update", "delete"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents/status"] - verbs: ["update"] - - apiGroups: ["apiextensions.k8s.io"] - resources: ["customresourcedefinitions"] - verbs: ["create", "list", "watch", "delete", "get", "update"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots/status"] - verbs: ["update"] - - apiGroups: [""] - resources: ["persistentvolumeclaims/status"] - verbs: ["update", "patch"] - - apiGroups: [""] - resources: ["configmaps"] - verbs: ["get"] - - apiGroups: ["replication.storage.openshift.io"] - resources: ["volumereplications", "volumereplicationclasses"] - verbs: ["create", "delete", "get", "list", "patch", "update", "watch"] - - apiGroups: ["replication.storage.openshift.io"] - resources: ["volumereplications/finalizers"] - verbs: ["update"] - - apiGroups: ["replication.storage.openshift.io"] - resources: ["volumereplications/status"] - verbs: ["get", "patch", "update"] - - apiGroups: ["replication.storage.openshift.io"] - resources: ["volumereplicationclasses/status"] - verbs: ["get"] - - apiGroups: [""] - resources: ["serviceaccounts"] - verbs: ["get"] -# OLM: END CSI RBD CLUSTER ROLE -# OLM: BEGIN CSI RBD CLUSTER ROLEBINDING + - apiGroups: + - "" + resources: + - pods + - configmaps + - services + verbs: + - get + - list + - watch + - patch + - create + - update + - delete + - apiGroups: + - apps + - extensions + resources: + - daemonsets + - statefulsets + - deployments + verbs: + - get + - list + - watch + - create + - update + - delete + - apiGroups: + - batch + resources: + - cronjobs + verbs: + - delete --- +kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding metadata: - name: rook-csi-rbd-plugin-sa-psp + name: cephfs-csi-provisioner-role-cfg + namespace: rook-ceph # namespace:operator +subjects: + - kind: ServiceAccount + name: rook-csi-cephfs-provisioner-sa + namespace: rook-ceph # namespace:operator roleRef: + kind: Role + name: cephfs-external-provisioner-cfg apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: "psp:rook" +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rbd-csi-nodeplugin-role-cfg + namespace: rook-ceph # namespace:operator subjects: - kind: ServiceAccount name: rook-csi-rbd-plugin-sa namespace: rook-ceph # namespace:operator +roleRef: + kind: Role + name: rbd-csi-nodeplugin + apiGroup: rbac.authorization.k8s.io --- +kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding metadata: - name: rook-csi-rbd-provisioner-sa-psp + name: rbd-csi-provisioner-role-cfg + namespace: rook-ceph # namespace:operator +subjects: + - kind: ServiceAccount + name: rook-csi-rbd-provisioner-sa + namespace: rook-ceph # namespace:operator +roleRef: + kind: Role + name: rbd-external-provisioner-cfg + apiGroup: rbac.authorization.k8s.io +--- +# Allow the operator to create resources in this cluster's namespace +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rook-ceph-cluster-mgmt + namespace: rook-ceph # namespace:cluster roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: "psp:rook" + name: rook-ceph-cluster-mgmt subjects: - kind: ServiceAccount - name: rook-csi-rbd-provisioner-sa + name: rook-ceph-system namespace: rook-ceph # namespace:operator --- -kind: ClusterRoleBinding +kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: - name: rbd-csi-nodeplugin + name: rook-ceph-cmd-reporter + namespace: rook-ceph # namespace:cluster +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: rook-ceph-cmd-reporter subjects: - kind: ServiceAccount - name: rook-csi-rbd-plugin-sa - namespace: rook-ceph # namespace:operator + name: rook-ceph-cmd-reporter + namespace: rook-ceph # namespace:cluster +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: rook-ceph-cmd-reporter-psp + namespace: rook-ceph # namespace:cluster roleRef: + apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: rbd-csi-nodeplugin + name: psp:rook +subjects: + - kind: ServiceAccount + name: rook-ceph-cmd-reporter + namespace: rook-ceph # namespace:cluster +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: rook-ceph-default-psp + namespace: rook-ceph # namespace:cluster + labels: + operator: rook + storage-backend: ceph + app.kubernetes.io/part-of: rook-ceph-operator +roleRef: apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: psp:rook +subjects: + - kind: ServiceAccount + name: default + namespace: rook-ceph # namespace:cluster --- -kind: ClusterRoleBinding +# Allow the ceph mgr to access resources scoped to the CephCluster namespace necessary for mgr modules +kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: - name: rbd-csi-provisioner-role + name: rook-ceph-mgr + namespace: rook-ceph # namespace:cluster +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: rook-ceph-mgr subjects: - kind: ServiceAccount - name: rook-csi-rbd-provisioner-sa - namespace: rook-ceph # namespace:operator + name: rook-ceph-mgr + namespace: rook-ceph # namespace:cluster +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: rook-ceph-mgr-psp + namespace: rook-ceph # namespace:cluster roleRef: - kind: ClusterRole - name: rbd-external-provisioner-runner apiGroup: rbac.authorization.k8s.io -# OLM: END CSI RBD CLUSTER ROLEBINDING + kind: ClusterRole + name: psp:rook +subjects: + - kind: ServiceAccount + name: rook-ceph-mgr + namespace: rook-ceph # namespace:cluster --- -# Aspects of ceph osd purge job that require access to the operator/cluster namespace -kind: Role +# Allow the ceph mgr to access resources in the Rook operator namespace necessary for mgr modules +kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: - name: rook-ceph-purge-osd + name: rook-ceph-mgr-system namespace: rook-ceph # namespace:operator -rules: - - apiGroups: [""] - resources: ["configmaps"] - verbs: ["get"] - - apiGroups: ["apps"] - resources: ["deployments"] - verbs: ["get", "delete"] - - apiGroups: ["batch"] - resources: ["jobs"] - verbs: ["get", "list", "delete"] - - apiGroups: [""] - resources: ["persistentvolumeclaims"] - verbs: ["delete"] +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: rook-ceph-mgr-system +subjects: + - kind: ServiceAccount + name: rook-ceph-mgr + namespace: rook-ceph # namespace:cluster +--- +# Allow the osd pods in this namespace to work with configmaps +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rook-ceph-osd + namespace: rook-ceph # namespace:cluster +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: rook-ceph-osd +subjects: + - kind: ServiceAccount + name: rook-ceph-osd + namespace: rook-ceph # namespace:cluster +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: rook-ceph-osd-psp + namespace: rook-ceph # namespace:cluster +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: psp:rook +subjects: + - kind: ServiceAccount + name: rook-ceph-osd + namespace: rook-ceph # namespace:cluster --- # Allow the osd purge job to run in this namespace kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: rook-ceph-purge-osd - namespace: rook-ceph # namespace:operator + namespace: rook-ceph # namespace:cluster roleRef: apiGroup: rbac.authorization.k8s.io kind: Role @@ -1233,10 +1234,120 @@ roleRef: subjects: - kind: ServiceAccount name: rook-ceph-purge-osd + namespace: rook-ceph # namespace:cluster +--- +# Grant the operator, agent, and discovery agents access to resources in the rook-ceph-system namespace +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rook-ceph-system + namespace: rook-ceph # namespace:operator + labels: + operator: rook + storage-backend: ceph + app.kubernetes.io/part-of: rook-ceph-operator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: rook-ceph-system +subjects: + - kind: ServiceAccount + name: rook-ceph-system namespace: rook-ceph # namespace:operator --- +# Service account for the job that reports the Ceph version in an image +apiVersion: v1 +kind: ServiceAccount +metadata: + name: rook-ceph-cmd-reporter + namespace: rook-ceph # namespace:cluster + labels: + operator: rook + storage-backend: ceph + app.kubernetes.io/part-of: rook-ceph-operator +# imagePullSecrets: +# - name: my-registry-secret +--- +# Service account for Ceph mgrs +apiVersion: v1 +kind: ServiceAccount +metadata: + name: rook-ceph-mgr + namespace: rook-ceph # namespace:cluster + labels: + operator: rook + storage-backend: ceph + app.kubernetes.io/part-of: rook-ceph-operator +# imagePullSecrets: +# - name: my-registry-secret +--- +# Service account for Ceph OSDs +apiVersion: v1 +kind: ServiceAccount +metadata: + name: rook-ceph-osd + namespace: rook-ceph # namespace:cluster + labels: + operator: rook + storage-backend: ceph + app.kubernetes.io/part-of: rook-ceph-operator +# imagePullSecrets: +# - name: my-registry-secret +--- +# Service account for job that purges OSDs from a Rook-Ceph cluster apiVersion: v1 kind: ServiceAccount metadata: name: rook-ceph-purge-osd + namespace: rook-ceph # namespace:cluster +# imagePullSecrets: +# - name: my-registry-secret +--- +# Service account for the Rook-Ceph operator +apiVersion: v1 +kind: ServiceAccount +metadata: + name: rook-ceph-system namespace: rook-ceph # namespace:operator + labels: + operator: rook + storage-backend: ceph + app.kubernetes.io/part-of: rook-ceph-operator +# imagePullSecrets: +# - name: my-registry-secret +--- +# Service account for the CephFS CSI driver +apiVersion: v1 +kind: ServiceAccount +metadata: + name: rook-csi-cephfs-plugin-sa + namespace: rook-ceph # namespace:operator +# imagePullSecrets: +# - name: my-registry-secret +--- +# Service account for the CephFS CSI provisioner +apiVersion: v1 +kind: ServiceAccount +metadata: + name: rook-csi-cephfs-provisioner-sa + namespace: rook-ceph # namespace:operator +# imagePullSecrets: +# - name: my-registry-secret +--- +# Service account for the RBD CSI driver +apiVersion: v1 +kind: ServiceAccount +metadata: + name: rook-csi-rbd-plugin-sa + namespace: rook-ceph # namespace:operator +# imagePullSecrets: +# - name: my-registry-secret +--- +# Service account for the RBD CSI provisioner +apiVersion: v1 +kind: ServiceAccount +metadata: + name: rook-csi-rbd-provisioner-sa + namespace: rook-ceph # namespace:operator +# imagePullSecrets: +# - name: my-registry-secret diff --git a/rook-ceph/install/crds.yaml b/rook-ceph/install/crds.yaml index 814e5e7..745411e 100644 --- a/rook-ceph/install/crds.yaml +++ b/rook-ceph/install/crds.yaml @@ -33,11 +33,10 @@ spec: metadata: type: object spec: - description: PoolSpec represents the spec of ceph pool + description: NamedBlockPoolSpec allows a block pool to be created with a non-default name. This is more specific than the NamedPoolSpec so we get schema validation on the allowed pool names that can be specified. properties: compressionMode: - default: none - description: 'The inline compression mode in Bluestore OSD to set to (options are: none, passive, aggressive, force)' + description: 'DEPRECATED: use Parameters instead, e.g., Parameters["compression_mode"] = "force" The inline compression mode in Bluestore OSD to set to (options are: none, passive, aggressive, force) Do NOT set a default value for kubebuilder as this will override the Parameters' enum: - none - passive @@ -64,13 +63,11 @@ spec: description: The algorithm for erasure coding type: string codingChunks: - description: Number of coding chunks per object in an erasure coded storage pool (required for erasure-coded pool type) - maximum: 9 + description: Number of coding chunks per object in an erasure coded storage pool (required for erasure-coded pool type). This is the number of OSDs that can be lost simultaneously before data cannot be recovered. minimum: 0 type: integer dataChunks: - description: Number of data chunks per object in an erasure coded storage pool (required for erasure-coded pool type) - maximum: 9 + description: Number of data chunks per object in an erasure coded storage pool (required for erasure-coded pool type). The number of chunks required to recover an object when any single OSD is lost is the same as dataChunks so be aware that the larger the number of data chunks, the higher the cost of recovery. minimum: 0 type: integer required: @@ -116,6 +113,12 @@ spec: type: object type: array type: object + name: + description: The desired name of the pool if different from the CephBlockPool CR name. + enum: + - device_health_metrics + - .nfs + type: string parameters: additionalProperties: type: string @@ -360,6 +363,283 @@ status: --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.5.1-0.20210420220833-f284e2e8098c + creationTimestamp: null + name: cephbucketnotifications.ceph.rook.io +spec: + group: ceph.rook.io + names: + kind: CephBucketNotification + listKind: CephBucketNotificationList + plural: cephbucketnotifications + singular: cephbucketnotification + scope: Namespaced + versions: + - name: v1 + schema: + openAPIV3Schema: + description: CephBucketNotification represents a Bucket Notifications + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: BucketNotificationSpec represent the spec of a Bucket Notification + properties: + events: + description: List of events that should trigger the notification + items: + description: BucketNotificationSpec represent the event type of the bucket notification + enum: + - s3:ObjectCreated:* + - s3:ObjectCreated:Put + - s3:ObjectCreated:Post + - s3:ObjectCreated:Copy + - s3:ObjectCreated:CompleteMultipartUpload + - s3:ObjectRemoved:* + - s3:ObjectRemoved:Delete + - s3:ObjectRemoved:DeleteMarkerCreated + type: string + type: array + filter: + description: Spec of notification filter + properties: + keyFilters: + description: Filters based on the object's key + items: + description: NotificationKeyFilterRule represent a single key rule in the Notification Filter spec + properties: + name: + description: Name of the filter - prefix/suffix/regex + enum: + - prefix + - suffix + - regex + type: string + value: + description: Value to filter on + type: string + required: + - name + - value + type: object + type: array + metadataFilters: + description: Filters based on the object's metadata + items: + description: NotificationFilterRule represent a single rule in the Notification Filter spec + properties: + name: + description: Name of the metadata or tag + minLength: 1 + type: string + value: + description: Value to filter on + type: string + required: + - name + - value + type: object + type: array + tagFilters: + description: Filters based on the object's tags + items: + description: NotificationFilterRule represent a single rule in the Notification Filter spec + properties: + name: + description: Name of the metadata or tag + minLength: 1 + type: string + value: + description: Value to filter on + type: string + required: + - name + - value + type: object + type: array + type: object + topic: + description: The name of the topic associated with this notification + minLength: 1 + type: string + required: + - topic + type: object + status: + description: Status represents the status of an object + properties: + phase: + type: string + type: object + x-kubernetes-preserve-unknown-fields: true + required: + - metadata + - spec + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.5.1-0.20210420220833-f284e2e8098c + creationTimestamp: null + name: cephbuckettopics.ceph.rook.io +spec: + group: ceph.rook.io + names: + kind: CephBucketTopic + listKind: CephBucketTopicList + plural: cephbuckettopics + singular: cephbuckettopic + scope: Namespaced + versions: + - name: v1 + schema: + openAPIV3Schema: + description: CephBucketTopic represents a Ceph Object Topic for Bucket Notifications + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: BucketTopicSpec represent the spec of a Bucket Topic + properties: + endpoint: + description: Contains the endpoint spec of the topic + properties: + amqp: + description: Spec of AMQP endpoint + properties: + ackLevel: + default: broker + description: The ack level required for this topic (none/broker/routeable) + enum: + - none + - broker + - routeable + type: string + disableVerifySSL: + description: Indicate whether the server certificate is validated by the client or not + type: boolean + exchange: + description: Name of the exchange that is used to route messages based on topics + minLength: 1 + type: string + uri: + description: The URI of the AMQP endpoint to push notification to + minLength: 1 + type: string + required: + - exchange + - uri + type: object + http: + description: Spec of HTTP endpoint + properties: + disableVerifySSL: + description: Indicate whether the server certificate is validated by the client or not + type: boolean + uri: + description: The URI of the HTTP endpoint to push notification to + minLength: 1 + type: string + required: + - uri + type: object + kafka: + description: Spec of Kafka endpoint + properties: + ackLevel: + default: broker + description: The ack level required for this topic (none/broker) + enum: + - none + - broker + type: string + disableVerifySSL: + description: Indicate whether the server certificate is validated by the client or not + type: boolean + uri: + description: The URI of the Kafka endpoint to push notification to + minLength: 1 + type: string + useSSL: + description: Indicate whether to use SSL when communicating with the broker + type: boolean + required: + - uri + type: object + type: object + objectStoreName: + description: The name of the object store on which to define the topic + minLength: 1 + type: string + objectStoreNamespace: + description: The namespace of the object store on which to define the topic + minLength: 1 + type: string + opaqueData: + description: Data which is sent in each event + type: string + persistent: + description: Indication whether notifications to this endpoint are persistent or not + type: boolean + required: + - endpoint + - objectStoreName + - objectStoreNamespace + type: object + status: + description: BucketTopicStatus represents the Status of a CephBucketTopic + properties: + ARN: + description: The ARN of the topic generated by the RGW + nullable: true + type: string + phase: + type: string + type: object + x-kubernetes-preserve-unknown-fields: true + required: + - metadata + - spec + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.5.1-0.20210420220833-f284e2e8098c @@ -743,7 +1023,106 @@ spec: - port type: object terminationGracePeriodSeconds: - description: Optional duration in seconds the pod needs to terminate gracefully upon probe failure. The grace period is the duration in seconds after the processes running in the pod are sent a termination signal and the time when the processes are forcibly halted with a kill signal. Set this value longer than the expected cleanup time for your process. If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this value overrides the value provided by the pod spec. Value must be non-negative integer. The value zero indicates stop immediately via the kill signal (no opportunity to shut down). This is an alpha field and requires enabling ProbeTerminationGracePeriod feature gate. + description: Optional duration in seconds the pod needs to terminate gracefully upon probe failure. The grace period is the duration in seconds after the processes running in the pod are sent a termination signal and the time when the processes are forcibly halted with a kill signal. Set this value longer than the expected cleanup time for your process. If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this value overrides the value provided by the pod spec. Value must be non-negative integer. The value zero indicates stop immediately via the kill signal (no opportunity to shut down). This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + type: object + type: object + description: LivenessProbe allows changing the livenessProbe configuration for a given daemon + type: object + startupProbe: + additionalProperties: + description: ProbeSpec is a wrapper around Probe so it can be enabled or disabled for a Ceph daemon + properties: + disabled: + description: Disabled determines whether probe is disable or not + type: boolean + probe: + description: Probe describes a health check to be performed against a container to determine whether it is alive or ready to receive traffic. + properties: + exec: + description: One and only one of the following should be specified. Exec specifies the action to take. + properties: + command: + description: Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + type: object + failureThreshold: + description: Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1. + format: int32 + type: integer + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: Host name to connect to, defaults to the pod IP. You probably want to set "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header to be used in HTTP probes + properties: + name: + description: The header field name + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting to the host. Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: 'Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + periodSeconds: + description: How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook' + properties: + host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: Optional duration in seconds the pod needs to terminate gracefully upon probe failure. The grace period is the duration in seconds after the processes running in the pod are sent a termination signal and the time when the processes are forcibly halted with a kill signal. Set this value longer than the expected cleanup time for your process. If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this value overrides the value provided by the pod spec. Value must be non-negative integer. The value zero indicates stop immediately via the kill signal (no opportunity to shut down). This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. format: int64 type: integer timeoutSeconds: @@ -752,7 +1131,7 @@ spec: type: integer type: object type: object - description: LivenessProbe allows to change the livenessprobe configuration for a given daemon + description: StartupProbe allows changing the startupProbe configuration for a given daemon type: object type: object labels: @@ -812,6 +1191,7 @@ spec: type: boolean count: description: Count is the number of Ceph monitors + maximum: 9 minimum: 0 type: integer stretchCluster: @@ -872,7 +1252,23 @@ spec: type: string type: array dataSource: - description: 'This field can be used to specify either: * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) * An existing PVC (PersistentVolumeClaim) * An existing custom resource that implements data population (Alpha) In order to use custom resource types that implement data population, the AnyVolumeDataSource feature gate must be enabled. If the provisioner or an external controller can support the specified data source, it will create a new volume based on the contents of the specified data source.' + description: 'This field can be used to specify either: * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) * An existing PVC (PersistentVolumeClaim) If the provisioner or an external controller can support the specified data source, it will create a new volume based on the contents of the specified data source. If the AnyVolumeDataSource feature gate is enabled, this field will always have the same contents as the DataSourceRef field.' + properties: + apiGroup: + description: APIGroup is the group for the resource being referenced. If APIGroup is not specified, the specified Kind must be in the core API group. For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource being referenced + type: string + name: + description: Name is the name of resource being referenced + type: string + required: + - kind + - name + type: object + dataSourceRef: + description: 'Specifies the object from which to populate the volume with data, if a non-empty volume is desired. This may be any local object from a non-empty API group (non core object) or a PersistentVolumeClaim object. When this field is specified, volume binding will only succeed if the type of the specified object matches some installed volume populator or dynamic provisioner. This field will replace the functionality of the DataSource field and as such if both fields are non-empty, they must have the same value. For backwards compatibility, both fields (DataSource and DataSourceRef) will be set to the same value automatically if one of them is empty and the other is non-empty. There are two important differences between DataSource and DataSourceRef: * While DataSource only allows two specific types of objects, DataSourceRef allows any non-core object, as well as PersistentVolumeClaim objects. * While DataSource ignores disallowed values (dropping them), DataSourceRef preserves all values, and generates an error if a disallowed value is specified. (Alpha) Using this field requires the AnyVolumeDataSource feature gate to be enabled.' properties: apiGroup: description: APIGroup is the group for the resource being referenced. If APIGroup is not specified, the specified Kind must be in the core API group. For any other third-party types, APIGroup is required. @@ -1043,7 +1439,23 @@ spec: type: string type: array dataSource: - description: 'This field can be used to specify either: * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) * An existing PVC (PersistentVolumeClaim) * An existing custom resource that implements data population (Alpha) In order to use custom resource types that implement data population, the AnyVolumeDataSource feature gate must be enabled. If the provisioner or an external controller can support the specified data source, it will create a new volume based on the contents of the specified data source.' + description: 'This field can be used to specify either: * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) * An existing PVC (PersistentVolumeClaim) If the provisioner or an external controller can support the specified data source, it will create a new volume based on the contents of the specified data source. If the AnyVolumeDataSource feature gate is enabled, this field will always have the same contents as the DataSourceRef field.' + properties: + apiGroup: + description: APIGroup is the group for the resource being referenced. If APIGroup is not specified, the specified Kind must be in the core API group. For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource being referenced + type: string + name: + description: Name is the name of resource being referenced + type: string + required: + - kind + - name + type: object + dataSourceRef: + description: 'Specifies the object from which to populate the volume with data, if a non-empty volume is desired. This may be any local object from a non-empty API group (non core object) or a PersistentVolumeClaim object. When this field is specified, volume binding will only succeed if the type of the specified object matches some installed volume populator or dynamic provisioner. This field will replace the functionality of the DataSource field and as such if both fields are non-empty, they must have the same value. For backwards compatibility, both fields (DataSource and DataSourceRef) will be set to the same value automatically if one of them is empty and the other is non-empty. There are two important differences between DataSource and DataSourceRef: * While DataSource only allows two specific types of objects, DataSourceRef allows any non-core object, as well as PersistentVolumeClaim objects. * While DataSource ignores disallowed values (dropping them), DataSourceRef preserves all values, and generates an error if a disallowed value is specified. (Alpha) Using this field requires the AnyVolumeDataSource feature gate to be enabled.' properties: apiGroup: description: APIGroup is the group for the resource being referenced. If APIGroup is not specified, the specified Kind must be in the core API group. For any other third-party types, APIGroup is required. @@ -1244,7 +1656,6 @@ spec: description: HostNetwork to enable host network type: boolean ipFamily: - default: IPv4 description: IPFamily is the single stack IPv6 or IPv4 protocol enum: - IPv4 @@ -1428,7 +1839,7 @@ spec: type: object type: object namespaceSelector: - description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. This field is alpha-level and is only honored when PodAffinityNamespaceSelector feature is enabled. + description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. This field is beta-level and is only honored when PodAffinityNamespaceSelector feature is enabled. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. @@ -1513,7 +1924,7 @@ spec: type: object type: object namespaceSelector: - description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. This field is alpha-level and is only honored when PodAffinityNamespaceSelector feature is enabled. + description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. This field is beta-level and is only honored when PodAffinityNamespaceSelector feature is enabled. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. @@ -1597,7 +2008,7 @@ spec: type: object type: object namespaceSelector: - description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. This field is alpha-level and is only honored when PodAffinityNamespaceSelector feature is enabled. + description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. This field is beta-level and is only honored when PodAffinityNamespaceSelector feature is enabled. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. @@ -1682,7 +2093,7 @@ spec: type: object type: object namespaceSelector: - description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. This field is alpha-level and is only honored when PodAffinityNamespaceSelector feature is enabled. + description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. This field is beta-level and is only honored when PodAffinityNamespaceSelector feature is enabled. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. @@ -1887,6 +2298,7 @@ spec: config: additionalProperties: type: string + nullable: true type: object x-kubernetes-preserve-unknown-fields: true fullpath: @@ -1921,6 +2333,7 @@ spec: config: additionalProperties: type: string + nullable: true type: object x-kubernetes-preserve-unknown-fields: true fullpath: @@ -2000,7 +2413,23 @@ spec: type: string type: array dataSource: - description: 'This field can be used to specify either: * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) * An existing PVC (PersistentVolumeClaim) * An existing custom resource that implements data population (Alpha) In order to use custom resource types that implement data population, the AnyVolumeDataSource feature gate must be enabled. If the provisioner or an external controller can support the specified data source, it will create a new volume based on the contents of the specified data source.' + description: 'This field can be used to specify either: * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) * An existing PVC (PersistentVolumeClaim) If the provisioner or an external controller can support the specified data source, it will create a new volume based on the contents of the specified data source. If the AnyVolumeDataSource feature gate is enabled, this field will always have the same contents as the DataSourceRef field.' + properties: + apiGroup: + description: APIGroup is the group for the resource being referenced. If APIGroup is not specified, the specified Kind must be in the core API group. For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource being referenced + type: string + name: + description: Name is the name of resource being referenced + type: string + required: + - kind + - name + type: object + dataSourceRef: + description: 'Specifies the object from which to populate the volume with data, if a non-empty volume is desired. This may be any local object from a non-empty API group (non core object) or a PersistentVolumeClaim object. When this field is specified, volume binding will only succeed if the type of the specified object matches some installed volume populator or dynamic provisioner. This field will replace the functionality of the DataSource field and as such if both fields are non-empty, they must have the same value. For backwards compatibility, both fields (DataSource and DataSourceRef) will be set to the same value automatically if one of them is empty and the other is non-empty. There are two important differences between DataSource and DataSourceRef: * While DataSource only allows two specific types of objects, DataSourceRef allows any non-core object, as well as PersistentVolumeClaim objects. * While DataSource ignores disallowed values (dropping them), DataSourceRef preserves all values, and generates an error if a disallowed value is specified. (Alpha) Using this field requires the AnyVolumeDataSource feature gate to be enabled.' properties: apiGroup: description: APIGroup is the group for the resource being referenced. If APIGroup is not specified, the specified Kind must be in the core API group. For any other third-party types, APIGroup is required. @@ -2320,7 +2749,7 @@ spec: type: object type: object namespaceSelector: - description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. This field is alpha-level and is only honored when PodAffinityNamespaceSelector feature is enabled. + description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. This field is beta-level and is only honored when PodAffinityNamespaceSelector feature is enabled. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. @@ -2405,7 +2834,7 @@ spec: type: object type: object namespaceSelector: - description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. This field is alpha-level and is only honored when PodAffinityNamespaceSelector feature is enabled. + description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. This field is beta-level and is only honored when PodAffinityNamespaceSelector feature is enabled. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. @@ -2489,7 +2918,7 @@ spec: type: object type: object namespaceSelector: - description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. This field is alpha-level and is only honored when PodAffinityNamespaceSelector feature is enabled. + description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. This field is beta-level and is only honored when PodAffinityNamespaceSelector feature is enabled. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. @@ -2574,7 +3003,7 @@ spec: type: object type: object namespaceSelector: - description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. This field is alpha-level and is only honored when PodAffinityNamespaceSelector feature is enabled. + description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. This field is beta-level and is only honored when PodAffinityNamespaceSelector feature is enabled. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. @@ -2860,7 +3289,7 @@ spec: type: object type: object namespaceSelector: - description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. This field is alpha-level and is only honored when PodAffinityNamespaceSelector feature is enabled. + description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. This field is beta-level and is only honored when PodAffinityNamespaceSelector feature is enabled. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. @@ -2945,7 +3374,7 @@ spec: type: object type: object namespaceSelector: - description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. This field is alpha-level and is only honored when PodAffinityNamespaceSelector feature is enabled. + description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. This field is beta-level and is only honored when PodAffinityNamespaceSelector feature is enabled. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. @@ -3029,7 +3458,7 @@ spec: type: object type: object namespaceSelector: - description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. This field is alpha-level and is only honored when PodAffinityNamespaceSelector feature is enabled. + description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. This field is beta-level and is only honored when PodAffinityNamespaceSelector feature is enabled. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. @@ -3114,7 +3543,7 @@ spec: type: object type: object namespaceSelector: - description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. This field is alpha-level and is only honored when PodAffinityNamespaceSelector feature is enabled. + description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. This field is beta-level and is only honored when PodAffinityNamespaceSelector feature is enabled. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. @@ -3306,7 +3735,23 @@ spec: type: string type: array dataSource: - description: 'This field can be used to specify either: * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) * An existing PVC (PersistentVolumeClaim) * An existing custom resource that implements data population (Alpha) In order to use custom resource types that implement data population, the AnyVolumeDataSource feature gate must be enabled. If the provisioner or an external controller can support the specified data source, it will create a new volume based on the contents of the specified data source.' + description: 'This field can be used to specify either: * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) * An existing PVC (PersistentVolumeClaim) If the provisioner or an external controller can support the specified data source, it will create a new volume based on the contents of the specified data source. If the AnyVolumeDataSource feature gate is enabled, this field will always have the same contents as the DataSourceRef field.' + properties: + apiGroup: + description: APIGroup is the group for the resource being referenced. If APIGroup is not specified, the specified Kind must be in the core API group. For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource being referenced + type: string + name: + description: Name is the name of resource being referenced + type: string + required: + - kind + - name + type: object + dataSourceRef: + description: 'Specifies the object from which to populate the volume with data, if a non-empty volume is desired. This may be any local object from a non-empty API group (non core object) or a PersistentVolumeClaim object. When this field is specified, volume binding will only succeed if the type of the specified object matches some installed volume populator or dynamic provisioner. This field will replace the functionality of the DataSource field and as such if both fields are non-empty, they must have the same value. For backwards compatibility, both fields (DataSource and DataSourceRef) will be set to the same value automatically if one of them is empty and the other is non-empty. There are two important differences between DataSource and DataSourceRef: * While DataSource only allows two specific types of objects, DataSourceRef allows any non-core object, as well as PersistentVolumeClaim objects. * While DataSource ignores disallowed values (dropping them), DataSourceRef preserves all values, and generates an error if a disallowed value is specified. (Alpha) Using this field requires the AnyVolumeDataSource feature gate to be enabled.' properties: apiGroup: description: APIGroup is the group for the resource being referenced. If APIGroup is not specified, the specified Kind must be in the core API group. For any other third-party types, APIGroup is required. @@ -3487,7 +3932,23 @@ spec: type: string type: array dataSource: - description: 'This field can be used to specify either: * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) * An existing PVC (PersistentVolumeClaim) * An existing custom resource that implements data population (Alpha) In order to use custom resource types that implement data population, the AnyVolumeDataSource feature gate must be enabled. If the provisioner or an external controller can support the specified data source, it will create a new volume based on the contents of the specified data source.' + description: 'This field can be used to specify either: * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) * An existing PVC (PersistentVolumeClaim) If the provisioner or an external controller can support the specified data source, it will create a new volume based on the contents of the specified data source. If the AnyVolumeDataSource feature gate is enabled, this field will always have the same contents as the DataSourceRef field.' + properties: + apiGroup: + description: APIGroup is the group for the resource being referenced. If APIGroup is not specified, the specified Kind must be in the core API group. For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource being referenced + type: string + name: + description: Name is the name of resource being referenced + type: string + required: + - kind + - name + type: object + dataSourceRef: + description: 'Specifies the object from which to populate the volume with data, if a non-empty volume is desired. This may be any local object from a non-empty API group (non core object) or a PersistentVolumeClaim object. When this field is specified, volume binding will only succeed if the type of the specified object matches some installed volume populator or dynamic provisioner. This field will replace the functionality of the DataSource field and as such if both fields are non-empty, they must have the same value. For backwards compatibility, both fields (DataSource and DataSourceRef) will be set to the same value automatically if one of them is empty and the other is non-empty. There are two important differences between DataSource and DataSourceRef: * While DataSource only allows two specific types of objects, DataSourceRef allows any non-core object, as well as PersistentVolumeClaim objects. * While DataSource ignores disallowed values (dropping them), DataSourceRef preserves all values, and generates an error if a disallowed value is specified. (Alpha) Using this field requires the AnyVolumeDataSource feature gate to be enabled.' properties: apiGroup: description: APIGroup is the group for the resource being referenced. If APIGroup is not specified, the specified Kind must be in the core API group. For any other third-party types, APIGroup is required. @@ -3986,7 +4447,7 @@ spec: type: object type: object namespaceSelector: - description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. This field is alpha-level and is only honored when PodAffinityNamespaceSelector feature is enabled. + description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. This field is beta-level and is only honored when PodAffinityNamespaceSelector feature is enabled. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. @@ -4071,7 +4532,7 @@ spec: type: object type: object namespaceSelector: - description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. This field is alpha-level and is only honored when PodAffinityNamespaceSelector feature is enabled. + description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. This field is beta-level and is only honored when PodAffinityNamespaceSelector feature is enabled. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. @@ -4155,7 +4616,7 @@ spec: type: object type: object namespaceSelector: - description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. This field is alpha-level and is only honored when PodAffinityNamespaceSelector feature is enabled. + description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. This field is beta-level and is only honored when PodAffinityNamespaceSelector feature is enabled. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. @@ -4240,7 +4701,7 @@ spec: type: object type: object namespaceSelector: - description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. This field is alpha-level and is only honored when PodAffinityNamespaceSelector feature is enabled. + description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. This field is beta-level and is only honored when PodAffinityNamespaceSelector feature is enabled. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. @@ -4449,13 +4910,12 @@ spec: description: FilesystemSpec represents the spec of a file system properties: dataPools: - description: The data pool settings + description: The data pool settings, with optional predefined pool name. items: - description: PoolSpec represents the spec of ceph pool + description: NamedPoolSpec represents the named ceph pool spec properties: compressionMode: - default: none - description: 'The inline compression mode in Bluestore OSD to set to (options are: none, passive, aggressive, force)' + description: 'DEPRECATED: use Parameters instead, e.g., Parameters["compression_mode"] = "force" The inline compression mode in Bluestore OSD to set to (options are: none, passive, aggressive, force) Do NOT set a default value for kubebuilder as this will override the Parameters' enum: - none - passive @@ -4482,13 +4942,11 @@ spec: description: The algorithm for erasure coding type: string codingChunks: - description: Number of coding chunks per object in an erasure coded storage pool (required for erasure-coded pool type) - maximum: 9 + description: Number of coding chunks per object in an erasure coded storage pool (required for erasure-coded pool type). This is the number of OSDs that can be lost simultaneously before data cannot be recovered. minimum: 0 type: integer dataChunks: - description: Number of data chunks per object in an erasure coded storage pool (required for erasure-coded pool type) - maximum: 9 + description: Number of data chunks per object in an erasure coded storage pool (required for erasure-coded pool type). The number of chunks required to recover an object when any single OSD is lost is the same as dataChunks so be aware that the larger the number of data chunks, the higher the cost of recovery. minimum: 0 type: integer required: @@ -4534,6 +4992,9 @@ spec: type: object type: array type: object + name: + description: Name of the pool + type: string parameters: additionalProperties: type: string @@ -4622,8 +5083,7 @@ spec: nullable: true properties: compressionMode: - default: none - description: 'The inline compression mode in Bluestore OSD to set to (options are: none, passive, aggressive, force)' + description: 'DEPRECATED: use Parameters instead, e.g., Parameters["compression_mode"] = "force" The inline compression mode in Bluestore OSD to set to (options are: none, passive, aggressive, force) Do NOT set a default value for kubebuilder as this will override the Parameters' enum: - none - passive @@ -4650,13 +5110,11 @@ spec: description: The algorithm for erasure coding type: string codingChunks: - description: Number of coding chunks per object in an erasure coded storage pool (required for erasure-coded pool type) - maximum: 9 + description: Number of coding chunks per object in an erasure coded storage pool (required for erasure-coded pool type). This is the number of OSDs that can be lost simultaneously before data cannot be recovered. minimum: 0 type: integer dataChunks: - description: Number of data chunks per object in an erasure coded storage pool (required for erasure-coded pool type) - maximum: 9 + description: Number of data chunks per object in an erasure coded storage pool (required for erasure-coded pool type). The number of chunks required to recover an object when any single OSD is lost is the same as dataChunks so be aware that the larger the number of data chunks, the higher the cost of recovery. minimum: 0 type: integer required: @@ -4974,7 +5432,7 @@ spec: type: object type: object namespaceSelector: - description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. This field is alpha-level and is only honored when PodAffinityNamespaceSelector feature is enabled. + description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. This field is beta-level and is only honored when PodAffinityNamespaceSelector feature is enabled. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. @@ -5059,7 +5517,7 @@ spec: type: object type: object namespaceSelector: - description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. This field is alpha-level and is only honored when PodAffinityNamespaceSelector feature is enabled. + description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. This field is beta-level and is only honored when PodAffinityNamespaceSelector feature is enabled. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. @@ -5143,7 +5601,7 @@ spec: type: object type: object namespaceSelector: - description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. This field is alpha-level and is only honored when PodAffinityNamespaceSelector feature is enabled. + description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. This field is beta-level and is only honored when PodAffinityNamespaceSelector feature is enabled. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. @@ -5228,7 +5686,7 @@ spec: type: object type: object namespaceSelector: - description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. This field is alpha-level and is only honored when PodAffinityNamespaceSelector feature is enabled. + description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. This field is beta-level and is only honored when PodAffinityNamespaceSelector feature is enabled. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. @@ -5454,6 +5912,28 @@ spec: status: description: CephFilesystemStatus represents the status of a Ceph Filesystem properties: + conditions: + items: + description: Condition represents a status condition on any Rook-Ceph Custom Resource. + properties: + lastHeartbeatTime: + format: date-time + type: string + lastTransitionTime: + format: date-time + type: string + message: + type: string + reason: + description: ConditionReason is a reason for a condition + type: string + status: + type: string + type: + description: ConditionType represent a resource's status + type: string + type: object + type: array info: additionalProperties: type: string @@ -5617,6 +6097,72 @@ status: --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.5.1-0.20210420220833-f284e2e8098c + creationTimestamp: null + name: cephfilesystemsubvolumegroups.ceph.rook.io +spec: + group: ceph.rook.io + names: + kind: CephFilesystemSubVolumeGroup + listKind: CephFilesystemSubVolumeGroupList + plural: cephfilesystemsubvolumegroups + singular: cephfilesystemsubvolumegroup + scope: Namespaced + versions: + - name: v1 + schema: + openAPIV3Schema: + description: CephFilesystemSubVolumeGroup represents a Ceph Filesystem SubVolumeGroup + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: Spec represents the specification of a Ceph Filesystem SubVolumeGroup + properties: + filesystemName: + description: FilesystemName is the name of Ceph Filesystem SubVolumeGroup volume name. Typically it's the name of the CephFilesystem CR. If not coming from the CephFilesystem CR, it can be retrieved from the list of Ceph Filesystem volumes with `ceph fs volume ls`. To learn more about Ceph Filesystem abstractions see https://docs.ceph.com/en/latest/cephfs/fs-volumes/#fs-volumes-and-subvolumes + type: string + required: + - filesystemName + type: object + status: + description: Status represents the status of a CephFilesystem SubvolumeGroup + properties: + info: + additionalProperties: + type: string + nullable: true + type: object + phase: + description: ConditionType represent a resource's status + type: string + type: object + x-kubernetes-preserve-unknown-fields: true + required: + - metadata + - spec + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.5.1-0.20210420220833-f284e2e8098c @@ -5651,16 +6197,14 @@ spec: properties: rados: description: RADOS is the Ganesha RADOS specification + nullable: true properties: namespace: - description: Namespace is the RADOS namespace where NFS client recovery data is stored. + description: The namespace inside the Ceph pool (set by 'pool') where shared NFS-Ganesha config is stored. This setting is required for Ceph v15 and ignored for Ceph v16. As of Ceph Pacific v16+, this is internally set to the name of the CephNFS. type: string pool: - description: Pool is the RADOS pool where NFS client recovery data is stored. + description: The Ceph pool used store the shared configuration for NFS-Ganesha daemons. This setting is required for Ceph v15 and ignored for Ceph v16. As of Ceph Pacific 16.2.7+, this is internally hardcoded to ".nfs". type: string - required: - - namespace - - pool type: object server: description: Server is the Ganesha Server specification @@ -5850,7 +6394,7 @@ spec: type: object type: object namespaceSelector: - description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. This field is alpha-level and is only honored when PodAffinityNamespaceSelector feature is enabled. + description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. This field is beta-level and is only honored when PodAffinityNamespaceSelector feature is enabled. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. @@ -5935,7 +6479,7 @@ spec: type: object type: object namespaceSelector: - description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. This field is alpha-level and is only honored when PodAffinityNamespaceSelector feature is enabled. + description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. This field is beta-level and is only honored when PodAffinityNamespaceSelector feature is enabled. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. @@ -6019,7 +6563,7 @@ spec: type: object type: object namespaceSelector: - description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. This field is alpha-level and is only honored when PodAffinityNamespaceSelector feature is enabled. + description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. This field is beta-level and is only honored when PodAffinityNamespaceSelector feature is enabled. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. @@ -6104,7 +6648,7 @@ spec: type: object type: object namespaceSelector: - description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. This field is alpha-level and is only honored when PodAffinityNamespaceSelector feature is enabled. + description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. This field is beta-level and is only honored when PodAffinityNamespaceSelector feature is enabled. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. @@ -6253,7 +6797,6 @@ spec: - active type: object required: - - rados - server type: object status: @@ -6315,6 +6858,7 @@ spec: description: PullSpec represents the pulling specification of a Ceph Object Storage Gateway Realm properties: endpoint: + pattern: ^https*:// type: string required: - endpoint @@ -6380,8 +6924,7 @@ spec: nullable: true properties: compressionMode: - default: none - description: 'The inline compression mode in Bluestore OSD to set to (options are: none, passive, aggressive, force)' + description: 'DEPRECATED: use Parameters instead, e.g., Parameters["compression_mode"] = "force" The inline compression mode in Bluestore OSD to set to (options are: none, passive, aggressive, force) Do NOT set a default value for kubebuilder as this will override the Parameters' enum: - none - passive @@ -6408,13 +6951,11 @@ spec: description: The algorithm for erasure coding type: string codingChunks: - description: Number of coding chunks per object in an erasure coded storage pool (required for erasure-coded pool type) - maximum: 9 + description: Number of coding chunks per object in an erasure coded storage pool (required for erasure-coded pool type). This is the number of OSDs that can be lost simultaneously before data cannot be recovered. minimum: 0 type: integer dataChunks: - description: Number of data chunks per object in an erasure coded storage pool (required for erasure-coded pool type) - maximum: 9 + description: Number of data chunks per object in an erasure coded storage pool (required for erasure-coded pool type). The number of chunks required to recover an object when any single OSD is lost is the same as dataChunks so be aware that the larger the number of data chunks, the higher the cost of recovery. minimum: 0 type: integer required: @@ -6777,7 +7318,7 @@ spec: type: object type: object namespaceSelector: - description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. This field is alpha-level and is only honored when PodAffinityNamespaceSelector feature is enabled. + description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. This field is beta-level and is only honored when PodAffinityNamespaceSelector feature is enabled. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. @@ -6862,7 +7403,7 @@ spec: type: object type: object namespaceSelector: - description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. This field is alpha-level and is only honored when PodAffinityNamespaceSelector feature is enabled. + description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. This field is beta-level and is only honored when PodAffinityNamespaceSelector feature is enabled. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. @@ -6946,7 +7487,7 @@ spec: type: object type: object namespaceSelector: - description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. This field is alpha-level and is only honored when PodAffinityNamespaceSelector feature is enabled. + description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. This field is beta-level and is only honored when PodAffinityNamespaceSelector feature is enabled. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. @@ -7031,7 +7572,7 @@ spec: type: object type: object namespaceSelector: - description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. This field is alpha-level and is only honored when PodAffinityNamespaceSelector feature is enabled. + description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. This field is beta-level and is only honored when PodAffinityNamespaceSelector feature is enabled. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. @@ -7197,27 +7738,219 @@ spec: description: The annotations-related configuration to add/set on each rgw service. nullable optional type: object type: object - sslCertificateRef: - description: The name of the secret that stores the ssl certificate for secure rgw connections - nullable: true - type: string - type: object - healthCheck: - description: The rgw Bucket healthchecks and liveness probe - nullable: true - properties: - bucket: - description: HealthCheckSpec represents the health check of an object store bucket - properties: - disabled: - type: boolean - interval: - description: Interval is the internal in second or minute for the health check to run like 60s for 60 seconds - type: string - timeout: - type: string - type: object - livenessProbe: + sslCertificateRef: + description: The name of the secret that stores the ssl certificate for secure rgw connections + nullable: true + type: string + type: object + healthCheck: + description: The rgw Bucket healthchecks and liveness probe + nullable: true + properties: + bucket: + description: HealthCheckSpec represents the health check of an object store bucket + properties: + disabled: + type: boolean + interval: + description: Interval is the internal in second or minute for the health check to run like 60s for 60 seconds + type: string + timeout: + type: string + type: object + livenessProbe: + description: ProbeSpec is a wrapper around Probe so it can be enabled or disabled for a Ceph daemon + properties: + disabled: + description: Disabled determines whether probe is disable or not + type: boolean + probe: + description: Probe describes a health check to be performed against a container to determine whether it is alive or ready to receive traffic. + properties: + exec: + description: One and only one of the following should be specified. Exec specifies the action to take. + properties: + command: + description: Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + type: object + failureThreshold: + description: Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1. + format: int32 + type: integer + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: Host name to connect to, defaults to the pod IP. You probably want to set "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header to be used in HTTP probes + properties: + name: + description: The header field name + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting to the host. Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: 'Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + periodSeconds: + description: How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook' + properties: + host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: Optional duration in seconds the pod needs to terminate gracefully upon probe failure. The grace period is the duration in seconds after the processes running in the pod are sent a termination signal and the time when the processes are forcibly halted with a kill signal. Set this value longer than the expected cleanup time for your process. If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this value overrides the value provided by the pod spec. Value must be non-negative integer. The value zero indicates stop immediately via the kill signal (no opportunity to shut down). This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + type: object + type: object + readinessProbe: + description: ProbeSpec is a wrapper around Probe so it can be enabled or disabled for a Ceph daemon + properties: + disabled: + description: Disabled determines whether probe is disable or not + type: boolean + probe: + description: Probe describes a health check to be performed against a container to determine whether it is alive or ready to receive traffic. + properties: + exec: + description: One and only one of the following should be specified. Exec specifies the action to take. + properties: + command: + description: Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + type: object + failureThreshold: + description: Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1. + format: int32 + type: integer + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: Host name to connect to, defaults to the pod IP. You probably want to set "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header to be used in HTTP probes + properties: + name: + description: The header field name + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting to the host. Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: 'Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + periodSeconds: + description: How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook' + properties: + host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: Optional duration in seconds the pod needs to terminate gracefully upon probe failure. The grace period is the duration in seconds after the processes running in the pod are sent a termination signal and the time when the processes are forcibly halted with a kill signal. Set this value longer than the expected cleanup time for your process. If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this value overrides the value provided by the pod spec. Value must be non-negative integer. The value zero indicates stop immediately via the kill signal (no opportunity to shut down). This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + type: object + type: object + startupProbe: description: ProbeSpec is a wrapper around Probe so it can be enabled or disabled for a Ceph daemon properties: disabled: @@ -7304,7 +8037,7 @@ spec: - port type: object terminationGracePeriodSeconds: - description: Optional duration in seconds the pod needs to terminate gracefully upon probe failure. The grace period is the duration in seconds after the processes running in the pod are sent a termination signal and the time when the processes are forcibly halted with a kill signal. Set this value longer than the expected cleanup time for your process. If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this value overrides the value provided by the pod spec. Value must be non-negative integer. The value zero indicates stop immediately via the kill signal (no opportunity to shut down). This is an alpha field and requires enabling ProbeTerminationGracePeriod feature gate. + description: Optional duration in seconds the pod needs to terminate gracefully upon probe failure. The grace period is the duration in seconds after the processes running in the pod are sent a termination signal and the time when the processes are forcibly halted with a kill signal. Set this value longer than the expected cleanup time for your process. If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this value overrides the value provided by the pod spec. Value must be non-negative integer. The value zero indicates stop immediately via the kill signal (no opportunity to shut down). This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. format: int64 type: integer timeoutSeconds: @@ -7319,8 +8052,7 @@ spec: nullable: true properties: compressionMode: - default: none - description: 'The inline compression mode in Bluestore OSD to set to (options are: none, passive, aggressive, force)' + description: 'DEPRECATED: use Parameters instead, e.g., Parameters["compression_mode"] = "force" The inline compression mode in Bluestore OSD to set to (options are: none, passive, aggressive, force) Do NOT set a default value for kubebuilder as this will override the Parameters' enum: - none - passive @@ -7347,13 +8079,11 @@ spec: description: The algorithm for erasure coding type: string codingChunks: - description: Number of coding chunks per object in an erasure coded storage pool (required for erasure-coded pool type) - maximum: 9 + description: Number of coding chunks per object in an erasure coded storage pool (required for erasure-coded pool type). This is the number of OSDs that can be lost simultaneously before data cannot be recovered. minimum: 0 type: integer dataChunks: - description: Number of data chunks per object in an erasure coded storage pool (required for erasure-coded pool type) - maximum: 9 + description: Number of data chunks per object in an erasure coded storage pool (required for erasure-coded pool type). The number of chunks required to recover an object when any single OSD is lost is the same as dataChunks so be aware that the larger the number of data chunks, the higher the cost of recovery. minimum: 0 type: integer required: @@ -7614,9 +8344,76 @@ spec: spec: description: ObjectStoreUserSpec represent the spec of an Objectstoreuser properties: + capabilities: + description: Additional admin-level capabilities for the Ceph object store user + nullable: true + properties: + bucket: + description: Admin capabilities to read/write Ceph object store buckets. Documented in https://docs.ceph.com/en/latest/radosgw/admin/?#add-remove-admin-capabilities + enum: + - '*' + - read + - write + - read, write + type: string + metadata: + description: Admin capabilities to read/write Ceph object store metadata. Documented in https://docs.ceph.com/en/latest/radosgw/admin/?#add-remove-admin-capabilities + enum: + - '*' + - read + - write + - read, write + type: string + usage: + description: Admin capabilities to read/write Ceph object store usage. Documented in https://docs.ceph.com/en/latest/radosgw/admin/?#add-remove-admin-capabilities + enum: + - '*' + - read + - write + - read, write + type: string + user: + description: Admin capabilities to read/write Ceph object store users. Documented in https://docs.ceph.com/en/latest/radosgw/admin/?#add-remove-admin-capabilities + enum: + - '*' + - read + - write + - read, write + type: string + zone: + description: Admin capabilities to read/write Ceph object store zones. Documented in https://docs.ceph.com/en/latest/radosgw/admin/?#add-remove-admin-capabilities + enum: + - '*' + - read + - write + - read, write + type: string + type: object displayName: description: The display name for the ceph users type: string + quotas: + description: ObjectUserQuotaSpec can be used to set quotas for the object store user to limit their usage. See the [Ceph docs](https://docs.ceph.com/en/latest/radosgw/admin/?#quota-management) for more + nullable: true + properties: + maxBuckets: + description: Maximum bucket limit for the ceph user + nullable: true + type: integer + maxObjects: + description: Maximum number of objects across all the user's buckets + format: int64 + nullable: true + type: integer + maxSize: + anyOf: + - type: integer + - type: string + description: Maximum size limit of all objects across all the user's buckets See https://pkg.go.dev/k8s.io/apimachinery/pkg/api/resource#Quantity for more info. + nullable: true + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object store: description: The store the user will be created in type: string @@ -7745,8 +8542,7 @@ spec: nullable: true properties: compressionMode: - default: none - description: 'The inline compression mode in Bluestore OSD to set to (options are: none, passive, aggressive, force)' + description: 'DEPRECATED: use Parameters instead, e.g., Parameters["compression_mode"] = "force" The inline compression mode in Bluestore OSD to set to (options are: none, passive, aggressive, force) Do NOT set a default value for kubebuilder as this will override the Parameters' enum: - none - passive @@ -7773,13 +8569,11 @@ spec: description: The algorithm for erasure coding type: string codingChunks: - description: Number of coding chunks per object in an erasure coded storage pool (required for erasure-coded pool type) - maximum: 9 + description: Number of coding chunks per object in an erasure coded storage pool (required for erasure-coded pool type). This is the number of OSDs that can be lost simultaneously before data cannot be recovered. minimum: 0 type: integer dataChunks: - description: Number of data chunks per object in an erasure coded storage pool (required for erasure-coded pool type) - maximum: 9 + description: Number of data chunks per object in an erasure coded storage pool (required for erasure-coded pool type). The number of chunks required to recover an object when any single OSD is lost is the same as dataChunks so be aware that the larger the number of data chunks, the higher the cost of recovery. minimum: 0 type: integer required: @@ -7911,8 +8705,7 @@ spec: nullable: true properties: compressionMode: - default: none - description: 'The inline compression mode in Bluestore OSD to set to (options are: none, passive, aggressive, force)' + description: 'DEPRECATED: use Parameters instead, e.g., Parameters["compression_mode"] = "force" The inline compression mode in Bluestore OSD to set to (options are: none, passive, aggressive, force) Do NOT set a default value for kubebuilder as this will override the Parameters' enum: - none - passive @@ -7939,13 +8732,11 @@ spec: description: The algorithm for erasure coding type: string codingChunks: - description: Number of coding chunks per object in an erasure coded storage pool (required for erasure-coded pool type) - maximum: 9 + description: Number of coding chunks per object in an erasure coded storage pool (required for erasure-coded pool type). This is the number of OSDs that can be lost simultaneously before data cannot be recovered. minimum: 0 type: integer dataChunks: - description: Number of data chunks per object in an erasure coded storage pool (required for erasure-coded pool type) - maximum: 9 + description: Number of data chunks per object in an erasure coded storage pool (required for erasure-coded pool type). The number of chunks required to recover an object when any single OSD is lost is the same as dataChunks so be aware that the larger the number of data chunks, the higher the cost of recovery. minimum: 0 type: integer required: @@ -8327,7 +9118,7 @@ spec: type: object type: object namespaceSelector: - description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. This field is alpha-level and is only honored when PodAffinityNamespaceSelector feature is enabled. + description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. This field is beta-level and is only honored when PodAffinityNamespaceSelector feature is enabled. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. @@ -8412,7 +9203,7 @@ spec: type: object type: object namespaceSelector: - description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. This field is alpha-level and is only honored when PodAffinityNamespaceSelector feature is enabled. + description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. This field is beta-level and is only honored when PodAffinityNamespaceSelector feature is enabled. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. @@ -8496,7 +9287,7 @@ spec: type: object type: object namespaceSelector: - description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. This field is alpha-level and is only honored when PodAffinityNamespaceSelector feature is enabled. + description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. This field is beta-level and is only honored when PodAffinityNamespaceSelector feature is enabled. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. @@ -8581,7 +9372,7 @@ spec: type: object type: object namespaceSelector: - description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. This field is alpha-level and is only honored when PodAffinityNamespaceSelector feature is enabled. + description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. This field is beta-level and is only honored when PodAffinityNamespaceSelector feature is enabled. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. @@ -8863,290 +9654,3 @@ spec: x-kubernetes-preserve-unknown-fields: true subresources: status: {} ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.5.1-0.20210420220833-f284e2e8098c - creationTimestamp: null - name: volumereplicationclasses.replication.storage.openshift.io -spec: - group: replication.storage.openshift.io - names: - kind: VolumeReplicationClass - listKind: VolumeReplicationClassList - plural: volumereplicationclasses - shortNames: - - vrc - singular: volumereplicationclass - scope: Cluster - versions: - - additionalPrinterColumns: - - jsonPath: .spec.provisioner - name: provisioner - type: string - name: v1alpha1 - schema: - openAPIV3Schema: - description: VolumeReplicationClass is the Schema for the volumereplicationclasses API - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: VolumeReplicationClassSpec specifies parameters that an underlying storage system uses when creating a volume replica. A specific VolumeReplicationClass is used by specifying its name in a VolumeReplication object. - properties: - parameters: - additionalProperties: - type: string - description: Parameters is a key-value map with storage provisioner specific configurations for creating volume replicas - type: object - provisioner: - description: Provisioner is the name of storage provisioner - type: string - required: - - provisioner - type: object - status: - description: VolumeReplicationClassStatus defines the observed state of VolumeReplicationClass - type: object - type: object - served: true - storage: true - subresources: - status: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.5.1-0.20210420220833-f284e2e8098c - creationTimestamp: null - name: volumereplications.replication.storage.openshift.io -spec: - group: replication.storage.openshift.io - names: - kind: VolumeReplication - listKind: VolumeReplicationList - plural: volumereplications - shortNames: - - vr - singular: volumereplication - scope: Namespaced - versions: - - additionalPrinterColumns: - - jsonPath: .metadata.creationTimestamp - name: Age - type: date - - jsonPath: .spec.volumeReplicationClass - name: volumeReplicationClass - type: string - - jsonPath: .spec.dataSource.name - name: pvcName - type: string - - jsonPath: .spec.replicationState - name: desiredState - type: string - - jsonPath: .status.state - name: currentState - type: string - name: v1alpha1 - schema: - openAPIV3Schema: - description: VolumeReplication is the Schema for the volumereplications API - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: VolumeReplicationSpec defines the desired state of VolumeReplication - properties: - dataSource: - description: DataSource represents the object associated with the volume - properties: - apiGroup: - description: APIGroup is the group for the resource being referenced. If APIGroup is not specified, the specified Kind must be in the core API group. For any other third-party types, APIGroup is required. - type: string - kind: - description: Kind is the type of resource being referenced - type: string - name: - description: Name is the name of resource being referenced - type: string - required: - - kind - - name - type: object - replicationState: - description: ReplicationState represents the replication operation to be performed on the volume. Supported operations are "primary", "secondary" and "resync" - enum: - - primary - - secondary - - resync - type: string - volumeReplicationClass: - description: VolumeReplicationClass is the VolumeReplicationClass name for this VolumeReplication resource - type: string - required: - - dataSource - - replicationState - - volumeReplicationClass - type: object - status: - description: VolumeReplicationStatus defines the observed state of VolumeReplication - properties: - conditions: - description: Conditions are the list of conditions and their status. - items: - description: "Condition contains details for one aspect of the current state of this API Resource. --- This struct is intended for direct use as an array at the field path .status.conditions. For example, type FooStatus struct{ // Represents the observations of a foo's current state. // Known .status.conditions.type are: \"Available\", \"Progressing\", and \"Degraded\" // +patchMergeKey=type // +patchStrategy=merge // +listType=map // +listMapKey=type Conditions []metav1.Condition `json:\"conditions,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"` \n // other fields }" - properties: - lastTransitionTime: - description: lastTransitionTime is the last time the condition transitioned from one status to another. This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: message is a human readable message indicating details about the transition. This may be an empty string. - maxLength: 32768 - type: string - observedGeneration: - description: observedGeneration represents the .metadata.generation that the condition was set based upon. For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date with respect to the current state of the instance. - format: int64 - minimum: 0 - type: integer - reason: - description: reason contains a programmatic identifier indicating the reason for the condition's last transition. Producers of specific condition types may define expected values and meanings for this field, and whether the values are considered a guaranteed API. The value should be a CamelCase string. This field may not be empty. - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: type of condition in CamelCase or in foo.example.com/CamelCase. --- Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be useful (see .node.status.conditions), the ability to deconflict is important. The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - type: array - lastCompletionTime: - format: date-time - type: string - lastStartTime: - format: date-time - type: string - message: - type: string - observedGeneration: - description: observedGeneration is the last generation change the operator has dealt with - format: int64 - type: integer - state: - description: State captures the latest state of the replication operation - type: string - type: object - type: object - served: true - storage: true - subresources: - status: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.5.1-0.20210420220833-f284e2e8098c - creationTimestamp: null - name: volumes.rook.io -spec: - group: rook.io - names: - kind: Volume - listKind: VolumeList - plural: volumes - shortNames: - - rv - singular: volume - scope: Namespaced - versions: - - name: v1alpha2 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - attachments: - items: - properties: - clusterName: - type: string - mountDir: - type: string - node: - type: string - podName: - type: string - podNamespace: - type: string - readOnly: - type: boolean - required: - - clusterName - - mountDir - - node - - podName - - podNamespace - - readOnly - type: object - type: array - kind: - description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - required: - - attachments - - metadata - type: object - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] diff --git a/rook-ceph/install/operator-openshift.yaml b/rook-ceph/install/operator-openshift.yaml index 1a54647..6f29f4c 100644 --- a/rook-ceph/install/operator-openshift.yaml +++ b/rook-ceph/install/operator-openshift.yaml @@ -11,12 +11,14 @@ apiVersion: security.openshift.io/v1 metadata: name: rook-ceph allowPrivilegedContainer: true -allowHostNetwork: true allowHostDirVolumePlugin: true +allowHostPID: false +# set to true if running rook with host networking enabled +allowHostNetwork: false +# set to true if running rook with the provider as host +allowHostPorts: false priority: -allowedCapabilities: [] -allowHostPorts: true -allowHostPID: true # remove this once we drop support for Nautilus +allowedCapabilities: ["MKNOD"] allowHostIPC: true readOnlyRootFilesystem: false requiredDropCapabilities: [] @@ -29,14 +31,10 @@ fsGroup: type: MustRunAs supplementalGroups: type: RunAsAny -allowedFlexVolumes: - - driver: "ceph.rook.io/rook" - - driver: "ceph.rook.io/rook-ceph" volumes: - configMap - downwardAPI - emptyDir - - flexVolume - hostPath - persistentVolumeClaim - projected @@ -55,17 +53,23 @@ kind: SecurityContextConstraints apiVersion: security.openshift.io/v1 metadata: name: rook-ceph-csi +# To allow running privilegedContainers allowPrivilegedContainer: true +# CSI daemonset pod needs hostnetworking allowHostNetwork: true +# This need to be set to true as we use HostPath allowHostDirVolumePlugin: true priority: -allowedCapabilities: ["*"] +# SYS_ADMIN is needed for rbd to execture rbd map command +allowedCapabilities: ["SYS_ADMIN"] +# Needed as we run liveness container on daemonset pods allowHostPorts: true +# Needed as we are setting this in RBD plugin pod allowHostPID: true +# Required for encryption allowHostIPC: true +# Set to false as we write to RootFilesystem inside csi containers readOnlyRootFilesystem: false -requiredDropCapabilities: [] -defaultAddCapabilities: [] runAsUser: type: RunAsAny seLinuxContext: @@ -74,10 +78,12 @@ fsGroup: type: RunAsAny supplementalGroups: type: RunAsAny -allowedFlexVolumes: - - driver: "ceph.rook.io/rook" - - driver: "ceph.rook.io/rook-ceph" -volumes: ["*"] +# The type of volumes which are mounted to csi pods +volumes: + - configMap + - projected + - emptyDir + - hostPath users: # A user needs to be added for each rook service account. # This assumes running in the default sample "rook-ceph" namespace. @@ -98,7 +104,7 @@ metadata: # should be in the namespace of the operator namespace: rook-ceph # namespace:operator data: - # The logging level for the operator: INFO | DEBUG + # The logging level for the operator: ERROR | WARNING | INFO | DEBUG ROOK_LOG_LEVEL: "INFO" # Enable the CSI driver. @@ -117,6 +123,9 @@ data: # Supported values from 0 to 5. 0 for general useful logs, 5 for trace level verbosity. # CSI_LOG_LEVEL: "0" + # Set replicas for csi provisioner deployment. + CSI_PROVISIONER_REPLICAS: "2" + # OMAP generator generates the omap mapping between the PV name and the RBD image # which helps CSI to identify the rbd images for CSI operations. # CSI_ENABLE_OMAP_GENERATOR need to be enabled when we are using rbd mirroring feature. @@ -132,7 +141,7 @@ data: # Enable Ceph Kernel clients on kernel < 4.17 which support quotas for Cephfs # If you disable the kernel client, your application may be disrupted during upgrade. - # See the upgrade guide: https://rook.io/docs/rook/master/ceph-upgrade.html + # See the upgrade guide: https://rook.io/docs/rook/latest/ceph-upgrade.html CSI_FORCE_CEPHFS_KERNEL_CLIENT: "true" # (Optional) policy for modifying a volume's ownership or permissions when the RBD PVC is being mounted. @@ -145,15 +154,19 @@ data: # (Optional) Allow starting unsupported ceph-csi image ROOK_CSI_ALLOW_UNSUPPORTED_VERSION: "false" + + # (Optional) control the host mount of /etc/selinux for csi plugin pods. + CSI_PLUGIN_ENABLE_SELINUX_HOST_MOUNT: "false" + # The default version of CSI supported by Rook will be started. To change the version # of the CSI driver to something other than what is officially supported, change # these images to the desired release of the CSI driver. - ROOK_CSI_CEPH_IMAGE: "${PROXY_REGISTRY}/cephcsi/cephcsi:v3.4.0" - ROOK_CSI_REGISTRAR_IMAGE: "${PROXY_REGISTRY}/sig-storage/csi-node-driver-registrar:v2.3.0" - ROOK_CSI_RESIZER_IMAGE: "${PROXY_REGISTRY}/sig-storage/csi-resizer:v1.3.0" - ROOK_CSI_PROVISIONER_IMAGE: "${PROXY_REGISTRY}/sig-storage/csi-provisioner:v3.0.0" - ROOK_CSI_SNAPSHOTTER_IMAGE: "${PROXY_REGISTRY}/sig-storage/csi-snapshotter:v4.2.0" - ROOK_CSI_ATTACHER_IMAGE: "${PROXY_REGISTRY}/sig-storage/csi-attacher:v3.3.0" + ROOK_CSI_CEPH_IMAGE: "${PROXY_REGISTRY}/cephcsi/cephcsi:v3.5.1" + ROOK_CSI_REGISTRAR_IMAGE: "${PROXY_REGISTRY}/sig-storage/csi-node-driver-registrar:v2.4.0" + ROOK_CSI_RESIZER_IMAGE: "${PROXY_REGISTRY}/sig-storage/csi-resizer:v1.4.0" + ROOK_CSI_PROVISIONER_IMAGE: "${PROXY_REGISTRY}/sig-storage/csi-provisioner:v3.1.0" + ROOK_CSI_SNAPSHOTTER_IMAGE: "${PROXY_REGISTRY}/sig-storage/csi-snapshotter:v5.0.1" + ROOK_CSI_ATTACHER_IMAGE: "${PROXY_REGISTRY}/sig-storage/csi-attacher:v3.4.0" # (Optional) set user created priorityclassName for csi plugin pods. # CSI_PLUGIN_PRIORITY_CLASSNAME: "system-node-critical" @@ -390,33 +403,24 @@ data: # Configure CSI RBD grpc and liveness metrics port # CSI_RBD_GRPC_METRICS_PORT: "9090" # CSI_RBD_LIVENESS_METRICS_PORT: "9080" + # CSIADDONS_PORT: "9070" # Whether the OBC provisioner should watch on the operator namespace or not, if not the namespace of the cluster will be used ROOK_OBC_WATCH_OPERATOR_NAMESPACE: "true" - # Whether to enable the flex driver. By default it is enabled and is fully supported, but will be deprecated in some future release - # in favor of the CSI driver. - ROOK_ENABLE_FLEX_DRIVER: "false" # Whether to start the discovery daemon to watch for raw storage devices on nodes in the cluster. # This daemon does not need to run if you are only going to create your OSDs based on StorageClassDeviceSets with PVCs. ROOK_ENABLE_DISCOVERY_DAEMON: "false" - # Enable volume replication controller + # Enable the volume replication controller + # Before enabling, ensure the Volume Replication CRDs are created. + # See https://rook.io/docs/rook/latest/ceph-csi-drivers.html#rbd-mirroring CSI_ENABLE_VOLUME_REPLICATION: "false" # The timeout value (in seconds) of Ceph commands. It should be >= 1. If this variable is not set or is an invalid value, it's default to 15. ROOK_CEPH_COMMANDS_TIMEOUT_SECONDS: "15" - # CSI_VOLUME_REPLICATION_IMAGE: "quay.io/csiaddons/volumereplication-operator:v0.1.0" - - # (Optional) Admission controller NodeAffinity. - # ADMISSION_CONTROLLER_NODE_AFFINITY: "role=storage-node; storage=rook, ceph" - # (Optional) Admission controller tolerations list. Put here list of taints you want to tolerate in YAML format. - # Admission controller would be best to start on the same nodes as other ceph daemons. - # ADMISSION_CONTROLLER_TOLERATIONS: | - # - effect: NoSchedule - # key: node-role.kubernetes.io/controlplane - # operator: Exists - # - effect: NoExecute - # key: node-role.kubernetes.io/etcd - # operator: Exists + # CSI_VOLUME_REPLICATION_IMAGE: "quay.io/csiaddons/volumereplication-operator:v0.3.0" + # Enable the csi addons sidecar. + CSI_ENABLE_CSIADDONS: "false" + # ROOK_CSIADDONS_IMAGE: "quay.io/csiaddons/k8s-sidecar:v0.2.1" --- # The deployment for the rook operator # OLM: BEGIN OPERATOR DEPLOYMENT @@ -428,6 +432,10 @@ metadata: labels: operator: rook storage-backend: ceph + app.kubernetes.io/name: rook-ceph + app.kubernetes.io/instance: rook-ceph + app.kubernetes.io/component: rook-ceph-operator + app.kubernetes.io/part-of: rook-ceph-operator spec: selector: matchLabels: @@ -441,45 +449,26 @@ spec: serviceAccountName: rook-ceph-system containers: - name: rook-ceph-operator - image: ${PROXY_REGISTRY}/rook/ceph:v1.7.7 + image: ${PROXY_REGISTRY}/rook/ceph:v1.8.4 args: ["ceph", "operator"] + securityContext: + runAsNonRoot: true + runAsUser: 2016 + runAsGroup: 2016 volumeMounts: - mountPath: /var/lib/rook name: rook-config - mountPath: /etc/ceph name: default-config-dir + - mountPath: /etc/webhook + name: webhook-cert + ports: + - containerPort: 9443 + name: https-webhook + protocol: TCP env: - name: ROOK_CURRENT_NAMESPACE_ONLY value: "false" - # Rook Agent toleration. Will tolerate all taints with all keys. - # Choose between NoSchedule, PreferNoSchedule and NoExecute: - # - name: AGENT_TOLERATION - # value: "NoSchedule" - # (Optional) Rook Agent toleration key. Set this to the key of the taint you want to tolerate - # - name: AGENT_TOLERATION_KEY - # value: "" - # (Optional) Rook Agent priority class name to set on the pod(s) - # - name: AGENT_PRIORITY_CLASS_NAME - # value: "" - # (Optional) Rook Agent NodeAffinity. - # - name: AGENT_NODE_AFFINITY - # value: "role=storage-node; storage=rook,ceph" - # (Optional) Rook Agent mount security mode. Can by `Any` or `Restricted`. - # `Any` uses Ceph admin credentials by default/fallback. - # For using `Restricted` you must have a Ceph secret in each namespace storage should be consumed from and - # set `mountUser` to the Ceph user, `mountSecret` to the Kubernetes secret name. - # to the namespace in which the `mountSecret` Kubernetes secret namespace. - # - name: AGENT_MOUNT_SECURITY_MODE - # value: "Any" - # Set the path where the Rook agent can find the flex volumes - - name: FLEXVOLUME_DIR_PATH - value: "/etc/kubernetes/kubelet-plugins/volume/exec" - # Set the path where kernel modules can be found - # - name: LIB_MODULES_DIR_PATH - # value: "" - # Mount any extra directories into the agent container - # - name: AGENT_MOUNTS - # value: "somemount=/host/path:/container/path,someothermount=/host/path2:/container/path2" # Rook Discover toleration. Will tolerate all taints with all keys. # Choose between NoSchedule, PreferNoSchedule and NoExecute: # - name: DISCOVER_TOLERATION @@ -577,4 +566,6 @@ spec: emptyDir: {} - name: default-config-dir emptyDir: {} + - name: webhook-cert + emptyDir: {} # OLM: END OPERATOR DEPLOYMENT