History: SOP

Source of version: 84 (current)

{maketoc} !Bootstrapping Expandable Encrypted Redundant Mass Storage !!Wipe New Disks {CODE(caption="Wipe New Disks")} root@debian:/dev/disk/by-path# for disk in pci-0000\:10\:00.0-sas-0x50014380057d*; do (for i in `seq 4001`; do echo "---> $disk @ $i">&2; openssl rand -rand /dev/urandom 1000000000 2>/dev/null; done) > $disk & done; wait {CODE} !!Create New RAID0 Arrays {CODE(caption="Create New RAID0 Arrays")} root@debian:/dev/disk/by-path# ls -l | grep lun | awk -F/ '{print $3}' > /tmp/devs root@debian:/dev/disk/by-path# cat /tmp/devs | (for i in `seq 24`; do read DEV; echo yes | mdadm -C /dev/md/drive$i -l 0 --force -n 1 -c 256 /dev/$DEV; done) root@debian:~# for i in `seq 24`; do UUID=`mdadm -D /dev/md/drive$i | grep UUID | awk '{print $3}'`; echo "ARRAY /dev/md/drive$i uuid=$UUID"; done >> /etc/mdadm/mdadm.conf {CODE} !!Encrypt RAID0 Arrays {CODE(caption="Encrypt RAID0 Arrays")} root@debian:~# touch /run/password root@debian:~# chmod go-rwx /run/password root@debian:~# vi -b /run/password # Use ":set noeol" in vi before saving the file, this will omit the trailing newline on the password root@debian:~# for i in `seq 24`; do echo YES | cryptsetup --key-file=/run/password luksFormat /dev/md/drive$i & done; wait {CODE} !!Open Encrypted Devices {CODE(caption="Open Encrypted Devices")} root@debian:~# for i in `seq 24`; do cryptsetup --key-file=/run/password luksOpen /dev/md/drive$i drive$i-crypt & done; wait root@debian:~# rm /run/password {CODE} !!Create ZFS Pool of Encrypted Devices {CODE(caption="Create ZFS Pool of Encrypted Devices")} root@debian:~# zpool create -o ashift=9 pool raidz3 /dev/mapper/drive1-crypt /dev/mapper/drive2-crypt /dev/mapper/drive3-crypt /dev/mapper/drive4-crypt /dev/mapper/drive5-crypt /dev/mapper/drive6-crypt /dev/mapper/drive7-crypt /dev/mapper/drive8-crypt /dev/mapper/drive9-crypt /dev/mapper/drive10-crypt /dev/mapper/drive11-crypt /dev/mapper/drive12-crypt /dev/mapper/drive13-crypt /dev/mapper/drive14-crypt /dev/mapper/drive15-crypt /dev/mapper/drive16-crypt /dev/mapper/drive17-crypt /dev/mapper/drive18-crypt /dev/mapper/drive19-crypt /dev/mapper/drive20-crypt /dev/mapper/drive21-crypt /dev/mapper/drive22-crypt /dev/mapper/drive23-crypt /dev/mapper/drive24-crypt {CODE} !!Create new ZFS Dataset {CODE(caption="Create new ZFS Dataset")} root@debian:~# zfs create -o acltype=posixacl -o atime=off -o xattr=sa pool/data {CODE} !Shutting Down Mass Storage {CODE(caption="Shutting Down Mass Storage")} root@debian:~# zpool export pool root@debian:~# for i in /dev/mapper/drive*-crypt; do cryptsetup luksClose $i; done root@debian:~# for i in `seq 24`; do mdadm -S /dev/md/drive$i; done # Alternatively: mdadm -S -s, but it may shut down arrays you won't want shut down. {CODE} !Mounting Mass Storage {CODE(caption="Mounting Mass Storage")} root@debian:~# mdadm -A -s root@debian:~# touch /run/password root@debian:~# chmod go-rwx /run/password root@debian:~# vim /run/password root@debian:~# for i in `seq 24`; do cryptsetup --key-file=/run/password luksOpen /dev/md/drive$i drive$i-crypt & done; wait root@debian:~# rm /run/password root@debian:~# zpool import pool {CODE} !Installing Debian onto Encrypted ZFS Root In general, follow: [https://github.com/zfsonlinux/pkg-zfs/wiki/HOWTO-install-Debian-GNU-Linux-to-a-Native-ZFS-Root-Filesystem] # Boot Debian Live CD which includes non-free firmware: [http://cdimage.debian.org/cdimage/unofficial/non-free/cd-including-firmware/7.6.0-live+nonfree/amd64/iso-hybrid/debian-live-7.6.0-amd64-kde-desktop+nonfree.iso] # In general, follow: [http://zfsonlinux.org/debian.html] # Refer to the following disk layout and boot sequence information: [http://www.pixelbeat.org/docs/disk/] # {CODE(caption="Partition each of the 3 drives with a boot and root")} apt-get install parted # Start of 2048s is the 1MB de facto standard offset # End of 4196351s is derived from ((1024*1024*1024*2) / 512) - 1 # Start of 4196352s is simply the previous end sector + 1 # 100% denotes the end of the disk for disk in /dev/disk/by-path/pci-0000:14:00.0-sas-0x443322110?000000-lun-0; do echo "---> $disk"; yes | parted $disk mklabel msdos mkpart primary 2048s 4196351s set 1 boot on mkpart primary 4196352s 100%; done {CODE} # {CODE(caption="MD-RAID the /boot partition")} apt-get install mdadm # Using metadata version 0.90 or 1.0 is critical to each device being usable by GRUB without mdraid support # This is accomplished some 0.90 and 1.0 metadata is placed at the end of each disk, leaving GRUB to detect # the underlying file system as though RAID wasn't involved. mdadm -C /dev/md/boot -l 1 -n 3 -e 1.0 /dev/disk/by-path/pci-0000:14:00.0-sas-0x443322110?000000-lun-0-part1 {CODE} # {CODE(caption="Format the /boot partition")} # Find the corresponding md device by examining /proc/mdstat mke2fs -m 0 -L BOOT /dev/md122 {CODE} # {CODE(caption="Create the root filesystem dm-crypt devices")} touch /run/password chmod go-rwx /run/password vi -b /run/password # Use ":set noeol" in vi before saving the file, this will omit the trailing newline on the password for disk in /dev/disk/by-path/pci-0000:14:00.0-sas-0x443322110?000000-lun-0-part2; do echo YES | cryptsetup --key-file=/run/password luksFormat $disk & done; wait i=0; for disk in /dev/disk/by-path/pci-0000:14:00.0-sas-0x443322110?000000-lun-0-part2; do cryptsetup --key-file=/run/password --allow-discards luksOpen $disk root$i-crypt; i=`expr $i + 1`; done i=0; for disk in /dev/disk/by-path/pci-0000:14:00.0-sas-0x443322110?000000-lun-0-part2; do if [ $i -ne 0 ]; then /lib/cryptsetup/scripts/decrypt_derived root0-crypt | cryptsetup --key-file=/run/password luksAddKey $disk; fi; i=`expr $i + 1`; done rm /run/password {CODE} # {CODE(caption="Install the ZFS utilities")} wget 'http://archive.zfsonlinux.org/debian/pool/main/z/zfsonlinux/zfsonlinux_3~wheezy_all.deb' dpkg -i 'zfsonlinux_3~wheezy_all.deb' apt-get update apt-get install debian-zfs modprobe zfs {CODE} # {CODE(caption="Create the root ZFS filesystem")} zpool create rpool mirror /dev/mapper/root?-crypt zfs create -o acltype=posixacl -o atime=off -o xattr=sa rpool/ROOT zfs create -o acltype=posixacl -o atime=off -o xattr=sa rpool/ROOT/debian-1 zfs umount -a zfs set mountpoint=/ rpool/ROOT/debian-1 zpool set bootfs=rpool/ROOT/debian-1 rpool zpool export rpool {CODE} # {CODE(caption="Prepare the disks for installation")} zpool import -d /dev/mapper -R /mnt rpool mkdir /mnt/boot mount /dev/md122 /mnt/boot {CODE} # {CODE(caption="Bootstrap Debian")} apt-get install debootstrap debootstrap testing /mnt http://ftp.us.debian.org/debian {CODE} # {CODE(caption="Prepare the target and chroot")} cp /etc/hostname /mnt/etc/ cp /etc/hosts /mnt/etc/ echo 'LABEL="BOOT" /boot ext2 auto defaults 0 1' >> /mnt/etc/fstab echo '/dev/mapper/root1-crypt / none none 0 0' >> /etc/fstab # Yes, this is a hack: http://markus.heberling.net/tag/zfs/ # The above must be a vdev of the rpool which is a derived device, and not the primary key device, # otherwise update-initramfs will not include the decrypt_derived script in the initramfs output. echo 'deb http://ftp.us.debian.org/debian testing main contrib non-free' > /mnt/etc/apt/sources.list echo "auto lo iface lo inet loopback auto eth0 iface eth0 inet dhcp" >> /mnt/etc/network/interfaces mount --bind /dev /mnt/dev mount --bind /proc /mnt/proc mount --bind /sys /mnt/sys chroot /mnt /bin/bash --login {CODE} # {CODE(caption="Work around a grub-probe bug triggered when it interacts with ZFS VDEVs that are in /dev/mapper instead of /dev")} echo 'ENV{DM_NAME}=="root[0-9]*-crypt" ACTION=="add|change" SYMLINK+="root%n-crypt dm-name-root%n-crypt"' > /etc/udev/rules.d/99-grub-zfs.rules udevadm control -R udevadm test /sys/block/dm-0 udevadm test /sys/block/dm-1 udevadm test /sys/block/dm-2 {CODE} # {CODE(caption="Configure the target")} apt-get install locales vi /etc/locale.gen # Uncomment the "en_US.UTF-8" locale locale-gen wget 'http://archive.zfsonlinux.org/debian/pool/main/z/zfsonlinux/zfsonlinux_3~jessie_all.deb' dpkg -i 'zfsonlinux_3~jessie_all.deb' apt-get update apt-get install mdadm cryptsetup i=0; for disk in /dev/disk/by-path/pci-0000:14:00.0-sas-0x443322110?000000-lun-0-part2; do UUID=`cryptsetup luksDump $disk | grep UUID | awk '{print $2}'`; if [ $i -eq 0 ]; then echo "root$i-crypt UUID=$UUID none luks,discard"; else echo "root$i-crypt UUID=$UUID root0-crypt keyscript=/lib/cryptsetup/scripts/decrypt_derived,luks,discard"; fi; i=`expr $i + 1`; done > /etc/crypttab i=0; for disk in /dev/disk/by-path/pci-0000:14:00.0-sas-0x443322110?000000-lun-0-part2; do UUID=`cryptsetup luksDump $disk | grep UUID | awk '{print $2}'`; if [ $i -eq 0 ]; then echo "target=root$i-crypt,source=UUID=$UUID,key=none,rootdev,discard"; else echo "target=root$i-crypt,source=UUID=$UUID,key=root0-crypt,keyscript=/lib/cryptsetup/scripts/decrypt_derived,rootdev,discard"; fi; i=`expr $i + 1`; done > /etc/initramfs-tools/conf.d/cryptroot apt-get install linux-image-amd64 debian-zfs apt-get install firmware-bnx2 grub2-common grub-pc zfs-initramfs apt-get dist-upgrade vi /etc/default/grub # Add "boot=zfs" to GRUB_CMDLINE_LINUX (NOT to GRUB_CMDLINE_LINUX_DEFAULT!) sed -i 's|modprobe|/sbin/modprobe|' /usr/share/initramfs-tools/scripts/zfs # Apply bugfix: https://github.com/zfsonlinux/pkg-zfs/issues/119 # Skip the above step if at some point the bugfix is applied directly to the published .deb files update-grub update-initramfs -u passwd root {CODE} #{CODE(caption="Clean up and reboot")} umount /mnt/boot umount /mnt/dev umount /mnt/proc umount /mnt/sys zfs umount -a zpool export rpool reboot {CODE} !Installing a Virtual Machine onto a ZVol #Look at the following for details about 9p: [http://www.slideshare.net/ericvh/9p-overview] #Look at the following for qemu-kvm virtio performance tuning: [http://events.linuxfoundation.org/sites/events/files/slides/CloudOpen2013_Khoa_Huynh_v3.pdf] #{CODE(caption="Prepare host networking for VMs")} apt-get install bridge-utils echo "source-directory /etc/network/interfaces.d auto lo br0 iface lo inet loopback iface eth0 inet manual iface br0 inet static bridge_ports eth0 address 192.168.42.57 netmask 255.255.255.0 gateway 192.168.42.252 bridge_stp off bridge_waitport 0 bridge_fd 0 " > /etc/network/interfaces /etc/init.d/networking restart # Use "ip a del" to remove any IPs still remaining on eth0 {CODE} #{CODE(caption="Prepare host for VMs")} apt-get install qemu-kvm libvirt-bin {CODE} #{CODE(caption="Use the following make-vm.sh script")} VM_NAME=$1 zfs create -o volblocksize=128k -o sync=disabled -V 32G pool/${VM_NAME}-vm virt-install -n $VM_NAME --memory=2048 --vcpus=2 --cpu host-model-only --pxe --boot=network,hd --os-variant=debianwheezy --disk path=/dev/zvol/pool/${VM_NAME}-vm,device=disk,bus=virtio,sparse=false,cache=none,format=raw,io=native --filesystem mode=squash,source=/pool/data/Adults/Security,target=security -w bridge=br0 --graphics spice,listen=192.168.42.57 --sound ac97 --video=qxl --autostart --noautoconsole virsh destroy $VM_NAME /etc/init.d/libvirtd restart {CODE} #{CODE(caption="Mount the shared filesystem inside the VM")} mkdir /security echo "security /security 9p trans=virtio,version=9p2000.L,msize=512k 0 0" >> /etc/fstab mount /security {CODE} #{CODE(caption="Use the following remove-vm.sh script")} VM_NAME=$1 virsh destroy $VM_NAME virsh undefine $VM_NAME /etc/init.d/libvirtd restart zfs destroy pool/${VM_NAME}-vm {CODE} !Replacing a dead drive #Ensure the associated dm-crypt and md-raid devices are stopped #Wipe the new drive, either with zeros or entropy (documented above) #{CODE(caption="Find the old RAID0 array UUID")} grep driveN /etc/mdadm/mdadm.conf {CODE} #{CODE(caption="Create a new RAID0 array from the drive")} mdadm -C /dev/md/driveN-new -l 0 --force -n 1 -c 256 --uuid=<UUID> -N driveN --homehost=debian /dev/<NEW DRIVE> {CODE} #{CODE(caption="Encrypt the new array")} cryptsetup luksFormat /dev/md/driveN {CODE} #{CODE(caption="Add a derived key slot to the new array")} # Instead of using the clipboard below, you can also use a named pipe (mkfifo) or a tmpfs file to store the "derived key" # The below shows the "derived key" - PROTECT THIS - MASTER KEY OF root0-crypt! dmsetup table --showkeys root0-crypt | awk {'print $5'} # Enter the existing password, then paste in the "derived key" cryptsetup luksAddKey /dev/md/driveN {CODE} #{CODE(caption="Start the encrypted container")} cryptsetup luksOpen /dev/md/driveN driveN-crypt {CODE} #{CODE(caption="Replace the drive in ZFS")} zpool replace pool driveN-crypt {CODE} #{CODE(caption="Speed up the resilver as per http://broken.net/uncategorized/zfs-performance-tuning-for-scrubs-and-resilvers/")} echo 0 > /sys/module/zfs/parameters/zfs_resilver_delay echo 0 > /sys/module/zfs/parameters/zfs_scrub_delay echo 256 > /sys/module/zfs/parameters/zfs_top_maxinflight echo 5000 > /sys/module/zfs/parameters/zfs_resilver_min_time_ms {CODE} !Make a Docker Container Start Automatically #{CODE(caption="Make it always restart")} docker run -d --restart=always <image name> {CODE} !Make a Guest VM Shutdown With Host #{CODE(caption="Make sure acpid is available and running")} emerge acpid rc-config add acpid default {CODE} !Get sabnzbdplus To Start On Boot #[https://forums.sabnzbd.org/viewtopic.php?f=16&t=18846] In short: set no_ipv6=1 in sabnzbd.ini

---