Bootstrapping Expandable Encrypted Redundant Mass Storage

Wipe New Disks

root@debian:/dev/disk/by-path# for disk in pci-0000\:10\:00.0-sas-0x50014380057d*; do (for i in `seq 4001`; do echo "---> $disk @ $i">&2; openssl rand -rand /dev/urandom 1000000000 2>/dev/null; done) > $disk & done; wait

Create New RAID0 Arrays

root@debian:/dev/disk/by-path# ls -l | grep lun | awk -F/ '{print $3}' > /tmp/devs
root@debian:/dev/disk/by-path# cat /tmp/devs | (for i in `seq 24`; do read DEV; echo yes | mdadm -C /dev/md/drive$i -l 0 --force -n 1 -c 256 /dev/$DEV; done)
root@debian:~# for i in `seq 24`; do UUID=`mdadm -D /dev/md/drive$i | grep UUID | awk '{print $3}'`; echo "ARRAY /dev/md/drive$i uuid=$UUID"; done >> /etc/mdadm/mdadm.conf

Encrypt RAID0 Arrays

root@debian:~# touch /run/password
root@debian:~# chmod go-rwx /run/password
root@debian:~# vi -b /run/password # Use ":set noeol" in vi before saving the file, this will omit the trailing newline on the password
root@debian:~# for i in `seq 24`; do echo YES | cryptsetup --key-file=/run/password luksFormat /dev/md/drive$i & done; wait

Open Encrypted Devices

root@debian:~# for i in `seq 24`; do cryptsetup --key-file=/run/password luksOpen /dev/md/drive$i drive$i-crypt & done; wait
root@debian:~# rm /run/password

Create ZFS Pool of Encrypted Devices

root@debian:~# zpool create -o ashift=9 pool raidz3 /dev/mapper/drive1-crypt /dev/mapper/drive2-crypt /dev/mapper/drive3-crypt /dev/mapper/drive4-crypt /dev/mapper/drive5-crypt /dev/mapper/drive6-crypt /dev/mapper/drive7-crypt /dev/mapper/drive8-crypt /dev/mapper/drive9-crypt /dev/mapper/drive10-crypt /dev/mapper/drive11-crypt /dev/mapper/drive12-crypt /dev/mapper/drive13-crypt /dev/mapper/drive14-crypt /dev/mapper/drive15-crypt /dev/mapper/drive16-crypt /dev/mapper/drive17-crypt /dev/mapper/drive18-crypt /dev/mapper/drive19-crypt /dev/mapper/drive20-crypt /dev/mapper/drive21-crypt /dev/mapper/drive22-crypt /dev/mapper/drive23-crypt /dev/mapper/drive24-crypt

Create new ZFS Dataset

root@debian:~# zfs create -o acltype=posixacl -o atime=off -o xattr=sa pool/data

Shutting Down Mass Storage

root@debian:~# zpool export pool
root@debian:~# for i in /dev/mapper/drive*-crypt; do cryptsetup luksClose $i; done
root@debian:~# for i in `seq 24`; do mdadm -S /dev/md/drive$i; done
# Alternatively: mdadm -S -s, but it may shut down arrays you won't want shut down.

Mounting Mass Storage

root@debian:~# mdadm -A -s
root@debian:~# touch /run/password
root@debian:~# chmod go-rwx /run/password
root@debian:~# vim /run/password
root@debian:~# for i in `seq 24`; do cryptsetup --key-file=/run/password luksOpen /dev/md/drive$i drive$i-crypt & done; wait
root@debian:~# rm /run/password
root@debian:~# zpool import pool

Installing Debian onto Encrypted ZFS Root

In general, follow: https://github.com/zfsonlinux/pkg-zfs/wiki/HOWTO-install-Debian-GNU-Linux-to-a-Native-ZFS-Root-Filesystem

1. Boot Debian Live CD which includes non-free firmware: http://cdimage.debian.org/cdimage/unofficial/non-free/cd-including-firmware/7.6.0-live+nonfree/amd64/iso-hybrid/debian-live-7.6.0-amd64-kde-desktop+nonfree.iso
2. In general, follow: http://zfsonlinux.org/debian.html
3. Refer to the following disk layout and boot sequence information: http://www.pixelbeat.org/docs/disk/
4. Partition each of the 3 drives with a boot and root

   apt-get install parted
   
   # Start of 2048s is the 1MB de facto standard offset
   # End of 4196351s is derived from ((1024*1024*1024*2) / 512) - 1
   # Start of 4196352s is simply the previous end sector + 1
   # 100% denotes the end of the disk
   for disk in /dev/disk/by-path/pci-0000:14:00.0-sas-0x443322110?000000-lun-0; do echo "---> $disk"; yes | parted $disk mklabel msdos mkpart primary 2048s 4196351s set 1 boot on mkpart primary 4196352s 100%; done

5. MD-RAID the /boot partition

   apt-get install mdadm
   
   # Using metadata version 0.90 or 1.0 is critical to each device being usable by GRUB without mdraid support
   # This is accomplished some 0.90 and 1.0 metadata is placed at the end of each disk, leaving GRUB to detect
   # the underlying file system as though RAID wasn't involved.
   mdadm -C /dev/md/boot -l 1 -n 3 -e 1.0 /dev/disk/by-path/pci-0000:14:00.0-sas-0x443322110?000000-lun-0-part1

6. Format the /boot partition

   # Find the corresponding md device by examining /proc/mdstat
   mke2fs -m 0 -L BOOT /dev/md122

7. Create the root filesystem dm-crypt devices

   touch /run/password
   
   chmod go-rwx /run/password
   
   vi -b /run/password # Use ":set noeol" in vi before saving the file, this will omit the trailing newline on the password
   
   for disk in /dev/disk/by-path/pci-0000:14:00.0-sas-0x443322110?000000-lun-0-part2; do echo YES | cryptsetup --key-file=/run/password luksFormat $disk & done; wait
   
   i=0; for disk in /dev/disk/by-path/pci-0000:14:00.0-sas-0x443322110?000000-lun-0-part2; do cryptsetup --key-file=/run/password --allow-discards luksOpen $disk root$i-crypt; i=`expr $i + 1`; done
   
   i=0; for disk in /dev/disk/by-path/pci-0000:14:00.0-sas-0x443322110?000000-lun-0-part2; do if [ $i -ne 0 ]; then /lib/cryptsetup/scripts/decrypt_derived root0-crypt | cryptsetup --key-file=/run/password luksAddKey $disk; fi; i=`expr $i + 1`; done
   
   rm /run/password

8. Install the ZFS utilities

   wget 'http://archive.zfsonlinux.org/debian/pool/main/z/zfsonlinux/zfsonlinux_3~wheezy_all.deb'
   
   dpkg -i 'zfsonlinux_3~wheezy_all.deb'
   
   apt-get update
   
   apt-get install debian-zfs
   
   modprobe zfs

9. Create the root ZFS filesystem

   zpool create rpool mirror /dev/mapper/root?-crypt
   
   zfs create -o acltype=posixacl -o atime=off -o xattr=sa rpool/ROOT
   
   zfs create -o acltype=posixacl -o atime=off -o xattr=sa rpool/ROOT/debian-1
   
   zfs umount -a
   
   zfs set mountpoint=/ rpool/ROOT/debian-1
   
   zpool set bootfs=rpool/ROOT/debian-1 rpool
   
   zpool export rpool

10. Prepare the disks for installation

    zpool import -d /dev/mapper -R /mnt rpool
    
    mkdir /mnt/boot
    
    mount /dev/md122 /mnt/boot

11. Bootstrap Debian

    apt-get install debootstrap
    
    debootstrap testing /mnt http://ftp.us.debian.org/debian

12. Prepare the target and chroot

    cp /etc/hostname /mnt/etc/
    
    cp /etc/hosts /mnt/etc/
    
    echo 'LABEL="BOOT" /boot ext2 auto defaults 0 1' >> /mnt/etc/fstab
    
    echo '/dev/mapper/root1-crypt / none none 0 0' >> /etc/fstab # Yes, this is a hack: http://markus.heberling.net/tag/zfs/
    # The above must be a vdev of the rpool which is a derived device, and not the primary key device,
    # otherwise update-initramfs will not include the decrypt_derived script in the initramfs output.
    
    echo 'deb http://ftp.us.debian.org/debian testing main contrib non-free' > /mnt/etc/apt/sources.list
    
    echo "auto lo
    iface lo inet loopback
    
    auto eth0
    iface eth0 inet dhcp" >> /mnt/etc/network/interfaces
    
    mount --bind /dev  /mnt/dev
    
    mount --bind /proc /mnt/proc
    
    mount --bind /sys  /mnt/sys
    
    chroot /mnt /bin/bash --login

13. Work around a grub-probe bug triggered when it interacts with ZFS VDEVs that are in /dev/mapper instead of /dev

    echo 'ENV{DM_NAME}=="root[0-9]*-crypt" ACTION=="add|change" SYMLINK+="root%n-crypt dm-name-root%n-crypt"' > /etc/udev/rules.d/99-grub-zfs.rules
    
    udevadm control -R
    
    udevadm test /sys/block/dm-0
    
    udevadm test /sys/block/dm-1
    
    udevadm test /sys/block/dm-2

14. Configure the target

    apt-get install locales
    
    vi /etc/locale.gen # Uncomment the "en_US.UTF-8" locale
    
    locale-gen
    
    wget 'http://archive.zfsonlinux.org/debian/pool/main/z/zfsonlinux/zfsonlinux_3~jessie_all.deb'
    
    dpkg -i 'zfsonlinux_3~jessie_all.deb'
    
    apt-get update
    
    apt-get install mdadm cryptsetup
    
    i=0; for disk in /dev/disk/by-path/pci-0000:14:00.0-sas-0x443322110?000000-lun-0-part2; do UUID=`cryptsetup luksDump $disk | grep UUID | awk '{print $2}'`; if [ $i -eq 0 ]; then echo "root$i-crypt UUID=$UUID none luks,discard"; else echo "root$i-crypt UUID=$UUID root0-crypt keyscript=/lib/cryptsetup/scripts/decrypt_derived,luks,discard"; fi; i=`expr $i + 1`; done > /etc/crypttab
    
    i=0; for disk in /dev/disk/by-path/pci-0000:14:00.0-sas-0x443322110?000000-lun-0-part2; do UUID=`cryptsetup luksDump $disk | grep UUID | awk '{print $2}'`; if [ $i -eq 0 ]; then echo "target=root$i-crypt,source=UUID=$UUID,key=none,rootdev,discard"; else echo "target=root$i-crypt,source=UUID=$UUID,key=root0-crypt,keyscript=/lib/cryptsetup/scripts/decrypt_derived,rootdev,discard"; fi; i=`expr $i + 1`; done > /etc/initramfs-tools/conf.d/cryptroot
    
    apt-get install linux-image-amd64 debian-zfs
    
    apt-get install firmware-bnx2 grub2-common grub-pc zfs-initramfs
    
    apt-get dist-upgrade
    
    vi /etc/default/grub # Add "boot=zfs" to GRUB_CMDLINE_LINUX (NOT to GRUB_CMDLINE_LINUX_DEFAULT!)
    
    sed -i 's|modprobe|/sbin/modprobe|' /usr/share/initramfs-tools/scripts/zfs # Apply bugfix: https://github.com/zfsonlinux/pkg-zfs/issues/119
    # Skip the above step if at some point the bugfix is applied directly to the published .deb files
    
    update-grub
    
    update-initramfs -u
    
    passwd root

15. Clean up and reboot

    umount /mnt/boot
    
    umount /mnt/dev
    
    umount /mnt/proc
    
    umount /mnt/sys
    
    zfs umount -a
    
    zpool export rpool
    
    reboot

Installing a Virtual Machine onto a ZVol

1. Look at the following for details about 9p: http://www.slideshare.net/ericvh/9p-overview
2. Look at the following for qemu-kvm virtio performance tuning: http://events.linuxfoundation.org/sites/events/files/slides/CloudOpen2013_Khoa_Huynh_v3.pdf
3. Prepare host networking for VMs

   apt-get install bridge-utils
   echo "source-directory /etc/network/interfaces.d
   auto lo br0
   
   iface lo inet loopback
   
   iface eth0 inet manual
   
   iface br0 inet static
           bridge_ports eth0
           address 192.168.42.57
           netmask 255.255.255.0
           gateway 192.168.42.252
           bridge_stp off
           bridge_waitport 0
           bridge_fd 0
   " > /etc/network/interfaces
   /etc/init.d/networking restart
   # Use "ip a del" to remove any IPs still remaining on eth0

4. Prepare host for VMs

   apt-get install qemu-kvm libvirt-bin

5. Use the following make-vm.sh script

   VM_NAME=$1
   
   zfs create -o volblocksize=128k -o sync=disabled -V 32G pool/${VM_NAME}-vm
   
   virt-install -n $VM_NAME --memory=2048 --vcpus=2 --cpu host-model-only --pxe --boot=network,hd --os-variant=debianwheezy --disk path=/dev/zvol/pool/${VM_NAME}-vm,device=disk,bus=virtio,sparse=false,cache=none,format=raw,io=native --filesystem mode=squash,source=/pool/data/Adults/Security,target=security -w bridge=br0 --graphics spice,listen=192.168.42.57 --sound ac97 --video=qxl --autostart --noautoconsole
   
   virsh destroy $VM_NAME
   
   /etc/init.d/libvirtd restart

6. Mount the shared filesystem inside the VM

   mkdir /security
   echo "security /security 9p trans=virtio,version=9p2000.L,msize=512k 0 0" >> /etc/fstab
   mount /security

7. Use the following remove-vm.sh script

   VM_NAME=$1
   virsh destroy $VM_NAME
   virsh undefine $VM_NAME
   /etc/init.d/libvirtd restart
   zfs destroy pool/${VM_NAME}-vm

Replacing a dead drive

1. Ensure the associated dm-crypt and md-raid devices are stopped
2. Wipe the new drive, either with zeros or entropy (documented above)
3. Find the old RAID0 array UUID

   grep driveN /etc/mdadm/mdadm.conf

4. Create a new RAID0 array from the drive

   mdadm -C /dev/md/driveN-new -l 0 --force -n 1 -c 256 --uuid=<UUID> -N driveN --homehost=debian /dev/<NEW DRIVE>

5. Encrypt the new array

   cryptsetup luksFormat /dev/md/driveN

6. Add a derived key slot to the new array

   # Instead of using the clipboard below, you can also use a named pipe (mkfifo) or a tmpfs file to store the "derived key"
   # The below shows the "derived key" - PROTECT THIS - MASTER KEY OF root0-crypt!
   dmsetup table --showkeys root0-crypt | awk {'print $5'}
   
   # Enter the existing password, then paste in the "derived key"
   cryptsetup luksAddKey /dev/md/driveN

7. Start the encrypted container

   cryptsetup luksOpen /dev/md/driveN driveN-crypt

8. Replace the drive in ZFS

   zpool replace pool driveN-crypt

9. Speed up the resilver as per http://broken.net/uncategorized/zfs-performance-tuning-for-scrubs-and-resilvers/

   echo 0 > /sys/module/zfs/parameters/zfs_resilver_delay
   echo 0 > /sys/module/zfs/parameters/zfs_scrub_delay
   echo 256 > /sys/module/zfs/parameters/zfs_top_maxinflight
   echo 5000 > /sys/module/zfs/parameters/zfs_resilver_min_time_ms

Make a Docker Container Start Automatically

1. Make it always restart

   docker run -d --restart=always <image name>

Make a Guest VM Shutdown With Host

1. Make sure acpid is available and running

   emerge acpid
   rc-config add acpid default

Get sabnzbdplus To Start On Boot

1. https://forums.sabnzbd.org/viewtopic.php?f=16&t=18846 In short: set no_ipv6=1 in sabnzbd.ini