Loading...
 
  • Page actions

SOP

Contents  »  Computers  »  SOP

Bootstrapping Expandable Encrypted Redundant Mass Storage

Wipe New Disks

Wipe New Disks
root@debian:/dev/disk/by-path# for disk in pci-0000\:10\:00.0-sas-0x50014380057d*; do (for i in `seq 4001`; do echo "---> $disk @ $i">&2; openssl rand -rand /dev/urandom 1000000000 2>/dev/null; done) > $disk & done; wait

Create New RAID0 Arrays

Create New RAID0 Arrays
root@debian:/dev/disk/by-path# ls -l | grep lun | awk -F/ '{print $3}' > /tmp/devs
root@debian:/dev/disk/by-path# cat /tmp/devs | (for i in `seq 24`; do read DEV; echo yes | mdadm -C /dev/md/drive$i -l 0 --force -n 1 -c 256 /dev/$DEV; done)
root@debian:~# for i in `seq 24`; do UUID=`mdadm -D /dev/md/drive$i | grep UUID | awk '{print $3}'`; echo "ARRAY /dev/md/drive$i uuid=$UUID"; done >> /etc/mdadm/mdadm.conf

Encrypt RAID0 Arrays

Encrypt RAID0 Arrays
root@debian:~# touch /run/password
root@debian:~# chmod go-rwx /run/password
root@debian:~# vi -b /run/password # Use ":set noeol" in vi before saving the file, this will omit the trailing newline on the password
root@debian:~# for i in `seq 24`; do echo YES | cryptsetup --key-file=/run/password luksFormat /dev/md/drive$i & done; wait

Open Encrypted Devices

Open Encrypted Devices
root@debian:~# for i in `seq 24`; do cryptsetup --key-file=/run/password luksOpen /dev/md/drive$i drive$i-crypt & done; wait
root@debian:~# rm /run/password

Create ZFS Pool of Encrypted Devices

Create ZFS Pool of Encrypted Devices
root@debian:~# zpool create -o ashift=9 pool raidz3 /dev/mapper/drive1-crypt /dev/mapper/drive2-crypt /dev/mapper/drive3-crypt /dev/mapper/drive4-crypt /dev/mapper/drive5-crypt /dev/mapper/drive6-crypt /dev/mapper/drive7-crypt /dev/mapper/drive8-crypt /dev/mapper/drive9-crypt /dev/mapper/drive10-crypt /dev/mapper/drive11-crypt /dev/mapper/drive12-crypt /dev/mapper/drive13-crypt /dev/mapper/drive14-crypt /dev/mapper/drive15-crypt /dev/mapper/drive16-crypt /dev/mapper/drive17-crypt /dev/mapper/drive18-crypt /dev/mapper/drive19-crypt /dev/mapper/drive20-crypt /dev/mapper/drive21-crypt /dev/mapper/drive22-crypt /dev/mapper/drive23-crypt /dev/mapper/drive24-crypt

Create new ZFS Dataset

Create new ZFS Dataset
root@debian:~# zfs create -o acltype=posixacl -o atime=off -o xattr=sa pool/data

Shutting Down Mass Storage

Shutting Down Mass Storage
root@debian:~# zpool export pool
root@debian:~# for i in /dev/mapper/drive*-crypt; do cryptsetup luksClose $i; done
root@debian:~# for i in `seq 24`; do mdadm -S /dev/md/drive$i; done
# Alternatively: mdadm -S -s, but it may shut down arrays you won't want shut down.

Mounting Mass Storage

Mounting Mass Storage
root@debian:~# mdadm -A -s
root@debian:~# touch /run/password
root@debian:~# chmod go-rwx /run/password
root@debian:~# vim /run/password
root@debian:~# for i in `seq 24`; do cryptsetup --key-file=/run/password luksOpen /dev/md/drive$i drive$i-crypt & done; wait
root@debian:~# rm /run/password
root@debian:~# zpool import pool

Installing Debian onto Encrypted ZFS Root

In general, follow: https://github.com/zfsonlinux/pkg-zfs/wiki/HOWTO-install-Debian-GNU-Linux-to-a-Native-ZFS-Root-Filesystem

  1. Boot Debian Live CD which includes non-free firmware: http://cdimage.debian.org/cdimage/unofficial/non-free/cd-including-firmware/7.6.0-live+nonfree/amd64/iso-hybrid/debian-live-7.6.0-amd64-kde-desktop+nonfree.iso
  2. In general, follow: http://zfsonlinux.org/debian.html
  3. Refer to the following disk layout and boot sequence information: http://www.pixelbeat.org/docs/disk/
  4. Partition each of the 3 drives with a boot and root
    apt-get install parted

# Start of 2048s is the 1MB de facto standard offset
# End of 4196351s is derived from ((1024*1024*1024*2) / 512) - 1
# Start of 4196352s is simply the previous end sector + 1
# 100% denotes the end of the disk
for disk in /dev/disk/by-path/pci-0000:14:00.0-sas-0x443322110?000000-lun-0; do echo "---> $disk"; yes | parted $disk mklabel msdos mkpart primary 2048s 4196351s set 1 boot on mkpart primary 4196352s 100%; done
  5. MD-RAID the /boot partition
    apt-get install mdadm

# Using metadata version 0.90 or 1.0 is critical to each device being usable by GRUB without mdraid support
# This is accomplished some 0.90 and 1.0 metadata is placed at the end of each disk, leaving GRUB to detect
# the underlying file system as though RAID wasn't involved.
mdadm -C /dev/md/boot -l 1 -n 3 -e 1.0 /dev/disk/by-path/pci-0000:14:00.0-sas-0x443322110?000000-lun-0-part1
  6. Format the /boot partition
    # Find the corresponding md device by examining /proc/mdstat
mke2fs -m 0 -L BOOT /dev/md122
  7. Create the root filesystem dm-crypt devices
    touch /run/password

chmod go-rwx /run/password

vi -b /run/password # Use ":set noeol" in vi before saving the file, this will omit the trailing newline on the password

for disk in /dev/disk/by-path/pci-0000:14:00.0-sas-0x443322110?000000-lun-0-part2; do echo YES | cryptsetup --key-file=/run/password luksFormat $disk & done; wait

i=0; for disk in /dev/disk/by-path/pci-0000:14:00.0-sas-0x443322110?000000-lun-0-part2; do cryptsetup --key-file=/run/password --allow-discards luksOpen $disk root$i-crypt; i=`expr $i + 1`; done

i=0; for disk in /dev/disk/by-path/pci-0000:14:00.0-sas-0x443322110?000000-lun-0-part2; do if [ $i -ne 0 ]; then /lib/cryptsetup/scripts/decrypt_derived root0-crypt | cryptsetup --key-file=/run/password luksAddKey $disk; fi; i=`expr $i + 1`; done

rm /run/password
  8. Install the ZFS utilities
    wget 'http://archive.zfsonlinux.org/debian/pool/main/z/zfsonlinux/zfsonlinux_3~wheezy_all.deb'

dpkg -i 'zfsonlinux_3~wheezy_all.deb'

apt-get update

apt-get install debian-zfs

modprobe zfs
  9. Create the root ZFS filesystem
    zpool create rpool mirror /dev/mapper/root?-crypt

zfs create -o acltype=posixacl -o atime=off -o xattr=sa rpool/ROOT

zfs create -o acltype=posixacl -o atime=off -o xattr=sa rpool/ROOT/debian-1

zfs umount -a

zfs set mountpoint=/ rpool/ROOT/debian-1

zpool set bootfs=rpool/ROOT/debian-1 rpool

zpool export rpool
  10. Prepare the disks for installation
    zpool import -d /dev/mapper -R /mnt rpool

mkdir /mnt/boot

mount /dev/md122 /mnt/boot
  11. Bootstrap Debian
    apt-get install debootstrap

debootstrap testing /mnt http://ftp.us.debian.org/debian
  12. Prepare the target and chroot
    cp /etc/hostname /mnt/etc/

cp /etc/hosts /mnt/etc/

echo 'LABEL="BOOT" /boot ext2 auto defaults 0 1' >> /mnt/etc/fstab

echo '/dev/mapper/root1-crypt / none none 0 0' >> /etc/fstab # Yes, this is a hack: http://markus.heberling.net/tag/zfs/
# The above must be a vdev of the rpool which is a derived device, and not the primary key device,
# otherwise update-initramfs will not include the decrypt_derived script in the initramfs output.

echo 'deb http://ftp.us.debian.org/debian testing main contrib non-free' > /mnt/etc/apt/sources.list

echo "auto lo
iface lo inet loopback

auto eth0
iface eth0 inet dhcp" >> /mnt/etc/network/interfaces

mount --bind /dev  /mnt/dev

mount --bind /proc /mnt/proc

mount --bind /sys  /mnt/sys

chroot /mnt /bin/bash --login
  13. Work around a grub-probe bug triggered when it interacts with ZFS VDEVs that are in /dev/mapper instead of /dev
    echo 'ENV{DM_NAME}=="root[0-9]*-crypt" ACTION=="add|change" SYMLINK+="root%n-crypt dm-name-root%n-crypt"' > /etc/udev/rules.d/99-grub-zfs.rules

udevadm control -R

udevadm test /sys/block/dm-0

udevadm test /sys/block/dm-1

udevadm test /sys/block/dm-2
  14. Configure the target
    apt-get install locales

vi /etc/locale.gen # Uncomment the "en_US.UTF-8" locale

locale-gen

wget 'http://archive.zfsonlinux.org/debian/pool/main/z/zfsonlinux/zfsonlinux_3~jessie_all.deb'

dpkg -i 'zfsonlinux_3~jessie_all.deb'

apt-get update

apt-get install mdadm cryptsetup

i=0; for disk in /dev/disk/by-path/pci-0000:14:00.0-sas-0x443322110?000000-lun-0-part2; do UUID=`cryptsetup luksDump $disk | grep UUID | awk '{print $2}'`; if [ $i -eq 0 ]; then echo "root$i-crypt UUID=$UUID none luks,discard"; else echo "root$i-crypt UUID=$UUID root0-crypt keyscript=/lib/cryptsetup/scripts/decrypt_derived,luks,discard"; fi; i=`expr $i + 1`; done > /etc/crypttab

i=0; for disk in /dev/disk/by-path/pci-0000:14:00.0-sas-0x443322110?000000-lun-0-part2; do UUID=`cryptsetup luksDump $disk | grep UUID | awk '{print $2}'`; if [ $i -eq 0 ]; then echo "target=root$i-crypt,source=UUID=$UUID,key=none,rootdev,discard"; else echo "target=root$i-crypt,source=UUID=$UUID,key=root0-crypt,keyscript=/lib/cryptsetup/scripts/decrypt_derived,rootdev,discard"; fi; i=`expr $i + 1`; done > /etc/initramfs-tools/conf.d/cryptroot

apt-get install linux-image-amd64 debian-zfs

apt-get install firmware-bnx2 grub2-common grub-pc zfs-initramfs

apt-get dist-upgrade

vi /etc/default/grub # Add "boot=zfs" to GRUB_CMDLINE_LINUX (NOT to GRUB_CMDLINE_LINUX_DEFAULT!)

sed -i 's|modprobe|/sbin/modprobe|' /usr/share/initramfs-tools/scripts/zfs # Apply bugfix: https://github.com/zfsonlinux/pkg-zfs/issues/119
# Skip the above step if at some point the bugfix is applied directly to the published .deb files

update-grub

update-initramfs -u

passwd root
  15. Clean up and reboot
    umount /mnt/boot

umount /mnt/dev

umount /mnt/proc

umount /mnt/sys

zfs umount -a

zpool export rpool

reboot

Installing a Virtual Machine onto a ZVol

  1. Look at the following for details about 9p: http://www.slideshare.net/ericvh/9p-overview
  2. Look at the following for qemu-kvm virtio performance tuning: http://events.linuxfoundation.org/sites/events/files/slides/CloudOpen2013_Khoa_Huynh_v3.pdf
  3. Prepare host networking for VMs
    apt-get install bridge-utils
echo "source-directory /etc/network/interfaces.d
auto lo br0

iface lo inet loopback

iface eth0 inet manual

iface br0 inet static
        bridge_ports eth0
        address 192.168.42.57
        netmask 255.255.255.0
        gateway 192.168.42.252
        bridge_stp off
        bridge_waitport 0
        bridge_fd 0
" > /etc/network/interfaces
/etc/init.d/networking restart
# Use "ip a del" to remove any IPs still remaining on eth0
  4. Prepare host for VMs
    apt-get install qemu-kvm libvirt-bin
  5. Use the following make-vm.sh script
    VM_NAME=$1

zfs create -o volblocksize=128k -o sync=disabled -V 32G pool/${VM_NAME}-vm

virt-install -n $VM_NAME --memory=2048 --vcpus=2 --cpu host-model-only --pxe --boot=network,hd --os-variant=debianwheezy --disk path=/dev/zvol/pool/${VM_NAME}-vm,device=disk,bus=virtio,sparse=false,cache=none,format=raw,io=native --filesystem mode=squash,source=/pool/data/Adults/Security,target=security -w bridge=br0 --graphics spice,listen=192.168.42.57 --sound ac97 --video=qxl --autostart --noautoconsole

virsh destroy $VM_NAME

/etc/init.d/libvirtd restart
  6. Mount the shared filesystem inside the VM
    mkdir /security
echo "security /security 9p trans=virtio,version=9p2000.L,msize=512k 0 0" >> /etc/fstab
mount /security
  7. Use the following remove-vm.sh script
    VM_NAME=$1
virsh destroy $VM_NAME
virsh undefine $VM_NAME
/etc/init.d/libvirtd restart
zfs destroy pool/${VM_NAME}-vm

Replacing a dead drive

  1. Ensure the associated dm-crypt and md-raid devices are stopped
  2. Wipe the new drive, either with zeros or entropy (documented above)
  3. Find the old RAID0 array UUID
    grep driveN /etc/mdadm/mdadm.conf
  4. Create a new RAID0 array from the drive
    mdadm -C /dev/md/driveN-new -l 0 --force -n 1 -c 256 --uuid=<UUID> -N driveN --homehost=debian /dev/<NEW DRIVE>
  5. Encrypt the new array
    cryptsetup luksFormat /dev/md/driveN
  6. Add a derived key slot to the new array
    # Instead of using the clipboard below, you can also use a named pipe (mkfifo) or a tmpfs file to store the "derived key"
# The below shows the "derived key" - PROTECT THIS - MASTER KEY OF root0-crypt!
dmsetup table --showkeys root0-crypt | awk {'print $5'}

# Enter the existing password, then paste in the "derived key"
cryptsetup luksAddKey /dev/md/driveN
  7. Start the encrypted container
    cryptsetup luksOpen /dev/md/driveN driveN-crypt
  8. Replace the drive in ZFS
    zpool replace pool driveN-crypt
  9. Speed up the resilver as per http://broken.net/uncategorized/zfs-performance-tuning-for-scrubs-and-resilvers/
    echo 0 > /sys/module/zfs/parameters/zfs_resilver_delay
echo 0 > /sys/module/zfs/parameters/zfs_scrub_delay
echo 256 > /sys/module/zfs/parameters/zfs_top_maxinflight
echo 5000 > /sys/module/zfs/parameters/zfs_resilver_min_time_ms

Make a Docker Container Start Automatically

  1. Make it always restart
    docker run -d --restart=always <image name>

Make a Guest VM Shutdown With Host

  1. Make sure acpid is available and running
    emerge acpid
rc-config add acpid default

Get sabnzbdplus To Start On Boot

  1. https://forums.sabnzbd.org/viewtopic.php?f=16&t=18846 In short: set no_ipv6=1 in sabnzbd.ini

Source Comments