Sysadmin Cookbook

#!/bin/bash # # lxc: linux Container library apt-get install debootstrap # Authors: # Daniel Lezcano <daniel.lezcano@free.fr> # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public # License as published by the Free Software Foundation; either # version 2.1 of the License, or (at your option) any later version. # This library is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # Lesser General Public License for more details. # You should have received a copy of the GNU Lesser General Public # License along with this library; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA configure_debian() { rootfs=$1 hostname=$2 # configure the inittab cat <<EOF > $rootfs/etc/inittab id:3:initdefault: si::sysinit:/etc/init.d/rcS l0:0:wait:/etc/init.d/rc 0 l1:1:wait:/etc/init.d/rc 1 l2:2:wait:/etc/init.d/rc 2 l3:3:wait:/etc/init.d/rc 3 l4:4:wait:/etc/init.d/rc 4 l5:5:wait:/etc/init.d/rc 5 l6:6:wait:/etc/init.d/rc 6 # Normally not reached, but fallthrough in case of emergency. z6:6:respawn:/sbin/sulogin 1:2345:respawn:/sbin/getty 38400 console c1:12345:respawn:/sbin/getty 38400 tty1 linux c2:12345:respawn:/sbin/getty 38400 tty2 linux c3:12345:respawn:/sbin/getty 38400 tty3 linux c4:12345:respawn:/sbin/getty 38400 tty4 linux EOF # disable selinux in debian mkdir -p $rootfs/selinux echo 0 > $rootfs/selinux/enforce # by default setup root password with no password cat <<EOF > $rootfs/etc/ssh/sshd_config Port 22 Protocol 2 HostKey /etc/ssh/ssh_host_rsa_key HostKey /etc/ssh/ssh_host_dsa_key UsePrivilegeSeparation yes KeyRegenerationInterval 3600 ServerKeyBits 768 SyslogFacility AUTH LogLevel INFO LoginGraceTime 120 PermitRootLogin yes StrictModes yes RSAAuthentication yes PubkeyAuthentication yes IgnoreRhosts yes RhostsRSAAuthentication no HostbasedAuthentication no PermitEmptyPasswords yes ChallengeResponseAuthentication no EOF # configure the network using the dhcp cat <<EOF > $rootfs/etc/network/interfaces auto lo iface lo inet loopback auto eth0 iface eth0 inet dhcp EOF # set the hostname cat <<EOF > $rootfs/etc/hostname $hostname EOF # reconfigure some services chroot $rootfs /usr/sbin/dpkg-reconfigure locales # remove pointless services in a container chroot $rootfs /usr/sbin/update-rc.d -f umountfs remove chroot $rootfs /usr/sbin/update-rc.d -f hwclock.sh remove chroot $rootfs /usr/sbin/update-rc.d -f hwclockfirst.sh remove } arch=$(arch) download_debian() { packages=\ ifupdown,\ locales,\ libui-dialog-perl,\ dialog,\ netbase,\ net-tools,\ iproute,\ openssh-server cache=$1 # check the mini debian was not already downloaded mkdir -p "$cache/partial-$arch" if [ $? -ne 0 ]; then echo "Failed to create '$cache/partial-$arch' directory" return 1 fi # download a mini debian into a cache echo "Downloading debian minimal ..." debootstrap --verbose --variant=minbase --arch=$arch \ --include $packages \ wheezy $cache/partial-$arch http://ftp.debian.org/debian if [ $? -ne 0 ]; then echo "Failed to download the rootfs, aborting." return 1 fi mv "$1/partial-$arch" "$1/rootfs-$arch" echo "Download complete." return 0 } copy_debian() { cache=$1 rootfs=$3 # make a local copy of the minidebian echo -n "Copying rootfs to $rootfs..." cp -a $cache/rootfs-$arch $rootfs || return 1 return 0 } install_debian() { cache="/var/cache/lxc/debian" rootfs=$1 mkdir -p /var/lock/subsys/ ( flock -n -x 200 if [ $? -ne 0 ]; then echo "Cache repository is busy." return 1 fi if [ "$arch" == "x86_64" ]; then arch=amd64 fi if [ "$arch" == "i686" ]; then arch=i386 fi echo "Checking cache download in $cache/rootfs-$arch ... " if [ ! -e "$cache/rootfs-$arch" ]; then download_debian $cache $arch if [ $? -ne 0 ]; then echo "Failed to download 'debian base'" return 1 fi fi copy_debian $cache $arch $rootfs if [ $? -ne 0 ]; then echo "Failed to copy rootfs" return 1 fi return 0 ) 200>/var/lock/subsys/lxc return $? } copy_configuration() { path=$1 rootfs=$2 name=$3 cat <<EOF >> $path/config lxc.tty = 4 lxc.pts = 1024 lxc.rootfs = $rootfs lxc.cgroup.devices.deny = a # /dev/null and zero lxc.cgroup.devices.allow = c 1:3 rwm lxc.cgroup.devices.allow = c 1:5 rwm # consoles lxc.cgroup.devices.allow = c 5:1 rwm lxc.cgroup.devices.allow = c 5:0 rwm lxc.cgroup.devices.allow = c 4:0 rwm lxc.cgroup.devices.allow = c 4:1 rwm # /dev/{,u}random lxc.cgroup.devices.allow = c 1:9 rwm lxc.cgroup.devices.allow = c 1:8 rwm lxc.cgroup.devices.allow = c 136:* rwm lxc.cgroup.devices.allow = c 5:2 rwm # rtc lxc.cgroup.devices.allow = c 254:0 rwm EOF if [ $? -ne 0 ]; then echo "Failed to add configuration" return 1 fi return 0 } clean() { cache="/var/cache/lxc/debian" if [ ! -e $cache ]; then exit 0 fi # lock, so we won't purge while someone is creating a repository ( flock -n -x 200 if [ $? != 0 ]; then echo "Cache repository is busy." exit 1 fi echo -n "Purging the download cache..." rm --preserve-root --one-file-system -rf $cache && echo "Done." || exit 1 exit 0 ) 200>/var/lock/subsys/lxc } usage() { cat <<EOF $1 -h|--help -p|--path=<path> -a|--arch=stable --clean EOF return 0 } options=$(getopt -o hp:n:ca: -l help,path:,name:,clean,arch: -- "$@") if [ $? -ne 0 ]; then usage $(basename $0) exit 1 fi eval set -- "$options" while true do case "$1" in -h|--help) usage $0 && exit 0;; -p|--path) path=$2; shift 2;; -n|--name) name=$2; shift 2;; -c|--clean) clean=$2; shift 2;; -a|--arch) arch=$2; shift 2;; --) shift 1; break ;; *) break ;; esac done if [ ! -z "$clean" -a -z "$path" ]; then clean || exit 1 exit 0 fi type debootstrap if [ $? -ne 0 ]; then echo "'debootstrap' command is missing" exit 1 fi if [ -z "$path" ]; then echo "'path' parameter is required" exit 1 fi if [ "$(id -u)" != "0" ]; then echo "This script should be run as 'root'" exit 1 fi rootfs=$path/rootfs install_debian $rootfs if [ $? -ne 0 ]; then echo "failed to install debian" exit 1 fi configure_debian $rootfs $name if [ $? -ne 0 ]; then echo "failed to configure debian for a container" exit 1 fi copy_configuration $path $rootfs if [ $? -ne 0 ]; then echo "failed write configuration file" exit 1 fi if [ ! -z $clean ]; then clean || exit 1 exit 0 fi

lxc watchdog

#! /bin/sh ### BEGIN INIT INFO # Provides: lxc-watchdog # Required-Start: $remote_fs $named $network $time # Required-Stop: $remote_fs $named $network # Required-Start: # Required-Stop: # Default-Start: 2 3 4 5 # Default-Stop: 0 1 6 # Short-Description: Manage Linux Containers startup/shutdown # Description: Uses clever inotify hack to monitor container's # halt/reboot events watching /var/run/utmp ### END INIT INFO # Author: Dobrica Pavlinusic <dpavlin@rot13.org> # # based on Tony Risinger post to lxc-users mailing list # http://www.mail-archive.com/lxc-users@lists.sourceforge.net/msg00074.html # # Install with: # ln -sf /srv/sysadmin-cookbook/recepies/lxc/lxc-watchdog.sh /etc/init.d/lxc-watchdog # update-rc.d lxc-watchdog defaults which inotifywait >/dev/null || apt-get install inotify-tools lxc_exists() { name=$1 if [ ! -e /var/lib/lxc/$name/config ] ; then echo "Usage: $0 name" lxc_status exit 1 fi } lxc_rootfs() { grep '^ *lxc\.rootfs *=' "/var/lib/lxc/$1/config" | cut -d= -f2 | sed 's/^ *//' } lxc_hostname() { inside=`cat $(lxc_rootfs $1)/etc/hostname` config=`grep lxc.utsname /var/lib/lxc/$name/config | cut -d= -f2` echo "$config [$inside]"; } lxc_ip() { ( grep lxc.network.ipv4 /var/lib/lxc/$name/config | grep -v '^#' | cut -d= -f2 || \ grep address $(lxc_rootfs $name)/etc/network/interfaces | grep -v '^#' | sed 's/.*address //' ) | \ head -1 | \ sed -e 's/ *//g' -e 's/\/.*$//' } lxc_status() { ( find /var/lib/lxc/ -name "config" | cut -d/ -f5 | sort -u | while read name ; do status=`lxc-info -n $name 2>/dev/null | grep state: | cut -d: -f2` # 0.7.5 test -z "$status" && status=`lxc-info -n $name | sed -e 's/^.* is //'` # 0.7.2 boot="-" test -s /var/lib/lxc/$name/on_boot && boot="boot" echo "$name $status $boot $(lxc_rootfs $name) $(lxc_ip $name) $(lxc_hostname $name)" done ) | column -t } cleanup_init_scripts() { rootfs=$(lxc_rootfs $1) ls \ $rootfs/etc/rc?.d/*checkroot* \ $rootfs/etc/rc?.d/*umountfs \ $rootfs/etc/rc?.d/*umountroot \ $rootfs/etc/rc?.d/*hwclock* \ $rootfs/etc/rc?.d/*udev* \ $rootfs/etc/rc?.d/*checkfs* \ 2>/dev/null | xargs -i rm -v {} echo $1 > $rootfs/etc/hostname grep $1 $rootfs/etc/hosts || echo "$(lxc_ip $1) $1" >> $rootfs/etc/hosts } setup_inittab() { rootfs=$(lxc_rootfs $1) remove=$2 add=$3 # let container respond to kill -SIGPWR inittab=$rootfs/etc/inittab if test -e $inittab && ! grep "$add" ${inittab} >/dev/null ; then grep -v "$remove" ${inittab} > ${inittab}.new echo $add >> ${inittab}.new mv ${inittab}.new ${inittab} echo "$inittab modified with $add" fi } lxc_log() { echo `date +%Y-%m-%dT%H:%M:%S` $* } lxc_kill() { name=$1 sig=$2 ver=`lxc-version | cut -d: -f2 | sed 's/\.//g'` opts='' test $ver -ge 075 && opts='--' init_pid=`lxc-ps $opts -C init -o pid | grep "^$name" | cut -d" " -f2-` if [ -z "$init_pid" ] ; then lxc-info -n $name exit 1 fi lxc_log "$name kill $sig $init_pid" /bin/kill $sig $init_pid } lxc_stop() { lxc_log "$name stop" lxc_kill $name -SIGPWR lxc-wait -n $name -s STOPPED lxc_log "$name stoped" # rm -f /var/lib/lxc/${name}/on_boot } lxc_start() { name=$1 rootfs=$(lxc_rootfs $1) if [ ! -e $rootfs ] ; then echo "ERROR $name rootfs $rootfs not found" return fi if ! lxc-info -n $name | grep RUNNING ; then lxc_log "$name start" dev=`df -P $rootfs | tail -1 | cut -d" " -f1` mount $dev -o remount,rw # fix debian upgrade which remounts dir ro lxc-start -n $name -o /tmp/${name}.log -d lxc-wait -n $name -s RUNNING lxc-info -n $name test -f /var/lib/lxc/${name}/on_boot || echo $name > /var/lib/lxc/${name}/on_boot fi } lxc_watchdog() { name=$1 cgroup=$(mount -t cgroup | awk '{ print $3 }') test -d "$cgroup/lxc/$1" && cgroup="$cgroup/lxc" rootfs=$(lxc_rootfs $1) run=$rootfs/var/run test -L $run && run=$rootfs/`readlink $run` # recent Debian have symlink to /run cd $run || echo "can't cd watchdog into $run" while true; do tasks=`wc -l < $cgroup/${name}/tasks` stop_on=1 # init sulogins=`lxc-ps --name $name | grep sulogin | wc -l` if [ "$sulogins" -gt 0 ] ; then stop_on=`expr $stop_on + $sulogins` fi test -z "$tasks" && exit 1 if [ "$tasks" -eq $stop_on ]; then runlevel="$(runlevel utmp)" lxc_log "$name runlevel $runlevel" case $runlevel in N*) # nothing for new boot state ;; ??0|unknown) lxc_log "$name halt" lxc-stop -n "${name}" lxc-wait -n ${name} -s STOPPED break ;; ??6) lxc_log "$name reboot"; lxc-stop -n ${name} lxc-wait -n ${name} -s STOPPED lxc-start -d -n ${name} -o /tmp/${name}.log ;; *) # make sure vps is still running state="$(lxc-info -n "${name}" | sed -e 's/.* is //')" [ "$state" = "RUNNING" ] || break ;; esac else lxc_log "$name $tasks tasks $sulogins console" fi # time of 5 minutes on it JUST IN CASE... inotifywait -qqt 300 utmp done lxc_log "$name watchdog exited" } usage() { echo "Usage: $0 {start|stop|restart|status|boot|disable} [name name ... ]" >&2 exit 3 } command_on_lxc() { command=$1 shift echo "# $command $1" case "$command" in start) lxc_exists $1 cleanup_init_scripts $1 setup_inittab $1 ":respawn:/sbin/getty.*tty1" "c1:12345:respawn:/sbin/getty 38400 tty1 linux" setup_inittab $1 "::power" "p0::powerfail:/sbin/init 0" setup_inittab $1 "::ctrlaltdel" "p6::ctrlaltdel:/sbin/init 6" lxc_start $1 # give container 5 seconds to start more than one process ( sleep 5 ; nohup $0 watchdog $1 >> /tmp/$1.log 2>/dev/null ) & ;; stop|halt) lxc_exists $1 lxc_stop $1 ;; reload|force-reload|restart|reboot) lxc_kill $1 -SIGINT ;; watchdog) lxc_watchdog $1 ;; boot) echo $1 > /var/lib/lxc/$1/on_boot ;; disable) echo -n > /var/lib/lxc/$1/on_boot ;; *) usage ;; esac } command=$1 test -z "$command" && usage test "$command" = "status" && lxc_status && exit shift if [ -z "$1" ] ; then ls /var/lib/lxc/*/on_boot | while read path ; do name=`echo $path | cut -d/ -f5` if [ "$command" != "start" -o "$command" = "start" -a -s $path ] ; then command_on_lxc $command $name else echo "# skip $command $name" fi done else while [ ! -z "$1" ] ; do command_on_lxc $command $1 shift done fi

ve2lxc

#!/bin/sh -x test -z "$1" && echo "usage: $0 /path/to/ve/private [10.60.0.253 [hostname]]" && exit dir=$1 ip=$2 hostname=$3 netmask=`grep netmask /etc/network/interfaces | head -1 | sed 's/^.*netmask *//'` gateway=`grep gateway /etc/network/interfaces | head -1 | sed 's/^.*gateway *//'` test -z "$ip" && ip=10.60.0.252 test -z "$hostname" && hostname=ve2lxc path=/$dir/etc/inittab tmp=/tmp/inittab cp $path $tmp || exit append() { if ! grep "$1" $path ; then echo "$1" >> $tmp fi } append "z6:6:respawn:/sbin/sulogin" append "1:2345:respawn:/sbin/getty 38400 console" append "c1:12345:respawn:/sbin/getty 38400 tty1 linux" append "c2:12345:respawn:/sbin/getty 38400 tty2 linux" append "c3:12345:respawn:/sbin/getty 38400 tty3 linux" append "c4:12345:respawn:/sbin/getty 38400 tty4 linux" if ! diff -uw $path $tmp ; then cp $path $path.old && mv $tmp $path fi lxc-stop -n $hostname lxc-destroy -n $hostname test -d /cgroup || mkdir /cgroup grep /cgroup /etc/fstab || echo "cgroup /cgroup cgroup rw 0 0" >> /etc/fstab grep eth0 $dir/etc/network/interfaces || cat << __interfaces__ > $dir/etc/network/interfaces auto eth0 lo iface lo inet loopback iface eth0 inet static address $ip netmask $netmask gateway $gateway __interfaces__ echo $hostname > $dir/etc/hostname echo "$ip $hostname" >> $dir/etc/hosts conf=/tmp/$hostname.conf cat << __lxc__ > $conf lxc.utsname = $hostname lxc.tty = 4 lxc.pts = 1024 lxc.network.type = veth lxc.network.flags = up lxc.network.link = br0 lxc.network.name = eth0 lxc.network.mtu = 1500 #lxc.network.hwaddr = AC:DE:48:00:00:01 # interface visible on host, part of bridge #lxc.network.veth.pair = veth0 #lxc.mount = $MNTFILE lxc.rootfs = $dir # lxc.mount.entry=/opt /virtual/lxc/rootfs/opt none ro,bind 0 0 lxc.cgroup.devices.deny = a # /dev/null and zero lxc.cgroup.devices.allow = c 1:3 rwm lxc.cgroup.devices.allow = c 1:5 rwm # consoles lxc.cgroup.devices.allow = c 5:1 rwm lxc.cgroup.devices.allow = c 5:0 rwm lxc.cgroup.devices.allow = c 4:0 rwm lxc.cgroup.devices.allow = c 4:1 rwm # /dev/{,u}random lxc.cgroup.devices.allow = c 1:9 rwm lxc.cgroup.devices.allow = c 1:8 rwm lxc.cgroup.devices.allow = c 136:* rwm lxc.cgroup.devices.allow = c 5:2 rwm # rtc lxc.cgroup.devices.allow = c 254:0 rwm __lxc__ cp -v /etc/resolv.conf /$dir/etc/resolv.conf mount | grep /cgroup || mount /cgroup || exit lxc-create -n $hostname -f $conf && lxc-start -n $hostname

bitmap

#!/bin/sh -x # https://raid.wiki.kernel.org/index.php/Bitmap mdadm --grow --bitmap=internal $1

scrub

#!/bin/sh -x echo check > /sys/block/$1/md/sync_action watch cat /proc/mdstat

munin

munin plugins

gearman

#!/usr/bin/perl use warnings; use strict; # 2012-05-27 Dobrica Pavlinusic <dpavlin@rot13.org> use IO::Socket::INET; use Data::Dump qw(dump); my $arg = shift @ARGV; $arg ||= ''; my $sock = IO::Socket::INET->new( PeerAddr => '127.0.0.1', PeerPort => 4730, Proto => 'tcp' ) || die $1; if ( $arg eq 'autoconf' ) { print $sock ? "yes\n" : "no\n"; exit 0; } print $sock "STATUS\n"; my $stats; while ( my $line = <$sock> ) { chomp $line; # warn "# [$line]\n"; last if $line eq '.'; next if $line =~ m/\t0$/; # ignore functions which don't have active workers my ( $name, $queued, $running, $workers ) = split(/\t/,$line,4); $stats->{queued}->{$name} = $queued; $stats->{running}->{$name} = $running; $stats->{workers}->{$name} = $workers; } if ( $arg eq 'config' ) { foreach my $multigraph ( keys %$stats ) { print "multigraph $multigraph\n"; print "graph_category gearman\n"; print "graph_title Gearman $multigraph\n"; foreach my $name ( keys %{ $stats->{$multigraph} } ) { my $label = $name; $name =~ s/\W+/_/g; print "$name.label $label\n"; } } } else { foreach my $multigraph ( keys %$stats ) { print "multigraph $multigraph\n"; while ( my ($name,$value) = each %{ $stats->{$multigraph} } ) { $name =~ s/\W+/_/g; print "$name.value $value\n"; } } }

log

#!/usr/bin/perl use warnings; use strict; use lib $ENV{'MUNIN_LIBDIR'}; use Munin::Plugin; # -*- perl -*- =head1 NAME log - Plugin to monitor log files =head1 CONFIGURATION add munin user to adm group if needed to read C</var/log/*.log> [log] group adm =head1 MAGIC MARKERS #%# family=auto #%# capabilities=autoconf =cut my $log = '/var/log/daemon.log'; if ( $ARGV[0] ) { if ( $ARGV[0] eq 'autoconf' ) { print -r $log ? "yes\n" : "no\n"; } elsif ( $ARGV[0] eq 'config' ) { print <<EOM; multigraph stunnel graph_category log graph_title stunnel clients clients.label clients multigraph stunnel_transfer graph_category log graph_title stunnel transfer graph_vlabel bytes/\${graph_period} ssl.label SSL ssl.draw AREA socket.label Socket socket.draw LINE2 multigraph n2n graph_category log graph_title n2n peers pending.label Pending peers operational.label Operational peers multigraph lines graph_category log graph_title log lines stunnel.label stunnel n2n.label n2n ignored.label ignored EOM } exit 0; } my ( $pos ) = restore_state(); $pos = -s $log unless defined $pos; my ($fh,$reset) = tail_open($log,$pos); my $ip; my $stat; while(<$fh>) { if ( m/stunnel/ ) { $stat->{lines}->{stunnel}++; if ( m/accepted connection from (.+):\d+/ ) { $ip->{$1}++; } elsif ( m/Connection closed: (\d+) bytes sent to SSL, (\d+) bytes sent to socket/i ) { $stat->{stunnel_transfer}->{ssl} += $1; $stat->{stunnel_transfer}->{socket} += $2; } } elsif ( m/n2n/ ) { $stat->{lines}->{n2n}++; if ( m/Pending peers list size=(\d+)/ ) { $stat->{n2n}->{pending} = $1; # latest } elsif ( m/Operational peers list size=(\d+)/ ) { $stat->{n2n}->{operational} = $1; # latest } elsif ( m/pending=(\d+), operational=(\d+)/ ) { $stat->{n2n} = { pending => 1, operational => 2 }; } } else { $stat->{lines}->{ignored}++; } } $stat->{stunnel}->{clients} = scalar keys %$ip if $ip; foreach my $graph ( keys %$stat ) { print "multigraph $graph\n"; print "$_.value $stat->{$graph}->{$_}\n" foreach keys %{ $stat->{$graph} }; } $pos = tail_close($fh); save_state($pos);

nbd

install server

root@opr:~# apt-get install -y nbd-server

create /etc/nbd server/config

[generic] user = nbd group = disk [export] exportname = /dev/sda port = 1234

start nbd server

root@opr:~# /etc/init.d/nbd-server start nbd-server.

install client

root@opl:~# apt-get install -y nbd-client

start client

root@opl:/srv/sysadmin-cookbook/recepies/nbd# nbd-client 10.60.0.91 1234 /dev/nbd0 Negotiation: ..size = 244140625KB bs=1024, sz=244140625 root@opl:/srv/sysadmin-cookbook/recepies/nbd# fdisk -l /dev/nbd0 Disk /dev/nbd0: 250.0 GB, 250000000000 bytes 255 heads, 63 sectors/track, 30394 cylinders Units = cylinders of 16065 * 512 = 8225280 bytes Disk identifier: 0x00000000 Device Boot Start End Blocks Id System /dev/nbd0p1 1 12159 97667136 83 Linux

netpipe tcp

install

root@opl:~# apt-get install netpipe-tcp

Makefile

graph: find . -name "*.np" -size 0 -exec rm {} \; ./np2graphviz.pl | dot -Tpng -o graph.png && qiv graph.png gnuplot: ls */*.np | xargs -i ./gnuplot.sh {} xlax: mkxlax `cat hosts` install: cat hosts | xargs -i ssh {} apt-get install -y netpipe-tcp

collect

#!/bin/sh cat hosts | xargs -i sh -c "test -d {} || mkdir {} ; rsync -v {}:/tmp/*.np {}/" ls -al */*.np

gnuplot

#!/bin/sh echo "creating $1.png" cat << __gnuplot__ | gnuplot set term png set output "$1.png" set grid set ylabel "Throughput In Mbps" set xlabel "Message Size" set size 1,0.8 plot "$1" using 1:2 title "$1" with lines linewidth 3 __gnuplot__

hosts

mjesec.ffzg.hr koha-hw.ffzg.hr 10.60.0.9 10.60.0.10 10.60.0.90 10.60.0.91 10.60.0.92 10.60.0.93 10.60.0.200

np2graphviz

#!/usr/bin/perl use warnings; use strict; my $graph; my ($max,$min); foreach my $file ( glob '*/*.np' ) { my $direction = $file; $direction =~ s/\.np$//; my ( $from, $to ) = split(m{/},$direction,2); my $line = `tail -1 $file`; $line =~ s{^\s+}{}; $line =~ s{\s+$}{}; my ( $size, $speed, $rtt ) = split(/\s+/, $line); warn "$from -> $to | $size | $speed | $rtt\n"; my $len = int($speed / 100); my $rev = qq|"$to" -> "$from"|; # make edge bi-directional if speed difference is less then 10% if ( $graph->{$rev} && abs($graph->{$rev}->{speed}->[0] - $speed) < ($speed/10) ) { $graph->{$rev}->{speed}->[1] = int($speed); $graph->{$rev}->{dir} = 'both'; } else { $graph->{ qq|"$from" -> "$to"| } = { size => $size, speed => [ int($speed) ], rtt => $rtt, dir => 'forward', }; } $min ||= $speed; $min = $speed if $speed < $min; $max ||= $speed; $max = $speed if $speed > $max; } warn "# speed $min ... $max\n"; print qq| digraph "netpipe" { |, join("\n", map { my $node = $_; my @speed = @{ $graph->{$node}->{speed} }; my $speed = 0; $speed += $_ foreach @speed; $speed /= $#speed + 1; my $c = 'ff0000'; $c = '00ff00' if ( $speed / 100 ) > 5; $c = '0000ff' if ( $speed / 1000 ) > 1; $c = '8888ff' if ( $speed / 1000 ) > 2; my $label = qq|labelfontsize=10,weight=$speed,|; $label .= qq|headlabel=$speed[0],| if $speed[0]; $label .= qq|taillabel=$speed[1],| if $speed[1]; $label .= qq|style=dashed,| if $graph->{$node}->{dir} eq 'both'; qq|$node [ $label color="#$c",dir=$graph->{$node}->{dir} ]|; } keys %$graph), qq| } |;

ssh

#!/bin/sh -x cat hosts | xargs -i ssh {} $*

test all

#!/usr/bin/perl # usage: test-all.pl hosts use warnings; use strict; use autodie; use File::Slurp; use Data::Dump qw(dump); chdir '/srv/sysadmin-cookbook/recepies/netpipe-tcp/'; my @hosts = read_file 'hosts'; @hosts = map { chomp; $_ } @hosts; warn "hosts = ",dump(@hosts); foreach my $host ( @hosts ) { chomp($host); my @test; foreach my $to ( @hosts ) { next if -s "$host/$to.np"; warn "start NPtcp on $to\n"; system "ssh $to NPtcp &"; push @test, $to; } warn "# missing ", dump(@test); open(my $ssh, '|-', "ssh $host xargs -i NPtcp -h {} -u 1048576 -o /tmp/{}.np"); foreach my $to ( @test ) { warn "TEST from $host to $to\n"; print $ssh "$to\n"; } close($ssh); system "rsync -v $host:/tmp/*.np $host/"; }

test

#!/bin/sh cd /srv/sysadmin-cookbook/recepies/netpipe-tcp/ while read host ; do echo "TEST `hostname` $host" NPtcp -h $host -u 1048576 -o /tmp/$host.np done < hosts

ntpdate

install

# install on hardware root@koha-hw:~# apt-get -y install ntpdate

nvidia

README

http://tutanhamon.com.ua/technovodstvo/NVIDIA-UNIX-driver/

info

#!/bin/sh get() { echo -n $* " " sudo nvidia-settings -q $* -t } get GPUCurrentPerfLevel get GPUCurrentClockFreqs get GPUPowerSource

openvpn

install

apt-get install openvpn

on server.generate static key

#!/bin/sh -x KEY=/etc/openvpn/`hostname`.key test -f $KEY && ls -al $KEY || openvpn --genkey --secret $KEY echo "transfer $KEY to client"

on client.create /etc/openvpn/tap_home

remote 123.123.123.123 secret prod.key comp-lzo dev tap_home up "/etc/openvpn/tap_home.sh"

on server

dev tap_home secret prod.key comp-lzo up /etc/openvpn/tap_home.sh

on client.create /etc/openvpn/tap_home

#!/bin/sh -x ifconfig $1 10.60.0.81 netmask 255.255.254.0 up iptables -t nat -A POSTROUTING -s 192.168.1.0/24 -o $1 -j MASQUERADE

on server

#!/bin/sh -x ifconfig $1 up brctl addif br0 $1

append /etc/network/interfaces

auto tap_home iface tap_home inet manual openvpn tap_home

openvz

create ve 1 config

root@opl:/etc/vz/conf# vzsplit -n 1 -f 1 -s 2048 Config /etc/vz/conf/ve-1.conf-sample was created

set ve on clone

root@opl:/etc/vz/conf# vzctl set 60017 --private /zfs/vz-60017-clone/private/212226/ --root /zfs/vz-60017-clone/root/212226/ --ipadd 10.60.0.17 --hostname koha-clone --applyconfig 1 --diskspace 60G:70G --save

pppoe server

install

apt-get install pppoe

create /etc/ppp/pppoe server options

# PPPoE server #nologin mru 1492 noreplacedefaultroute proxyarp ms-dns 192.168.1.2

append /etc/ppp/pap secrets

# PPPoE server #client hostname <password> IP test * "test" *

append /etc/ppp/chap secrets

# PPPoE server # client server secret IP addresses test * test *

configure private net

ifconfig eth0:10 10.0.0.1 up

PADI

pppoe-relay -B eth0:1 -C eth0 -n 1 -F

pppoe server

pppoe-server -I eth0:1 -T 60 -C fake -S fake -L 10.0.0.2 -R 10.0.0.10 -N 1 -F

NAT

iptables -t nat -A POSTROUTING -s 10.0.0.0/24 -j SNAT --to-source 192.168.1.90

tshark dump

tshark -i eth0 -f '!port 80'

ps3

ps3 xserver xorg video spu

install

apt-get source xserver-xorg-video-spu apt-get build-dep xserver-xorg-video-spu apt-get install build-essential fakeroot dpkg-dev

build

cd xserver-xorg-video-spu* && dpkg-buildpackage -rfakeroot -b

pxe

install

root@klin:/srv/sysadmin-cookbook/recepies/pxe# apt-get install dnsmasq

create /etc/dnsmasq.d/pxe

dhcp-boot=pxelinux.0 dhcp-range=192.168.2.50,192.168.2.150,12h enable-tftp tftp-root=/srv/sysadmin-cookbook/recepies/pxe/tftpboot/ dhcp-boot=pxelinux.0

enable conf dir

#!/bin/sh -x grep '^conf-dif=/etc/dnsmasq.d' /etc/dnsmasq.conf || ( echo 'conf-dir=/etc/dnsmasq.d' >> /etc/dnsmasq.conf && /etc/init.d/dnsmasq restart )

create tftpboot

#!/bin/sh url=ftp.hr.debian.org/debian/dists/lenny/main/installer-i386/current/images/netboot/netboot.tar.gz url=http://people.debian.org/~joeyh/d-i/images/daily/netboot/netboot.tar.gz test -d tftpboot || wget -nc $url && mkdir tftpboot && cd tftpboot && tar xvfz ../netboot.tar.gz

nat 192.168.2.0 wlan0

sudo iptables -t nat -F sudo iptables -t nat -A POSTROUTING -s 192.168.2.0/24 -o wlan0 -j MASQUERADE sudo sh -c "echo 1 > /proc/sys/net/ipv4/ip_forward"

qr wifi

#/bin/sh -xe # Generate QR code for WIFI network test -z "$1" && echo -n "ssid: " && read ssid || ssid=$1 test -z "$2" && echo -n "password: " && read pass || pass=$2 qrencode -o /tmp/qr-${ssid}.png "WIFI:S:$ssid;T:WPA;P:$pass;" ; qiv -m /tmp/qr-${ssid}.png

rsync

clone directory

rsync -ravHAX /var/lib/vz/ /zfs/vz/

rsync pull backup over ssh

pull backup from snapshot ../../lvm/create lvm snapshot

root@opl:~# rsync -ravHC --progress koha-hw:/mnt/vz-backup/ /zfs/vz/

pull backup ../../lvm/remove lvm snapshot

rsync -ravH koha-hw:/var/lib/vz/private/ /zfs/vz/private/

rsync clone

#!/bin/sh -x rsync -ravH --numeric-ids --sparse --delete $*

screen

install

apt-get install screen

screen sharing

create student

useradd -d `pwd`/student -s `pwd`/student/screen.sh student

screen sharing perms

chmod u+s /usr/bin/screen chmod 755 /var/run/screen

screen restore perms

chmod 755 /var/run/screen chmod 2755 /usr/bin/screen

Makefile

screen: sudo sh -x 2.screen-sharing-perms sudo -u dpavlin xterm +rv -n perl -e screen -S perl -c screenrc & test: xterm -fg grey -e ssh student@localhost &

screenrc

multiuser on acladd student #aclchg student -w "#" aclumask student-w

screen sharing student

screen

#!/bin/sh exec screen -x dpavlin/perl

smart

debian install

sudo apt-get install sudo smartmontools git-core

README

Controller status dump: 3ware Inc 9750 SAS2/SATA-II RAID http://www.lsi.com/channel/support/products/Pages/3wareSAS9750-8i.aspx http://kb.lsi.com/Download15800.aspx Symbios Logic SAS2008 http://www.supermicro.com/support/faqs/data_lib/FAQ_9633_SAS2IRCU_Phase_5.0-5.00.00.00.zip

count drives

dump smart

#!/bin/sh -x tw_cli=/srv/3ware/x86_64/tw_cli sas2ircu=/srv/sas2ircu_linux_x86_rel/sas2ircu cd /srv/smart dir=`hostname -s` test -d $dir || mkdir $dir cd $dir #ls -d /sys/block/sd* | cut -d/ -f4 | xargs -i sh -cx "/usr/sbin/smartctl -a /dev/{} > smart.{}" if [ -z "$DEBUG" ] ; then ls /dev/disk/by-id/scsi-* | grep -v part[0-9]* | while read dev ; do echo $dev name=`echo $dev | sed 's/^.*scsi-//'` sudo smartctl -a $dev > smart.$name done fi test -x $sas2ircu && $sas2ircu LIST | grep 1000h | awk '{ print $1 }' | xargs -i $sas2ircu {} DISPLAY > controller.lsi_sysbios if [ -x $tw_cli ] ; then controller=`$tw_cli show | grep ^c[0-9] | cut -d" " -f1` if [ ! -z "$controller" ] ; then $tw_cli /$controller show all > controller.3ware $tw_cli /$controller/bbu show all >> controller.3ware fi fi test ! -z "$DEBUG" && exit 1 git add smart.* controller.* git commit -m `date +%Y-%m-%d.%H:%M:%S` -a

dump

#!/bin/sh if [ ! -e /etc/cron.d/smart ] ; then echo '*/30 * * * * /srv/sysadmin-cookbook/recepies/smart/dump.sh' > /etc/cron.d/smart fi cd /srv/sysadmin-cookbook/recepies/smart # awk '{ if ( $3 >= 2930266584 ) print $4 }' driver="-d megaraid,0" cat /proc/partitions | grep ' sd.$' | awk '{ print $4 }' | xargs -i sudo sh -cx "smartctl $* $driver -a /dev/{} > smart.{}" git commit -m `date +%Y-%m-%d.%H:%M:%S` -a

failure

sed='s!/smart.! !' egrep '(Raw_Read_Error_Rate|Reallocated_Sector_Ct|Seek_Error_Rate|Spin_Retry_Count|Reallocated_Event_Count|Current_Pending_Sector|Offline_Uncorrectable|UDMA_CRC_Error_Count)' */smart.* | awk '{ if ( $10 > 0 ) print $1 " " $2 " " $10 }' | sed -e "$sed" -e 's/:[0-9][0-9]* / /' | sort | while read host serial status count ; do echo $host $serial $status $count egrep -A 1 '(failure|progress)' $host/smart.*$serial | sed -e "$sed" done

smart test all

ls -d /sys/block/sd* | cut -d/ -f4 | xargs -i sh -cx "/usr/sbin/smartctl -t long /dev/{}"

smart test relocate

#!/usr/bin/perl use warnings; use strict; my $drive = $ARGV[0] || die "usage: $0 /dev/sda\n"; my $delay = 15; my $rewrite_sectors = 16; my $test_started = 0; sub write_sector { my $s = shift; system "hdparm --write-sector $s --yes-i-know-what-i-am-doing $drive"; } sub smart_test { my $sector = shift; my $cmd = "smartctl -t select,$sector-max $drive"; warn "$cmd\n"; system $cmd; $test_started = 1; } sub smart { my $cmd = "smartctl -l selective $drive"; warn "$cmd\n"; open(my $fh, '-|', $cmd); while(<$fh>) { chomp; print "# $_\n"; if ( m/Completed_read_failure.*$(\d+)-\d+$/ ) { my $sector = $1; print "rewrite sector: $sector\n"; foreach my $s ( $sector .. $sector + $rewrite_sectors ) { write_sector $s; } smart_test $sector; return 1; } elsif ( m/Self_test_in_progress/ ) { $test_started = 1; return 1; } elsif ( m/Not_testing/ ) { if ( $test_started ) { smart_last_error(); return 1 if $test_started; } smart_test 0; # first-time invocation } } return 0; } sub smart_last_error { my $cmd = "smartctl -l selftest $drive"; warn "$cmd\n"; open(my $fh, '-|', $cmd); while(<$fh>) { chomp; print "# $_\n"; if (/^#\s+1.+Completed: read failure\s+\S+\s+\S+\s+(\d+)/) { write_sector $1; smart_test $1; return 1; } } } smart_last_error || smart_test 0; while ( smart ) { warn "sleep $delay s", ( $test_started ? " smart test running..." : "idle" ), "\n"; sleep $delay; }

smtp

install

apt-get install swaks

ssd

trim test

#!/bin/sh -xe # mount /dev/sda2 /mnt -o rw,noatime,discard seq 1 1000 > testfile hdparm --fibmap testfile | tee out.hdparm sector=`cat out.hdparm | tail -1 | awk '{ print $2 }'` hdparm --read-sector $sector /dev/sda > out.$sector.1 rm testfile sync hdparm --read-sector $sector /dev/sda > out.$sector.2 md5sum out.$sector.[12]

ssh

ssh login without password

generate root ssh key

root@opl:~# test -f /root/.ssh/id_rsa || ssh-keygen -f /root/.ssh/id_rsa -N ''

copy root identity

root@opl:~# ssh-copy-id -i /root/.ssh/id_rsa llin

strace

strace count

#!/bin/sh -x trace=/tmp/strace strace -c -o $trace $* && ls -al $trace && cat $trace

systemtap

kernel source

apt-get install linux-source-`uname -r | cut -d\- -f1` kernel-package fakeroot

kernel build

r=`uname -r | cut -d\- -f1` cd /usr/src test -d linux-source-$r || tar xjf linux-source-$r.tar.bz2 cd linux-source-$r cat /boot/config-`uname -r` | sed \ -e 's/^# CONFIG_DEBUG_INFO.*/CONFIG_DEBUG_INFO=y/' \ -e 's/^# CONFIG_KPROBES.*/CONFIG_KPROBES=y/' \ > .config make oldconfig fakeroot make-kpkg --initrd --append-to-version=-systemtap kernel_image kernel_headers kernel_debug

usb sniff

usbmon

#!/bin/sh -x modprobe usbmon mount -t debugfs / /sys/kernel/debug

vblade

client install

apt-get install aoetools

server install

apt-get install vblade

aoe module

rmmod aoe modprobe aoe aoe_iflist="eth0 virtual" dmesg | tail -1

client info

aoe-discover aoe-stat

server lvblade

aoe-interfaces eth0 vblade 0 1 eth0 /dev/vg/lvblade

vde2

install

apt-get install vde2

append /etc/network/interfaces

iface tap0 inet static address 172.25.25.1 netmask 255.255.255.0 vde2-switch -

add to group vde2 net

#!/bin/sh -x USER=$1 usermod -a -G vde2-net $USER

eth0 nat

#!/bin/sh -x sysctl -w net.ipv4.ip_forward=1 iptables -t nat -A POSTROUTING -o eth0 -j MASQUERADE

kvm

#!/bin/sh #kvm-img create -f qcow2 faitest.img 5G vdeq kvm -m 256 -net nic,vlan=1 -net vde,vlan=1,sock=/var/run/vde2/tap0.ctl/ctl -boot n # -hda faitest.img

vde2 virtualnetmanager

build deb

#!/bin/sh -x test -d trunk && cd trunk && svn update || svn co https://virtualnetmgr.svn.sourceforge.net/svnroot/virtualnetmgr/trunk && cd trunk sudo checkinstall --requires graphviz --requires python --pkgname virtualnetmanager ./install.sh

vim

install

root@opl:~# apt-get -y install vim

create /home/dpavlin/

web

stress test http

url='http://10.60.0.17:81/cgi-bin/koha/opac-search.pl?idx=&q=' cat /usr/share/dict/words | grep '^[a-z]*$' | xargs -i time wget -O /dev/null $url{}

yukon

install

sudo apt-get install libx11-dev libxv-dev x11proto-xext-dev mesa-common-dev libgl1-mesa-dev

checkout

svn co https://devel.neopsis.com/svn/seom/branches/packetized-stream seom svn co https://devel.neopsis.com/svn/yukon/branches/rewrite yukon

README

Yukon is a set of libraries and applications that are designed to capture realtime videos of OpenGL applications (games). https://devel.neopsis.com/projects/yukon/

zfs

install

root@opl:~# apt-get install libfuse2 fuse-utils libaio1 root@opl:~# dpkg -i zfs-fuse_20090430-1_i386.deb Selecting previously deselected package zfs-fuse. (Reading database ... 21076 files and directories currently installed.) Unpacking zfs-fuse (from zfs-fuse_20090430-1_i386.deb) ... Setting up zfs-fuse (20090430-1) ...

create pool

# http://docs.sun.com/app/docs/doc/819-5461/gaynr?a=view root@opl:~# lvcreate -n zfs -L 100G raid0 Logical volume "zfs" created root@opl:~# zpool create zfs /dev/raid0/zfs root@opl:~# zpool list NAME SIZE USED AVAIL CAP HEALTH ALTROOT zfs 99.5G 6.92G 92.6G 6% ONLINE -

create file system

root@opl:~# zfs create zfs/vz root@opl:~# zfs list /zfs/vz NAME USED AVAIL REFER MOUNTPOINT zfs/vz 72.2G 18.8G 72.2G /zfs/vz

create compressed file system

# http://docs.sun.com/app/docs/doc/819-5461/gayns?a=view root@opl:~# zfs create zfs/install root@opl:~# zfs set compression=on zfs/install root@opl:~# zfs list zfs/install NAME USED AVAIL REFER MOUNTPOINT zfs/install 18K 18.8G 18K /zfs/install

attach device

root@opl:/srv/sysadmin-cookbook/recepies/zfs# fdisk -l /dev/nbd0 Disk /dev/nbd0: 250.0 GB, 250000000000 bytes 255 heads, 63 sectors/track, 30394 cylinders Units = cylinders of 16065 * 512 = 8225280 bytes Disk identifier: 0x00000000 Device Boot Start End Blocks Id System /dev/nbd0p1 1 13374 107426623+ 83 Linux root@opl:~# zpool status pool: zfs state: ONLINE scrub: scrub completed after 0h19m with 0 errors on Thu Apr 30 22:46:09 2009 config: NAME STATE READ WRITE CKSUM zfs ONLINE 0 0 0 raid0/zfs ONLINE 0 0 0 errors: No known data errors root@opl:~# zpool attach zfs /dev/raid0/zfs /dev/nbd0p1 root@opl:~# zpool status pool: zfs state: ONLINE status: One or more devices is currently being resilvered. The pool will continue to function, possibly in a degraded state. action: Wait for the resilver to complete. scrub: resilver in progress for 0h2m, 3.47% done, 1h5m to go config: NAME STATE READ WRITE CKSUM zfs ONLINE 0 0 0 mirror ONLINE 0 0 0 raid0/zfs ONLINE 0 0 0 nbd0p1 ONLINE 0 0 0 errors: No known data errors

create snapshot

# http://docs.sun.com/app/docs/doc/819-5461/gbcya?a=view root@opl:~# zfs snapshot zfs/vz@`date +%Y-%m-%d_%H:%M:%S` root@opl:~# zfs list NAME USED AVAIL REFER MOUNTPOINT zfs 85.5G 12.4G 6.92G /zfs zfs/install 6.40G 12.4G 6.40G /zfs/install zfs/vz 72.2G 12.4G 72.2G /zfs/vz zfs/vz@2009-05-01_14:41:50 0 - 72.2G -

create writable clone

root@opl:/zfs/vz# zfs clone zfs/vz@60017 zfs/vz-60017-clone root@opl:/zfs/vz# zfs list /zfs/vz-60017-clone NAME USED AVAIL REFER MOUNTPOINT zfs/vz-60017-clone 0 88.2G 73.1G /zfs/vz-60017-clone

nc zfs receive

root@opl:~# nc -l -p 8888 | dd_rescue - - | zfs receive opl/backup/212052

nc zfs send

root@opr:/etc/vz/conf# zfs send opr/vz/private/212052@2009-05-01 | dd_rescue - - | nc -w 1 10.60.0.90 8888

create /etc/cron.d/zfs cron backup

SHELL=/bin/sh PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin # m h dom mon dow user command 15 21 * * * root /srv/sysadmin-cookbook/recepies/zfs/pull-snapshot-backup.sh

append /etc/rc

echo -n "Starting ZFS-fuse: " pidof zfs-fuse || ( /usr/local/sbin/zfs-fuse && sleep 1 && /usr/local/sbin/zfs mount -a && /usr/local/sbin/zpool status -x )

Makefile

# use newer zfs-fuse branch #upstream=http://www.wizy.org/mercurial/zfs-fuse/trunk #upstream=http://git.rudd-o.com/zfs/ upstream=http://git.zfs-fuse.net/official/ all: echo "make [checkout deb clean]" checkout: git clone $(upstream) zfs-fuse install: uninstall cd zfs-fuse/src && scons install #install_dir=/usr/sbin/ uninstall: rm -vf /usr/local/sbin/zdb /usr/local/sbin/ztest /usr/local/sbin/zpool /usr/local/sbin/zfs /usr/local/sbin/zfs-fuse deb: uninstall echo "ZFS on FUSE/Linux" > zfs-fuse/description-pak echo "install:" > zfs-fuse/Makefile echo " cd src && scons install" >> zfs-fuse/Makefile cd zfs-fuse && sudo checkinstall \ --pkgname zfs-fuse --pkgversion `hg log --limit 1 | cut -d: -f2 | head -1` \ --pkglicense CDDL --pkggroup contrib/non-free \ --pkgsource $(upstream) --maintainer dpavlin@rot13.org \ --provides zfs --requires libfuse2,fuse-utils,libaio1 \ --exclude /rest/cvs/zfs-fuse/src/.sconsign.dblite \ depends: sudo apt-get install checkinstall libfuse-dev fuse-utils libaio-dev clean: rm -Rf zfs-fuse/

README

Upstream source with fixes: http://git.rudd-o.com/zfs/

disks

#!/usr/bin/perl use warnings; use strict; use autodie; use Data::Dump qw(dump); my ($before, $delimiter, $after) = ( ''=>' | '=> "\n" ); # double space for display ($before, $delimiter, $after) = ( '"'=>'","'=> '"'."\n" ) if @ARGV; my $zfs; open(my $zpool, '-|', 'zpool status'); my $name = 'zfs'; my $vdev = ''; while(<$zpool>) { chomp; $name = $1 if /^\s+pool:\s(\S+)/; $vdev = $1 if /^\s+(raid\S+|spare|log|cache)/; $zfs->{$1} = join(' ', $name, $vdev) if /^\s+(sd\S+)/; } warn "# zfs ",dump($zfs); my $md; open(my $mdstat, '<', '/proc/mdstat'); while(<$mdstat>) { chomp; if ( m/(md\d+)\s+:\s+active\s+(\S+)\s+(.+)/ ) { my ( $nr, $raid ) = ( $1, $2 ); foreach my $dev ( split(/\s+/, $3) ) { my $d = $1 if $dev =~ m{(sd\w+)\d+}; $md->{$d} = "$nr $raid $dev"; } } } warn "# md ",dump($md); open(my $lsscsi, '-|', 'lsscsi --size -v'); print $before, join($delimiter, qw(id type name dev size pert_of path) ), $after; while(my $line = <$lsscsi>) { chomp($line); # warn "## $line\n"; my @l = $line =~ m{(^\S+)\s+(\S+)\s+(.+)\s+(\S+)\s+(\S+)}; my $dev = $l[3]; $dev =~ s{/dev/}{}; warn "# dev $dev\n"; push @l, exists $zfs->{$dev} ? $zfs->{$dev} : exists $md->{$dev} ? $md->{$dev} : '-'; my $v = <$lsscsi>; chomp($v); push @l, $1 if $v =~ m/\[(.+)\]/; print $before,join($delimiter,@l),$after; };

zfs enlarge pool

attach block device to pool ../../lvm/lvcreate

root@opl:/srv/sysadmin-cookbook/recepies/lvm# zpool status pool: zfs state: ONLINE scrub: resilver completed after 0h45m with 0 errors on Fri May 1 01:56:58 2009 config: NAME STATE READ WRITE CKSUM zfs ONLINE 0 0 0 mirror ONLINE 0 0 0 raid0/zfs ONLINE 0 0 0 nbd0p1 ONLINE 0 0 0 root@opl:/srv/sysadmin-cookbook/recepies/lvm# zpool attach zfs /dev/raid0/zfs /dev/raid0/zfs-200 root@opl:/srv/sysadmin-cookbook/recepies/zfs/enlarge-pool# zpool status zfs pool: zfs state: ONLINE status: One or more devices is currently being resilvered. The pool will continue to function, possibly in a degraded state. action: Wait for the resilver to complete. scrub: resilver in progress for 0h6m, 10.14% done, 1h0m to go config: NAME STATE READ WRITE CKSUM zfs ONLINE 0 0 0 mirror ONLINE 0 0 0 raid0/zfs ONLINE 0 0 0 nbd0p1 ONLINE 0 0 0 raid0/zfs-200 ONLINE 0 0 0 errors: No known data errors

detach old block device

# this step is optional but will move read load from local block device to network mirror device root@opl:/srv/sysadmin-cookbook/recepies/zfs/enlarge-pool# zpool detach zfs /dev/raid0/zfs root@opl:/srv/sysadmin-cookbook/recepies/zfs/enlarge-pool# zpool status pool: zfs state: ONLINE status: One or more devices is currently being resilvered. The pool will continue to function, possibly in a degraded state. action: Wait for the resilver to complete. scrub: resilver in progress for 0h0m, 0.00% done, 265h17m to go config: NAME STATE READ WRITE CKSUM zfs ONLINE 0 0 0 mirror ONLINE 0 0 0 nbd0p1 ONLINE 0 0 0 raid0/zfs-200 ONLINE 0 0 0 errors: No known data errors

remove small remote block device

root@opl:~# zfs list NAME USED AVAIL REFER MOUNTPOINT zfs 97.7G 288M 22K /zfs zfs/install 6.40G 288M 6.40G /zfs/install zfs/vz 91.3G 288M 72.8G /zfs/vz zfs/vz@2009-05-01_14:41:50 18.5G - 72.2G - root@opl:~# zpool status pool: zfs state: ONLINE scrub: resilver completed after 0h28m with 0 errors on Fri May 1 18:13:31 2009 config: NAME STATE READ WRITE CKSUM zfs ONLINE 0 0 0 mirror ONLINE 0 0 0 nbd0p1 ONLINE 0 0 0 raid0/zfs-200 ONLINE 0 0 0 errors: No known data errors root@opl:~# zpool detach zfs nbd0p1 root@opl:~# zpool status pool: zfs state: ONLINE scrub: resilver completed after 0h28m with 0 errors on Fri May 1 18:13:31 2009 config: NAME STATE READ WRITE CKSUM zfs ONLINE 0 0 0 raid0/zfs-200 ONLINE 0 0 0 errors: No known data errors root@opl:~# zfs list NAME USED AVAIL REFER MOUNTPOINT zfs 97.7G 98.7G 22K /zfs zfs/install 6.40G 98.7G 6.40G /zfs/install zfs/vz 91.3G 98.7G 72.8G /zfs/vz zfs/vz@2009-05-01_14:41:50 18.5G - 72.2G - r

pull rsync snap

#!/bin/sh -x from=10.60.0.200 pool=`zpool list -o name -H` log=/$pool/log/ exclude='--exclude var/cache --exclude var/lib/koha/zebradb/biblios --exclude data/webpac2/var/' function rsync_veid() { veid=$1 host=$2 date_time=`rsync $from::mnt/$host/.snap/ | tail -1 | awk '{ print $5 }'` date=`echo $date_time | cut -dT -f1` ( rsync $exclude -ravHz --numeric-ids --delete --force --modify-window=2 \ $from::mnt/$host/.snap/$date_time/ \ /$pool/backup/$veid/ \ && zfs snapshot $pool/backup/$veid@$date 2>&1 ) | tee -a $log/$date.log } rsync_veid 212052 koha-dev rsync_veid 212056 webpac2 rsync_veid 212226 koha #zfs list -r $pool/backup | tee -a $log/$date.log

pull snapshot backup

#!/bin/sh -x # prod.vbz.ffzg.hr from=10.60.0.200 vg=/dev/raid5 date=`date +%Y-%m-%d` pool=`zpool list -o name -H` log=/$pool/log/ exclude='--exclude var/cache --exclude var/lib/koha/zebradb/biblios --exclude data/webpac2/var/ --exclude tmp/' test -d $log || mkdir $log || exit rsync_veid() { ssh $from "umount /mnt/backup/$2 ; lvremove -f $vg/$2-backup" ssh $from "sync && sync && lvcreate -s -L 10G -n $2-backup $vg/$2 && mount $vg/$2-backup /mnt/backup/$2" || exit test -d /$pool/backup/$1 || zfs create $pool/backup/$1 || exit echo "## rsync $1" rsync $exclude -ravHz --numeric-ids --delete --force --modify-window=2 $from::mnt/backup/$2/rootfs/ /$pool/backup/$1/ && zfs snapshot $pool/backup/$1@$date # ssh $from "umount /mnt/backup/$2 && lvremove -f $vg/$2-backup" } ( zfs list -r $pool/backup rsync_veid 212226 koha rsync_veid 212052 koha-dev rsync_veid 212056 webpac2 zfs list -r $pool/backup 2>&1 ) | tee -a $log/$date.log

replace failing drive

#!/bin/sh -x pool=t1 # destroy existing pool zpool status $pool && zpool destroy $pool mkdisk() { dd if=/dev/zero of=disk/$1 bs=1M count=64 } test -d disk || mkdir disk mkdisk 1 mkdisk 2 mkdisk 3 mkdisk 4 mkdisk 5 mkdisk 6 mkdisk 7 mkdisk 8 mkdisk 9 mkdisk 10 mkdisk 11 mkdisk spare d=`pwd`/disk zpool create $pool raidz1 $d/1 $d/2 $d/3 $d/4 $d/5 $d/6 $d/7 $d/8 $d/9 $d/10 $d/11 spare $d/spare zpool status $pool dd if=/dev/zero of=/$pool/foo bs=1M count=500 zfs list $pool dd if=/dev/urandom of=$d/11 bs=1M count=20 zpool status $pool zpool scrub $pool zpool status $pool zpool replace $pool $d/11 $d/spare zpool status $pool zpool detach $pool $d/11 zpool status $pool mkdisk 11.replaced zpool replace $pool $d/spare $d/11.replaced zpool status $pool

zfs expire snapshot

#!/usr/bin/perl use warnings; use strict; use DateTime; use Data::Dump qw/dump/; my $debug = $ENV{DEBUG} || 0; my $config = { 'default' => { 21 => 5, 30 => 10, 90 => 30, }, '212052' => { # koha-dev 7 => 10, 14 => 30, }, '212056' => { # webpac2 7 => 5, 14 => 30, } }; my $now = DateTime->now(); my $last_backup; open(my $fs, '-|', 'zfs list -t snapshot -H'); while(<$fs>) { chomp; my ( $name, $used, $avail, $refer, $mountpoint ) = split(/\t/,$_,6); next unless $name =~ m{(.+)@(\d\d\d\d)-(\d\d)-(\d\d)}; my $host = $1; my $date = DateTime->new( year => $2, month => $3, day => $4 ); my $age = $now->delta_days( $date )->delta_days; my $op = ' '; my $last = 0; my $c = (grep { $host =~ m{\Q$_\E} } keys %$config)[0]; $c = 'default' unless defined $c; warn "# config: $c\n" if $debug; my $h = $host; $h =~ s{,+/([^/]+)$}{}; # just hostname without path $c = $config->{$c} || die "can't find config for $c"; warn "# c = ",dump($c) if $debug; my $keep_every_days; my $older_than_days; foreach ( sort { $b <=> $a } keys %$c ) { $older_than_days = $_; $keep_every_days = $c->{$_}; warn "## $host $age > $older_than_days" if $debug; last if $age > $older_than_days; } my $config_applied = ''; if ( $age > $older_than_days ) { $config_applied = "> $older_than_days keep $keep_every_days"; $last_backup->{$host} ||= $date; $last = $last_backup->{$host}->delta_days( $date )->delta_days; if ( $last && $last < $keep_every_days ) { $op = 'D'; } else { $op = ' '; $last_backup->{$host} = $date; } } else { $config_applied = 'none'; } print "$op $name\t$used\t$refer\t$age\t$last\t$config_applied\n"; system "zfs destroy $name" if $op eq 'D' && @ARGV; }

zfs receive snaphost

#!/bin/sh -x from=10.60.0.90 fs=`ssh $from zfs list -t snapshot | grep @ | iselect -t "select snapshot to pull" -a | sed 's/ .*$//'` if [ -z "$fs" ] ; then exit; fi veid=`echo $fs | cut -d/ -f3 | cut -d@ -f1` date=`echo $fs | cut -d/ -f3 | cut -d@ -f2` pool=`echo $fs | cut -d/ -f1` echo "pull $pool / $veid @ $date" local=`zfs list | grep $veid | cut -d" " -f1` if [ -z "$local" ] ; then local_pool=`zfs list | grep /backup/ | head -1 | cut -d/ -f1` local="$local_pool/backup/$veid" zfs create $local || exit fi echo "clone $fs -- $veid to $local"; ssh $from "zfs send $fs | nc -w 5 -l -p 8888" & sleep 1 nc $from 8888 | dd_rescue -w -y 0 -l /tmp/$veid@$data.log - - | zfs receive -F $local && zfs snapshot $local@$date zfs list -t snapshot | grep $veid

zfs replicate pool

#!/usr/bin/perl use warnings; use strict; use Net::OpenSSH; use Data::Dump qw(dump); use List::Util qw(first); use Time::HiRes; my $compress = '| lzop -c'; my $decompress = 'lzop -d |'; my $arh = Net::OpenSSH->new('root@10.60.0.204'); my $dev = Net::OpenSSH->new('root@10.60.0.202'); sub on { my ($ssh,$command) = @_; warn "## ", $ssh->get_host, "> $command\n" if $ENV{DEBUG}; if ( $command =~ m/zfs list/ ) { map { chomp; $_; } $ssh->capture($command); } else { $ssh->capture($command); } } print on $arh => 'zpool status'; print on $dev => 'zpool status'; my @arh = on $arh => 'zfs list -H -o name'; my @dev = on $dev => 'zfs list -H -o name'; warn "# ",dump( \@arh, \@dev ); my $from_pool = $arh[0]; my $to_pool = $dev[0]; sub snapshots_from { my ($ssh) = @_; my $host = $ssh->get_host; my $snapshot; my @snapshots = on $ssh => 'zfs list -H -t snapshot -o name'; die $ssh->error if $ssh->error; foreach my $s (@snapshots) { my ($fs,$name) = split(/\@/,$s); push @{ $snapshot->{$fs} }, $name; } # warn "snapshots_from $host ",dump($snapshot),$/; return $snapshot; } foreach my $fs ( @arh ) { my $name = $fs; $name =~ s{^$from_pool/}{} || next; # FIXME skip top-level fs warn "? $name"; my $arh_snapshot = snapshots_from $arh; if ( ! exists( $arh_snapshot->{$fs} ) ) { my $snapshot = $fs . '@send'; print on $arh => "zfs snapshot $snapshot"; die $arh->error if $arh->error; $arh_snapshot = snapshots_from $arh; } my $max_snapshot = $#{ $arh_snapshot->{$fs} }; warn "$fs has ",$max_snapshot+1," snapshots\n"; my $to_dev = "$to_pool/$name"; foreach my $i ( 0 .. $max_snapshot ) { my $snap = $arh_snapshot->{$fs}->[$i] || die "no snap"; my $dev_snapshot = snapshots_from $dev; if ( exists $dev_snapshot->{$to_dev} ) { if ( first { /^\Q$snap\E$/ } @{ $dev_snapshot->{$to_dev} } ) { warn "+ $name @ $snap exists\n"; next; } else { warn "- $name @ $snap missing\n"; } } else { warn "$name not found on target yet"; } my $snapshot; if ( $i == 0 ) { $snapshot = "$from_pool/$name\@$snap"; } else { my $prev = $arh_snapshot->{$fs}->[$i-1] || die "no prev"; $snapshot = "-i $from_pool/$name\@$prev $from_pool/$name\@$snap"; } warn "zfs transfer $snapshot -> $to_dev\n"; my $t = time(); my $recv = "nc -w 3 -l -p 8888 | $decompress zfs receive $to_dev"; warn ">> $recv\n"; my ($rin1,$pid1) = $dev->pipe_in($recv); warn ">> pid: $pid1"; sleep 1; # FIXME wait for netcat to start my $send = "zfs send $snapshot $compress | nc -q 0 -w 2 10.60.0.202 8888"; warn "<< $send\n"; $arh->system($send); die $arh->error if $arh->error; $t = time() - $t; warn "took $t seconds to complete\n"; $dev->system("zfs set readonly=on $to_pool/$name") if $i == 0; warn "ERROR: ",$dev->error if $dev->error; sleep 1; $dev_snapshot = snapshots_from $dev; die "can't find new snapshot $snap" unless $dev_snapshot->{$to_dev}; } }

acct

install

lastcomm duration

amt

install

amt openamt

heci

lms

lms

serial console

apache2

deflate test

deflate

server status

apt

apt import key

bookreader

README

btrfs

install

create snapshot

README

btrfs progs unstable checkinstall

btrfs snap expire

mksnap

pull snapshot backup

compcache

download

compile

load modules

crm114

download

cryo

checkout and compile

debian

apt sources

debian cpan

README

dell

install

README

flash bios

make mirror

deploy cookbook

install subversion

append /root/.ssh/config

perms

checkout ../ssh/login without password/2.copy root identity

setup PATH

etherpuppet

install

README

build arch

firefox

install

userChrome

ganeti

gnt ill

ganeti migrate lxc

ganeti

host

lxc

inspect container ip

lxc mount point

ganeti root lvm

lxc snapshot create

lxc rsync snapshot

lxc snapshot remove

lxc stop container

A ganeti fix root

B ganeti network interfaces

C ganeti swap

D ganeti create instance

E ganeti network local

F ganeti drdb

G ganeti start

H host console

I acpi restart

restore to zfs vol

gearman