[Qemu-devel] mini-HOWTO: Using qemu to lock down tetrinetx server

[Rusty Russell]

Hi all,

        I recently set up a tetrinet server inside a qemu image, for a
layer of additional security.  The image is under 2MB, and below is a
mini-howto in setting up such a server in case anyone else is
interested.

You will need:

1) A qemu-compatible disk image: I have a simple script to make one,
  (see attachment).

#! /bin/sh
# Script to make image of given size (in MB)

if [ $# -ne 2 ]; then
    echo Usage: "$0 size-in-MB image" >&2
    echo "eg $0 150 qemu-disk" >&2
    exit 1
fi

SIZE=$1
IMAGE=$2

HEADS=16
SECTORS=63
# 512 bytes in a sector: cancel the 512 with one of the 1024s...
CYLINDERS=$(( $SIZE * 1024 * 2 / ($HEADS * $SECTORS) ))

# Create a filesystem: one track for partition table.
dd bs=$(($SECTORS * 512)) if=/dev/zero of=$IMAGE.raw count=$(($CYLINDERS * 
$HEADS - 1))
mke2fs -q -m1 -F -j $IMAGE.raw

# Prepend partiation table
# Create file with partition table.
uudecode -o- << "EOF" | gunzip > $IMAGE
begin 664 partition-table.gz
M'XL("*_<##\"`W!A<G1I=&EO;BUT86)L90#LT#$-`"`0!,&']D6A`D6XP1T&
M"%B@))FIMKGF(OA9C;%;EENYZO.Z3P\"````!P``__\:!0````#__QH%````
M`/__&@4`````__\:!0````#__QH%`````/__&@4`````__\:!0````#__QH%
M`````/__&@4`````__\:!0````#__QH%`````/__&@4`````__\:!0````#_
M_QH%`````/__&@4`````__\:!0````#__QH%`````/__&@4`````__\:!0``
M``#__QH%`````/__&@4`````__\:!0````#__QH%`````/__&@4`````__\:
M!0````#__QH%`````/__&@4`````__\:!0````#__QH%`````/__&@4`````
M__\:!0````#__QH%`````/__&@4`````__\:!0````#__QH%`````/__&@4`
M````__\:!0````#__QH%`````/__&@4`````__\:!0````#__QH%`````/__
M&@4`````__\:!0````#__QH%`````/__&@4`````__\:!0````#__QH%````
M`/__&@4`````__\:!0````#__QH%`````/__&@4`````__\:!0````#__QH%
M`````/__&@4`````__\:!0````#__QH%`````/__&@4`````__\:!0````#_
M_QH%`````/__&@4`````__\:!0````#__QH%`````/__&@4`````__\:!0``
M``#__QH%`````/__&@4`````__\:!0````#__QH%`````/__&@4`````__\:
M!0````#__QH%`````/__&@4`````__\:!0````#__QH%`````/__&@4`````
M__\:!0````#__QH%`````/__&@4`````__\:!0````#__QH%`````/__&@4`
M````__\:!0````#__QH%`````/__&@4`````__\:!0````#__QH%`````/__
M&@4`````__\:!0````#__QH%`````/__&@4`````__\:!0````#__QH%````
M`/__&@4`````__\:!0````#__QH%`````/__&@4`````__\:!0````#__QH%
M`````/__&@4`````__\:!0````#__QH%`````/__&@4`````__\:!0````#_
M_QH%`````/__&@4`````__\:!0````#__QH%`````/address@hidden&_
&<address@hidden
`
end
EOF
cat $IMAGE.raw >> $IMAGE
rm $IMAGE.raw

# Repartition so one partition covers entire disk.
echo '63,' | sfdisk -uS -H$HEADS -S$SECTORS -C$CYLINDERS $IMAGE

2) The "vl" binary, and a qemu-compatible kernel bzImage.
   I used qemu 0.4.3, modified with the -tun-fd patch and the closed
   stdin patch.  If you use the softmm version of qemu, any kernel
   bzImage should work.

3) The root-running version of the tundev program (see attachment)

#define _GNU_SOURCE /* asprintf */
#include <stdio.h>
#include <string.h>
#include <sys/types.h>
#include <pwd.h>
#include <grp.h>
#include <sys/stat.h>
#include <sys/ioctl.h>
#include <fcntl.h>
#include <unistd.h>
#include <stdlib.h>
#include <net/if.h>
#include <linux/if_tun.h>

/* Tiny code to open tap/tun device, and hand the fd to vl.
   Run as root, drops to given user. */
int main(int argc, char *argv[])
{
        struct ifreq ifr;
        struct passwd *p;
        unsigned int i;
        char *newargs[argc];
        int fd;

        if (argc < 4) {
                fprintf(stderr,
                        "Usage: switch user logfile vl <vl options>...\n");
                exit(1);
        }

        fd = open("/dev/net/tun", O_RDWR);
        if (fd < 0) {
                perror("Could not open /dev/net/tun");
                exit(1);
        }

        memset(&ifr, 0, sizeof(ifr));
        ifr.ifr_flags = IFF_TAP | IFF_NO_PI;
        strncpy(ifr.ifr_name, "tun%d", IFNAMSIZ);
        if (ioctl(fd, TUNSETIFF, (void *) &ifr) != 0) {
                perror("Could not get tun device");
                exit(1);
        }

        /* Set userid. */
        p = getpwnam(argv[1]);
        if (!p) {
                fprintf(stderr, "No user '%s'\n", argv[1]);
                exit(1);
        }
        setgroups(0, NULL);
        setgid(p->pw_gid);
        if (setuid(p->pw_uid) != 0) {
                perror("setting uid");
                exit(1);
        }

        /* Insert -tun-fd, delete other args */
        newargs[0] = argv[3];
        asprintf(&newargs[1], "-tun-fd=%d", fd);
        for (i = 4; i <= argc; i++)
                newargs[i-2] = argv[i];
                
        if (strcmp(argv[2], "-") == 0) {
                execvp(newargs[0], newargs);
                exit(1);
        }

        switch (fork()) {
        case 0: {
                close(1);
                close(2);
                open(argv[2], O_WRONLY|O_APPEND);
                open(argv[2], O_WRONLY|O_APPEND);
                close(0);
                execvp(newargs[0], newargs);
                exit(1);
        }
        case -1:
                perror("fork failed");
                exit(1);
        }
        printf("%s\n", ifr.ifr_name);
        exit(0);
}

Steps:
1) Mount the filesystem.  If you've already prepended a partition
   table, use
        losetup -o $((63 * 512)) /dev/loop0 <FILE>
        mount /dev/loop0 /mnt/qemu
   Otherwise you can just do:
        mount -o loop <FILE> /mnt/qemu

2) Remove unneccessary files, add files you need, and lock down
   directories.  This is best done by creating an oldroot dir and
   moving things into that.  For tetrinet, I ended up under 2MB
   (see attachment)

/mnt/qemu:
total 222
dr-x--x--x    2 root     root         1024 Sep 17 21:10 dev
dr-x--x--x    3 root     root         1024 Sep 17 20:48 etc
-rwx------    1 root     root        53516 Jun 28 12:09 ifconfig
d--x--xr-x    2 root     root         1024 Sep 17 21:08 lib
drwx------    2 root     root        12288 Sep 18 13:30 lost+found
-rwx------    1 root     root        41848 Jun 28 12:09 route
-rwxrwxr-x    1 rusty    rusty        6748 Sep 18 13:27 run
-rwx--x---    1 root     games      104184 Jan 25  2003 tetrinetx
dr-x--x--x    3 root     root         1024 Sep 17 20:54 var

/mnt/qemu/dev:
total 0
crw-------    1 root     root       5,   1 Jul 10 19:32 console
crw-rw----    1 root     games      4,  64 Sep 16 18:48 ttyS0

/mnt/qemu/etc:
total 6
-rw-r-----    1 root     games          41 Sep 17 20:48 hosts
-rw-r-----    1 root     games         465 Mar 12  1999 nsswitch.conf
-rw-r-----    1 root     games        2030 Jun 16 03:40 protocols
-rw-r-----    1 root     games          64 Jul 10 10:49 resolv.conf
dr-x--x--x    2 root     root         1024 Sep 17 20:34 tetrinetx

/mnt/qemu/etc/tetrinetx:
total 7
-rw-r-----    1 root     games        4042 Sep 17 20:34 game.conf
-rw-r-----    1 root     games          87 Jan 25  2003 game.motd
-rw-r-----    1 root     games         180 Jan 25  2003 game.pmotd
-rw-r-----    1 root     games         768 Jan 25  2003 game.secure

/mnt/qemu/lib:
total 1574
-rwxr-xr-x    1 root     root        82456 Apr 20 04:57 ld-2.3.1.so
lrwxrwxrwx    1 root     root           11 Jul 10 19:32 ld-linux.so.2 -> 
ld-2.3.1.so
lrwxr-xr-x    1 root     root           14 Sep 16 17:27 libadns.so.1 -> 
libadns.so.1.0
-rwxr-xr-x    1 root     root        61044 Sep  9  2002 libadns.so.1.0
-rwxr-xr-x    1 root     root      1103880 Apr 20 04:57 libc-2.3.1.so
lrwxrwxrwx    1 root     root           13 Jul 10 19:32 libc.so.6 -> 
libc-2.3.1.so
-rwxr-xr-x    1 root     root         7992 Apr 20 04:57 libdl-2.3.1.so
lrwxrwxrwx    1 root     root           14 Jul 10 19:32 libdl.so.2 -> 
libdl-2.3.1.so
lrwxrwxrwx    1 root     root           17 Jul 10 19:32 libncurses.so.5 -> 
libncurses.so.5.3
-rwxr-xr-x    1 root     root       238160 Jun 15 02:13 libncurses.so.5.3
-rwxr-xr-x    1 root     root        12828 Apr 20 04:57 libnss_dns-2.3.1.so
lrwxrwxrwx    1 root     root           19 Jul 10 19:32 libnss_dns.so.2 -> 
libnss_dns-2.3.1.so
-rwxr-xr-x    1 root     root        32204 Apr 20 04:57 libnss_files-2.3.1.so
lrwxrwxrwx    1 root     root           21 Jul 10 19:32 libnss_files.so.2 -> 
libnss_files-2.3.1.so
-rwxr-xr-x    1 root     root        56652 Apr 20 04:57 libresolv-2.3.1.so
lrwxrwxrwx    1 root     root           18 Jul 10 19:32 libresolv.so.2 -> 
libresolv-2.3.1.so

/mnt/qemu/lost+found:
total 0

/mnt/qemu/var:
total 1
dr-x--x--x    3 root     root         1024 Sep 17 20:54 log

/mnt/qemu/var/log:
total 1
dr-x--x--x    2 root     root         1024 Sep 17 20:54 tetrinetx

/mnt/qemu/var/log/tetrinetx:
total 0
lrwxrwxrwx    1 root     root           10 Sep 17 20:54 game.log -> /dev/ttyS0

3) I wanted the filesystem to be read-only (too bad about the tetrinet
   high scores file: I don't care), but I wanted the logs, so I made
   /var/log/tetrinetx/game.log a symlink to /dev/ttyS0, and allowed
   the games group to write to it.

4) I wrote a simply launcher program called run to avoid the need for
   a shell (see attachment)

/* Simply C program to act as init. */
#include <stdio.h>
#include <stdarg.h>
#include <stdlib.h>
#include <string.h>
#include <errno.h>
#include <unistd.h>
#include <grp.h>
#include <sys/reboot.h>
#include <linux/reboot.h>
#include <sys/types.h>
#include <sys/wait.h>
#include <sys/stat.h>
#include <fcntl.h>

static int wait_for(const char *prog)
{
        int status;

        wait(&status);
        if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) {
                fprintf(stderr, "run: %s failed\n", prog);
                return 0;
        }
        return 1;
}

int main(int argc, char *argv[])
{
        if (fork() == 0) {
                execl("/ifconfig", "ifconfig", "eth0", "192.168.1.2", NULL);
                fprintf(stderr, "run: ifconfig exec failed\n");
                exit(1);
        }
        if (!wait_for("ifconfig"))
                goto reboot;

        if (fork() == 0) {
                execl("/route", "route", "add", "default", "gw", "192.168.1.1",
                      NULL);
                fprintf(stderr, "run: route exec failed\n");
                exit(1);
        }
        if (!wait_for("route"))
                goto reboot;

        /* tetrinet does DNS lookup on own hostname */
        sethostname("ozlabs.org", strlen("ozlabs.org"));

        if (fork() == 0) {
                setgroups(0, NULL);
                /* Games/games */
                setgid(60);
                setuid(5);
                close(0);
                close(1);
                close(2);
                open("/dev/null", O_RDWR);
                open("/dev/null", O_RDWR);
                open("/dev/null", O_RDWR);
                execl("/tetrinetx", "tetrinetx", NULL);
                exit(1);
        }
        if (!wait_for("tetrinetx"))
                goto reboot;

        /* Drop privs, why not? */
        setgroups(0, NULL);
        setgid(65534);
        setuid(65534);
        close(0);
        close(1);
        close(2);

        while (wait(NULL) >= 0);

reboot:
        sync();
        reboot(LINUX_REBOOT_CMD_RESTART);
        sleep(60);
        return 1;
}

5) Unmount the qemu image.  Once I'd trimmed out most of the code, I
   created a new smaller one one, mounted that, and used tar to
   transfer everything across.

6) My startup script looks like so:

        #! /bin/sh
        # Set up forwarding and masquerading, and make sure tun module is 
loaded.

        set -e

        INTERFACE=`./tundev games /var/log/qemu-tetrinet ./vl -hda 
tetrinet-filesystem bzImage root=/dev/hda1 ide1=noprobe ide2=noprobe 
ide3=noprobe ide4=noprobe ide5=noprobe init=/run`

        ifdown $INTERFACE 2>/dev/null || true
        ifup $INTERFACE

   You need to reconfigure the tun devices everytime they get opened,
   hence the ifdown/ifup stuff.

8) You will also need to forward incoming connections to the server.
   In this case, the server makes no outgoing connections, so this
   works:

   iptables -t nat -A PREROUTING -p tcp -i eth0 --dport 31457 -j DNAT --to 
192.168.1.2
   iptables -A FORWARD -o tun0 -m state --state NEW,INVALID -j DROP
   iptables -N LOGDROP
   iptables -A LOGDROP -j LOG
   iptables -A LOGDROP -j DROP
   iptables -A FORWARD -i tun0 -m state --state NEW,INVALID -j LOGDROP

Good luck!
Rusty.
--
  Anyone who quotes me in their sig is an idiot. -- Rusty Russell.