[taler-grid5k] 99/141: add docker to build g5k image

[gnunet]

This is an automated email from the git hooks/post-receive script.

marco-boss pushed a commit to branch master
in repository grid5k.

commit 1f05a24d0afbe9c18b8025f0368f20704483492c
Author: Boss Marco <bossm8@bfh.ch>
AuthorDate: Thu Nov 4 09:37:15 2021 +0100

    add docker to build g5k image
---
 experiment/.gitignore => .gitignore |   0
 docker/Dockerfile                   |  70 +++++++++++++++++++++++
 docker/README.md                    |  88 +++++++++++++++++++++++++++++
 docker/docker-compose.yaml          |  30 ++++++++++
 docker/entrypoint.sh                | 108 ++++++++++++++++++++++++++++++++++++
 image/taler-debian11.yaml           |   5 +-
 6 files changed, 298 insertions(+), 3 deletions(-)

diff --git a/experiment/.gitignore b/.gitignore
similarity index 100%
rename from experiment/.gitignore
rename to .gitignore
diff --git a/docker/Dockerfile b/docker/Dockerfile
new file mode 100644
index 0000000..5871b47
--- /dev/null
+++ b/docker/Dockerfile
@@ -0,0 +1,70 @@
+FROM debian:11
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+ENV GRID5K_DEST=lyon,lille
+ENV GNUNET_COMMIT_SHA=master
+ENV EXCHANGE_COMMIT_SHA=master
+ENV MEERCHANT_COMMIT_SHA=master
+ENV GRID5K_COMMIT_SHA=master
+
+RUN apt update && \
+    apt upgrade -y && \
+    apt install -y \
+        recutils \
+        autoconf \
+        uncrustify \
+        autopoint \
+        libtool \
+        python3-pip \
+        libgcrypt20-dev \
+        libjansson-dev \
+        libcurl4-gnutls-dev \
+        libsodium-dev \
+        libidn2-dev \
+        libunistring-dev \
+        libmicrohttpd-dev \
+        libsqlite3-dev \
+        libqrencode-dev \
+        pkg-config \
+        gettext \
+        postgresql \
+        libpq-dev \
+        texinfo \
+        postgresql-contrib \
+        curl \
+        jq \
+        git \
+        make \
+        ruby-dev \
+       ruby-childprocess \
+       zstd \
+       libguestfs-tools \
+       linux-headers-amd64 \
+       socat \
+       qemu-utils \
+               ruby-dev \
+       ruby-childprocess \
+               pigz \
+       netcat \
+       eatmydata \
+       dirmngr \
+       gnupg \
+       gnupg-agent \
+        expect \
+       rsync && \
+    pip install jinja2 && \
+    gem install kameleon-builder
+
+RUN mkdir taler && cd taler && \
+    git clone https://git.gnunet.org/gnunet.git && \
+    git clone https://git.taler.net/exchange.git && \
+    git clone https://git.taler.net/merchant.git && \
+    git clone https://git.taler.net/grid5k.git
+
+COPY entrypoint.sh /entrypoint.sh
+RUN chmod +x entrypoint.sh
+
+VOLUME ["/taler"]
+
+CMD ["/bin/bash", "-c", "/entrypoint.sh"]
diff --git a/docker/README.md b/docker/README.md
new file mode 100644
index 0000000..51b5586
--- /dev/null
+++ b/docker/README.md
@@ -0,0 +1,88 @@
+# Taler Grid5000 Build Image
+
+This docker image can be used to build the Grid5000 image for the taler 
performance experiments
+
+## Build
+
+Build this image with `docker build . -t taler:build`
+Or alternatively with `docker-compose up --build` **NOTE** this will also run 
the image, read below
+
+## Run the build
+
+Running the image will build GNUnet, Taler and the Grid5000 image from the 
specified commits. 
+
+The image will then be uploaded to the specified nodes Grid5000 public 
directory using the certificate provided.
+
+### docker
+
+```bash
+docker run -it --rm \
+           --device=/dev/kvm:/dev/kvm \
+           --device=/dev/fuse:/dev/fuse \
+           --device=/dev/net/tun:/dev/net/tun \
+           -p 5900:5900 \
+           --cap-add NET_ADMIN \
+           --cap-add SYS_ADMIN \
+           --security-opt apparmor:unconfined \
+           -e GRID5K_USER=<user> \
+           -e GRID5K_CERT=<cert_path> \
+           -e GRID5K_CERT_PASSWD=<cert_passwd> \
+           taler:build
+```
+
+### docker-compose
+
+Assuming an env file `.env` with the following contents:
+
+```bash
+GRID5K_USER=<user>
+GRID5k_CERT=<cert_path>
+```
+
+the build can be started with:
+
+```bash
+GRID5K_CERT_PASSWD=<cert_passwd> docker-compose up --build
+```
+
+### Notes
+
+#### Environment Variables
+
+**GRID5K_USER`**: the user which GRID5K_CERT belongs to
+**GRID5K_CERT**: the certificate which is used to login to the Grid5000 nodes
+**GRID5K_CERT_PASSWD**: the password to decrypt GRID5K_CERT
+**GRID5K_DEST**: comma separated list of where to copy the image to in the 
grid (lille,lyon)
+
+##### Additional
+
+**GNUNET_COMMIT_SHA**: Which commit to use of gnunet (master)
+**EXCHANGE_COMMIT_SHA**: Which commit to use of taler-exchange (master)
+**MERCHANT_COMMIT_SHA**: Which commit to use of taler-merchant (master)
+**GRID5K_COMMIT_SHA**: Which commit to use of this repo (master)
+
+#### VNC
+
+What happens during the build can be inspected via the logs from docker. 
+However, kameleon uses qemu which can be inspected with `vncviewer` in certain 
steps and when port 5900 is mapped:
+
+```bash
+apt install -y tigervnc-viewer
+```
+
+```bash
+vncviewer :0
+```
+
+#### Output
+
+The image will be published to the Grid5000's public directory on a specified 
node.
+Additionally the generated image can also be mounted to the host by specifying 
`-v <some_path>:/root/output`
+or 
+
+```yaml
+volumes:
+  - <some_path>:/root/output
+```
+
+respectively
diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml
new file mode 100644
index 0000000..fcca896
--- /dev/null
+++ b/docker/docker-compose.yaml
@@ -0,0 +1,30 @@
+version: "3"
+
+services:
+  taler-build:
+    image: taler:build
+    build: .
+    ports:
+      - "5900:5900"
+    cap_add:
+      - NET_ADMIN
+      - SYS_ADMIN
+    devices:
+      - /dev/kvm
+      - /dev/fuse
+      - /dev/net/tun
+    environment:
+      GNUNET_COMMIT_SHA: dda40766b9b6df3644c945429e12f28cc9e7e0c8
+      EXCHANGE_COMMIT_SHA: b338ce6f88a6f34a663ef55b740086ed4272362b
+      MERCHANT_COMMIT_SHA: 99e7729ede69e218198a8c846e8096ad83127f6b
+      GRID5K_COMMIT_SHA: node-setup
+      GRID5K_USER: ${GRID5K_USER}
+      GRID5K_CERT_PASSWD: ${GRID5K_CERT_PASSWD}
+    security_opt:
+      - apparmor:unconfined
+    volumes:
+      - build:/taler
+      - ${GRID5K_CERT}:/root/cert.pem
+
+volumes:
+  build:
diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh
new file mode 100644
index 0000000..ed492cc
--- /dev/null
+++ b/docker/entrypoint.sh
@@ -0,0 +1,108 @@
+#!/bin/bash
+set -e
+
+TALER_HOME=/taler
+
+cd ${TALER_HOME}/gnunet
+git checout master && pull
+git checkout ${GNUNET_COMMIT_SHA} 
+
+./bootstrap && ./configure --enable-logging=verbose --prefix=/usr
+if [ "$CLEAN" = true ]; then
+  make clean
+fi
+make -j 6
+make install
+make -j 6 dist
+
+
+cd ${TALER_HOME}/exchange
+git checkout master && git pull
+git checkout ${EXCHANGE_COMMIT_SHA} 
+
+./bootstrap && ./configure --enable-logging=verbose --prefix=/usr
+if [ "$CLEAN" = true ]; then
+  make clean
+fi
+make -j 6
+make install
+find . -name "*Makefile*" -exec sed -i '/x-taler-bank.fee/d' {} \;
+make -j 6 dist
+
+
+cd ${TALER_HOME}/merchant
+git checkout master && git pull
+git checkout ${MERCHANT_COMMIT_SHA}
+
+./bootstrap && ./configure --enable-logging=verbose --prefix=/usr
+if [ "$CLEAN" = true ]; then
+  make clean
+fi
+make -j 6
+make -j 6 dist
+
+
+cd ${TALER_HOME}/grid5k
+git checkout master && git pull
+git checkout ${GRID5K_COMMIT_SHA} 
+
+cd image
+
+mv ${TALER_HOME}/gnunet/gnunet*.tar.gz grid5000/steps/data/gnunet.tar.gz
+mv ${TALER_HOME}/exchange/taler-exchange*.tar.gz 
grid5000/steps/data/exchange.tar.gz
+mv ${TALER_HOME}/merchant/taler-merchant*.tar.gz 
grid5000/steps/data/merchant.tar.gz
+
+rm -rf build
+
+yes r | kameleon build taler-debian11.yaml
+
+cd build/taler-debian11
+
+if [ -f "/root/cert.pem" ]; then
+
+  mkdir -p /root/.ssh/
+  
+  echo "
+  Host g5k
+    User ${GRID5K_USER}
+    HostName access.grid5000.fr
+    IdentityFile /root/cert.pem
+    ForwardAgent no
+  
+  Host *.g5k
+    User ${GRID5K_USER}
+    ProxyCommand ssh g5k -W \"\$(basename %h .g5k):%p\"
+    ForwardAgent no
+  " >> /root/.ssh/config
+  
+  eval $(ssh-agent)
+  
+  if [[ -f "/root/cert.pem" ]]; then
+    if [[ "${GRID5K_CERT_PASSWD}" != "" ]]; then
+      /usr/bin/expect -c "
+      spawn ssh-add /root/cert.pem;
+      expect \"Enter passphrase for /root/cert.pem: \";
+      send ${GRID5K_CERT_PASSWD}\n;
+      expect \"Identity added: /root/cert.pem (/root/cert.pem)\";
+      interact"
+    else 
+      ssh-add /root/cert.pem
+    fi
+  fi
+  
+  mv taler-debian11.dsc taler-debian11.dsc.bak
+  
+  IFS=, read -ra G5K_HOSTS <<< "${GRID5K_DEST}" 
+  for G5K_HOST in "${G5K_HOSTS[@]}"; do
+    echo "Copying image to ${G5K_HOST}"
+    sed "s/<G5K_HOST>/${G5K_HOST}/g; s/<G5K_USER>/${GRID5K_USER}/g" \
+        taler-debian11.dsc.bak > taler-debian11.dsc
+    scp -o StrictHostKeyChecking=no taler-debian11.tar.zst taler-debian11.dsc \
+        g5k:"${G5K_HOST}"/public/
+  done 
+fi
+
+if [ -d "/root/output" ]; then
+  mv taler-debian11.dsc.bak taler-debian11.dsc || true
+  cp taler-debian11.tar.zst taler-debian11.dsc /root/output/ 
+fi
diff --git a/image/taler-debian11.yaml b/image/taler-debian11.yaml
index 808d948..ed2cc7b 100644
--- a/image/taler-debian11.yaml
+++ b/image/taler-debian11.yaml
@@ -24,9 +24,8 @@ global:
   ## Version
   g5k_version: 2
   ## Environment image path and compression
-  # Note: unknown why this is not propagated to the ".dsc" file,
-  #       So we must adjust it manually!
-  g5k_tar_path: 
"http://public.lille.grid5000.fr/~bfhch01/taler-debian11.tar.zst";
+  # Note: setup for the docker build image - please replace G5K_HOST and 
G5K_USER for manual builds
+  g5k_tar_path: 
"http://public.<G5K_HOST>.grid5000.fr/~<G5K_USER>/taler-debian11.tar.zst"
   # g5k_tar_compression: "zstd"
   ## Environment postinstall path, compression, and script command
   # g5k_postinst_path: server:///grid5000/postinstalls/g5k-postinstall.tgz

-- 
To stop receiving notification emails like this one, please contact
gnunet@gnunet.org.