This note includes all commands I typed when I set up Arch Linux on my new bare metal server.

Why I choose Arch Linux

  • Simple as it should be
  • Outstanding community efforts to maintaining package registry
  • Well organized wiki resources


wipe whole disk

wipefs -a /dev/sda

create partition

select /dev/sda
mktable gpt
mkpart EFI fat32 0 512MB # EFI
mkpart Arch ext4 512MB 100% # Arch
set 1 esp on # flag partition 1 as ESP

install file-system

mkfs.vfat -F 32 /dev/sda1 # EFI
mkfs.ext4 /dev/sda2 # Arch

mount disk

mkdir -p /mnt/boot
mount /dev/sda2 /mnt
mount /dev/sda1 /mnt/boot

install base & Linux kernel

reflector --protocol https --latest 30 --sort rate --save /etc/pacman.d/mirrorlist --verbose # optimize mirror list

# choose between 'linux' or 'linux-lts'
pacstrap /mnt base linux linux-firmware
# base-devel need to be included as well?
genfstab -U /mnt >> /mnt/etc/fstab
arch-chroot /mnt
pacman -Syu # upgrade
pacman -Qe # list explicitly installed pkgs
pacman -Rs # remove pkg and its deps
pacman -Qtd # list orphans

pacman -S man-db man-pages git informant

add fstab entries

# backup UUID=<UUID> /mnt/backup ext4 defaults 0 2 # archive (do not prevent boot even if fsck fails) UUID=<UUID> /mnt/archive ext4 defaults,nofail,x-systemd.device-timeout=4 0 2

Find <UUID> from the output of lsblk -f.

findmnt --verify --verbose # verify fstab


pacman -S \
  grub \
  efibootmgr \
  amd-ucode # AMD microcode
grub-install --target=x86_64-efi --efi-directory=/boot --bootloader-id=GRUB

vim /etc/default/grub
grub-mkconfig -o /boot/grub/grub.cfg


sed -i -e 's/#NTP=/' -e 's/#Fall/Fall/' /etc/systemd/timesyncd.conf
systemctl enable --now systemd-timesyncd


ln -sf /usr/share/zoneinfo/Asia/Tokyo /etc/localtime
hwclock --systohc
vim /etc/locale.gen & locale-gen
echo "LANG=en_US.UTF-8" > /etc/locale.conf


hostnamectl set-hostname polka
hostnamectl set-chassis server
/etc/hosts localhost ::1 localhost polka


[Match] Name=enp5s0 [Network] #DHCP=yes Address= Gateway= DNS= # self-hosted DNS resolver DNS= # Cloudflare for the fallback DNS server MACVLAN=dns-shim # to handle local dns lookup to which is managed by Docker macvlan driver
# to handle local dns lookup to [NetDev] Name=dns-shim Kind=macvlan [MACVLAN] Mode=bridge
# to handle local dns lookup to [Match] Name=dns-shim [Network] IPForward=yes [Address] Address= Scope=link [Route] Destination=

ip equivalent to the above settings:

ip link add dns-shim link enp5s0 type macvlan mode bridge # add macvlan shim
ip a add dev dns-shim # assign host ip to shim defined in docker-compose.yml
ip link set dns-shim up # enable interface
ip route add dev dns-shim # route macvlan subnet to shim interface
systemctl enable --now systemd-networkd
networkctl status

# for self-hosted dns resolver
sed -r -i -e 's/#?DNSStubListener=yes/DNSStubListener=no/g' -e 's/#DNS=/DNS=' /etc/systemd/resolved.conf
ln -sf /run/systemd/resolve/resolv.conf /etc/resolv.conf

systemctl enable --now systemd-resolved
resolvectl status
resolvectl query
drill @

pacman -S wpa_supplicant
vim /etc/wpa_supplicant/wpa_supplicant.conf
# ctrl_interface=/run/wpa_supplicant
# update_config=1
wpa_supplicant -B -i wlp8s0 -c /etc/wpa_supplicant/wpa_supplicant.conf
wpa_cli # default control socket -> /var/run/wpa_supplicant
modinfo iwlwifi

If networkctl keep showing enp5s0 as degraded, then run ip addr add dev enp5s0 to manually assign static IP address for the workaround.


pacman -S firewalld

See also Introduction to Netfilter – To Linux and beyond !


pacman -S zsh
chsh -s /bin/zsh


passwd # change root passwd

useradd -m -s /bin/zsh uetchy # add local user
passwd uetchy # change local user password

userdbctl # verify users

pacman -S sudo
echo "%sudo ALL=(ALL) NOPASSWD:/usr/bin/pacman" > /etc/sudoers.d/pacman # allow users in sudo group to run pacman without password (optional)
usermod -aG sudo uetchy # add local user to sudo group
visudo -c


pacman -S openssh
vim /etc/ssh/sshd_config
systemctl enable --now sshd

on the host machine:

ssh-copy-id [email protected]


git clone
cd yay
makepkg -si


exit # leave chroot
umount -R /mnt

Additional setup


pacman -S nvidia # 'nvidia-lts' for linux-lts
cat /var/lib/modprobe.d/nvidia.conf # ensure having 'blacklist nouveau'

nvidia-smi # test runtime


pacman -S docker docker-compose
yay -S nvidia-container-runtime
{ "log-driver": "json-file", "log-opts": { "max-size": "10m", // default: -1 (unlimited) "max-file": "3" // default: 1 }, "exec-opts": ["native.cgroupdriver=systemd"], // for kubernetes "runtimes": { // for docker-compose "nvidia": { "path": "/usr/bin/nvidia-container-runtime", "runtimeArgs": [] } } }
systemctl enable --now docker

groupadd docker
usermod -aG docker user

GPU_OPTS=(--gpus all --device /dev/nvidia0 --device /dev/nvidiactl --device /dev/nvidia-modeset --device /dev/nvidia-uvm --device /dev/nvidia-uvm-tools)
docker run --rm -it ${GPU_OPTS} nvidia/cuda:10.2-cudnn7-runtime nvidia-smi
docker run --rm -it ${GPU_OPTS} tensorflow/tensorflow:1.14.0-gpu-py3 bash

Use journald log driver in Docker Compose

      driver: "journald"
        tag: "{{.ImageName}}/{{.Name}}/{{.ID}}" # default: "{{.ID}}"


yay -S telegraf
vim /etc/telegraf/telegraf.conf
Cmnd_Alias FAIL2BAN = /usr/bin/fail2ban-client status, /usr/bin/fail2ban-client status * telegraf ALL=(root) NOEXEC: NOPASSWD: FAIL2BAN Defaults!FAIL2BAN !logfile, !syslog, !pam_session
chmod 440 /etc/sudoers.d/telegraf


pacman -S fail2ban
[INCLUDES] before = common.conf [Definition] failregex = .* client login failed: .+ client:\ <HOST> ignoreregex =
[DEFAULT] bantime = 120m ignoreip = [sshd] enabled = true port = 22,10122 maxretry = 3 mode = aggressive # [mailu] enabled = true backend = systemd journalmatch = CONTAINER_NAME=mail_front_1 filter = bad-auth findtime = 1h maxretry = 3 bantime = 3d banaction = iptables-allports chain = DOCKER-USER
- iptables.service firewalld.service ip6tables.service ipset.service nftables.service + iptables.service firewalld.service ip6tables.service ipset.service nftables.service docker.service
systemctl enable --now fail2ban
fail2ban-client status mailu


Dynamic DNS for Cloudflare.

yay -S cfddns sendmail
token: <token>
systemctl enable --now cfddns


pacman -S smartmontools
systemctl enable --now smartd

smartctl -t short /dev/sdc
smartctl -l selftest /dev/sdc


pacman -S borg
[Unit] Description=Borg Daily Backup Service [Service] Type=simple Nice=19 IOSchedulingClass=2 IOSchedulingPriority=7 ExecStart=/etc/backups/
[Unit] Description=Borg Daily Backup Timer [Timer] WakeSystem=false OnCalendar=*-*-* 03:00 RandomizedDelaySec=10min [Install]
#!/bin/bash -ue # The udev rule is not terribly accurate and may trigger our service before # the kernel has finished probing partitions. Sleep for a bit to ensure # the kernel is done. # # This can be avoided by using a more precise udev rule, e.g. matching # a specific hardware path and partition. sleep 5 # # Script configuration # export BORG_PASSPHRASE="<secret>" MOUNTPOINT=/mnt/backup TARGET=$MOUNTPOINT/borg # Archive name schema DATE=$(date --iso-8601) # # Create backups # # Options for borg create BORG_OPTS="--stats --compression lz4 --checkpoint-interval 86400" # No one can answer if Borg asks these questions, it is better to just fail quickly # instead of hanging. export BORG_RELOCATED_REPO_ACCESS_IS_OK=no export BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK=no # Log Borg version borg --version echo "Starting backup for $DATE" echo "# system" borg create $BORG_OPTS \ --exclude /var/cache \ --exclude /var/lib/docker/devicemapper \ --exclude /root/.cache \ --exclude /root/.pyenv \ --exclude /root/.vscode-server \ --exclude /root/.local/share/TabNine \ --exclude 'sh:/home/*/.cache' \ --exclude 'sh:/home/*/.cargo' \ --exclude 'sh:/home/*/.pyenv' \ --exclude 'sh:/home/*/.vscode-server' \ --exclude 'sh:/home/*/.local/share/TabNine' \ --one-file-system \ $TARGET::'{hostname}-system-{now}' \ / /boot echo "# data" borg create $BORG_OPTS \ --exclude 'sh:/mnt/data/nextcloud/appdata_*/preview' \ --exclude 'sh:/mnt/data/nextcloud/appdata_*/dav-photocache' \ $TARGET::'{hostname}-data-{now}' \ /mnt/data echo "# archive" borg create $BORG_OPTS \ $TARGET::'{hostname}-archive-{now}' \ /mnt/archive echo "# ftl" borg create $BORG_OPTS \ $TARGET::'{hostname}-ftl-{now}' \ /mnt/ftl echo "Start pruning" BORG_PRUNE_OPTS_NORMAL="--list --stats --keep-daily 7 --keep-weekly 3 --keep-monthly 2" BORG_PRUNE_OPTS_LESS="--list --stats --keep-daily 3 --keep-weekly 1 --keep-monthly 1" borg prune $BORG_PRUNE_OPTS_NORMAL --prefix '{hostname}-system-' $TARGET borg prune $BORG_PRUNE_OPTS_NORMAL --prefix '{hostname}-archive-' $TARGET borg prune $BORG_PRUNE_OPTS_LESS --prefix '{hostname}-data-' $TARGET borg prune $BORG_PRUNE_OPTS_LESS --prefix '{hostname}-ftl-' $TARGET echo "Completed backup for $DATE" # Just to be completely paranoid sync
ln -sf /etc/backups/borg.{service,timer} /etc/systemd/system/
systemctl enable --now borg


pacman -S kubeadm kubelet kubectl
systemctl enable --now kubelet
kubeadm init --pod-network-cidr=''
cp /etc/kubernetes/admin.conf ~/.kube/config

kubectl taint nodes --all # to allow allocating pods to the master node

# setup flannel network manager
kubectl apply -f

# setup nginx ingress controller

kubectl cluster-info
kubectl get nodes
kubectl get pods -A
kubectl get cm -n kube-system kubeadm-config -o yaml

wildcard certs

pacman -S certbot certbot-dns-cloudflare
echo "dns_cloudflare_api_token = <token>" > ~/.secrets/certbot/cloudflare.ini
chmod 600 ~/.secrets/certbot/cloudflare.ini
certbot certonly \
  --email [email protected] \
  --agree-tos \
  --dns-cloudflare \
  --dns-cloudflare-credentials ~/.secrets/certbot/cloudflare.ini \
  -d "*"
openssl x509 -in /etc/letsencrypt/live/ -text
certbot certificates
[Unit] Description=Let's Encrypt renewal [Service] Type=oneshot ExecStart=/usr/bin/certbot renew --quiet --agree-tos --deploy-hook "docker exec nginx-proxy-le /app/signal_le_service"
[Unit] Description=Twice daily renewal of Let's Encrypt's certificates [Timer] OnCalendar=0/12:00:00 RandomizedDelaySec=1h Persistent=true [Install]


pacman -S alsa-utils # maybe requires reboot
usermod -aG audio uetchy

# list devices as root
aplay -l
arecord -L
cat /proc/asound/cards

# test speaker
speaker-test -c2

# test mic
arecord -vv -Dhw:2,0 -fS32_LE mic.wav
aplay mic.wav

# gui mixer

# for
pacman -S pulseaudio pulsemixer
pulseaudio --start
pacmd list-cards
# INPUT/RECORD load-module module-alsa-source device="default" tsched=1 # OUTPUT/PLAYBACK load-module module-alsa-sink device="default" tsched=1 # Accept clients -- very important load-module module-native-protocol-unix load-module module-native-protocol-tcp
pcm.mic { type hw card M96k rate 44100 format S32_LE } pcm.speaker { type plug slave { pcm "hw:1,0" } } pcm.!default { type asym capture.pcm "mic" playback.pcm "speaker" } #defaults.pcm.card 1 #defaults.ctl.card 1


system healthcheck

systemctl --failed # show failed units
free -h # show memory usage
lsblk -f # show disk usage
networkctl status # show network status
userdbctl # show users
nvidia-smi # verify nvidia cards
htop # show task overview
neofetch # show system info

analyzing logs

journalctl -p err -b-1 -r # show error logs from previous boot in reverse order
journalctl CONTAINER_NAME=service_web_1 # show error from docker container named 'service_web_1'
journalctl -u docker -f # tail docker logs

Common Issues

Longer SSH login (D-bus glitch)

systemctl restart systemd-logind
systemctl restart polkit

Annoying systemd-homed is not available log messages

Move pam_unix before pam_systemd_home.

#%PAM-1.0 auth required preauth # Optionally use requisite above if you do not want to prompt for the password # on locked accounts. auth [success=2 default=ignore] try_first_pass nullok -auth [success=1 default=ignore] auth [default=die] authfail auth optional auth required auth required authsucc # If you drop the above call to the lock will be done also # on non-consecutive authentication failures. account [success=1 default=ignore] -account required account optional account required password [success=1 default=ignore] try_first_pass nullok shadow -password required password optional session required session required session optional

Annoying systemd-journald-audit log


Missing /dev/nvidia-{uvm*,modeset}

This occurs after updating linux kernel.

  1. Reinstall nvidia-container-runtime.
  2. Run docker --rm --gpus all --device /dev/nvidia0 --device /dev/nvidiactl --device /dev/nvidia-modeset --device /dev/nvidia-uvm --device /dev/nvidia-uvm-tools -it nvidia/cuda:10.2-cudnn7-runtime nvidia-smi once.

Useful links