Debugging

# Attach to VM console
noid console my-vm

# Press Ctrl+] to exit

# View boot messages
noid exec my-vm -- dmesg

# Check system logs
noid exec my-vm -- journalctl -xe

# Kernel messages
noid exec my-vm -- tail -f /var/log/kern.log

# List all processes
noid exec my-vm -- ps aux

# Process tree
noid exec my-vm -- pstree -p

# Top processes by CPU
noid exec my-vm -- top -b -n 1 | head -20

# Top processes by memory
noid exec my-vm -- ps aux --sort=-%mem | head -10

# Attach to process with strace
noid exec my-vm -- strace -p <pid>

# Get stack trace
noid exec my-vm -- pstack <pid>

# Memory map
noid exec my-vm -- pmap <pid>

# Open files
noid exec my-vm -- lsof -p <pid>

# Test external connectivity
noid exec my-vm -- ping -c 3 8.8.8.8

# DNS resolution
noid exec my-vm -- nslookup google.com

# Test specific port
noid exec my-vm -- nc -zv example.com 443

# Trace route
noid exec my-vm -- traceroute google.com

# Check IP address
noid exec my-vm -- ip addr show

# Routing table
noid exec my-vm -- ip route show

# ARP cache
noid exec my-vm -- arp -a

# Network interfaces
noid exec my-vm -- ifconfig -a

# Capture packets
noid exec my-vm -- tcpdump -i eth0 -n

# HTTP traffic
noid exec my-vm -- tcpdump -i eth0 -A 'tcp port 80'

# DNS queries
noid exec my-vm -- tcpdump -i eth0 'udp port 53'

# Network statistics
noid exec my-vm -- netstat -s

# Filesystem usage
noid exec my-vm -- df -h

# Directory sizes
noid exec my-vm -- du -sh /*

# Find large files
noid exec my-vm -- find / -type f -size +100M -exec ls -lh {} \\;

# Inode usage
noid exec my-vm -- df -i

# Find recently modified files
noid exec my-vm -- find /var/log -mtime -1 -type f

# Search file content
noid exec my-vm -- grep -r "error" /var/log/

# Check file permissions
noid exec my-vm -- ls -la /path/to/file

# File type
noid exec my-vm -- file /path/to/file

# Memory statistics
noid exec my-vm -- free -h

# Detailed memory info
noid exec my-vm -- cat /proc/meminfo

# Process memory
noid exec my-vm -- ps aux --sort=-%mem | head -20

# Memory map
noid exec my-vm -- cat /proc/<pid>/maps

# Install valgrind
noid exec my-vm -- apt-get install -y valgrind

# Check for memory leaks
noid exec my-vm -- valgrind --leak-check=full ./your-app

# Monitor memory over time
noid exec my-vm -- sh -c "
  while true; do
    free -m | grep Mem
    sleep 5
  done
"

# Application logs
noid exec my-vm -- tail -f /var/log/app.log

# Filter errors
noid exec my-vm -- grep ERROR /var/log/app.log

# Last 100 errors
noid exec my-vm -- grep ERROR /var/log/app.log | tail -100

# Real-time error monitoring
noid exec my-vm -- tail -f /var/log/app.log | grep --color=always ERROR

# List all environment variables
noid exec my-vm -- printenv

# Check specific variable
noid exec my-vm -- echo \$PATH

# Set and test variable
noid exec my-vm -e DEBUG=true -- ./run-app

# Enable core dumps
noid exec my-vm -- ulimit -c unlimited

# Core dump location
noid exec my-vm -- cat /proc/sys/kernel/core_pattern

# Analyze core dump with gdb
noid exec my-vm -- gdb ./app core

# CPU usage over time
noid exec my-vm -- mpstat 1 10

# Per-CPU statistics
noid exec my-vm -- mpstat -P ALL

# I/O wait time
noid exec my-vm -- iostat -x 1 10

# Profile Python application
noid exec my-vm -- python3 -m cProfile app.py

# Profile Node.js application
noid exec my-vm -- node --prof app.js

# Profile with perf
noid exec my-vm -- perf record -g ./app
noid exec my-vm -- perf report

# View Noid server logs
sudo journalctl -u noid-server -f

# Filter errors
sudo journalctl -u noid-server | grep ERROR

# Last 100 lines
sudo journalctl -u noid-server -n 100

# Specific time range
sudo journalctl -u noid-server --since "1 hour ago"

# List Firecracker processes
ps aux | grep firecracker

# Check Firecracker logs
sudo ls -la /var/lib/noid/vms/*/firecracker.log

# Monitor Firecracker API calls
sudo strace -p $(pgrep firecracker) -e trace=network

# System resource usage
htop

# Disk I/O
iotop

# Network bandwidth
iftop

# All processes resource usage
systemd-cgtop

# Check if VM already exists
noid list | grep my-vm

# Check server logs
sudo journalctl -u noid-server -n 50

# Verify Firecracker
firecracker --version

# Check KVM availability
lsmod | grep kvm

# Check if using btrfs
mount | grep noid

# Disk performance
sudo iostat -x 1 10

# Check if golden snapshot exists
ls -la /var/lib/noid/checkpoints/

# Check bridge
ip link show noid0

# Verify NAT rules
sudo iptables -t nat -L

# Check IP pool
# (check server logs for IP allocation)

# Test VM connectivity
noid exec my-vm -- ping -c 3 8.8.8.8

# Create checkpoint before debugging
noid checkpoint my-vm before-debug

# Make changes, test, experiment
noid exec my-vm -- gdb ./app

# If things break, restore
noid restore my-vm before-debug

# If fix works, create new checkpoint
noid checkpoint my-vm fixed

# Create checkpoints at key points
noid checkpoint my-vm checkpoint-1
# ... make changes ...
noid checkpoint my-vm checkpoint-2
# ... make changes ...
noid checkpoint my-vm checkpoint-3

# Issue found, bisect by restoring
noid restore my-vm checkpoint-2
# Test if issue exists

# Narrow down to specific change

# Start debug server in VM
noid exec my-vm -- gdbserver :2345 ./app

# Get VM IP
VM_IP=$(noid list | grep my-vm | awk '{print $4}')

# Connect from host
gdb ./app
(gdb) target remote $VM_IP:2345
(gdb) continue

# Essential debugging tools
noid exec my-vm -- apt-get install -y \
  strace \
  ltrace \
  gdb \
  valgrind \
  tcpdump \
  netcat \
  curl \
  wget \
  vim \
  htop \
  iotop \
  lsof

#!/bin/bash
# debug-vm.sh

VM_NAME=$1

echo "=== System Info ==="
noid exec $VM_NAME -- uname -a

echo -e "\n=== Memory Usage ==="
noid exec $VM_NAME -- free -h

echo -e "\n=== Disk Usage ==="
noid exec $VM_NAME -- df -h

echo -e "\n=== Network ==="
noid exec $VM_NAME -- ip addr show

echo -e "\n=== Running Processes ==="
noid exec $VM_NAME -- ps aux | head -20

echo -e "\n=== Recent Errors ==="
noid exec $VM_NAME -- journalctl -p err -n 10

./debug-vm.sh my-vm