# System update
sudo apt update && sudo apt upgrade -y

# Install essential build tools
sudo apt install -y build-essential gcc g++ make cmake

# Install kernel headers (required for NVIDIA driver build)
sudo apt install -y linux-headers-$(uname -r)

# Network tools
sudo apt install -y net-tools curl wget git vim htop

# Set timezone
sudo timedatectl set-timezone Asia/Seoul

# Check Secure Boot status
mokutil --sb-state

# Completely remove existing NVIDIA-related packages
sudo apt-get purge nvidia* -y
sudo apt-get autoremove -y
sudo apt-get autoclean

# Check available driver list
sudo ubuntu-drivers devices

# Auto-install recommended driver
sudo ubuntu-drivers autoinstall

# Add NVIDIA official repository key and source
sudo apt install -y software-properties-common
sudo add-apt-repository ppa:graphics-drivers/ppa -y
sudo apt update

# Install specific version (e.g., version 550)
sudo apt install -y nvidia-driver-550

# Reboot
sudo reboot

nvidia-smi

# Disable Nouveau driver
sudo bash -c "echo 'blacklist nouveau' >> /etc/modprobe.d/blacklist-nouveau.conf"
sudo bash -c "echo 'options nouveau modeset=0' >> /etc/modprobe.d/blacklist-nouveau.conf"
sudo update-initramfs -u
sudo reboot

# Stop GUI environment (if desktop environment is installed on the server)
sudo systemctl stop gdm3
# or
sudo systemctl stop lightdm

# Run .run file
chmod +x NVIDIA-Linux-x86_64-550.120.run
sudo ./NVIDIA-Linux-x86_64-550.120.run

# Check available server drivers
apt list --installed 2>/dev/null | grep nvidia-driver
apt-cache search nvidia-driver | grep server

# Verify CUDA-capable GPU
lspci | grep -i nvidia

# Verify gcc installation
gcc --version

# Verify kernel headers
uname -r
sudo apt install -y linux-headers-$(uname -r)

# Install CUDA keyring package
wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.1-1_all.deb
sudo dpkg -i cuda-keyring_1.1-1_all.deb
sudo apt-get update

# Install CUDA Toolkit (latest version)
sudo apt-get install -y cuda-toolkit

# Or install a specific version
sudo apt-get install -y cuda-toolkit-12-6

# Add to ~/.bashrc or ~/.zshrc
export PATH=/usr/local/cuda/bin:$PATH
export LD_LIBRARY_PATH=/usr/local/cuda/lib64:$LD_LIBRARY_PATH

# Apply
source ~/.bashrc

# Install multiple versions (e.g., 12.4 and 12.6)
sudo apt-get install -y cuda-toolkit-12-4
sudo apt-get install -y cuda-toolkit-12-6

# Verify installations under /usr/local/
ls /usr/local/ | grep cuda
# cuda -> cuda-12.6 (symbolic link)
# cuda-12.4
# cuda-12.6

# Register version switching with update-alternatives
sudo update-alternatives --install /usr/local/cuda cuda /usr/local/cuda-12.4 10
sudo update-alternatives --install /usr/local/cuda cuda /usr/local/cuda-12.6 20

# Switch versions
sudo update-alternatives --config cuda

# Verify installation
nvcc --version

# Check the CUDA version supported by the driver
nvidia-smi | head -3

# Check the actually installed CUDA Toolkit version
nvcc --version

# If CUDA keyring is already installed, you can install the cuDNN package directly
sudo apt-get install -y cudnn

# Install cuDNN for a specific CUDA version
sudo apt-get install -y cudnn-cuda-12

# After downloading from the NVIDIA Developer site
tar -xvf cudnn-linux-x86_64-9.x.x.x_cudaXX-archive.tar.xz

# Copy files
sudo cp cudnn-linux-x86_64-9.x.x.x_cudaXX-archive/include/cudnn*.h /usr/local/cuda/include
sudo cp -P cudnn-linux-x86_64-9.x.x.x_cudaXX-archive/lib/libcudnn* /usr/local/cuda/lib64
sudo chmod a+r /usr/local/cuda/include/cudnn*.h /usr/local/cuda/lib64/libcudnn*

# Check cuDNN version (when installed via apt)
dpkg -l | grep cudnn

# Or check directly from the header file
cat /usr/local/cuda/include/cudnn_version.h | grep CUDNN_MAJOR -A 2

# Install Miniforge (includes Mamba)
wget https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Linux-x86_64.sh
bash Miniforge3-Linux-x86_64.sh -b -p $HOME/miniforge3

# Set environment variables
$HOME/miniforge3/bin/mamba init bash
source ~/.bashrc

# Create deep learning environment
mamba create -n dl python=3.11 -y
mamba activate dl

# Install PyTorch (for CUDA 12.4 - check the official site for the latest command)
pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu124

# Verify installation
import torch
print(f"PyTorch version: {torch.__version__}")
print(f"CUDA available: {torch.cuda.is_available()}")
print(f"CUDA version: {torch.version.cuda}")
print(f"cuDNN version: {torch.backends.cudnn.version()}")
print(f"GPU: {torch.cuda.get_device_name(0)}")

# Install TensorFlow (with GPU support)
pip install tensorflow[and-cuda]

# Verify installation
import tensorflow as tf
print(f"TensorFlow version: {tf.__version__}")
print(f"GPU devices: {tf.config.list_physical_devices('GPU')}")

# Set up Docker official repository
sudo apt-get update
sudo apt-get install -y ca-certificates curl gnupg

sudo install -m 0755 -d /etc/apt/keyrings
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg
sudo chmod a+r /etc/apt/keyrings/docker.gpg

echo \
  "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \
  $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \
  sudo tee /etc/apt/sources.list.d/docker.list > /dev/null

sudo apt-get update
sudo apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin

# Add current user to docker group (to use without sudo)
sudo usermod -aG docker $USER
newgrp docker

# Set up NVIDIA Container Toolkit repository
curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey | \
  sudo gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg

curl -s -L https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list | \
  sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g' | \
  sudo tee /etc/apt/sources.list.d/nvidia-container-toolkit.list

# Install packages
sudo apt-get update
sudo apt-get install -y nvidia-container-toolkit

# Configure Docker runtime
sudo nvidia-ctk runtime configure --runtime=docker

# Restart Docker
sudo systemctl restart docker

# Test GPU access
docker run --rm --gpus all nvidia/cuda:12.6.0-base-ubuntu22.04 nvidia-smi

# Use specific GPU only
docker run --rm --gpus '"device=0"' nvidia/cuda:12.6.0-base-ubuntu22.04 nvidia-smi

# Run PyTorch container with all GPUs
docker run --rm --gpus all -it pytorch/pytorch:2.5.1-cuda12.4-cudnn9-runtime python -c \
  "import torch; print(torch.cuda.is_available()); print(torch.cuda.get_device_name(0))"

# docker-compose.yml
services:
  training:
    image: pytorch/pytorch:2.5.1-cuda12.4-cudnn9-runtime
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: all # or a specific number: 1, 2, etc.
              capabilities: [gpu]
    volumes:
      - ./data:/workspace/data
      - ./models:/workspace/models
    shm_size: '8g' # PyTorch DataLoader shared memory

# Basic status check
nvidia-smi

# Real-time monitoring (1-second interval)
watch -n 1 nvidia-smi

# Query specific information
nvidia-smi --query-gpu=name,temperature.gpu,utilization.gpu,memory.used,memory.total --format=csv

# Per-process GPU usage
nvidia-smi pmon -s um -d 1

# Enable Persistence Mode (recommended for server environments)
sudo nvidia-smi -pm 1

# Install
sudo apt install -y nvtop

# Run
nvtop

pip install gpustat

# Real-time monitoring
gpustat -i 1 --color

# Install OpenSSH server
sudo apt install -y openssh-server

# Start SSH service and enable auto-start
sudo systemctl enable ssh
sudo systemctl start ssh

# ~/.ssh/config
Host gpu-server
    HostName 192.168.1.100
    User username
    Port 22
    IdentityFile ~/.ssh/id_ed25519
    ForwardAgent yes

# Install tmux
sudo apt install -y tmux

# Create new session
tmux new -s training

# Detach session: Ctrl+b, d

# Reattach to session
tmux attach -t training

# List sessions
tmux ls

# Jupyter Notebook port forwarding (remote 8888 -> local 8888)
ssh -L 8888:localhost:8888 gpu-server

# TensorBoard port forwarding (remote 6006 -> local 6006)
ssh -L 6006:localhost:6006 gpu-server

# Forward multiple ports simultaneously
ssh -L 8888:localhost:8888 -L 6006:localhost:6006 gpu-server

# Generate key on client
ssh-keygen -t ed25519 -C "your_email@example.com"

# Copy public key to server
ssh-copy-id -i ~/.ssh/id_ed25519.pub user@gpu-server

# Disable password authentication on server
sudo vim /etc/ssh/sshd_config

PasswordAuthentication no
PubkeyAuthentication yes
PermitRootLogin no

# Restart SSH service
sudo systemctl restart sshd

# Install and enable UFW
sudo apt install -y ufw

# Default policy: deny incoming, allow outgoing
sudo ufw default deny incoming
sudo ufw default allow outgoing

# Allow SSH
sudo ufw allow ssh

# Allow SSH from specific IP only (more secure)
sudo ufw allow from 192.168.1.0/24 to any port 22

# Enable firewall
sudo ufw enable

# Check status
sudo ufw status verbose

# Install
sudo apt install -y fail2ban

# Copy configuration file
sudo cp /etc/fail2ban/jail.conf /etc/fail2ban/jail.local

# Modify [sshd] section in /etc/fail2ban/jail.local
sudo vim /etc/fail2ban/jail.local

[sshd]
enabled = true
port = ssh
filter = sshd
logpath = /var/log/auth.log
maxretry = 3
bantime = 3600
findtime = 600

# Start fail2ban
sudo systemctl enable fail2ban
sudo systemctl start fail2ban

# Check status
sudo fail2ban-client status sshd

# inventory.ini
[gpu_servers]
gpu-server-01 ansible_host=192.168.1.101
gpu-server-02 ansible_host=192.168.1.102

[gpu_servers:vars]
ansible_user=ubuntu
ansible_ssh_private_key_file=~/.ssh/id_ed25519

# gpu_server_setup.yml
---
- name: GPU Server Initial Setup
  hosts: gpu_servers
  become: true
  vars:
    nvidia_driver_version: '550'
    cuda_keyring_url: 'https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.1-1_all.deb'

  tasks:
    # ===== System Basic Setup =====
    - name: Update and upgrade apt packages
      apt:
        update_cache: yes
        upgrade: dist
        cache_valid_time: 3600

    - name: Install essential packages
      apt:
        name:
          - build-essential
          - gcc
          - g++
          - make
          - cmake
          - linux-headers-{{ ansible_kernel }}
          - curl
          - wget
          - git
          - vim
          - htop
          - tmux
          - nvtop
          - net-tools
          - software-properties-common
        state: present

    - name: Set timezone to Asia/Seoul
      community.general.timezone:
        name: Asia/Seoul

    # ===== NVIDIA Driver Installation =====
    - name: Purge existing NVIDIA drivers
      apt:
        name: 'nvidia*'
        state: absent
        purge: yes
      ignore_errors: yes

    - name: Add NVIDIA PPA
      apt_repository:
        repo: ppa:graphics-drivers/ppa
        state: present

    - name: Install NVIDIA driver
      apt:
        name: 'nvidia-driver-{{ nvidia_driver_version }}'
        state: present
        update_cache: yes

    # ===== CUDA Toolkit Installation =====
    - name: Download CUDA keyring
      get_url:
        url: '{{ cuda_keyring_url }}'
        dest: /tmp/cuda-keyring.deb

    - name: Install CUDA keyring
      apt:
        deb: /tmp/cuda-keyring.deb

    - name: Install CUDA Toolkit
      apt:
        name: cuda-toolkit
        state: present
        update_cache: yes

    # ===== cuDNN Installation =====
    - name: Install cuDNN
      apt:
        name: cudnn-cuda-12
        state: present

    # ===== Docker Installation =====
    - name: Install Docker prerequisites
      apt:
        name:
          - ca-certificates
          - curl
          - gnupg
        state: present

    - name: Add Docker GPG key
      shell: |
        install -m 0755 -d /etc/apt/keyrings
        curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg
        chmod a+r /etc/apt/keyrings/docker.gpg
      args:
        creates: /etc/apt/keyrings/docker.gpg

    - name: Add Docker repository
      shell: |
        echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu $(. /etc/os-release && echo $VERSION_CODENAME) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null
      args:
        creates: /etc/apt/sources.list.d/docker.list

    - name: Install Docker
      apt:
        name:
          - docker-ce
          - docker-ce-cli
          - containerd.io
          - docker-buildx-plugin
          - docker-compose-plugin
        state: present
        update_cache: yes

    - name: Add user to docker group
      user:
        name: '{{ ansible_user }}'
        groups: docker
        append: yes

    # ===== NVIDIA Container Toolkit Installation =====
    - name: Add NVIDIA Container Toolkit GPG key
      shell: |
        curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey | gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg
      args:
        creates: /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg

    - name: Add NVIDIA Container Toolkit repository
      shell: |
        curl -s -L https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list | sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g' | tee /etc/apt/sources.list.d/nvidia-container-toolkit.list
      args:
        creates: /etc/apt/sources.list.d/nvidia-container-toolkit.list

    - name: Install NVIDIA Container Toolkit
      apt:
        name: nvidia-container-toolkit
        state: present
        update_cache: yes

    - name: Configure Docker runtime for NVIDIA
      command: nvidia-ctk runtime configure --runtime=docker

    - name: Restart Docker
      systemd:
        name: docker
        state: restarted

    # ===== Security Settings =====
    - name: Install security packages
      apt:
        name:
          - ufw
          - fail2ban
        state: present

    - name: Configure UFW - deny incoming
      ufw:
        direction: incoming
        policy: deny

    - name: Configure UFW - allow outgoing
      ufw:
        direction: outgoing
        policy: allow

    - name: Configure UFW - allow SSH
      ufw:
        rule: allow
        name: OpenSSH

    - name: Enable UFW
      ufw:
        state: enabled

    - name: Enable fail2ban
      systemd:
        name: fail2ban
        enabled: yes
        state: started

    # ===== GPU Persistence Mode =====
    - name: Enable NVIDIA Persistence Mode
      command: nvidia-smi -pm 1
      ignore_errors: yes

  handlers:
    - name: Reboot server
      reboot:
        reboot_timeout: 300

# Install Ansible
pip install ansible

# Run Playbook
ansible-playbook -i inventory.ini gpu_server_setup.yml

# Dry run (check without actually executing)
ansible-playbook -i inventory.ini gpu_server_setup.yml --check

1. Hardware assembly and BIOS setup (disable Secure Boot)
    |
2. Ubuntu 22.04/24.04 installation and basic package setup
    |
3. NVIDIA driver installation (apt method recommended)
    |
4. CUDA Toolkit installation (multiple versions can be installed)
    |
5. cuDNN installation
    |
6. Conda/Mamba environment setup -> PyTorch/TensorFlow installation
    |
7. Docker + NVIDIA Container Toolkit installation
    |
8. Security settings (SSH Key, UFW, fail2ban)
    |
9. Monitoring tools installation (nvtop, gpustat)
    |
10. Remote development environment setup (VS Code Remote, tmux)