From 19bb9605dae635b28691c4773c9c41e83ba8a928 Mon Sep 17 00:00:00 2001 From: CallMeR <9463297+callmer@user.noreply.gitee.com> Date: Sun, 25 Jun 2023 15:33:49 +0800 Subject: [PATCH] =?UTF-8?q?=E6=9B=B4=E6=96=B0=20PVE=20=E7=B3=BB=E7=BB=9F?= =?UTF-8?q?=E8=B0=83=E6=95=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- 03.PVE系统调整.md | 91 ++++++++++++++------------- 08.PVE开启硬件直通功能.md | 127 -------------------------------------- 2 files changed, 47 insertions(+), 171 deletions(-) delete mode 100644 08.PVE开启硬件直通功能.md diff --git a/03.PVE系统调整.md b/03.PVE系统调整.md index 54f0b27..ea51632 100644 --- a/03.PVE系统调整.md +++ b/03.PVE系统调整.md @@ -384,10 +384,10 @@ Jun 25 14:35:06 node01 systemd[1]: Started apt-daily-upgrade.timer - Daily apt u ```bash ## 编辑 Grub 配置文件 -nano /etc/default/grub +vim /etc/default/grub ``` -在 `nano` 编辑器对话框中修改 `GRUB_CMDLINE_LINUX_DEFAULT` 参数，注意命令中间的空格。 +在 `vim` 编辑器对话框中修改 `GRUB_CMDLINE_LINUX_DEFAULT` 参数，注意命令中间的空格。 ```bash ## Intel 处理器添加参数 @@ -414,19 +414,20 @@ update-grub ```bash ## 编辑系统配置文件 -nano /etc/modules -``` +vim /etc/modules -在配置文件末尾输入以下内容。 - -```txt +## 添加以下系统模块 vfio vfio_iommu_type1 vfio_pci -vfio_virqfd ``` -将配置文件保存，并重启 PVE 服务器。 +使用以下命令更新 `initramfs` ，更新完成后，建议重启 PVE 服务器。 + +```bash +## 更新 initramfs +update-initramfs -u -k all +``` ### 5.3.检查硬件直通 @@ -439,38 +440,38 @@ dmesg | grep -e DMAR -e IOMMU -e AMD-Vi 检查 `IOMMU` 、 `Directed I/O` 或 `Interrupt Remapping` 的启用状态。 -N5105 处理器示例输出如下： +N6005 处理器示例输出如下： ```txt -[ 0.016437] ACPI: DMAR 0x00000000787BF000 000088 (v02 INTEL EDK2 00000002 01000013) -[ 0.016472] ACPI: Reserving DMAR table memory at [mem 0x787bf000-0x787bf087] -[ 0.052437] DMAR: IOMMU enabled -[ 0.144431] DMAR: Host address width 39 -[ 0.144432] DMAR: DRHD base: 0x000000fed90000 flags: 0x0 -[ 0.144439] DMAR: dmar0: reg_base_addr fed90000 ver 4:0 cap 1c0000c40660462 ecap 49e2ff0505e -[ 0.144441] DMAR: DRHD base: 0x000000fed91000 flags: 0x1 -[ 0.144446] DMAR: dmar1: reg_base_addr fed91000 ver 1:0 cap d2008c40660462 ecap f050da -[ 0.144449] DMAR: RMRR base: 0x0000007b800000 end: 0x0000007fbfffff -[ 0.144452] DMAR-IR: IOAPIC id 2 under DRHD base 0xfed91000 IOMMU 1 -[ 0.144453] DMAR-IR: HPET id 0 under DRHD base 0xfed91000 -[ 0.144454] DMAR-IR: Queued invalidation will be enabled to support x2apic and Intr-remapping. -[ 0.146335] DMAR-IR: Enabled IRQ remapping in x2apic mode -[ 0.333509] pci 0000:00:02.0: DMAR: Skip IOMMU disabling for graphics -[ 0.410824] DMAR: No ATSR found -[ 0.410824] DMAR: No SATC found -[ 0.410826] DMAR: IOMMU feature fl1gp_support inconsistent -[ 0.410827] DMAR: IOMMU feature pgsel_inv inconsistent -[ 0.410829] DMAR: IOMMU feature nwfs inconsistent -[ 0.410829] DMAR: IOMMU feature pds inconsistent -[ 0.410830] DMAR: IOMMU feature eafs inconsistent -[ 0.410831] DMAR: IOMMU feature prs inconsistent -[ 0.410831] DMAR: IOMMU feature nest inconsistent -[ 0.410832] DMAR: IOMMU feature mts inconsistent -[ 0.410832] DMAR: IOMMU feature sc_support inconsistent -[ 0.410833] DMAR: IOMMU feature dev_iotlb_support inconsistent -[ 0.410835] DMAR: dmar0: Using Queued invalidation -[ 0.410838] DMAR: dmar1: Using Queued invalidation -[ 0.412755] DMAR: Intel(R) Virtualization Technology for Directed I/O +[ 0.017937] ACPI: DMAR 0x00000000746C2000 000088 (v02 INTEL EDK2 00000002 01000013) +[ 0.017965] ACPI: Reserving DMAR table memory at [mem 0x746c2000-0x746c2087] +[ 0.047812] DMAR: IOMMU enabled +[ 0.116031] DMAR: Host address width 39 +[ 0.116032] DMAR: DRHD base: 0x000000fed90000 flags: 0x0 +[ 0.116038] DMAR: dmar0: reg_base_addr fed90000 ver 4:0 cap 1c0000c40660462 ecap 49e2ff0505e +[ 0.116041] DMAR: DRHD base: 0x000000fed91000 flags: 0x1 +[ 0.116046] DMAR: dmar1: reg_base_addr fed91000 ver 1:0 cap d2008c40660462 ecap f050da +[ 0.116048] DMAR: RMRR base: 0x0000007b800000 end: 0x0000007fbfffff +[ 0.116051] DMAR-IR: IOAPIC id 2 under DRHD base 0xfed91000 IOMMU 1 +[ 0.116052] DMAR-IR: HPET id 0 under DRHD base 0xfed91000 +[ 0.116053] DMAR-IR: Queued invalidation will be enabled to support x2apic and Intr-remapping. +[ 0.117774] DMAR-IR: Enabled IRQ remapping in x2apic mode +[ 1.746431] pci 0000:00:02.0: DMAR: Skip IOMMU disabling for graphics +[ 2.290116] DMAR: No ATSR found +[ 2.290117] DMAR: No SATC found +[ 2.290118] DMAR: IOMMU feature fl1gp_support inconsistent +[ 2.290119] DMAR: IOMMU feature pgsel_inv inconsistent +[ 2.290120] DMAR: IOMMU feature nwfs inconsistent +[ 2.290121] DMAR: IOMMU feature pds inconsistent +[ 2.290122] DMAR: IOMMU feature eafs inconsistent +[ 2.290123] DMAR: IOMMU feature prs inconsistent +[ 2.290123] DMAR: IOMMU feature nest inconsistent +[ 2.290124] DMAR: IOMMU feature mts inconsistent +[ 2.290124] DMAR: IOMMU feature sc_support inconsistent +[ 2.290125] DMAR: IOMMU feature dev_iotlb_support inconsistent +[ 2.290126] DMAR: dmar0: Using Queued invalidation +[ 2.290129] DMAR: dmar1: Using Queued invalidation +[ 2.290568] DMAR: Intel(R) Virtualization Technology for Directed I/O ``` 检查系统 `IOMMU` 分组。 @@ -487,22 +488,24 @@ find /sys/kernel/iommu_groups/ -type l /sys/kernel/iommu_groups/15/devices/0000:04:00.0 /sys/kernel/iommu_groups/5/devices/0000:00:17.0 /sys/kernel/iommu_groups/13/devices/0000:02:00.0 -/sys/kernel/iommu_groups/3/devices/0000:00:14.2 -/sys/kernel/iommu_groups/3/devices/0000:00:14.0 +/sys/kernel/iommu_groups/3/devices/0000:00:15.2 +/sys/kernel/iommu_groups/3/devices/0000:00:15.0 /sys/kernel/iommu_groups/11/devices/0000:00:1f.0 /sys/kernel/iommu_groups/11/devices/0000:00:1f.5 /sys/kernel/iommu_groups/11/devices/0000:00:1f.3 /sys/kernel/iommu_groups/11/devices/0000:00:1f.4 -/sys/kernel/iommu_groups/1/devices/0000:00:02.0 +/sys/kernel/iommu_groups/1/devices/0000:00:00.0 /sys/kernel/iommu_groups/8/devices/0000:00:1c.5 /sys/kernel/iommu_groups/16/devices/0000:05:00.0 /sys/kernel/iommu_groups/6/devices/0000:00:1c.0 /sys/kernel/iommu_groups/14/devices/0000:03:00.0 /sys/kernel/iommu_groups/4/devices/0000:00:16.0 /sys/kernel/iommu_groups/12/devices/0000:01:00.0 -/sys/kernel/iommu_groups/2/devices/0000:00:04.0 +/sys/kernel/iommu_groups/2/devices/0000:00:14.5 +/sys/kernel/iommu_groups/2/devices/0000:00:14.2 +/sys/kernel/iommu_groups/2/devices/0000:00:14.0 /sys/kernel/iommu_groups/10/devices/0000:00:1c.7 -/sys/kernel/iommu_groups/0/devices/0000:00:00.0 +/sys/kernel/iommu_groups/0/devices/0000:00:02.0 /sys/kernel/iommu_groups/9/devices/0000:00:1c.6 ``` diff --git a/08.PVE开启硬件直通功能.md b/08.PVE开启硬件直通功能.md deleted file mode 100644 index fb6fc0f..0000000 --- a/08.PVE开启硬件直通功能.md +++ /dev/null @@ -1,127 +0,0 @@ -## 1.修改系统Grub参数 - -参考官方文档 [qm_pci_passthrough](https://pve.proxmox.com/pve-docs/pve-admin-guide.html#qm_pci_passthrough) 和 [Pci passthrough](https://pve.proxmox.com/wiki/Pci_passthrough) 开启 PVE 硬件直通功能。 - -使用 SSH 工具登录到 PVE 服务器，编辑系统 Grub 的配置文件 `/etc/default/grub` ： - -```bash -## 编辑 Grub 配置文件 -nano /etc/default/grub -``` - -在 `nano` 编辑器对话框中修改 `GRUB_CMDLINE_LINUX_DEFAULT` 参数，注意命令中间的空格： - -```bash -## Intel 处理器添加参数 -GRUB_CMDLINE_LINUX_DEFAULT="quiet intel_iommu=on iommu=pt" -``` - -根据官方文档的说明，AMD 处理器下硬件直通功能将会自动打开，否则需要手动修改 Grub 配置文件： - -```bash -## AMD 处理器添加参数 -GRUB_CMDLINE_LINUX_DEFAULT="quiet amd_iommu=on iommu=pt" -``` - -修改并保存后，需要更新系统 Grub ： - -```bash -## 更新系统 Grub -update-grub -``` - -## 2.加载系统内核模块 - -修改系统 `/etc/modules` 配置文件，增加必要的系统模块： - -```bash -## 编辑系统配置文件 -nano /etc/modules -``` - -在配置文件末尾输入以下内容： - -```txt -vfio -vfio_iommu_type1 -vfio_pci -vfio_virqfd -``` - -将配置文件保存，并重启 PVE 服务器。 - -PVE 服务器重启完成后，再次使用 SSH 工具登录，并使用以下命令检查硬件直通状态： - -```bash -## 检查系统硬件直通状态 -dmesg | grep -e DMAR -e IOMMU -e AMD-Vi -``` - -检查 `IOMMU` 、 `Directed I/O` 或 `Interrupt Remapping` 的启用状态。 - -N5105 处理器示例输出如下： - -```txt -[ 0.016437] ACPI: DMAR 0x00000000787BF000 000088 (v02 INTEL EDK2 00000002 01000013) -[ 0.016472] ACPI: Reserving DMAR table memory at [mem 0x787bf000-0x787bf087] -[ 0.052437] DMAR: IOMMU enabled -[ 0.144431] DMAR: Host address width 39 -[ 0.144432] DMAR: DRHD base: 0x000000fed90000 flags: 0x0 -[ 0.144439] DMAR: dmar0: reg_base_addr fed90000 ver 4:0 cap 1c0000c40660462 ecap 49e2ff0505e -[ 0.144441] DMAR: DRHD base: 0x000000fed91000 flags: 0x1 -[ 0.144446] DMAR: dmar1: reg_base_addr fed91000 ver 1:0 cap d2008c40660462 ecap f050da -[ 0.144449] DMAR: RMRR base: 0x0000007b800000 end: 0x0000007fbfffff -[ 0.144452] DMAR-IR: IOAPIC id 2 under DRHD base 0xfed91000 IOMMU 1 -[ 0.144453] DMAR-IR: HPET id 0 under DRHD base 0xfed91000 -[ 0.144454] DMAR-IR: Queued invalidation will be enabled to support x2apic and Intr-remapping. -[ 0.146335] DMAR-IR: Enabled IRQ remapping in x2apic mode -[ 0.333509] pci 0000:00:02.0: DMAR: Skip IOMMU disabling for graphics -[ 0.410824] DMAR: No ATSR found -[ 0.410824] DMAR: No SATC found -[ 0.410826] DMAR: IOMMU feature fl1gp_support inconsistent -[ 0.410827] DMAR: IOMMU feature pgsel_inv inconsistent -[ 0.410829] DMAR: IOMMU feature nwfs inconsistent -[ 0.410829] DMAR: IOMMU feature pds inconsistent -[ 0.410830] DMAR: IOMMU feature eafs inconsistent -[ 0.410831] DMAR: IOMMU feature prs inconsistent -[ 0.410831] DMAR: IOMMU feature nest inconsistent -[ 0.410832] DMAR: IOMMU feature mts inconsistent -[ 0.410832] DMAR: IOMMU feature sc_support inconsistent -[ 0.410833] DMAR: IOMMU feature dev_iotlb_support inconsistent -[ 0.410835] DMAR: dmar0: Using Queued invalidation -[ 0.410838] DMAR: dmar1: Using Queued invalidation -[ 0.412755] DMAR: Intel(R) Virtualization Technology for Directed I/O -``` - -检查系统 `IOMMU` 分组： - -```bash -## 检查 IOMMU group -find /sys/kernel/iommu_groups/ -type l -``` - -示例输出如下： - -```txt -/sys/kernel/iommu_groups/7/devices/0000:00:1c.4 -/sys/kernel/iommu_groups/15/devices/0000:04:00.0 -/sys/kernel/iommu_groups/5/devices/0000:00:17.0 -/sys/kernel/iommu_groups/13/devices/0000:02:00.0 -/sys/kernel/iommu_groups/3/devices/0000:00:14.2 -/sys/kernel/iommu_groups/3/devices/0000:00:14.0 -/sys/kernel/iommu_groups/11/devices/0000:00:1f.0 -/sys/kernel/iommu_groups/11/devices/0000:00:1f.5 -/sys/kernel/iommu_groups/11/devices/0000:00:1f.3 -/sys/kernel/iommu_groups/11/devices/0000:00:1f.4 -/sys/kernel/iommu_groups/1/devices/0000:00:02.0 -/sys/kernel/iommu_groups/8/devices/0000:00:1c.5 -/sys/kernel/iommu_groups/16/devices/0000:05:00.0 -/sys/kernel/iommu_groups/6/devices/0000:00:1c.0 -/sys/kernel/iommu_groups/14/devices/0000:03:00.0 -/sys/kernel/iommu_groups/4/devices/0000:00:16.0 -/sys/kernel/iommu_groups/12/devices/0000:01:00.0 -/sys/kernel/iommu_groups/2/devices/0000:00:04.0 -/sys/kernel/iommu_groups/10/devices/0000:00:1c.7 -/sys/kernel/iommu_groups/0/devices/0000:00:00.0 -/sys/kernel/iommu_groups/9/devices/0000:00:1c.6 -``` \ No newline at end of file