64-bit dma for drm(4) on amd64

classic Classic list List threaded Threaded
4 messages Options
Reply | Threaded
Open this post in threaded view
|

64-bit dma for drm(4) on amd64

Mark Kettenis
As a result of a recent discussion with jsg@, I realized that the
graphics drivers are (mostly) allocating memory from the dma region.
Since the the graphics stack can potentially gobble up large amounts
of memory, this means we can run out of dma memory which makes other
parts of our kernel quite unhappy.  Most of the supported hardware
actually supports 64-bit DMA just fine, and the drivers already have
code to handle the exceptions.  The diff below makes use of this
knowledge to (hopefully) safely allocate from "high" memory when
possible.  One big change is that this makes bus_dma(9) 64-bit DMA
aware in the sense that if the BUS_DMA_64BIT flag is used, we skip the
"not dma-reachable" panic.

It seems to work fine on my Intel Broadwell laptop.  I haven't tested
this on radeon(4) yet.  So further testing, especially on systems with
4GB of memory or more is necessary.

Please test.


Index: arch/amd64/amd64/bus_dma.c
===================================================================
RCS file: /cvs/src/sys/arch/amd64/amd64/bus_dma.c,v
retrieving revision 1.50
diff -u -p -r1.50 bus_dma.c
--- arch/amd64/amd64/bus_dma.c 14 Oct 2017 04:44:43 -0000 1.50
+++ arch/amd64/amd64/bus_dma.c 6 Jun 2019 21:19:21 -0000
@@ -319,7 +319,8 @@ _bus_dmamap_load_raw(bus_dma_tag_t t, bu
  if (plen < sgsize)
  sgsize = plen;
 
- if (paddr > dma_constraint.ucr_high)
+ if (paddr > dma_constraint.ucr_high &&
+    (map->_dm_flags & BUS_DMA_64BIT) == 0)
  panic("Non dma-reachable buffer at paddr %#lx(raw)",
     paddr);
 
@@ -583,7 +584,8 @@ _bus_dmamap_load_buffer(bus_dma_tag_t t,
  */
  pmap_extract(pmap, vaddr, (paddr_t *)&curaddr);
 
- if (curaddr > dma_constraint.ucr_high)
+ if (curaddr > dma_constraint.ucr_high &&
+    (map->_dm_flags & BUS_DMA_64BIT) == 0)
  panic("Non dma-reachable buffer at curaddr %#lx(raw)",
     curaddr);
 
Index: dev/pci/drm/drm_linux.c
===================================================================
RCS file: /cvs/src/sys/dev/pci/drm/drm_linux.c,v
retrieving revision 1.37
diff -u -p -r1.37 drm_linux.c
--- dev/pci/drm/drm_linux.c 4 Jun 2019 12:08:22 -0000 1.37
+++ dev/pci/drm/drm_linux.c 6 Jun 2019 21:19:21 -0000
@@ -293,16 +293,19 @@ struct vm_page *
 alloc_pages(unsigned int gfp_mask, unsigned int order)
 {
  int flags = (gfp_mask & M_NOWAIT) ? UVM_PLA_NOWAIT : UVM_PLA_WAITOK;
+ struct uvm_constraint_range *constraint = &no_constraint;
  struct pglist mlist;
 
  if (gfp_mask & M_CANFAIL)
  flags |= UVM_PLA_FAILOK;
  if (gfp_mask & M_ZERO)
  flags |= UVM_PLA_ZERO;
+ if (gfp_mask & __GFP_DMA32)
+ constraint = &dma_constraint;
 
  TAILQ_INIT(&mlist);
- if (uvm_pglistalloc(PAGE_SIZE << order, dma_constraint.ucr_low,
-    dma_constraint.ucr_high, PAGE_SIZE, 0, &mlist, 1, flags))
+ if (uvm_pglistalloc(PAGE_SIZE << order, constraint->ucr_low,
+    constraint->ucr_high, PAGE_SIZE, 0, &mlist, 1, flags))
  return NULL;
  return TAILQ_FIRST(&mlist);
 }
Index: dev/pci/drm/include/linux/gfp.h
===================================================================
RCS file: /cvs/src/sys/dev/pci/drm/include/linux/gfp.h,v
retrieving revision 1.1
diff -u -p -r1.1 gfp.h
--- dev/pci/drm/include/linux/gfp.h 14 Apr 2019 10:14:53 -0000 1.1
+++ dev/pci/drm/include/linux/gfp.h 6 Jun 2019 21:19:21 -0000
@@ -7,24 +7,25 @@
 #include <sys/malloc.h>
 #include <uvm/uvm_extern.h>
 
-#define GFP_ATOMIC M_NOWAIT
-#define GFP_NOWAIT M_NOWAIT
-#define GFP_KERNEL (M_WAITOK | M_CANFAIL)
-#define GFP_USER (M_WAITOK | M_CANFAIL)
-#define GFP_TEMPORARY (M_WAITOK | M_CANFAIL)
-#define GFP_HIGHUSER 0
-#define GFP_DMA32 0
-#define __GFP_NOWARN 0
-#define __GFP_NORETRY 0
-#define __GFP_ZERO M_ZERO
+#define __GFP_ZERO M_ZERO
+#define __GFP_DMA32 0x80000000
+#define __GFP_NOWARN 0
+#define __GFP_NORETRY 0
 #define __GFP_RETRY_MAYFAIL 0
 #define __GFP_MOVABLE 0
 #define __GFP_COMP 0
-#define GFP_TRANSHUGE_LIGHT 0
 #define __GFP_KSWAPD_RECLAIM 0
 #define __GFP_HIGHMEM 0
 #define __GFP_RECLAIMABLE 0
-#define __GFP_DMA32 0
+
+#define GFP_ATOMIC M_NOWAIT
+#define GFP_NOWAIT M_NOWAIT
+#define GFP_KERNEL (M_WAITOK | M_CANFAIL)
+#define GFP_USER (M_WAITOK | M_CANFAIL)
+#define GFP_TEMPORARY (M_WAITOK | M_CANFAIL)
+#define GFP_HIGHUSER 0
+#define GFP_DMA32 __GFP_DMA32
+#define GFP_TRANSHUGE_LIGHT 0
 
 static inline bool
 gfpflags_allow_blocking(const unsigned int flags)
Index: dev/pci/drm/ttm/ttm_tt.c
===================================================================
RCS file: /cvs/src/sys/dev/pci/drm/ttm/ttm_tt.c,v
retrieving revision 1.8
diff -u -p -r1.8 ttm_tt.c
--- dev/pci/drm/ttm/ttm_tt.c 14 Apr 2019 10:14:54 -0000 1.8
+++ dev/pci/drm/ttm/ttm_tt.c 6 Jun 2019 21:19:22 -0000
@@ -261,6 +261,7 @@ int ttm_dma_tt_init(struct ttm_dma_tt *t
     uint32_t page_flags)
 {
  struct ttm_tt *ttm = &ttm_dma->ttm;
+ int flags = BUS_DMA_WAITOK;
 
  ttm_tt_init_fields(ttm, bo, page_flags);
 
@@ -276,8 +277,10 @@ int ttm_dma_tt_init(struct ttm_dma_tt *t
 
  ttm_dma->dmat = bo->bdev->dmat;
 
+ if ((page_flags & TTM_PAGE_FLAG_DMA32) == 0)
+ flags |= BUS_DMA_64BIT;
  if (bus_dmamap_create(ttm_dma->dmat, ttm->num_pages << PAGE_SHIFT,
-    ttm->num_pages, ttm->num_pages << PAGE_SHIFT, 0, BUS_DMA_WAITOK,
+    ttm->num_pages, ttm->num_pages << PAGE_SHIFT, 0, flags,
     &ttm_dma->map)) {
  free(ttm_dma->segs, M_DRM, 0);
  ttm_tt_destroy(ttm);
@@ -293,6 +296,7 @@ int ttm_sg_tt_init(struct ttm_dma_tt *tt
    uint32_t page_flags)
 {
  struct ttm_tt *ttm = &ttm_dma->ttm;
+ int flags = BUS_DMA_WAITOK;
  int ret;
 
  ttm_tt_init_fields(ttm, bo, page_flags);
@@ -313,8 +317,10 @@ int ttm_sg_tt_init(struct ttm_dma_tt *tt
 
  ttm_dma->dmat = bo->bdev->dmat;
 
+ if ((page_flags & TTM_PAGE_FLAG_DMA32) == 0)
+ flags |= BUS_DMA_64BIT;
  if (bus_dmamap_create(ttm_dma->dmat, ttm->num_pages << PAGE_SHIFT,
-    ttm->num_pages, ttm->num_pages << PAGE_SHIFT, 0, BUS_DMA_WAITOK,
+    ttm->num_pages, ttm->num_pages << PAGE_SHIFT, 0, flags,
     &ttm_dma->map)) {
  free(ttm_dma->segs, M_DRM, 0);
  ttm_tt_destroy(ttm);

Reply | Threaded
Open this post in threaded view
|

Re: 64-bit dma for drm(4) on amd64

Thomas Frohwein-2
On Thu, Jun 06, 2019 at 11:41:07PM +0200, Mark Kettenis wrote:
[...]
> It seems to work fine on my Intel Broadwell laptop.  I haven't tested
> this on radeon(4) yet.  So further testing, especially on systems with
> 4GB of memory or more is necessary.
>
> Please test.

Tested on a desktop with 8GB RAM and amdgpu enabled (GPU is a Vega 64,
but that may not be relevant). I'm attaching dmesg below. Tested for
about 2 hours with some Chrome, some Firefox, mono... No regression
noticed. The situations where the display freezes were encountered as
before building the kernel with this patch, that is with Chrome after a
while with Google Maps (one of the most reliable triggers). Also with
`piglit run all results/all` after 39 tests (dmafence on wait channel
again).

If there's more that would help testing or reporting, I'm happy to.

>
>
> Index: arch/amd64/amd64/bus_dma.c
> ===================================================================
> RCS file: /cvs/src/sys/arch/amd64/amd64/bus_dma.c,v
> retrieving revision 1.50
> diff -u -p -r1.50 bus_dma.c
> --- arch/amd64/amd64/bus_dma.c 14 Oct 2017 04:44:43 -0000 1.50
> +++ arch/amd64/amd64/bus_dma.c 6 Jun 2019 21:19:21 -0000
> @@ -319,7 +319,8 @@ _bus_dmamap_load_raw(bus_dma_tag_t t, bu
>   if (plen < sgsize)
>   sgsize = plen;
>  
> - if (paddr > dma_constraint.ucr_high)
> + if (paddr > dma_constraint.ucr_high &&
> +    (map->_dm_flags & BUS_DMA_64BIT) == 0)
>   panic("Non dma-reachable buffer at paddr %#lx(raw)",
>      paddr);
>  
> @@ -583,7 +584,8 @@ _bus_dmamap_load_buffer(bus_dma_tag_t t,
>   */
>   pmap_extract(pmap, vaddr, (paddr_t *)&curaddr);
>  
> - if (curaddr > dma_constraint.ucr_high)
> + if (curaddr > dma_constraint.ucr_high &&
> +    (map->_dm_flags & BUS_DMA_64BIT) == 0)
>   panic("Non dma-reachable buffer at curaddr %#lx(raw)",
>      curaddr);
>  
> Index: dev/pci/drm/drm_linux.c
> ===================================================================
> RCS file: /cvs/src/sys/dev/pci/drm/drm_linux.c,v
> retrieving revision 1.37
> diff -u -p -r1.37 drm_linux.c
> --- dev/pci/drm/drm_linux.c 4 Jun 2019 12:08:22 -0000 1.37
> +++ dev/pci/drm/drm_linux.c 6 Jun 2019 21:19:21 -0000
> @@ -293,16 +293,19 @@ struct vm_page *
>  alloc_pages(unsigned int gfp_mask, unsigned int order)
>  {
>   int flags = (gfp_mask & M_NOWAIT) ? UVM_PLA_NOWAIT : UVM_PLA_WAITOK;
> + struct uvm_constraint_range *constraint = &no_constraint;
>   struct pglist mlist;
>  
>   if (gfp_mask & M_CANFAIL)
>   flags |= UVM_PLA_FAILOK;
>   if (gfp_mask & M_ZERO)
>   flags |= UVM_PLA_ZERO;
> + if (gfp_mask & __GFP_DMA32)
> + constraint = &dma_constraint;
>  
>   TAILQ_INIT(&mlist);
> - if (uvm_pglistalloc(PAGE_SIZE << order, dma_constraint.ucr_low,
> -    dma_constraint.ucr_high, PAGE_SIZE, 0, &mlist, 1, flags))
> + if (uvm_pglistalloc(PAGE_SIZE << order, constraint->ucr_low,
> +    constraint->ucr_high, PAGE_SIZE, 0, &mlist, 1, flags))
>   return NULL;
>   return TAILQ_FIRST(&mlist);
>  }
> Index: dev/pci/drm/include/linux/gfp.h
> ===================================================================
> RCS file: /cvs/src/sys/dev/pci/drm/include/linux/gfp.h,v
> retrieving revision 1.1
> diff -u -p -r1.1 gfp.h
> --- dev/pci/drm/include/linux/gfp.h 14 Apr 2019 10:14:53 -0000 1.1
> +++ dev/pci/drm/include/linux/gfp.h 6 Jun 2019 21:19:21 -0000
> @@ -7,24 +7,25 @@
>  #include <sys/malloc.h>
>  #include <uvm/uvm_extern.h>
>  
> -#define GFP_ATOMIC M_NOWAIT
> -#define GFP_NOWAIT M_NOWAIT
> -#define GFP_KERNEL (M_WAITOK | M_CANFAIL)
> -#define GFP_USER (M_WAITOK | M_CANFAIL)
> -#define GFP_TEMPORARY (M_WAITOK | M_CANFAIL)
> -#define GFP_HIGHUSER 0
> -#define GFP_DMA32 0
> -#define __GFP_NOWARN 0
> -#define __GFP_NORETRY 0
> -#define __GFP_ZERO M_ZERO
> +#define __GFP_ZERO M_ZERO
> +#define __GFP_DMA32 0x80000000
> +#define __GFP_NOWARN 0
> +#define __GFP_NORETRY 0
>  #define __GFP_RETRY_MAYFAIL 0
>  #define __GFP_MOVABLE 0
>  #define __GFP_COMP 0
> -#define GFP_TRANSHUGE_LIGHT 0
>  #define __GFP_KSWAPD_RECLAIM 0
>  #define __GFP_HIGHMEM 0
>  #define __GFP_RECLAIMABLE 0
> -#define __GFP_DMA32 0
> +
> +#define GFP_ATOMIC M_NOWAIT
> +#define GFP_NOWAIT M_NOWAIT
> +#define GFP_KERNEL (M_WAITOK | M_CANFAIL)
> +#define GFP_USER (M_WAITOK | M_CANFAIL)
> +#define GFP_TEMPORARY (M_WAITOK | M_CANFAIL)
> +#define GFP_HIGHUSER 0
> +#define GFP_DMA32 __GFP_DMA32
> +#define GFP_TRANSHUGE_LIGHT 0
>  
>  static inline bool
>  gfpflags_allow_blocking(const unsigned int flags)
> Index: dev/pci/drm/ttm/ttm_tt.c
> ===================================================================
> RCS file: /cvs/src/sys/dev/pci/drm/ttm/ttm_tt.c,v
> retrieving revision 1.8
> diff -u -p -r1.8 ttm_tt.c
> --- dev/pci/drm/ttm/ttm_tt.c 14 Apr 2019 10:14:54 -0000 1.8
> +++ dev/pci/drm/ttm/ttm_tt.c 6 Jun 2019 21:19:22 -0000
> @@ -261,6 +261,7 @@ int ttm_dma_tt_init(struct ttm_dma_tt *t
>      uint32_t page_flags)
>  {
>   struct ttm_tt *ttm = &ttm_dma->ttm;
> + int flags = BUS_DMA_WAITOK;
>  
>   ttm_tt_init_fields(ttm, bo, page_flags);
>  
> @@ -276,8 +277,10 @@ int ttm_dma_tt_init(struct ttm_dma_tt *t
>  
>   ttm_dma->dmat = bo->bdev->dmat;
>  
> + if ((page_flags & TTM_PAGE_FLAG_DMA32) == 0)
> + flags |= BUS_DMA_64BIT;
>   if (bus_dmamap_create(ttm_dma->dmat, ttm->num_pages << PAGE_SHIFT,
> -    ttm->num_pages, ttm->num_pages << PAGE_SHIFT, 0, BUS_DMA_WAITOK,
> +    ttm->num_pages, ttm->num_pages << PAGE_SHIFT, 0, flags,
>      &ttm_dma->map)) {
>   free(ttm_dma->segs, M_DRM, 0);
>   ttm_tt_destroy(ttm);
> @@ -293,6 +296,7 @@ int ttm_sg_tt_init(struct ttm_dma_tt *tt
>     uint32_t page_flags)
>  {
>   struct ttm_tt *ttm = &ttm_dma->ttm;
> + int flags = BUS_DMA_WAITOK;
>   int ret;
>  
>   ttm_tt_init_fields(ttm, bo, page_flags);
> @@ -313,8 +317,10 @@ int ttm_sg_tt_init(struct ttm_dma_tt *tt
>  
>   ttm_dma->dmat = bo->bdev->dmat;
>  
> + if ((page_flags & TTM_PAGE_FLAG_DMA32) == 0)
> + flags |= BUS_DMA_64BIT;
>   if (bus_dmamap_create(ttm_dma->dmat, ttm->num_pages << PAGE_SHIFT,
> -    ttm->num_pages, ttm->num_pages << PAGE_SHIFT, 0, BUS_DMA_WAITOK,
> +    ttm->num_pages, ttm->num_pages << PAGE_SHIFT, 0, flags,
>      &ttm_dma->map)) {
>   free(ttm_dma->segs, M_DRM, 0);
>   ttm_tt_destroy(ttm);
>

OpenBSD 6.5-current (GENERIC.MP) #72: Thu Jun  6 18:02:31 PDT 2019
    [hidden email]:/usr/src/sys/arch/amd64/compile/GENERIC.MP
real mem = 8521498624 (8126MB)
avail mem = 8250621952 (7868MB)
mpath0 at root
scsibus0 at mpath0: 256 targets
mainbus0 at root
bios0 at mainbus0: SMBIOS rev. 2.8 @ 0xe6cf0 (59 entries)
bios0: vendor American Megatrends Inc. version "1.90" date 03/07/2019
bios0: Micro-Star International Co., Ltd. MS-7B79
acpi0 at bios0: rev 2
acpi0: sleep states S0 S4 S5
acpi0: tables DSDT FACP APIC FPDT FIDT SSDT SSDT SSDT MCFG HPET SSDT UEFI IVRS SSDT CRAT CDIT SSDT SSDT WSMT SSDT
acpi0: wakeup devices GPP0(S4) GPP1(S4) GPP3(S4) GPP4(S4) GPP5(S4) GPP6(S4) GPP7(S4) GPP8(S4) GPP9(S4) GPPA(S4) GPPB(S4) GPPC(S4) GPPD(S4) GPPE(S4) GPPF(S4) GP17(S4) [...]
acpitimer0 at acpi0: 3579545 Hz, 32 bits
acpimadt0 at acpi0 addr 0xfee00000: PC-AT compat
cpu0 at mainbus0: apid 0 (boot processor)
cpu0: AMD Ryzen 7 2700 Eight-Core Processor, 3200.58 MHz, 17-08-02
cpu0: FPU,VME,DE,PSE,TSC,MSR,PAE,MCE,CX8,APIC,SEP,MTRR,PGE,MCA,CMOV,PAT,PSE36,CFLUSH,MMX,FXSR,SSE,SSE2,HTT,SSE3,PCLMUL,MWAIT,SSSE3,FMA3,CX16,SSE4.1,SSE4.2,MOVBE,POPCNT,AES,XSAVE,AVX,F16C,RDRAND,NXE,MMXX,FFXSR,PAGE1GB,RDTSCP,LONG,LAHF,CMPLEG,SVM,EAPICSP,AMCR8,ABM,SSE4A,MASSE,3DNOWP,OSVW,SKINIT,TCE,TOPEXT,CPCTR,DBKP,PCTRL3,MWAITX,ITSC,FSGSBASE,BMI1,AVX2,SMEP,BMI2,RDSEED,ADX,SMAP,CLFLUSHOPT,SHA,IBPB,XSAVEOPT,XSAVEC,XGETBV1,XSAVES
cpu0: 64KB 64b/line 4-way I-cache, 32KB 64b/line 8-way D-cache, 512KB 64b/line 8-way L2 cache, 16MB 64b/line 16-way L3 cache
cpu0: ITLB 64 4KB entries fully associative, 64 4MB entries fully associative
cpu0: DTLB 64 4KB entries fully associative, 64 4MB entries fully associative
cpu0: smt 0, core 0, package 0
mtrr: Pentium Pro MTRR support, 8 var ranges, 88 fixed ranges
cpu0: apic clock running at 99MHz
cpu0: mwait min=64, max=64, C-substates=1.1, IBE
cpu1 at mainbus0: apid 1 (application processor)
cpu1: AMD Ryzen 7 2700 Eight-Core Processor, 3199.99 MHz, 17-08-02
cpu1: FPU,VME,DE,PSE,TSC,MSR,PAE,MCE,CX8,APIC,SEP,MTRR,PGE,MCA,CMOV,PAT,PSE36,CFLUSH,MMX,FXSR,SSE,SSE2,HTT,SSE3,PCLMUL,MWAIT,SSSE3,FMA3,CX16,SSE4.1,SSE4.2,MOVBE,POPCNT,AES,XSAVE,AVX,F16C,RDRAND,NXE,MMXX,FFXSR,PAGE1GB,RDTSCP,LONG,LAHF,CMPLEG,SVM,EAPICSP,AMCR8,ABM,SSE4A,MASSE,3DNOWP,OSVW,SKINIT,TCE,TOPEXT,CPCTR,DBKP,PCTRL3,MWAITX,ITSC,FSGSBASE,BMI1,AVX2,SMEP,BMI2,RDSEED,ADX,SMAP,CLFLUSHOPT,SHA,IBPB,XSAVEOPT,XSAVEC,XGETBV1,XSAVES
cpu1: 64KB 64b/line 4-way I-cache, 32KB 64b/line 8-way D-cache, 512KB 64b/line 8-way L2 cache, 16MB 64b/line 16-way L3 cache
cpu1: ITLB 64 4KB entries fully associative, 64 4MB entries fully associative
cpu1: DTLB 64 4KB entries fully associative, 64 4MB entries fully associative
cpu1: smt 0, core 1, package 0
cpu2 at mainbus0: apid 2 (application processor)
cpu2: AMD Ryzen 7 2700 Eight-Core Processor, 3199.99 MHz, 17-08-02
cpu2: FPU,VME,DE,PSE,TSC,MSR,PAE,MCE,CX8,APIC,SEP,MTRR,PGE,MCA,CMOV,PAT,PSE36,CFLUSH,MMX,FXSR,SSE,SSE2,HTT,SSE3,PCLMUL,MWAIT,SSSE3,FMA3,CX16,SSE4.1,SSE4.2,MOVBE,POPCNT,AES,XSAVE,AVX,F16C,RDRAND,NXE,MMXX,FFXSR,PAGE1GB,RDTSCP,LONG,LAHF,CMPLEG,SVM,EAPICSP,AMCR8,ABM,SSE4A,MASSE,3DNOWP,OSVW,SKINIT,TCE,TOPEXT,CPCTR,DBKP,PCTRL3,MWAITX,ITSC,FSGSBASE,BMI1,AVX2,SMEP,BMI2,RDSEED,ADX,SMAP,CLFLUSHOPT,SHA,IBPB,XSAVEOPT,XSAVEC,XGETBV1,XSAVES
cpu2: 64KB 64b/line 4-way I-cache, 32KB 64b/line 8-way D-cache, 512KB 64b/line 8-way L2 cache, 16MB 64b/line 16-way L3 cache
cpu2: ITLB 64 4KB entries fully associative, 64 4MB entries fully associative
cpu2: DTLB 64 4KB entries fully associative, 64 4MB entries fully associative
cpu2: smt 0, core 2, package 0
cpu3 at mainbus0: apid 3 (application processor)
cpu3: AMD Ryzen 7 2700 Eight-Core Processor, 3199.99 MHz, 17-08-02
cpu3: FPU,VME,DE,PSE,TSC,MSR,PAE,MCE,CX8,APIC,SEP,MTRR,PGE,MCA,CMOV,PAT,PSE36,CFLUSH,MMX,FXSR,SSE,SSE2,HTT,SSE3,PCLMUL,MWAIT,SSSE3,FMA3,CX16,SSE4.1,SSE4.2,MOVBE,POPCNT,AES,XSAVE,AVX,F16C,RDRAND,NXE,MMXX,FFXSR,PAGE1GB,RDTSCP,LONG,LAHF,CMPLEG,SVM,EAPICSP,AMCR8,ABM,SSE4A,MASSE,3DNOWP,OSVW,SKINIT,TCE,TOPEXT,CPCTR,DBKP,PCTRL3,MWAITX,ITSC,FSGSBASE,BMI1,AVX2,SMEP,BMI2,RDSEED,ADX,SMAP,CLFLUSHOPT,SHA,IBPB,XSAVEOPT,XSAVEC,XGETBV1,XSAVES
cpu3: 64KB 64b/line 4-way I-cache, 32KB 64b/line 8-way D-cache, 512KB 64b/line 8-way L2 cache, 16MB 64b/line 16-way L3 cache
cpu3: ITLB 64 4KB entries fully associative, 64 4MB entries fully associative
cpu3: DTLB 64 4KB entries fully associative, 64 4MB entries fully associative
cpu3: smt 0, core 3, package 0
cpu4 at mainbus0: apid 8 (application processor)
cpu4: AMD Ryzen 7 2700 Eight-Core Processor, 3199.99 MHz, 17-08-02
cpu4: FPU,VME,DE,PSE,TSC,MSR,PAE,MCE,CX8,APIC,SEP,MTRR,PGE,MCA,CMOV,PAT,PSE36,CFLUSH,MMX,FXSR,SSE,SSE2,HTT,SSE3,PCLMUL,MWAIT,SSSE3,FMA3,CX16,SSE4.1,SSE4.2,MOVBE,POPCNT,AES,XSAVE,AVX,F16C,RDRAND,NXE,MMXX,FFXSR,PAGE1GB,RDTSCP,LONG,LAHF,CMPLEG,SVM,EAPICSP,AMCR8,ABM,SSE4A,MASSE,3DNOWP,OSVW,SKINIT,TCE,TOPEXT,CPCTR,DBKP,PCTRL3,MWAITX,ITSC,FSGSBASE,BMI1,AVX2,SMEP,BMI2,RDSEED,ADX,SMAP,CLFLUSHOPT,SHA,IBPB,XSAVEOPT,XSAVEC,XGETBV1,XSAVES
cpu4: 64KB 64b/line 4-way I-cache, 32KB 64b/line 8-way D-cache, 512KB 64b/line 8-way L2 cache, 16MB 64b/line 16-way L3 cache
cpu4: ITLB 64 4KB entries fully associative, 64 4MB entries fully associative
cpu4: DTLB 64 4KB entries fully associative, 64 4MB entries fully associative
cpu4: smt 0, core 8, package 0
cpu5 at mainbus0: apid 9 (application processor)
cpu5: AMD Ryzen 7 2700 Eight-Core Processor, 3199.99 MHz, 17-08-02
cpu5: FPU,VME,DE,PSE,TSC,MSR,PAE,MCE,CX8,APIC,SEP,MTRR,PGE,MCA,CMOV,PAT,PSE36,CFLUSH,MMX,FXSR,SSE,SSE2,HTT,SSE3,PCLMUL,MWAIT,SSSE3,FMA3,CX16,SSE4.1,SSE4.2,MOVBE,POPCNT,AES,XSAVE,AVX,F16C,RDRAND,NXE,MMXX,FFXSR,PAGE1GB,RDTSCP,LONG,LAHF,CMPLEG,SVM,EAPICSP,AMCR8,ABM,SSE4A,MASSE,3DNOWP,OSVW,SKINIT,TCE,TOPEXT,CPCTR,DBKP,PCTRL3,MWAITX,ITSC,FSGSBASE,BMI1,AVX2,SMEP,BMI2,RDSEED,ADX,SMAP,CLFLUSHOPT,SHA,IBPB,XSAVEOPT,XSAVEC,XGETBV1,XSAVES
cpu5: 64KB 64b/line 4-way I-cache, 32KB 64b/line 8-way D-cache, 512KB 64b/line 8-way L2 cache, 16MB 64b/line 16-way L3 cache
cpu5: ITLB 64 4KB entries fully associative, 64 4MB entries fully associative
cpu5: DTLB 64 4KB entries fully associative, 64 4MB entries fully associative
cpu5: smt 0, core 9, package 0
cpu6 at mainbus0: apid 10 (application processor)
cpu6: AMD Ryzen 7 2700 Eight-Core Processor, 3199.99 MHz, 17-08-02
cpu6: FPU,VME,DE,PSE,TSC,MSR,PAE,MCE,CX8,APIC,SEP,MTRR,PGE,MCA,CMOV,PAT,PSE36,CFLUSH,MMX,FXSR,SSE,SSE2,HTT,SSE3,PCLMUL,MWAIT,SSSE3,FMA3,CX16,SSE4.1,SSE4.2,MOVBE,POPCNT,AES,XSAVE,AVX,F16C,RDRAND,NXE,MMXX,FFXSR,PAGE1GB,RDTSCP,LONG,LAHF,CMPLEG,SVM,EAPICSP,AMCR8,ABM,SSE4A,MASSE,3DNOWP,OSVW,SKINIT,TCE,TOPEXT,CPCTR,DBKP,PCTRL3,MWAITX,ITSC,FSGSBASE,BMI1,AVX2,SMEP,BMI2,RDSEED,ADX,SMAP,CLFLUSHOPT,SHA,IBPB,XSAVEOPT,XSAVEC,XGETBV1,XSAVES
cpu6: 64KB 64b/line 4-way I-cache, 32KB 64b/line 8-way D-cache, 512KB 64b/line 8-way L2 cache, 16MB 64b/line 16-way L3 cache
cpu6: ITLB 64 4KB entries fully associative, 64 4MB entries fully associative
cpu6: DTLB 64 4KB entries fully associative, 64 4MB entries fully associative
cpu6: smt 0, core 10, package 0
cpu7 at mainbus0: apid 11 (application processor)
cpu7: AMD Ryzen 7 2700 Eight-Core Processor, 3199.99 MHz, 17-08-02
cpu7: FPU,VME,DE,PSE,TSC,MSR,PAE,MCE,CX8,APIC,SEP,MTRR,PGE,MCA,CMOV,PAT,PSE36,CFLUSH,MMX,FXSR,SSE,SSE2,HTT,SSE3,PCLMUL,MWAIT,SSSE3,FMA3,CX16,SSE4.1,SSE4.2,MOVBE,POPCNT,AES,XSAVE,AVX,F16C,RDRAND,NXE,MMXX,FFXSR,PAGE1GB,RDTSCP,LONG,LAHF,CMPLEG,SVM,EAPICSP,AMCR8,ABM,SSE4A,MASSE,3DNOWP,OSVW,SKINIT,TCE,TOPEXT,CPCTR,DBKP,PCTRL3,MWAITX,ITSC,FSGSBASE,BMI1,AVX2,SMEP,BMI2,RDSEED,ADX,SMAP,CLFLUSHOPT,SHA,IBPB,XSAVEOPT,XSAVEC,XGETBV1,XSAVES
cpu7: 64KB 64b/line 4-way I-cache, 32KB 64b/line 8-way D-cache, 512KB 64b/line 8-way L2 cache, 16MB 64b/line 16-way L3 cache
cpu7: ITLB 64 4KB entries fully associative, 64 4MB entries fully associative
cpu7: DTLB 64 4KB entries fully associative, 64 4MB entries fully associative
cpu7: smt 0, core 11, package 0
ioapic0 at mainbus0: apid 9 pa 0xfec00000, version 21, 24 pins
ioapic1 at mainbus0: apid 10 pa 0xfec01000, version 21, 32 pins
acpimcfg0 at acpi0
acpimcfg0: addr 0xf8000000, bus 0-63
acpihpet0 at acpi0: 14318180 Hz
acpiprt0 at acpi0: bus 0 (PCI0)
acpiprt1 at acpi0: bus 1 (GPP0)
acpiprt2 at acpi0: bus -1 (GPP1)
acpiprt3 at acpi0: bus -1 (GPP3)
acpiprt4 at acpi0: bus -1 (GPP4)
acpiprt5 at acpi0: bus -1 (GPP5)
acpiprt6 at acpi0: bus -1 (GPP6)
acpiprt7 at acpi0: bus -1 (GPP7)
acpiprt8 at acpi0: bus 29 (GPP8)
acpiprt9 at acpi0: bus -1 (GPP9)
acpiprt10 at acpi0: bus -1 (GPPA)
acpiprt11 at acpi0: bus -1 (GPPB)
acpiprt12 at acpi0: bus -1 (GPPC)
acpiprt13 at acpi0: bus -1 (GPPD)
acpiprt14 at acpi0: bus -1 (GPPE)
acpiprt15 at acpi0: bus -1 (GPPF)
acpiprt16 at acpi0: bus 32 (GP17)
acpiprt17 at acpi0: bus 33 (GP18)
acpiprt18 at acpi0: bus 3 (GPP2)
acpicpu0 at acpi0: C2(0@400 io@0x414), C1(0@1 mwait), PSS
acpicpu1 at acpi0: C2(0@400 io@0x414), C1(0@1 mwait), PSS
acpicpu2 at acpi0: C2(0@400 io@0x414), C1(0@1 mwait), PSS
acpicpu3 at acpi0: C2(0@400 io@0x414), C1(0@1 mwait), PSS
acpicpu4 at acpi0: C2(0@400 io@0x414), C1(0@1 mwait), PSS
acpicpu5 at acpi0: C2(0@400 io@0x414), C1(0@1 mwait), PSS
acpicpu6 at acpi0: C2(0@400 io@0x414), C1(0@1 mwait), PSS
acpicpu7 at acpi0: C2(0@400 io@0x414), C1(0@1 mwait), PSS
acpipci0 at acpi0 PCI0: 0x00000010 0x00000011 0x00000000
acpicmos0 at acpi0
acpibtn0 at acpi0: PWRB
"AMDI0030" at acpi0 not configured
"PNP0C14" at acpi0 not configured
"PNP0C14" at acpi0 not configured
"AMDIF030" at acpi0 not configured
"PNP0C14" at acpi0 not configured
cpu0: 3200 MHz: speeds: 3200 2800 1550 MHz
pci0 at mainbus0 bus 0
pchb0 at pci0 dev 0 function 0 "AMD AMD64 17h Root Complex" rev 0x00
"AMD AMD64 17h IOMMU" rev 0x00 at pci0 dev 0 function 2 not configured
pchb1 at pci0 dev 1 function 0 "AMD AMD64 17h PCIE" rev 0x00
ppb0 at pci0 dev 1 function 1 "AMD AMD64 17h PCIE" rev 0x00: msi
pci1 at ppb0 bus 1
nvme0 at pci1 dev 0 function 0 vendor "Phison", unknown product 0x5008 rev 0x01: msi, NVMe 1.2
nvme0: SBX, firmware E8FM11.4, serial 0385077C115D61009072
scsibus1 at nvme0: 1 targets
sd0 at scsibus1 targ 0 lun 0: <NVMe, SBX, E8FM> SCSI4 0/direct fixed
sd0: 488386MB, 512 bytes/sector, 1000215216 sectors
ppb1 at pci0 dev 1 function 3 "AMD AMD64 17h PCIE" rev 0x00: msi
pci2 at ppb1 bus 3
xhci0 at pci2 dev 0 function 0 "AMD 400 Series xHCI" rev 0x01: msi, xHCI 1.10
usb0 at xhci0: USB revision 3.0
uhub0 at usb0 configuration 1 interface 0 "AMD xHCI root hub" rev 3.00/1.00 addr 1
ahci0 at pci2 dev 0 function 1 "AMD 400 Series AHCI" rev 0x01: msi, AHCI 1.3.1
ahci0: port busy after first PMP probe FIS
ahci0: port busy after first PMP probe FIS
ahci0: port 0: 6.0Gb/s
ahci0: port busy after first PMP probe FIS
ahci0: port busy after first PMP probe FIS
ahci0: port 1: 3.0Gb/s
ahci0: port busy after first PMP probe FIS
ahci0: port busy after first PMP probe FIS
ahci0: port 2: 3.0Gb/s
ahci0: port busy after first PMP probe FIS
ahci0: port busy after first PMP probe FIS
ahci0: port 3: 6.0Gb/s
scsibus2 at ahci0: 32 targets
sd1 at scsibus2 targ 0 lun 0: <ATA, SK hynix SC308 S, 3000> SCSI3 0/direct fixed t10.ATA_SK_hynix_SC308_SATA_256GB_FJ67N008812307756_
sd1: 244198MB, 512 bytes/sector, 500118192 sectors, thin
sd2 at scsibus2 targ 1 lun 0: <ATA, Hitachi HUA72202, JKAO> SCSI3 0/direct fixed naa.5000cca222d7946d
sd2: 1907729MB, 512 bytes/sector, 3907029168 sectors
sd3 at scsibus2 targ 2 lun 0: <ATA, HITACHI HTS72321, EC1Z> SCSI3 0/direct fixed naa.5000cca61ac0734c
sd3: 152627MB, 512 bytes/sector, 312581808 sectors
sd4 at scsibus2 targ 3 lun 0: <ATA, KINGSTON SA400S3, SBFK> SCSI3 0/direct fixed naa.50026b778241eac5
sd4: 114473MB, 512 bytes/sector, 234441648 sectors, thin
ppb2 at pci2 dev 0 function 2 "AMD 400 Series PCIE" rev 0x01
pci3 at ppb2 bus 22
ppb3 at pci3 dev 0 function 0 "AMD 400 Series PCIE" rev 0x01: msi
pci4 at ppb3 bus 23
ppb4 at pci3 dev 1 function 0 "AMD 400 Series PCIE" rev 0x01: msi
pci5 at ppb4 bus 24
re0 at pci5 dev 0 function 0 "Realtek 8168" rev 0x15: RTL8168H/8111H (0x5400), msi, address 30:9c:23:e2:6e:fe
rgephy0 at re0 phy 7: RTL8251 PHY, rev. 0
ppb5 at pci3 dev 2 function 0 "AMD 400 Series PCIE" rev 0x01: msi
pci6 at ppb5 bus 25
ppb6 at pci3 dev 3 function 0 "AMD 400 Series PCIE" rev 0x01: msi
pci7 at ppb6 bus 26
ppb7 at pci3 dev 4 function 0 "AMD 400 Series PCIE" rev 0x01: msi
pci8 at ppb7 bus 27
ppb8 at pci3 dev 8 function 0 "AMD 400 Series PCIE" rev 0x01: msi
pci9 at ppb8 bus 28
pchb2 at pci0 dev 2 function 0 "AMD AMD64 17h PCIE" rev 0x00
pchb3 at pci0 dev 3 function 0 "AMD AMD64 17h PCIE" rev 0x00
ppb9 at pci0 dev 3 function 1 "AMD AMD64 17h PCIE" rev 0x00: msi
pci10 at ppb9 bus 29
ppb10 at pci10 dev 0 function 0 "AMD AMD64 17h PCIE" rev 0xc1
pci11 at ppb10 bus 30
ppb11 at pci11 dev 0 function 0 "AMD AMD64 17h PCIE" rev 0x00
pci12 at ppb11 bus 31
amdgpu0 at pci12 dev 0 function 0 "ATI Radeon Rx Vega" rev 0xc1
drm0 at amdgpu0
amdgpu0: msi
azalia0 at pci12 dev 0 function 1 "ATI Radeon Rx Vega HD Audio" rev 0x00: msi
azalia0: no supported codecs
pchb4 at pci0 dev 4 function 0 "AMD AMD64 17h PCIE" rev 0x00
pchb5 at pci0 dev 7 function 0 "AMD AMD64 17h PCIE" rev 0x00
ppb12 at pci0 dev 7 function 1 "AMD AMD64 17h PCIE" rev 0x00
pci13 at ppb12 bus 32
vendor "AMD", unknown product 0x145a (class instrumentation unknown subclass 0x00, rev 0x00) at pci13 dev 0 function 0 not configured
ccp0 at pci13 dev 0 function 2 "AMD AMD64 17h Crypto" rev 0x00
xhci1 at pci13 dev 0 function 3 "AMD AMD64 17h xHCI" rev 0x00: msi, xHCI 1.0
usb1 at xhci1: USB revision 3.0
uhub1 at usb1 configuration 1 interface 0 "AMD xHCI root hub" rev 3.00/1.00 addr 1
pchb6 at pci0 dev 8 function 0 "AMD AMD64 17h PCIE" rev 0x00
ppb13 at pci0 dev 8 function 1 "AMD AMD64 17h PCIE" rev 0x00
pci14 at ppb13 bus 33
vendor "AMD", unknown product 0x1455 (class instrumentation unknown subclass 0x00, rev 0x00) at pci14 dev 0 function 0 not configured
ahci1 at pci14 dev 0 function 2 "AMD FCH AHCI" rev 0x51: msi, AHCI 1.3.1
scsibus3 at ahci1: 32 targets
azalia1 at pci14 dev 0 function 3 "AMD AMD64 17h HD Audio" rev 0x00: apic 10 int 19
azalia1: codecs: Realtek/0x0892
audio0 at azalia1
"AMD FCH SMBus" rev 0x59 at pci0 dev 20 function 0 not configured
pcib0 at pci0 dev 20 function 3 "AMD FCH LPC" rev 0x51
pchb7 at pci0 dev 24 function 0 "AMD AMD64 17h Data Fabric" rev 0x00
pchb8 at pci0 dev 24 function 1 "AMD AMD64 17h Data Fabric" rev 0x00
pchb9 at pci0 dev 24 function 2 "AMD AMD64 17h Data Fabric" rev 0x00
pchb10 at pci0 dev 24 function 3 "AMD AMD64 17h Data Fabric" rev 0x00
pchb11 at pci0 dev 24 function 4 "AMD AMD64 17h Data Fabric" rev 0x00
pchb12 at pci0 dev 24 function 5 "AMD AMD64 17h Data Fabric" rev 0x00
pchb13 at pci0 dev 24 function 6 "AMD AMD64 17h Data Fabric" rev 0x00
pchb14 at pci0 dev 24 function 7 "AMD AMD64 17h Data Fabric" rev 0x00
isa0 at pcib0
isadma0 at isa0
com0 at isa0 port 0x3f8/8 irq 4: ns16550a, 16 byte fifo
pckbc0 at isa0 port 0x60/5 irq 1 irq 12
pckbd0 at pckbc0 (kbd slot)
wskbd0 at pckbd0: console keyboard
pcppi0 at isa0 port 0x61
spkr0 at pcppi0
lpt0 at isa0 port 0x378/4 irq 7
vmm0 at mainbus0: SVM/RVI
ugen0 at uhub0 port 19 "Broadcom Corp BCM20702A0" rev 2.00/1.12 addr 2
uhidev0 at uhub0 port 20 configuration 1 interface 0 "HOLTEK USB-HID MOUSE" rev 1.10/1.00 addr 3
uhidev0: iclass 3/1
ums0 at uhidev0: 5 buttons, Z and W dir
wsmouse0 at ums0 mux 0
uhidev1 at uhub0 port 20 configuration 1 interface 1 "HOLTEK USB-HID MOUSE" rev 1.10/1.00 addr 3
uhidev1: iclass 3/0
uhid0 at uhidev1: input=64, output=64, feature=0
uhidev2 at uhub0 port 20 configuration 1 interface 2 "HOLTEK USB-HID MOUSE" rev 1.10/1.00 addr 3
uhidev2: iclass 3/0, 6 report ids
ukbd0 at uhidev2 reportid 1: 8 variable keys, 6 key codes
wskbd1 at ukbd0 mux 1
uhid1 at uhidev2 reportid 2: input=1, output=0, feature=0
uhid2 at uhidev2 reportid 3: input=2, output=0, feature=0
ums1 at uhidev2 reportid 4: 5 buttons, Z and W dir
wsmouse1 at ums1 mux 0
uhid3 at uhidev2 reportid 6: input=7, output=0, feature=0
uhidev3 at uhub0 port 20 configuration 1 interface 3 "HOLTEK USB-HID MOUSE" rev 1.10/1.00 addr 3
uhidev3: iclass 3/0
uhid4 at uhidev3: input=64, output=64, feature=8
uhidev4 at uhub0 port 21 configuration 1 interface 0 "Holtek USB Keyboard" rev 2.00/1.05 addr 4
uhidev4: iclass 3/1
ukbd1 at uhidev4: 8 variable keys, 6 key codes
wskbd2 at ukbd1 mux 1
uhidev5 at uhub0 port 21 configuration 1 interface 1 "Holtek USB Keyboard" rev 2.00/1.05 addr 4
uhidev5: iclass 3/0, 7 report ids
uhid5 at uhidev5 reportid 2: input=1, output=0, feature=0
uhid6 at uhidev5 reportid 3: input=2, output=0, feature=0
ukbd2 at uhidev5 reportid 5: 112 variable keys, 0 key codes
wskbd3 at ukbd2 mux 1
uhid7 at uhidev5 reportid 6: input=2, output=0, feature=0
uhid8 at uhidev5 reportid 7: input=0, output=0, feature=7
uhidev6 at uhub0 port 21 configuration 1 interface 2 "Holtek USB Keyboard" rev 2.00/1.05 addr 4
uhidev6: iclass 3/0
uhid9 at uhidev6: input=8, output=8, feature=0
uhidev7 at uhub0 port 22 configuration 1 interface 0 "\M-)Microsoft Corporation Controller" rev 2.00/1.14 addr 5
uhidev7: iclass 255/93
uhid10 at uhidev7: input=20, output=0, feature=0
ugen1 at uhub0 port 22 configuration 1 "\M-)Microsoft Corporation Controller" rev 2.00/1.14 addr 5
uhub2 at uhub1 port 1 configuration 1 interface 0 "HTC CB USB2" rev 2.10/90.30 addr 2
uhub3 at uhub2 port 1 configuration 1 interface 0 "SMSC USB2137B" rev 2.10/60.80 addr 3
uhidev8 at uhub3 port 1 configuration 1 interface 0 "Valve Software Lighthouse FPGA RX" rev 1.10/1.01 addr 4
uhidev8: iclass 3/0, 32 report ids
uhid11 at uhidev8 reportid 1: input=0, output=63, feature=2
uhid12 at uhidev8 reportid 2: input=0, output=0, feature=33
uhid13 at uhidev8 reportid 3: input=0, output=0, feature=46
uhid14 at uhidev8 reportid 4: input=0, output=0, feature=4
uhid15 at uhidev8 reportid 5: input=0, output=0, feature=52
uhid16 at uhidev8 reportid 7: input=0, output=0, feature=4
uhid17 at uhidev8 reportid 8: input=0, output=0, feature=8
uhid18 at uhidev8 reportid 9: input=0, output=0, feature=63
uhid19 at uhidev8 reportid 10: input=0, output=0, feature=3
uhid20 at uhidev8 reportid 11: input=0, output=0, feature=3
uhid21 at uhidev8 reportid 12: input=0, output=0, feature=1
uhid22 at uhidev8 reportid 16: input=0, output=0, feature=2
uhid23 at uhidev8 reportid 17: input=0, output=0, feature=63
uhid24 at uhidev8 reportid 32: input=51, output=0, feature=0
uhidev9 at uhub3 port 1 configuration 1 interface 1 "Valve Software Lighthouse FPGA RX" rev 1.10/1.01 addr 4
uhidev9: iclass 3/0, 37 report ids
uhid25 at uhidev9 reportid 33: input=57, output=0, feature=0
uhid26 at uhidev9 reportid 37: input=63, output=0, feature=0
uvideo0 at uhub3 port 2 configuration 1 interface 0 "Alpha Imaging Tech HTC Vive" rev 2.00/1.00 addr 5
video0 at uvideo0
uaudio0 at uhub3 port 2 configuration 1 interface 3 "Alpha Imaging Tech HTC Vive" rev 2.00/1.00 addr 5
uaudio0: class v1, high-speed, sync, channels: 0 play, 1 rec, 2 ctls
audio1 at uaudio0
uaudio1 at uhub3 port 2 configuration 1 interface 5 "Alpha Imaging Tech HTC Vive" rev 2.00/1.00 addr 5
uaudio1: class v1, high-speed, sync, channels: 0 play, 1 rec, 2 ctls
audio2 at uaudio1
uhidev10 at uhub3 port 5 configuration 1 interface 0 "HTC HTC Vive" rev 2.00/2.00 addr 6
uhidev10: iclass 3/0, 4 report ids
uhid27 at uhidev10 reportid 1: input=63, output=63, feature=0
uhid28 at uhidev10 reportid 2: input=63, output=63, feature=0
uhid29 at uhidev10 reportid 3: input=63, output=0, feature=0
uhid30 at uhidev10 reportid 4: input=0, output=0, feature=63
uhidev11 at uhub3 port 6 configuration 1 interface 0 "Valve Software Watchman Dongle" rev 2.00/0.01 addr 7
uhidev11: iclass 3/0, 255 report ids
uhid31 at uhidev11 reportid 1: input=0, output=63, feature=2
uhid32 at uhidev11 reportid 2: input=0, output=0, feature=33
uhid33 at uhidev11 reportid 3: input=0, output=0, feature=46
uhid34 at uhidev11 reportid 5: input=0, output=0, feature=52
uhid35 at uhidev11 reportid 10: input=0, output=0, feature=3
uhid36 at uhidev11 reportid 11: input=0, output=0, feature=3
uhid37 at uhidev11 reportid 16: input=0, output=0, feature=2
uhid38 at uhidev11 reportid 17: input=0, output=0, feature=63
uhid39 at uhidev11 reportid 35: input=29, output=0, feature=0
uhid40 at uhidev11 reportid 36: input=58, output=0, feature=0
uhid41 at uhidev11 reportid 38: input=1, output=0, feature=0
uhid at uhidev11 not configured
uhidev12 at uhub3 port 7 configuration 1 interface 0 "Valve Software Watchman Dongle" rev 2.00/0.01 addr 8
uhidev12: iclass 3/0, 255 report ids
uhid42 at uhidev12 reportid 1: input=0, output=63, feature=2
uhid43 at uhidev12 reportid 2: input=0, output=0, feature=33
uhid44 at uhidev12 reportid 3: input=0, output=0, feature=46
uhid45 at uhidev12 reportid 5: input=0, output=0, feature=52
uhid46 at uhidev12 reportid 10: input=0, output=0, feature=3
uhid47 at uhidev12 reportid 11: input=0, output=0, feature=3
uhid48 at uhidev12 reportid 16: input=0, output=0, feature=2
uhid49 at uhidev12 reportid 17: input=0, output=0, feature=63
uhid50 at uhidev12 reportid 35: input=29, output=0, feature=0
uhid51 at uhidev12 reportid 36: input=58, output=0, feature=0
uhid52 at uhidev12 reportid 38: input=1, output=0, feature=0
uhid at uhidev12 not configured
ugen2 at uhub2 port 2 "Broadcom Corp BCM2045A0" rev 2.00/1.12 addr 9
vscsi0 at root
scsibus4 at vscsi0: 256 targets
softraid0 at root
scsibus5 at softraid0: 256 targets
sd5 at scsibus5 targ 1 lun 0: <OPENBSD, SR CRYPTO, 006> SCSI2 0/direct fixed
sd5: 488381MB, 512 bytes/sector, 1000206308 sectors
root on sd5a (f0e0a74479a62878.a) swap on sd5b dump on sd5b
initializing kernel modesetting (VEGA10 0x1002:0x687F 0x1002:0x6B76 0xC1).
amdgpu_device_resize_fb_bar: stub
amdgpu0: 1920x1080, 32bpp
wsdisplay0 at amdgpu0 mux 1: console (std, vt100 emulation), using wskbd0
wskbd1: connecting to wsdisplay0
wskbd2: connecting to wsdisplay0
wskbd3: connecting to wsdisplay0
wsdisplay0: screen 1-5 added (std, vt100 emulation)
wsmouse0 detached
ums0 detached
uhidev0 detached
uhid0 detached
uhidev1 detached
wskbd1: disconnecting from wsdisplay0
wskbd1 detached
ukbd0 detached
uhid1 detached
uhid2 detached
wsmouse1 detached
ums1 detached
uhid3 detached
uhidev2 detached
uhid4 detached
uhidev3 detached
uhidev0 at uhub0 port 20 configuration 1 interface 0 "HOLTEK USB-HID MOUSE" rev 1.10/1.00 addr 3
uhidev0: iclass 3/1
ums0 at uhidev0: 5 buttons, Z and W dir
wsmouse0 at ums0 mux 0
uhidev1 at uhub0 port 20 configuration 1 interface 1 "HOLTEK USB-HID MOUSE" rev 1.10/1.00 addr 3
uhidev1: iclass 3/0
uhid0 at uhidev1: input=64, output=64, feature=0
uhidev2 at uhub0 port 20 configuration 1 interface 2 "HOLTEK USB-HID MOUSE" rev 1.10/1.00 addr 3
uhidev2: iclass 3/0, 6 report ids
ukbd0 at uhidev2 reportid 1: 8 variable keys, 6 key codes
wskbd1 at ukbd0: console keyboard, using wsdisplay0
uhid1 at uhidev2 reportid 2: input=1, output=0, feature=0
uhid2 at uhidev2 reportid 3: input=2, output=0, feature=0
ums1 at uhidev2 reportid 4: 5 buttons, Z and W dir
wsmouse1 at ums1 mux 0
uhid3 at uhidev2 reportid 6: input=7, output=0, feature=0
uhidev3 at uhub0 port 20 configuration 1 interface 3 "HOLTEK USB-HID MOUSE" rev 1.10/1.00 addr 3
uhidev3: iclass 3/0
uhid4 at uhidev3: input=64, output=64, feature=8

Reply | Threaded
Open this post in threaded view
|

Re: 64-bit dma for drm(4) on amd64

Jonathan Gray-11
In reply to this post by Mark Kettenis
On Thu, Jun 06, 2019 at 11:41:07PM +0200, Mark Kettenis wrote:

> As a result of a recent discussion with jsg@, I realized that the
> graphics drivers are (mostly) allocating memory from the dma region.
> Since the the graphics stack can potentially gobble up large amounts
> of memory, this means we can run out of dma memory which makes other
> parts of our kernel quite unhappy.  Most of the supported hardware
> actually supports 64-bit DMA just fine, and the drivers already have
> code to handle the exceptions.  The diff below makes use of this
> knowledge to (hopefully) safely allocate from "high" memory when
> possible.  One big change is that this makes bus_dma(9) 64-bit DMA
> aware in the sense that if the BUS_DMA_64BIT flag is used, we skip the
> "not dma-reachable" panic.
>
> It seems to work fine on my Intel Broadwell laptop.  I haven't tested
> this on radeon(4) yet.  So further testing, especially on systems with
> 4GB of memory or more is necessary.
>
> Please test.

One of the ways dma32 is set in radeon/amdgpu is along the lines of

dma_bits = rdev->need_dma32 ? 32 : 40;
r = pci_set_dma_mask(rdev->pdev, DMA_BIT_MASK(dma_bits));
if (r) {
        rdev->need_dma32 = true;
        dma_bits = 32;

which we don't handle.  That is for the case where the card supports
64 bit dma but the system doesn't?  Not something we should be
concerned about?

Having __GFP_DMA32 set the sign bit on the flags seems like it is asking
for trouble.  How about 0x40000000 / bit 30 instead of
0x80000000 / bit 31.

Otherwise looks good.

>
>
> Index: arch/amd64/amd64/bus_dma.c
> ===================================================================
> RCS file: /cvs/src/sys/arch/amd64/amd64/bus_dma.c,v
> retrieving revision 1.50
> diff -u -p -r1.50 bus_dma.c
> --- arch/amd64/amd64/bus_dma.c 14 Oct 2017 04:44:43 -0000 1.50
> +++ arch/amd64/amd64/bus_dma.c 6 Jun 2019 21:19:21 -0000
> @@ -319,7 +319,8 @@ _bus_dmamap_load_raw(bus_dma_tag_t t, bu
>   if (plen < sgsize)
>   sgsize = plen;
>  
> - if (paddr > dma_constraint.ucr_high)
> + if (paddr > dma_constraint.ucr_high &&
> +    (map->_dm_flags & BUS_DMA_64BIT) == 0)
>   panic("Non dma-reachable buffer at paddr %#lx(raw)",
>      paddr);
>  
> @@ -583,7 +584,8 @@ _bus_dmamap_load_buffer(bus_dma_tag_t t,
>   */
>   pmap_extract(pmap, vaddr, (paddr_t *)&curaddr);
>  
> - if (curaddr > dma_constraint.ucr_high)
> + if (curaddr > dma_constraint.ucr_high &&
> +    (map->_dm_flags & BUS_DMA_64BIT) == 0)
>   panic("Non dma-reachable buffer at curaddr %#lx(raw)",
>      curaddr);
>  
> Index: dev/pci/drm/drm_linux.c
> ===================================================================
> RCS file: /cvs/src/sys/dev/pci/drm/drm_linux.c,v
> retrieving revision 1.37
> diff -u -p -r1.37 drm_linux.c
> --- dev/pci/drm/drm_linux.c 4 Jun 2019 12:08:22 -0000 1.37
> +++ dev/pci/drm/drm_linux.c 6 Jun 2019 21:19:21 -0000
> @@ -293,16 +293,19 @@ struct vm_page *
>  alloc_pages(unsigned int gfp_mask, unsigned int order)
>  {
>   int flags = (gfp_mask & M_NOWAIT) ? UVM_PLA_NOWAIT : UVM_PLA_WAITOK;
> + struct uvm_constraint_range *constraint = &no_constraint;
>   struct pglist mlist;
>  
>   if (gfp_mask & M_CANFAIL)
>   flags |= UVM_PLA_FAILOK;
>   if (gfp_mask & M_ZERO)
>   flags |= UVM_PLA_ZERO;
> + if (gfp_mask & __GFP_DMA32)
> + constraint = &dma_constraint;
>  
>   TAILQ_INIT(&mlist);
> - if (uvm_pglistalloc(PAGE_SIZE << order, dma_constraint.ucr_low,
> -    dma_constraint.ucr_high, PAGE_SIZE, 0, &mlist, 1, flags))
> + if (uvm_pglistalloc(PAGE_SIZE << order, constraint->ucr_low,
> +    constraint->ucr_high, PAGE_SIZE, 0, &mlist, 1, flags))
>   return NULL;
>   return TAILQ_FIRST(&mlist);
>  }
> Index: dev/pci/drm/include/linux/gfp.h
> ===================================================================
> RCS file: /cvs/src/sys/dev/pci/drm/include/linux/gfp.h,v
> retrieving revision 1.1
> diff -u -p -r1.1 gfp.h
> --- dev/pci/drm/include/linux/gfp.h 14 Apr 2019 10:14:53 -0000 1.1
> +++ dev/pci/drm/include/linux/gfp.h 6 Jun 2019 21:19:21 -0000
> @@ -7,24 +7,25 @@
>  #include <sys/malloc.h>
>  #include <uvm/uvm_extern.h>
>  
> -#define GFP_ATOMIC M_NOWAIT
> -#define GFP_NOWAIT M_NOWAIT
> -#define GFP_KERNEL (M_WAITOK | M_CANFAIL)
> -#define GFP_USER (M_WAITOK | M_CANFAIL)
> -#define GFP_TEMPORARY (M_WAITOK | M_CANFAIL)
> -#define GFP_HIGHUSER 0
> -#define GFP_DMA32 0
> -#define __GFP_NOWARN 0
> -#define __GFP_NORETRY 0
> -#define __GFP_ZERO M_ZERO
> +#define __GFP_ZERO M_ZERO
> +#define __GFP_DMA32 0x80000000
> +#define __GFP_NOWARN 0
> +#define __GFP_NORETRY 0
>  #define __GFP_RETRY_MAYFAIL 0
>  #define __GFP_MOVABLE 0
>  #define __GFP_COMP 0
> -#define GFP_TRANSHUGE_LIGHT 0
>  #define __GFP_KSWAPD_RECLAIM 0
>  #define __GFP_HIGHMEM 0
>  #define __GFP_RECLAIMABLE 0
> -#define __GFP_DMA32 0
> +
> +#define GFP_ATOMIC M_NOWAIT
> +#define GFP_NOWAIT M_NOWAIT
> +#define GFP_KERNEL (M_WAITOK | M_CANFAIL)
> +#define GFP_USER (M_WAITOK | M_CANFAIL)
> +#define GFP_TEMPORARY (M_WAITOK | M_CANFAIL)
> +#define GFP_HIGHUSER 0
> +#define GFP_DMA32 __GFP_DMA32
> +#define GFP_TRANSHUGE_LIGHT 0
>  
>  static inline bool
>  gfpflags_allow_blocking(const unsigned int flags)
> Index: dev/pci/drm/ttm/ttm_tt.c
> ===================================================================
> RCS file: /cvs/src/sys/dev/pci/drm/ttm/ttm_tt.c,v
> retrieving revision 1.8
> diff -u -p -r1.8 ttm_tt.c
> --- dev/pci/drm/ttm/ttm_tt.c 14 Apr 2019 10:14:54 -0000 1.8
> +++ dev/pci/drm/ttm/ttm_tt.c 6 Jun 2019 21:19:22 -0000
> @@ -261,6 +261,7 @@ int ttm_dma_tt_init(struct ttm_dma_tt *t
>      uint32_t page_flags)
>  {
>   struct ttm_tt *ttm = &ttm_dma->ttm;
> + int flags = BUS_DMA_WAITOK;
>  
>   ttm_tt_init_fields(ttm, bo, page_flags);
>  
> @@ -276,8 +277,10 @@ int ttm_dma_tt_init(struct ttm_dma_tt *t
>  
>   ttm_dma->dmat = bo->bdev->dmat;
>  
> + if ((page_flags & TTM_PAGE_FLAG_DMA32) == 0)
> + flags |= BUS_DMA_64BIT;
>   if (bus_dmamap_create(ttm_dma->dmat, ttm->num_pages << PAGE_SHIFT,
> -    ttm->num_pages, ttm->num_pages << PAGE_SHIFT, 0, BUS_DMA_WAITOK,
> +    ttm->num_pages, ttm->num_pages << PAGE_SHIFT, 0, flags,
>      &ttm_dma->map)) {
>   free(ttm_dma->segs, M_DRM, 0);
>   ttm_tt_destroy(ttm);
> @@ -293,6 +296,7 @@ int ttm_sg_tt_init(struct ttm_dma_tt *tt
>     uint32_t page_flags)
>  {
>   struct ttm_tt *ttm = &ttm_dma->ttm;
> + int flags = BUS_DMA_WAITOK;
>   int ret;
>  
>   ttm_tt_init_fields(ttm, bo, page_flags);
> @@ -313,8 +317,10 @@ int ttm_sg_tt_init(struct ttm_dma_tt *tt
>  
>   ttm_dma->dmat = bo->bdev->dmat;
>  
> + if ((page_flags & TTM_PAGE_FLAG_DMA32) == 0)
> + flags |= BUS_DMA_64BIT;
>   if (bus_dmamap_create(ttm_dma->dmat, ttm->num_pages << PAGE_SHIFT,
> -    ttm->num_pages, ttm->num_pages << PAGE_SHIFT, 0, BUS_DMA_WAITOK,
> +    ttm->num_pages, ttm->num_pages << PAGE_SHIFT, 0, flags,
>      &ttm_dma->map)) {
>   free(ttm_dma->segs, M_DRM, 0);
>   ttm_tt_destroy(ttm);
>

Reply | Threaded
Open this post in threaded view
|

Re: 64-bit dma for drm(4) on amd64

Mark Kettenis
> Date: Fri, 7 Jun 2019 13:06:22 +1000
> From: Jonathan Gray <[hidden email]>
>
> On Thu, Jun 06, 2019 at 11:41:07PM +0200, Mark Kettenis wrote:
> > As a result of a recent discussion with jsg@, I realized that the
> > graphics drivers are (mostly) allocating memory from the dma region.
> > Since the the graphics stack can potentially gobble up large amounts
> > of memory, this means we can run out of dma memory which makes other
> > parts of our kernel quite unhappy.  Most of the supported hardware
> > actually supports 64-bit DMA just fine, and the drivers already have
> > code to handle the exceptions.  The diff below makes use of this
> > knowledge to (hopefully) safely allocate from "high" memory when
> > possible.  One big change is that this makes bus_dma(9) 64-bit DMA
> > aware in the sense that if the BUS_DMA_64BIT flag is used, we skip the
> > "not dma-reachable" panic.
> >
> > It seems to work fine on my Intel Broadwell laptop.  I haven't tested
> > this on radeon(4) yet.  So further testing, especially on systems with
> > 4GB of memory or more is necessary.
> >
> > Please test.
>
> One of the ways dma32 is set in radeon/amdgpu is along the lines of
>
> dma_bits = rdev->need_dma32 ? 32 : 40;
> r = pci_set_dma_mask(rdev->pdev, DMA_BIT_MASK(dma_bits));
> if (r) {
> rdev->need_dma32 = true;
> dma_bits = 32;
>
> which we don't handle.  That is for the case where the card supports
> 64 bit dma but the system doesn't?  Not something we should be
> concerned about?

I think so.  At this point I'm not very concerned about this.

> Having __GFP_DMA32 set the sign bit on the flags seems like it is asking
> for trouble.  How about 0x40000000 / bit 30 instead of
> 0x80000000 / bit 31.

Shouldn't be an issue, but I've changed it to 0x00010000.

> Otherwise looks good.

I'll give it a couple of days for further testing before I move ahead.

> > Index: arch/amd64/amd64/bus_dma.c
> > ===================================================================
> > RCS file: /cvs/src/sys/arch/amd64/amd64/bus_dma.c,v
> > retrieving revision 1.50
> > diff -u -p -r1.50 bus_dma.c
> > --- arch/amd64/amd64/bus_dma.c 14 Oct 2017 04:44:43 -0000 1.50
> > +++ arch/amd64/amd64/bus_dma.c 6 Jun 2019 21:19:21 -0000
> > @@ -319,7 +319,8 @@ _bus_dmamap_load_raw(bus_dma_tag_t t, bu
> >   if (plen < sgsize)
> >   sgsize = plen;
> >  
> > - if (paddr > dma_constraint.ucr_high)
> > + if (paddr > dma_constraint.ucr_high &&
> > +    (map->_dm_flags & BUS_DMA_64BIT) == 0)
> >   panic("Non dma-reachable buffer at paddr %#lx(raw)",
> >      paddr);
> >  
> > @@ -583,7 +584,8 @@ _bus_dmamap_load_buffer(bus_dma_tag_t t,
> >   */
> >   pmap_extract(pmap, vaddr, (paddr_t *)&curaddr);
> >  
> > - if (curaddr > dma_constraint.ucr_high)
> > + if (curaddr > dma_constraint.ucr_high &&
> > +    (map->_dm_flags & BUS_DMA_64BIT) == 0)
> >   panic("Non dma-reachable buffer at curaddr %#lx(raw)",
> >      curaddr);
> >  
> > Index: dev/pci/drm/drm_linux.c
> > ===================================================================
> > RCS file: /cvs/src/sys/dev/pci/drm/drm_linux.c,v
> > retrieving revision 1.37
> > diff -u -p -r1.37 drm_linux.c
> > --- dev/pci/drm/drm_linux.c 4 Jun 2019 12:08:22 -0000 1.37
> > +++ dev/pci/drm/drm_linux.c 6 Jun 2019 21:19:21 -0000
> > @@ -293,16 +293,19 @@ struct vm_page *
> >  alloc_pages(unsigned int gfp_mask, unsigned int order)
> >  {
> >   int flags = (gfp_mask & M_NOWAIT) ? UVM_PLA_NOWAIT : UVM_PLA_WAITOK;
> > + struct uvm_constraint_range *constraint = &no_constraint;
> >   struct pglist mlist;
> >  
> >   if (gfp_mask & M_CANFAIL)
> >   flags |= UVM_PLA_FAILOK;
> >   if (gfp_mask & M_ZERO)
> >   flags |= UVM_PLA_ZERO;
> > + if (gfp_mask & __GFP_DMA32)
> > + constraint = &dma_constraint;
> >  
> >   TAILQ_INIT(&mlist);
> > - if (uvm_pglistalloc(PAGE_SIZE << order, dma_constraint.ucr_low,
> > -    dma_constraint.ucr_high, PAGE_SIZE, 0, &mlist, 1, flags))
> > + if (uvm_pglistalloc(PAGE_SIZE << order, constraint->ucr_low,
> > +    constraint->ucr_high, PAGE_SIZE, 0, &mlist, 1, flags))
> >   return NULL;
> >   return TAILQ_FIRST(&mlist);
> >  }
> > Index: dev/pci/drm/include/linux/gfp.h
> > ===================================================================
> > RCS file: /cvs/src/sys/dev/pci/drm/include/linux/gfp.h,v
> > retrieving revision 1.1
> > diff -u -p -r1.1 gfp.h
> > --- dev/pci/drm/include/linux/gfp.h 14 Apr 2019 10:14:53 -0000 1.1
> > +++ dev/pci/drm/include/linux/gfp.h 6 Jun 2019 21:19:21 -0000
> > @@ -7,24 +7,25 @@
> >  #include <sys/malloc.h>
> >  #include <uvm/uvm_extern.h>
> >  
> > -#define GFP_ATOMIC M_NOWAIT
> > -#define GFP_NOWAIT M_NOWAIT
> > -#define GFP_KERNEL (M_WAITOK | M_CANFAIL)
> > -#define GFP_USER (M_WAITOK | M_CANFAIL)
> > -#define GFP_TEMPORARY (M_WAITOK | M_CANFAIL)
> > -#define GFP_HIGHUSER 0
> > -#define GFP_DMA32 0
> > -#define __GFP_NOWARN 0
> > -#define __GFP_NORETRY 0
> > -#define __GFP_ZERO M_ZERO
> > +#define __GFP_ZERO M_ZERO
> > +#define __GFP_DMA32 0x80000000
> > +#define __GFP_NOWARN 0
> > +#define __GFP_NORETRY 0
> >  #define __GFP_RETRY_MAYFAIL 0
> >  #define __GFP_MOVABLE 0
> >  #define __GFP_COMP 0
> > -#define GFP_TRANSHUGE_LIGHT 0
> >  #define __GFP_KSWAPD_RECLAIM 0
> >  #define __GFP_HIGHMEM 0
> >  #define __GFP_RECLAIMABLE 0
> > -#define __GFP_DMA32 0
> > +
> > +#define GFP_ATOMIC M_NOWAIT
> > +#define GFP_NOWAIT M_NOWAIT
> > +#define GFP_KERNEL (M_WAITOK | M_CANFAIL)
> > +#define GFP_USER (M_WAITOK | M_CANFAIL)
> > +#define GFP_TEMPORARY (M_WAITOK | M_CANFAIL)
> > +#define GFP_HIGHUSER 0
> > +#define GFP_DMA32 __GFP_DMA32
> > +#define GFP_TRANSHUGE_LIGHT 0
> >  
> >  static inline bool
> >  gfpflags_allow_blocking(const unsigned int flags)
> > Index: dev/pci/drm/ttm/ttm_tt.c
> > ===================================================================
> > RCS file: /cvs/src/sys/dev/pci/drm/ttm/ttm_tt.c,v
> > retrieving revision 1.8
> > diff -u -p -r1.8 ttm_tt.c
> > --- dev/pci/drm/ttm/ttm_tt.c 14 Apr 2019 10:14:54 -0000 1.8
> > +++ dev/pci/drm/ttm/ttm_tt.c 6 Jun 2019 21:19:22 -0000
> > @@ -261,6 +261,7 @@ int ttm_dma_tt_init(struct ttm_dma_tt *t
> >      uint32_t page_flags)
> >  {
> >   struct ttm_tt *ttm = &ttm_dma->ttm;
> > + int flags = BUS_DMA_WAITOK;
> >  
> >   ttm_tt_init_fields(ttm, bo, page_flags);
> >  
> > @@ -276,8 +277,10 @@ int ttm_dma_tt_init(struct ttm_dma_tt *t
> >  
> >   ttm_dma->dmat = bo->bdev->dmat;
> >  
> > + if ((page_flags & TTM_PAGE_FLAG_DMA32) == 0)
> > + flags |= BUS_DMA_64BIT;
> >   if (bus_dmamap_create(ttm_dma->dmat, ttm->num_pages << PAGE_SHIFT,
> > -    ttm->num_pages, ttm->num_pages << PAGE_SHIFT, 0, BUS_DMA_WAITOK,
> > +    ttm->num_pages, ttm->num_pages << PAGE_SHIFT, 0, flags,
> >      &ttm_dma->map)) {
> >   free(ttm_dma->segs, M_DRM, 0);
> >   ttm_tt_destroy(ttm);
> > @@ -293,6 +296,7 @@ int ttm_sg_tt_init(struct ttm_dma_tt *tt
> >     uint32_t page_flags)
> >  {
> >   struct ttm_tt *ttm = &ttm_dma->ttm;
> > + int flags = BUS_DMA_WAITOK;
> >   int ret;
> >  
> >   ttm_tt_init_fields(ttm, bo, page_flags);
> > @@ -313,8 +317,10 @@ int ttm_sg_tt_init(struct ttm_dma_tt *tt
> >  
> >   ttm_dma->dmat = bo->bdev->dmat;
> >  
> > + if ((page_flags & TTM_PAGE_FLAG_DMA32) == 0)
> > + flags |= BUS_DMA_64BIT;
> >   if (bus_dmamap_create(ttm_dma->dmat, ttm->num_pages << PAGE_SHIFT,
> > -    ttm->num_pages, ttm->num_pages << PAGE_SHIFT, 0, BUS_DMA_WAITOK,
> > +    ttm->num_pages, ttm->num_pages << PAGE_SHIFT, 0, flags,
> >      &ttm_dma->map)) {
> >   free(ttm_dma->segs, M_DRM, 0);
> >   ttm_tt_destroy(ttm);
> >
>