/* * TODO * ==== * - implement mbuf, raw, uio * - check iommu_enter/iommu_remove * - check chainid stuff * - cujo bug workaround * - rename stuff according to feb96a6 * - convert extent to vmem * - iova sizing */ /* $NetBSD: uturn.c,v 1.6 2009/11/15 15:53:05 skrll Exp $ */ /* $OpenBSD: uturn.c,v 1.6 2007/12/29 01:26:14 kettenis Exp $ */ /* * Copyright (c) 2004 Michael Shalayeff * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR OR HIS RELATIVES BE LIABLE FOR ANY DIRECT, * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR * SERVICES; LOSS OF MIND, USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF * THE POSSIBILITY OF SUCH DAMAGE. */ /* * References: * 1. Hardware Cache Coherent Input/Output. Hewlett-Packard Journal, February * 1996. * 2. PA-RISC 1.1 Architecture and Instruction Set Reference Manual, * Hewlett-Packard, February 1994, Third Edition */ #include #include #include #include #include #include #include #include #include #include #include #include #define UTURNDEBUG #ifdef UTURNDEBUG #define DPRINTF(s) do { \ if (uturndebug) \ printf s; \ } while(0) int uturndebug = 0; #else #define DPRINTF(s) /* */ #endif /* * IOA Registers * * io_control * ========== * * The io_control register controls the forwarding of transactions. * * Bits Description * 0 - 13 HV * 14 - 15 TLB * 16 - 21 Reserved * 22 HV * 23 - 24 Mode * 25 - 31 Reserved * * Mode * ---- * * Address translcation of transactions forwarded from Runway to GSC+: * * 0 / Off Runway transactions which match the I/O range specified by the * IO_IO_LOW/IO_IO_HIGH registers will be ignored. * * 1 / Include All addresses within the I/O range specified by the IO_IO_LOW * and IO_IO_HIGH registers are transparently forwarded. * This is the I/O Adapter's normal operating mode. * * 3 / Peek used during system configuration to initialise the GSC+ bus. * Runway Write_Shorts in the address range specified by IO_IO_LOW * and IO_IO_HIGH are forwarded through the I/O Adapter AND the * GSC+ address is remapped to the Broadcast Physical Address space * by setting the 14 high order address bits of the 32 bit GSC+ * address to ones. * * TLB * ---- * * The TLB field affects transactions which are forwarded from GSC++ to Runway. * * 0 / Real No TLB translation. Address is directly mapped and the virutal * address is composed of selected phyiscal bits. * * 1 / Error Software fill the TLB manually * * 2 / Normal IOA fetches IO TLB misses from IO PDIR (in host memory) * */ struct uturn_regs { /* Runway Supervisory Set */ int32_t unused1[12]; uint32_t io_command; /* Offset 12 */ #define UTURN_CMD_TLB_PURGE 33 /* Purge I/O TLB entry */ #define UTURN_CMD_TLB_DIRECT_WRITE 35 /* I/O TLB Writes */ uint32_t io_status; /* Offset 13 */ uint32_t io_control; /* Offset 14 */ #define UTURN_IOCTRL_TLB_REAL 0x00000000 #define UTURN_IOCTRL_TLB_ERROR 0x00010000 #define UTURN_IOCTRL_TLB_NORMAL 0x00020000 #define UTURN_IOCTRL_MODE_OFF 0x00000000 #define UTURN_IOCTRL_MODE_INCLUDE 0x00000080 #define UTURN_IOCTRL_MODE_PEEK 0x00000180 #define UTURN_VIRTUAL_MODE \ (UTURN_IOCTRL_TLB_NORMAL | UTURN_IOCTRL_MODE_INCLUDE) int32_t unused2[1]; /* Runway Auxiliary Register Set */ uint32_t io_err_resp; /* Offset 0 */ uint32_t io_err_info; /* Offset 1 */ uint32_t io_err_req; /* Offset 2 */ uint32_t io_err_resp_hi; /* Offset 3 */ uint32_t io_tlb_entry_m; /* Offset 4 */ uint32_t io_tlb_entry_l; /* Offset 5 */ uint32_t unused3[1]; uint32_t io_pdir_base; /* Offset 7 */ uint32_t io_io_low_hv; /* Offset 8 */ uint32_t io_io_high_hv; /* Offset 9 */ uint32_t unused4[1]; uint32_t io_chain_id_mask; /* Offset 11 */ uint32_t unused5[2]; uint32_t io_io_low; /* Offset 14 */ uint32_t io_io_high; /* Offset 15 */ }; /* Uturn supports 256 TLB entries */ #define UTURN_CHAINID_SHIFT 8 #define UTURN_CHAINID_MASK 0xff #define UTURN_TLB_ENTRIES (1 << UTURN_CHAINID_SHIFT) #define UTURN_IOVP_SIZE PAGE_SIZE #define UTURN_IOVP_SHIFT PAGE_SHIFT #define UTURN_IOVP_MASK PAGE_MASK #define UTURN_IOVA(iovp, off) ((iovp) | (off)) #define UTURN_IOVP(iova) ((iova) & UTURN_IOVP_MASK) #define UTURN_IOVA_INDEX(iova) ((iova) >> UTURN_IOVP_SHIFT) struct uturn_softc { device_t sc_dv; bus_dma_tag_t sc_dmat; struct uturn_regs volatile *sc_regs; uint64_t *sc_pdir; uint32_t sc_chainid_shift; char sc_mapname[20]; struct extent *sc_map; /* XXXNH replace with something e.g.vmem? */ struct hppa_bus_dma_tag sc_dmatag; }; int uturnmatch(device_t, cfdata_t, void *); void uturnattach(device_t, device_t, void *); static void uturn_callback(device_t self, struct confargs *ca); CFATTACH_DECL_NEW(uturn, sizeof(struct uturn_softc), uturnmatch, uturnattach, NULL, NULL); extern struct cfdriver uturn_cd; int uturn_dmamap_create(void *, bus_size_t, int, bus_size_t, bus_size_t, int, bus_dmamap_t *); void uturn_dmamap_destroy(void *, bus_dmamap_t); int uturn_dmamap_load(void *, bus_dmamap_t, void *, bus_size_t, struct proc *, int); int uturn_dmamap_load_mbuf(void *, bus_dmamap_t, struct mbuf *, int); int uturn_dmamap_load_uio(void *, bus_dmamap_t, struct uio *, int); int uturn_dmamap_load_raw(void *, bus_dmamap_t, bus_dma_segment_t *, int, bus_size_t, int); void uturn_dmamap_unload(void *, bus_dmamap_t); void uturn_dmamap_sync(void *, bus_dmamap_t, bus_addr_t, bus_size_t, int); int uturn_dmamem_alloc(void *, bus_size_t, bus_size_t, bus_size_t, bus_dma_segment_t *, int, int *, int); void uturn_dmamem_free(void *, bus_dma_segment_t *, int); int uturn_dmamem_map(void *, bus_dma_segment_t *, int, size_t, void **, int); void uturn_dmamem_unmap(void *, void *, size_t); paddr_t uturn_dmamem_mmap(void *, bus_dma_segment_t *, int, off_t, int, int); static int uturn_iova_alloc(struct uturn_softc *, bus_dmamap_t, vaddr_t, bus_size_t, int, bus_addr_t *, bus_size_t *); const struct hppa_bus_dma_tag uturn_dmat = { NULL, uturn_dmamap_create, uturn_dmamap_destroy, uturn_dmamap_load, uturn_dmamap_load_mbuf, uturn_dmamap_load_uio, uturn_dmamap_load_raw, uturn_dmamap_unload, uturn_dmamap_sync, uturn_dmamem_alloc, uturn_dmamem_free, uturn_dmamem_map, uturn_dmamem_unmap, uturn_dmamem_mmap }; #define XXX_U2_IOA_RUNWAY 0x580 #define XXX_UTURN_IOA_RUNWAY 0x581 #define YYY_U2_BC_GSC 0x501 #define YYY_UTURN_BC_GSC 0x502 int uturnmatch(device_t parent, cfdata_t cf, void *aux) { struct confargs *ca = aux; /* there will be only one */ if (ca->ca_type.iodc_type != HPPA_TYPE_IOA || ca->ca_type.iodc_sv_model != HPPA_IOA_UTURN) return 0; if (ca->ca_type.iodc_model == 0x58 && ca->ca_type.iodc_revision >= 0x20) return 0; return 1; } void uturnattach(device_t parent, device_t self, void *aux) { struct confargs *ca = aux, nca; struct uturn_softc *sc = device_private(self); bus_space_handle_t ioh; volatile struct uturn_regs *r; struct pglist pglist; #if 0 struct vm_page *pg; #endif int iova_bits; vaddr_t va; #if 0 paddr_t pa; #endif psize_t size; int i; #if 0 #endif if (bus_space_map(ca->ca_iot, ca->ca_hpa, IOMOD_HPASIZE, 0, &ioh)) { aprint_error(": can't map IO space\n"); return; } sc->sc_dv = self; sc->sc_dmat = ca->ca_dmatag; sc->sc_regs = r = bus_space_vaddr(ca->ca_iot, ioh); aprint_normal(": %x-%x", r->io_io_low << 16, (r->io_io_high << 16) - 1); aprint_normal(": %x-%x", r->io_io_low_hv << 16, (r->io_io_high_hv << 16) - 1); aprint_normal(": %s rev %d\n", ca->ca_type.iodc_revision < 0x10? "U2" : "UTurn", ca->ca_type.iodc_revision & 0xf); /* * Setup the iommu. */ /* XXX minimum 1Mb, maximum 1Gb */ /* XXX 28 bits gives us 256Mb of iova space */ iova_bits = 28; /* size is # of pdir entries (64bits) in bytes. 1 entry per IOVA page */ size = (1 << (iova_bits - UTURN_IOVP_SHIFT)) * sizeof(uint64_t); /* * Chainid is the upper most bits of an IOVP used to determine which * TLB entry an IOVP will use. */ sc->sc_chainid_shift = iova_bits - UTURN_CHAINID_SHIFT; /* * Allocate memory for I/O pagetables. They need to be physically * contiguous. */ if (uvm_pglistalloc(size, 0, -1, PAGE_SIZE, 0, &pglist, 1, 0) != 0) panic("%s: no memory", __func__); #if 0 va = uvm_km_alloc(kernel_map, size, 0, UVM_KMF_VAONLY); if (va == 0) panic("%s: no memory", __func__); sc->sc_pdir = (int64_t *)va; /* Map the pages */ TAILQ_FOREACH(pg, &pglist, pageq.queue) { pa = VM_PAGE_TO_PHYS(pg); pmap_kenter_pa(va, pa, VM_PROT_READ | VM_PROT_WRITE, 0); va += PAGE_SIZE; } pmap_update(pmap_kernel()); #else va = (vaddr_t)VM_PAGE_TO_PHYS(TAILQ_FIRST(&pglist)); sc->sc_pdir = (int64_t *)va; #endif memset(sc->sc_pdir, 0, size); /* bus_space_write32 ??? */ r->io_chain_id_mask = UTURN_CHAINID_MASK << sc->sc_chainid_shift; r->io_pdir_base = VM_PAGE_TO_PHYS(TAILQ_FIRST(&pglist)); DPRINTF(("%s: pdir %p %lx\n", __func__, sc->sc_pdir, size)); DPRINTF(("%s: chain_id_mask (%p) %08x\n", __func__, &r->io_chain_id_mask, UTURN_CHAINID_MASK << sc->sc_chainid_shift)); DPRINTF(("%s: pdir_base (%p) %08x\n", __func__, &r->io_pdir_base, (int)VM_PAGE_TO_PHYS(TAILQ_FIRST(&pglist)))); r->io_tlb_entry_m = 0; r->io_tlb_entry_l = 0; /* for (i = UTURN_TLB_ENTRIES; i != 0; i--) { */ for (i = 0; i < UTURN_TLB_ENTRIES; i++) { #if 0 DPRINTF(("%s: sending UTURN_CMD_TLB_DIRECT_WRITE %08x\n", __func__, UTURN_CMD_TLB_DIRECT_WRITE | (i << sc->sc_chainid_shift))); #endif r->io_command = UTURN_CMD_TLB_DIRECT_WRITE | (i << sc->sc_chainid_shift); } /* * Go to "Virtual Mode" */ DPRINTF(("%s: setting io_control (%p) %08x\n", __func__, &r->io_control, UTURN_VIRTUAL_MODE)); r->io_control = UTURN_VIRTUAL_MODE; /* XXX ccio_init_resources */ #if 0 struct resource { resource_size_t start; resource_size_t end; const char *name; unsigned long flags; struct resource *parent, *sibling, *child; }; struct resource *res = ioc->mmio_region; char *name = kmalloc(14, GFP_KERNEL); snprintf(name, 14, "GSC Bus [%d/]", ioc->hw_path); ccio_init_resource(res, name, &ioc->ioc_regs->io_io_low); ccio_init_resource(res + 1, name, &ioc->ioc_regs->io_io_low_hv); #endif snprintf(sc->sc_mapname, sizeof(sc->sc_mapname), "%s_map", device_xname(sc->sc_dv)); sc->sc_map = extent_create(sc->sc_mapname, 0, (1 << iova_bits), M_DEVBUF, 0, 0, EX_NOWAIT); sc->sc_dmatag = uturn_dmat; sc->sc_dmatag._cookie = sc; /* * U2/UTurn is actually a combination of an Upper Bus Converter (UBC) * and a Lower Bus Converter (LBC). This driver attaches to the UBC; * the LBC isn't very interesting, so we skip it. This is easy, since * it always is module 63, hence the MAXMODBUS - 1 below. */ nca = *ca; nca.ca_hpabase = 0; nca.ca_dmatag = &sc->sc_dmatag; nca.ca_nmodules = MAXMODBUS - 1; pdc_scanbus(self, &nca, uturn_callback); /* XXX On some machines, PDC doesn't tell us about all devices. */ switch (cpu_hvers) { case HPPA_BOARD_HP809: case HPPA_BOARD_HP819: case HPPA_BOARD_HP829: case HPPA_BOARD_HP839: case HPPA_BOARD_HP849: case HPPA_BOARD_HP859: case HPPA_BOARD_HP869: case HPPA_BOARD_HP800D: case HPPA_BOARD_HP821: nca.ca_hpabase = r->io_io_low << 16; pdc_scanbus(self, &nca, uturn_callback); break; default: break; } } static void uturn_callback(device_t self, struct confargs *ca) { config_found_sm_loc(self, "gedoens", NULL, ca, mbprint, mbsubmatch); } /* XXX update with information from feb96a6.pdf */ /* * Given a virtual address (va) and space id, (sp), load the I/O PDIR entry * into the address pointed to by pdirp. Each I/O PDIR is encoded (HP bit * numbering) as below: * * WORD 0: * +------+----------------+-----------------------------------------------+ * | Phys | Virtual Index | Phys | * | 0:3 | 0:11 | 4:19 | * |4 bits| 12 bits | 16 bits | * +------+----------------+-----------------------------------------------+ * WORD 1: * +-----------------------+-----------------------------------------------+ * | Phys | Rsvd | Prefetch |Update |Rsvd |Lock |Safe |Valid | * | 20:39 | | Enable |Enable | |Enable|DMA | | * | 20 bits | 5 bits | 1 bit |1 bit |2 bits|1 bit |1 bit |1 bit | * +-----------------------+-----------------------------------------------+ * * The virtual index field is filled with the results of the LCI * (Load Coherence Index) instruction. The 8 bits used for the virtual * index are bits 12:19 of the value returned by LCI. */ #define UTURN_PENTRY_PREFETCH 0x40 #define UTURN_PENTRY_UPDATE 0x20 #define UTURN_PENTRY_LOCK 0x04 #define UTURN_PENTRY_SAFEDMA 0x02 /* use safe dma - for subcacheline */ #define UTURN_PENTRY_VALID 0x01 static void __inline uturn_iommu_enter(struct uturn_softc *sc, bus_addr_t iova, pa_space_t sp, vaddr_t va, paddr_t pa) { uint64_t pdir_entry; uint64_t *pdirp; uint32_t ci; /* coherent index */ // uint32_t *p0, *p1; pdirp = &sc->sc_pdir[UTURN_IOVA_INDEX(iova)]; DPRINTF(("%s: iova %lx pdir %p pdirp %p pa %lx", __func__, iova, sc->sc_pdir, pdirp, pa)); // p0 = (uint32_t *)pdirp; // p1 = p0++; /* * bottom 32 bits of pa page number, i.e last 12 bits 0. * hint goes into last 12 bits. */ pdir_entry = (UTURN_PENTRY_LOCK | UTURN_PENTRY_SAFEDMA | UTURN_PENTRY_VALID); pdir_entry |= (pa & ~PAGE_MASK); // *p1 = (pa & PAGE_MASK) | // UTURN_PENTRY_LOCK | UTURN_PENTRY_SAFEDMA | UTURN_PENTRY_VALID; /* // __LP64 // get bits 12:15 of physical address // shift bits 16:31 of physical address // and deposit them asm volatile ("extrd,u %1,15,4,%0" : "=r" (ci) : "r" (pa)); ci = pa[12:15] ; 4 bits asm volatile ("extrd,u %1,31,16,%0" : "+r" (pa) : "r" (pa)); pa = pa[16:31] ; 16 bits asm volatile ("depd %1,35,4,%0" : "+r" (pa) : "r" (ci)); pa[32:35] = ci ; 4 bits */ /* * * get CPU coherency index bits * Grab virtual index [0:11] * Deposit virt_idx bits into I/O PDIR word * * asm volatile ("lci %%r0(%%sr1, %1), %0" : "=r" (ci) : "r" (vba)); * asm volatile ("extru %1,19,12,%0" : "+r" (ci) : "r" (ci)); ci = ci[8:19] ; 12 bits * asm volatile ("depw %1,15,12,%0" : "+r" (pa) : "r" (ci)); pa[4:15] = ci ; 12 bits * * ((u32 *)pdir_ptr)[0] = (u32) pa; */ ci = lci(HPPA_SID_KERNEL, va); DPRINTF((": sp %lx, va %lx ci %x", (u_long)sp, va, ci)); // *p0 = ((ci >> 12) & 0xfff) << 16; pdir_entry &= 0x0000ffffffffffffUL; pdir_entry |= (((uint64_t)ci >> 12) << 48); pdir_entry |= ((((uint64_t)pa & 0x000f000000000000UL) >> 36) << 60); *pdirp = pdir_entry; DPRINTF((": pdir_entry %llx\n", pdir_entry)); /* * FIXME: PCX_W platforms don't need FDC/SYNC. (eg C360) * PCX-U/U+ do. (eg C200/C240) * PCX-T'? Don't know. (eg C110 or similar K-class) * * See PDC_MODEL/option 0/SW_CAP word for "Non-coherent IO-PDIR bit". * Hopefully we can patch (NOP) these out at boot time somehow. * * "Since PCX-U employs an offset hash that is incompatible with * the real mode coherence index generation of U2, the PDIR entry * must be flushed to memory to retain coherence." */ fdcache(HPPA_SID_KERNEL, (vaddr_t)pdirp, sizeof(uint64_t)); } static void __inline uturn_iommu_remove(struct uturn_softc *sc, bus_addr_t iova, bus_size_t size) { uint32_t chain_size = 1 << sc->sc_chainid_shift; bus_size_t len; KASSERT((iova & PAGE_MASK) == 0); KASSERT((size & PAGE_MASK) == 0); DPRINTF(("%s: sc %p iova %lx size %lx\n", __func__, sc, iova, size)); len = size; while (len != 0) { uint64_t *pdirp = &sc->sc_pdir[UTURN_IOVA_INDEX(iova)]; *pdirp = 0; /* XXX Just the valid bit??? */ /* Not needed for PCX-W - see PDC_MODEL */ fdcache(HPPA_SID_KERNEL, (vaddr_t)pdirp, sizeof(uint64_t)); iova += PAGE_SIZE; len -= PAGE_SIZE; } len = size + chain_size; while (len > chain_size) { /* bus_space_write4???*/ sc->sc_regs->io_command = UTURN_CMD_TLB_PURGE | iova; iova += chain_size; len -= chain_size; } } /* * Internal routine to allocate space in the IOMMU IOVA. */ static int uturn_iova_alloc(struct uturn_softc *sc, bus_dmamap_t map, vaddr_t va, bus_size_t len, int flags, bus_addr_t *iovap, bus_size_t *sgsizep) { bus_size_t sgsize; u_long /* align, */ voff, iovaddr; int s, err; DPRINTF(("%s: entered\n", __func__)); if (len > map->_dm_size) return EINVAL; /* * Remember page offset, then truncate the buffer address to * a page boundary. */ voff = va & PAGE_MASK; va = trunc_page(va); sgsize = round_page(len + voff); s = splhigh(); err = extent_alloc(sc->sc_map, sgsize, PAGE_SIZE /*map->_dm_align */, map->_dm_boundary, (flags & BUS_DMA_NOWAIT) == 0 ? EX_WAITOK : EX_NOWAIT, &iovaddr); splx(s); *iovap = (bus_addr_t)iovaddr; *sgsizep = sgsize; return err; } int uturn_dmamap_create(void *v, bus_size_t size, int nsegments, bus_size_t maxsegsz, bus_size_t boundary, int flags, bus_dmamap_t *dmamap) { struct uturn_softc *sc = v; int err; DPRINTF(("%s: size %lx nsegs %d maxsegsz %lx boundary %lx flags %d\n", __func__, size, nsegments, maxsegsz, boundary, flags)); err = bus_dmamap_create(sc->sc_dmat, size, nsegments, maxsegsz, boundary, flags, dmamap); if (err) return err; (*dmamap)->_dm_iovastart = 0; (*dmamap)->_dm_iovasize = 0; /* dmamap->_dm_align = PAGE_SIZE; */ return 0; } void uturn_dmamap_destroy(void *v, bus_dmamap_t map) { struct uturn_softc *sc = v; DPRINTF(("%s: map %p\n", __func__, map)); #if 0 KASSERT(map->_dm_iovastart == 0); KASSERT(map->_dm_iovasize == 0); #endif bus_dmamap_destroy(sc->sc_dmat, map); } int uturn_dmamap_load(void *v, bus_dmamap_t map, void *addr, bus_size_t size, struct proc *p, int flags) { struct uturn_softc *sc = v; bus_size_t sgsize; bus_addr_t iova; vaddr_t va = (vaddr_t)addr; pmap_t pmap; pa_space_t sp; int err; DPRINTF(("%s: map %p addr %p size %lx p %p flags %d\n", __func__, map, addr, size, p, flags)); /* * Make sure that on error condition we return "no valid mappings". */ map->dm_nsegs = 0; /* Allocate IOVA */ err = uturn_iova_alloc(sc, map, va, size, flags, &iova, &sgsize); if (err) return err; /* * We always use just one segment. */ map->dm_mapsize = size; map->dm_nsegs = 1; map->dm_segs[0].ds_addr = iova + (va & PAGE_MASK); map->dm_segs[0].ds_len = size; /* Set the active IOVA map */ map->_dm_iovastart = iova; map->_dm_iovasize = sgsize; if (p != NULL) pmap = p->p_vmspace->vm_map.pmap; else pmap = pmap_kernel(); sp = pmap->pm_space; for (; sgsize != 0; ) { paddr_t pa; /* * Get the physical address for this page. */ if (!pmap_extract(pmap, va, &pa)) { uturn_dmamap_unload(v, map); return EFAULT; } uturn_iommu_enter(sc, iova, sp, va, pa); iova += PAGE_SIZE; va += PAGE_SIZE; sgsize -= PAGE_SIZE; } return 0; } int uturn_dmamap_load_mbuf(void *v, bus_dmamap_t map, struct mbuf *m, int flags) { struct uturn_softc *sc = v; int err; int seg, s; u_long iovaddr; map->_dm_iovasize = 0; err = bus_dmamap_load_mbuf(sc->sc_dmat, map, m, flags); if (err) return err; /* becomes uturn_dmamap_load or similar */ for (seg = 0; seg < map->dm_nsegs; seg++) { struct hppa_bus_dma_segment *ds = &map->dm_segs[seg]; map->_dm_iovasize += round_page((ds->ds_addr & PAGE_MASK) + ds->ds_len); } /* 45678901234567890123456789012345678901234567890123456789012345678901234567890 */ s = splhigh(); err = extent_alloc(sc->sc_map, map->_dm_iovasize, PAGE_SIZE /* map->_dm_align */, map->_dm_boundary, (flags & BUS_DMA_NOWAIT) == 0 ? EX_WAITOK : EX_NOWAIT, &iovaddr); splx(s); map->_dm_iovastart = iovaddr; for (seg = 0; seg < map->dm_nsegs; seg++) { struct hppa_bus_dma_segment *ds = &map->dm_segs[seg]; vaddr_t va = ds->_ds_va & ~PAGE_MASK; paddr_t pa = ds->ds_addr & ~PAGE_MASK; vaddr_t eva = round_page(ds->_ds_va + ds->ds_len); DPRINTF(("%s: seg %2d va %lx(%lx) len %lx pa %lx(%lx) eva %lx\n", __func__, seg, ds->_ds_va, va, ds->ds_len, ds->ds_addr, pa, eva)); ds->ds_addr = iovaddr + (ds->_ds_va & PAGE_MASK); for (; va < eva;) { uturn_iommu_enter(sc, iovaddr, HPPA_SID_KERNEL, va, pa); va += PAGE_SIZE; pa += PAGE_SIZE; iovaddr += PAGE_SIZE; } } return 0; } int uturn_dmamap_load_uio(void *v, bus_dmamap_t map, struct uio *uio, int flags) { struct uturn_softc *sc = v; int ret; panic("not implemented"); ret = bus_dmamap_load_uio(sc->sc_dmat, map, uio, flags); return ret; } int uturn_dmamap_load_raw(void *v, bus_dmamap_t map, bus_dma_segment_t *segs, int nsegs, bus_size_t size, int flags) { struct uturn_softc *sc = v; struct vm_page *m; vaddr_t va; paddr_t pa; bus_addr_t iova; bus_size_t sgsize; struct pglist *mlist; int err; map->dm_nsegs = 0; va = segs[0]._ds_va; KASSERT((va & PAGE_MASK) == 0); /* Allocate IOMMU resources */ err = uturn_iova_alloc(sc, map, va, size, flags, &iova, &sgsize); if (err) return err; #if 0 /* * Note IOVA address in case bus_dmamem_map() is called later. * It can then insure cache coherency by choosing a KVA that * is aligned to `ds_addr'. */ #endif segs[0].ds_addr = iova; segs[0].ds_len = size; map->dm_segs[0].ds_addr = iova; map->dm_segs[0].ds_len = size; // map->dm_segs[0]._ds_sgsize = sgsize; /* Set the active IOVA map */ map->_dm_iovastart = iova; map->_dm_iovasize = sgsize; /* Map physical pages into IOMMU */ mlist = segs[0]._ds_mlist; TAILQ_FOREACH(m, mlist, pageq.queue) { if (sgsize == 0) panic("%s: size botch", __func__); pa = VM_PAGE_TO_PHYS(m); uturn_iommu_enter(sc, iova, HPPA_SID_KERNEL, va, pa); iova += PAGE_SIZE; va += PAGE_SIZE; sgsize -= PAGE_SIZE; } map->dm_nsegs = 1; map->dm_mapsize = size; return 0; } void uturn_dmamap_unload(void *v, bus_dmamap_t map) { struct uturn_softc *sc = v; bus_addr_t iova; bus_size_t len; int s, err; /* Flush the iommu */ iova = map->_dm_iovastart; len = map->_dm_iovasize; KASSERT(iova != 0); KASSERT(len > 0); /* Remove the IOMMU entries. */ uturn_iommu_remove(sc, iova, len); /* Mark the mappings as invalid. */ map->dm_mapsize = 0; map->dm_nsegs = 0; map->_dm_iovastart = 0; map->_dm_iovasize = 0; s = splhigh(); err = extent_free(sc->sc_map, iova, len, EX_NOWAIT); splx(s); if (err != 0) printf("warning: %ld of IOVA space lost\n", (long)len); } void uturn_dmamap_sync(void *v, bus_dmamap_t map, bus_addr_t off, bus_size_t len, int ops) { /* Nothing to do; DMA is cache-coherent. */ } int uturn_dmamem_alloc(void *v, bus_size_t size, bus_size_t alignment, bus_size_t boundary, bus_dma_segment_t *segs, int nsegs, int *rsegs, int flags) { struct uturn_softc *sc = v; int ret; DPRINTF(("%s: size %lx align %lx boundary %lx segs %p nsegs %d flags %d\n", __func__, size, alignment, boundary, segs, nsegs, flags)); ret = bus_dmamem_alloc(sc->sc_dmat, size, alignment, boundary, segs, nsegs, rsegs, flags); return ret; } void uturn_dmamem_free(void *v, bus_dma_segment_t *segs, int nsegs) { struct uturn_softc *sc = v; DPRINTF(("%s: segs %p nsegs %d\n", __func__, segs, nsegs)); bus_dmamem_free(sc->sc_dmat, segs, nsegs); } int uturn_dmamem_map(void *v, bus_dma_segment_t *segs, int nsegs, size_t size, void **kvap, int flags) { struct vm_page *pg; struct pglist *pglist; vaddr_t va; paddr_t pa; const uvm_flag_t kmflags = (flags & BUS_DMA_NOWAIT) != 0 ? UVM_KMF_NOWAIT : 0; DPRINTF(("%s: segs %p nsegs %d size %lx kvap %p flags %d\n", __func__, segs, nsegs, size, kvap, flags)); size = round_page(size); /* Get a chunk of kernel virtual space. */ va = uvm_km_alloc(kernel_map, size, 0, UVM_KMF_VAONLY | kmflags); if (va == 0) return ENOMEM; /* Stash that in the first segment. */ segs[0]._ds_va = va; *kvap = (void *)va; /* Map the allocated pages into the chunk. */ pglist = segs[0]._ds_mlist; TAILQ_FOREACH(pg, pglist, pageq.queue) { KASSERT(size != 0); pa = VM_PAGE_TO_PHYS(pg); pmap_kenter_pa(va, pa, VM_PROT_READ | VM_PROT_WRITE, 0); va += PAGE_SIZE; size -= PAGE_SIZE; } pmap_update(); return 0; } void uturn_dmamem_unmap(void *v, void *kva, size_t size) { DPRINTF(("%s: entered\n", __func__)); size = round_page(size); pmap_kremove((vaddr_t)kva, size); pmap_update(pmap_kernel()); uvm_km_free(kernel_map, (vaddr_t)kva, size, UVM_KMF_VAONLY); } paddr_t uturn_dmamem_mmap(void *v, bus_dma_segment_t *segs, int nsegs, off_t off, int prot, int flags) { struct uturn_softc *sc = v; int ret; DPRINTF(("%s: entered\n", __func__)); panic("boom"); /* Check for U-bit */ ret = bus_dmamem_mmap(sc->sc_dmat, segs, nsegs, off, prot, flags); return ret; }