patch-2.1.101 linux/arch/sparc64/mm/init.c

• Lines: 491
• Date: Fri May 8 00:11:29 1998
• Orig file: v2.1.100/linux/arch/sparc64/mm/init.c
• Orig date: Thu Apr 23 20:21:32 1998

diff -u --recursive --new-file v2.1.100/linux/arch/sparc64/mm/init.c linux/arch/sparc64/mm/init.c
@@ -1,4 +1,4 @@
-/*  $Id: init.c,v 1.71 1998/03/27 07:00:08 davem Exp $
+/*  $Id: init.c,v 1.81 1998/05/04 05:35:43 jj Exp $
  *  arch/sparc64/mm/init.c
  *
  *  Copyright (C) 1996,1997 David S. Miller (davem@caip.rutgers.edu)
@@ -8,6 +8,8 @@
 #include <linux/config.h>
 #include <linux/string.h>
 #include <linux/init.h>
+#include <linux/mm.h>
+#include <linux/malloc.h>
 #include <linux/blk.h>
 #include <linux/swap.h>
 #include <linux/swapctl.h>
@@ -52,11 +54,6 @@
 	__bfill64((void *)pmdp, &two_null_pte_table);
 }
 
-static __inline__ void __init_pgd(pgd_t *pgdp)
-{
-	__bfill64((void *)pgdp, &two_null_pmd_table);
-}
-
 /*
  * BAD_PAGE is the page that is used for page faults when linux
  * is out-of-memory. Older versions of linux just did a
@@ -125,6 +122,9 @@
 	printk("%d pages shared\n",shared);
 	printk("%d pages swap cached\n",cached);
 	printk("%ld pages in page table cache\n",pgtable_cache_size);
+#ifndef __SMP__
+	printk("%ld entries in page dir cache\n",pgd_cache_size);
+#endif	
 	show_buffers();
 #ifdef CONFIG_NET
 	show_net_buffers();
@@ -135,27 +135,47 @@
 
 /* This keeps track of pages used in sparc_alloc_dvma() invocations. */
 /* NOTE: All of these are inited to 0 in bss, don't need to make data segment bigger */
-static unsigned long dvma_map_pages[0x10000000 >> 16];
+#define DVMAIO_SIZE 0x2000000
+static unsigned long dvma_map_pages[DVMAIO_SIZE >> 16];
 static unsigned long dvma_pages_current_offset;
 static int dvma_pages_current_index;
+static unsigned long dvmaiobase = 0;
+static unsigned long dvmaiosz __initdata = 0;
 
 /* #define E3000_DEBUG */
 
-__initfunc(unsigned long iommu_init(int iommu_node, unsigned long memory_start,
-				    unsigned long memory_end, struct linux_sbus *sbus))
+__initfunc(void dvmaio_init(void))
+{
+	int i;
+	
+	if (!dvmaiobase) {
+		for (i = 0; sp_banks[i].num_bytes != 0; i++)
+			if (sp_banks[i].base_addr + sp_banks[i].num_bytes > dvmaiobase)
+				dvmaiobase = sp_banks[i].base_addr + sp_banks[i].num_bytes;
+		dvmaiobase = (dvmaiobase + DVMAIO_SIZE + 0x400000 - 1) & ~(0x400000 - 1);
+		for (i = 0; i < 6; i++)
+			if (dvmaiobase <= ((1024 * 64 * 1024) << i))
+				break;
+		dvmaiobase = ((1024 * 64 * 1024) << i) - DVMAIO_SIZE;
+		dvmaiosz = i;
+	}
+}
+
+__initfunc(void iommu_init(int iommu_node, struct linux_sbus *sbus))
 {
 	struct iommu_struct *iommu;
 	struct sysio_regs *sregs;
 	struct linux_prom64_registers rprop;
 	unsigned long impl, vers;
 	unsigned long control, tsbbase;
+	unsigned long tsbbases[32];
 	unsigned long *iopte;
-	u32 rlow, rhigh;
-	int err, i;
-
+	int err, i, j;
+	
+	dvmaio_init();
 #ifdef E3000_DEBUG
-	prom_printf("\niommu_init: [%x:%016lx:%016lx:%p] ",
-		    iommu_node, memory_start, memory_end, sbus);
+	prom_printf("\niommu_init: [%x:%p] ",
+		    iommu_node, sbus);
 #endif
 	err = prom_getproperty(iommu_node, "reg", (char *)&rprop,
 			       sizeof(rprop));
@@ -163,14 +183,8 @@
 		prom_printf("iommu_init: Cannot map SYSIO control registers.\n");
 		prom_halt();
 	}
-	rlow  = (rprop.phys_addr & 0xffffffff);
-	rhigh = (rprop.phys_addr >> 32);
-#ifdef E3000_DEBUG
-	prom_printf("rlow[%08x] rhigh[%08x] ", rlow, rhigh);
-#endif
-	sregs = (struct sysio_regs *) sparc_alloc_io(rlow, (void *)0,
-						     sizeof(struct sysio_regs),
-						     "SYSIO Regs", rhigh, 0x0);
+	sregs = (struct sysio_regs *) __va(rprop.phys_addr);
+
 #ifdef E3000_DEBUG
 	prom_printf("sregs[%p]\n");
 #endif
@@ -179,9 +193,7 @@
 		prom_halt();
 	}
 
-	memory_start = (memory_start + 7) & ~7;
-	iommu = (struct iommu_struct *) memory_start;
-	memory_start += sizeof(struct iommu_struct);
+	iommu = kmalloc(sizeof(struct iommu_struct), GFP_ATOMIC);
 
 #ifdef E3000_DEBUG
 	prom_printf("iommu_init: iommu[%p] ", iommu);
@@ -203,26 +215,54 @@
 	       (unsigned int) impl, (unsigned int)vers, (unsigned long) sregs);
 	
 	control &= ~(IOMMU_CTRL_TSBSZ);
-	control |= (IOMMU_TSBSZ_64K | IOMMU_CTRL_TBWSZ | IOMMU_CTRL_ENAB);
+	control |= ((IOMMU_TSBSZ_2K * dvmaiosz) | IOMMU_CTRL_TBWSZ | IOMMU_CTRL_ENAB);
 
 	/* Use only 64k pages, things are layed out in the 32-bit SBUS
 	 * address space like this:
 	 *
-	 * 0x00000000	----------------------------------------
-	 *		| Direct physical mappings for most    |
-	 *              | DVMA to paddr's within this range    |
-	 * 0xf0000000   ----------------------------------------
-	 * 		| For mappings requested via           |
-	 *              | sparc_alloc_dvma()		       |
-	 * 0xffffffff	----------------------------------------
-	 */
-	tsbbase = PAGE_ALIGN(memory_start);
-	memory_start = (tsbbase + ((64 * 1024) * 8));
+	 * 0x00000000	  ----------------------------------------
+	 *		  | Direct physical mappings for most    |
+	 *                | DVMA to paddr's within this range    |
+	 * dvmaiobase     ----------------------------------------
+	 * 		  | For mappings requested via           |
+	 *                | sparc_alloc_dvma()		         |
+	 * dvmaiobase+32M ----------------------------------------
+	 *
+	 * NOTE: we need to order 2 contiguous order 5, that's the largest
+	 *       chunk page_alloc will give us.   -JJ */
+	tsbbase = 0;
+	if (dvmaiosz == 6) {
+		memset (tsbbases, 0, sizeof(tsbbases));
+		for (i = 0; i < 32; i++) {
+			tsbbases[i] = __get_free_pages(GFP_DMA, 5);
+			for (j = 0; j < i; j++)
+				if (tsbbases[j] == tsbbases[i] + 32768*sizeof(iopte_t)) {
+					tsbbase = tsbbases[i];
+					break;
+				} else if (tsbbases[i] == tsbbases[j] + 32768*sizeof(iopte_t)) {
+					tsbbase = tsbbases[j];
+					break;
+				}
+			if (tsbbase) {
+				tsbbases[i] = 0;
+				tsbbases[j] = 0;
+				break;
+			}
+		}
+		for (i = 0; i < 32; i++)
+			if (tsbbases[i])
+				free_pages(tsbbases[i], 5);
+	} else
+		tsbbase = __get_free_pages(GFP_DMA, dvmaiosz);
+	if (!tsbbase) {
+		prom_printf("Strange. Could not allocate 512K of contiguous RAM.\n");
+		prom_halt();
+	}
 	iommu->page_table = (iopte_t *) tsbbase;
 	iopte = (unsigned long *) tsbbase;
 
 	/* Setup aliased mappings... */
-	for(i = 0; i < (65536 - 4096); i++) {
+	for(i = 0; i < (dvmaiobase >> 16); i++) {
 		*iopte  = (IOPTE_VALID | IOPTE_64K | IOPTE_STBUF |
 			   IOPTE_CACHE | IOPTE_WRITE);
 		*iopte |= (i << 16);
@@ -230,7 +270,7 @@
 	}
 
 	/* Clear all sparc_alloc_dvma() maps. */
-	for( ; i < 65536; i++)
+	for( ; i < ((dvmaiobase + DVMAIO_SIZE) >> 16); i++)
 		*iopte++ = 0;
 
 #ifdef E3000_DEBUG
@@ -252,23 +292,19 @@
 #endif
 	printk("IOMMU: Streaming Buffer IMPL[%x] REV[%x] ",
 	       (unsigned int)impl, (unsigned int)vers);
-	printk("FlushFLAG[%p,%016lx] ... ",
-	       (iommu->sbuf_flushflag_va = (unsigned int *)memory_start),
-	       (iommu->sbuf_flushflag_pa = __pa(memory_start)));
+	iommu->sbuf_flushflag_va = kmalloc(sizeof(unsigned long), GFP_DMA);
+	printk("FlushFLAG[%016lx] ... ", (iommu->sbuf_flushflag_pa = __pa(iommu->sbuf_flushflag_va)));
 	*(iommu->sbuf_flushflag_va) = 0;
-	memory_start += sizeof(unsigned long); /* yes, unsigned long, for alignment */
 
 	sregs->sbuf_control = (control | SYSIO_SBUFCTRL_SB_EN);
 
 #ifdef E3000_DEBUG
-	prom_printf("done, returning %016lx\n", memory_start);
+	prom_printf("done, returning\n");
 #endif
 	printk("ENABLED\n");
 
 	/* Finally enable DVMA arbitration for all devices, just in case. */
 	sregs->sbus_control |= SYSIO_SBCNTRL_AEN;
-
-	return memory_start;
 }
 
 void mmu_map_dma_area(unsigned long addr, int len, __u32 *dvma_addr,
@@ -300,7 +336,7 @@
 		}
 
 		/* Stick it in the IOMMU. */
-		i = (65536 - 4096) + i;
+		i = (dvmaiobase >> 16) + i;
 		for_each_sbus(sbus) {
 			struct iommu_struct *iommu = sbus->iommu;
 			unsigned long flags;
@@ -314,7 +350,7 @@
 	}
 
 	/* Get this out of the way. */
-	*dvma_addr = (__u32) ((0xf0000000) +
+	*dvma_addr = (__u32) ((dvmaiobase) +
 			      (dvma_pages_current_index << 16) +
 			      (dvma_pages_current_offset));
 
@@ -345,13 +381,17 @@
 {
 	__u32 sbus_addr = (__u32) __pa(vaddr);
 
-	if((sbus_addr < 0xf0000000) &&
-	   ((sbus_addr + len) < 0xf0000000))
+#ifndef DEBUG_IOMMU
+	return sbus_addr;
+#else
+	if((sbus_addr < dvmaiobase) &&
+	   ((sbus_addr + len) < dvmaiobase))
 		return sbus_addr;
 
 	/* "can't happen"... GFP_DMA assures this. */
 	panic("Very high scsi_one mappings should never happen.");
         return (__u32)0;
+#endif        
 }
 
 void mmu_release_scsi_one(u32 vaddr, unsigned long len, struct linux_sbus *sbus)
@@ -385,13 +425,17 @@
 {
 	while(sz >= 0) {
 		__u32 page = (__u32) __pa(((unsigned long) sg[sz].addr));
-		if((page < 0xf0000000) &&
-		   (page + sg[sz].len) < 0xf0000000) {
+#ifndef DEBUG_IOMMU
+		sg[sz].dvma_addr = page;
+#else		
+		if((page < dvmaiobase) &&
+		   (page + sg[sz].len) < dvmaiobase) {
 			sg[sz].dvma_addr = page;
 		} else {
 			/* "can't happen"... GFP_DMA assures this. */
 			panic("scsi_sgl high mappings should never happen.");
 		}
+#endif
 		sz--;
 	}
 }
@@ -485,10 +529,42 @@
 	}
 }
 
+static int prom_ditlb_set = 0;
 int prom_itlb_ent, prom_dtlb_ent;
 unsigned long prom_itlb_tag, prom_itlb_data;
 unsigned long prom_dtlb_tag, prom_dtlb_data;
 
+void prom_world(int enter)
+{
+	if (!prom_ditlb_set) return;
+	if (enter) {
+		/* Install PROM world. */
+		__asm__ __volatile__("stxa %0, [%1] %2"
+					: : "r" (prom_dtlb_tag), "r" (TLB_TAG_ACCESS),
+					"i" (ASI_DMMU));
+		membar("#Sync");
+		spitfire_put_dtlb_data(62, prom_dtlb_data);
+		membar("#Sync");
+		__asm__ __volatile__("stxa %0, [%1] %2"
+					: : "r" (prom_itlb_tag), "r" (TLB_TAG_ACCESS),
+					"i" (ASI_IMMU));
+		membar("#Sync");
+		spitfire_put_itlb_data(62, prom_itlb_data);
+		membar("#Sync");
+	} else {
+		__asm__ __volatile__("stxa %%g0, [%0] %1"
+					: : "r" (TLB_TAG_ACCESS), "i" (ASI_DMMU));
+		membar("#Sync");
+		spitfire_put_dtlb_data(62, 0x0UL);
+		membar("#Sync");
+		__asm__ __volatile__("stxa %%g0, [%0] %1"
+					: : "r" (TLB_TAG_ACCESS), "i" (ASI_IMMU));
+		membar("#Sync");
+		spitfire_put_itlb_data(62, 0x0UL);
+		membar("#Sync");
+	}
+}
+
 void inherit_locked_prom_mappings(int save_p)
 {
 	int i;
@@ -500,8 +576,7 @@
 	 * translations property.  The only ones that matter are
 	 * the locked PROM tlb entries, so we impose the following
 	 * irrecovable rule on the PROM, it is allowed 1 locked
-	 * entry in the ITLB and 1 in the DTLB.  We move those
-	 * (if necessary) up into tlb entry 62.
+	 * entry in the ITLB and 1 in the DTLB.
 	 *
 	 * Supposedly the upper 16GB of the address space is
 	 * reserved for OBP, BUT I WISH THIS WAS DOCUMENTED
@@ -510,7 +585,7 @@
 	 * systems to coordinate mmu mappings is also COMPLETELY
 	 * UNDOCUMENTED!!!!!! Thanks S(t)un!
 	 */
-	for(i = 0; i < 62; i++) {
+	for(i = 0; i < 63; i++) {
 		unsigned long data;
 
 		data = spitfire_get_dtlb_data(i);
@@ -528,13 +603,6 @@
 			spitfire_put_dtlb_data(i, 0x0UL);
 			membar("#Sync");
 
-			/* Re-install it. */
-			__asm__ __volatile__("stxa %0, [%1] %2"
-					     : : "r" (tag), "r" (TLB_TAG_ACCESS),
-					         "i" (ASI_DMMU));
-			membar("#Sync");
-			spitfire_put_dtlb_data(62, data);
-			membar("#Sync");
 			dtlb_seen = 1;
 			if(itlb_seen)
 				break;
@@ -566,6 +634,8 @@
 				break;
 		}
 	}
+	if (save_p)
+		prom_ditlb_set = 1;
 }
 
 /* Give PROM back his world, done during reboots... */
@@ -628,37 +698,67 @@
 			     : : "r" (pstate));
 }
 
-/* We are always protected by scheduler_lock under SMP. */
-void get_new_mmu_context(struct mm_struct *mm, unsigned long *ctx)
-{
-	unsigned int new_ctx = *ctx;
+unsigned long mmu_context_bmap[1UL << (CTX_VERSION_SHIFT - 6)];
 
-	if((new_ctx & ~(CTX_VERSION_MASK)) == 0) {
-		new_ctx += CTX_FIRST_VERSION;
-		if(new_ctx == 1)
-			new_ctx = CTX_FIRST_VERSION;
-		*ctx = new_ctx;
-		DO_LOCAL_FLUSH(smp_processor_id());
+/* We are always protected by scheduler_lock under SMP.
+ * Caller does TLB context flushing on local CPU if necessary.
+ */
+void get_new_mmu_context(struct mm_struct *mm)
+{
+	unsigned long ctx = (tlb_context_cache + 1) & ~(CTX_VERSION_MASK);
+	unsigned long new_ctx;
+	
+	if (mm->context != NO_CONTEXT && !((mm->context ^ tlb_context_cache) & CTX_VERSION_MASK))
+		clear_bit(mm->context & ~(CTX_VERSION_MASK), mmu_context_bmap);
+	new_ctx = find_next_zero_bit(mmu_context_bmap, 1UL << CTX_VERSION_SHIFT, ctx);
+	if (new_ctx >= (1UL << CTX_VERSION_SHIFT)) {
+		new_ctx = find_next_zero_bit(mmu_context_bmap, ctx, 1);
+		if (new_ctx >= ctx) {
+			new_ctx = (tlb_context_cache & CTX_VERSION_MASK) + CTX_FIRST_VERSION;
+			mmu_context_bmap[0] = 3;
+			memset(mmu_context_bmap + sizeof(long), 0, sizeof(mmu_context_bmap) - sizeof(long));
+			goto out;
+		}
 	}
+	set_bit(new_ctx, mmu_context_bmap);
+	new_ctx |= (tlb_context_cache & CTX_VERSION_MASK);
+out:	tlb_context_cache = new_ctx;
 	mm->context = new_ctx;
-	mm->cpu_vm_mask = 0;	/* Callers sets it properly. */
-	(*ctx)++;
+	mm->cpu_vm_mask = 0;
 }
 
-#ifndef __SMP__
-struct pgtable_cache_struct pgt_quicklists;
+#ifdef __SMP__
+spinlock_t user_page_lock = SPIN_LOCK_UNLOCKED;
 #endif
+struct upcache user_page_cache[2] __attribute__((aligned(32)));
 
-pgd_t *get_pgd_slow(void)
-{
-	pgd_t *pgd;
-
-	pgd = (pgd_t *) __get_free_page(GFP_KERNEL);
-	if(pgd)
-		__init_pgd(pgd);
-	return pgd;
+unsigned long get_user_page_slow(int which)
+{
+	unsigned long chunk;
+	struct upcache *up = &user_page_cache[!which];
+	struct page *p;
+
+	do { chunk = __get_free_pages(GFP_KERNEL, 1); } while(chunk==0);
+	p = mem_map + MAP_NR(chunk);
+	atomic_set(&p->count, 1);
+	atomic_set(&(p+1)->count, 1);
+	p->age = (p+1)->age = PAGE_INITIAL_AGE;
+	spin_lock(&user_page_lock);
+	if(up->count < USER_PAGE_WATER) {
+		struct page *new = p + !which;
+		new->next = up->list;
+		up->list = new;
+		up->count++;
+	} else
+		free_pages((chunk+(PAGE_SIZE*(!which))), 0);
+	spin_unlock(&user_page_lock);
+	return page_address(p + which);
 }
 
+#ifndef __SMP__
+struct pgtable_cache_struct pgt_quicklists;
+#endif
+
 pmd_t *get_pmd_slow(pgd_t *pgd, unsigned long offset)
 {
 	pmd_t *pmd;
@@ -747,7 +847,7 @@
 	pte_clear(ptep);
 }
 
-#if NOTUSED
+#ifdef NOTUSED
 void sparc_ultra_dump_itlb(void)
 {
         int slot;
@@ -796,6 +896,7 @@
 	unsigned long flags;
 	unsigned long shift = alias_base - ((unsigned long)&empty_zero_page);
 	
+	set_bit(0, mmu_context_bmap);
 	/* We assume physical memory starts at some 4mb multiple,
 	 * if this were not true we wouldn't boot up to this point
 	 * anyways.
@@ -889,6 +990,7 @@
 	unsigned long tmp = 0, paddr, endaddr;
 	unsigned long end = __pa(end_mem);
 
+	dvmaio_init();
 	for (paddr = __pa(start_mem); paddr < end; ) {
 		for (; sp_banks[tmp].num_bytes != 0; tmp++)
 			if (sp_banks[tmp].base_addr + sp_banks[tmp].num_bytes > paddr)
@@ -911,7 +1013,7 @@
 		endaddr = sp_banks[tmp].base_addr + sp_banks[tmp].num_bytes;
 		while (paddr < endaddr) {
 			mem_map[paddr>>PAGE_SHIFT].flags &= ~(1<<PG_reserved);
-			if (paddr >= 0xf0000000)
+			if (paddr >= dvmaiobase)
 				mem_map[paddr>>PAGE_SHIFT].flags &= ~(1<<PG_DMA);
 			paddr += PAGE_SIZE;
 		}