1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
2 * vim:expandtab:shiftwidth=8:tabstop=8:
4 * Copyright (C) 2002 Cluster File Systems, Inc.
5 * Author: Phil Schwan <phil@clusterfs.com>
7 * This file is part of Lustre, http://www.lustre.org.
9 * Lustre is free software; you can redistribute it and/or
10 * modify it under the terms of version 2 of the GNU General Public
11 * License as published by the Free Software Foundation.
13 * Lustre is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU General Public License for more details.
18 * You should have received a copy of the GNU General Public License
19 * along with Lustre; if not, write to the Free Software
20 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
22 * Darwin porting library
23 * Make things easy to port
25 #define DEBUG_SUBSYSTEM S_PORTALS
27 #include <mach/mach_types.h>
29 #include <netinet/in.h>
30 #include <netinet/tcp.h>
33 #include <sys/vnode.h>
35 #include <sys/filedesc.h>
36 #include <sys/namei.h>
37 #include <miscfs/devfs/devfs.h>
38 #include <kern/kalloc.h>
39 #include <kern/zalloc.h>
40 #include <kern/thread.h>
42 #include <libcfs/libcfs.h>
43 #include <libcfs/kp30.h>
46 * Definition of struct zone, copied from osfmk/kern/zalloc.h.
49 int count; /* Number of elements used now */
50 vm_offset_t free_elements;
51 vm_size_t cur_size; /* current memory utilization */
52 vm_size_t max_size; /* how large can this zone grow */
53 vm_size_t elem_size; /* size of an element */
54 vm_size_t alloc_size; /* size used for more memory */
55 char *zone_name; /* a name for the zone */
57 /* boolean_t */ exhaustible :1, /* (F) merely return if empty? */
58 /* boolean_t */ collectable :1, /* (F) garbage collect empty pages */
59 /* boolean_t */ expandable :1, /* (T) expand zone (with message)? */
60 /* boolean_t */ allows_foreign :1,/* (F) allow non-zalloc space */
61 /* boolean_t */ doing_alloc :1, /* is zone expanding now? */
62 /* boolean_t */ waiting :1, /* is thread waiting for expansion? */
63 /* boolean_t */ async_pending :1; /* asynchronous allocation pending? */
64 struct zone_hack * next_zone; /* Link for all-zones list */
66 * more fields follow, but we don't need them. We only need
67 * offset from the beginning of struct zone to ->next_zone
68 * field: it allows us to scan the list of all zones.
72 decl_simple_lock_data(extern, all_zones_lock)
75 * returns true iff zone with name @name already exists.
77 * XXX nikita: this function is defined in this file only because there is no
78 * better place to put it in.
80 zone_t cfs_find_zone(const char *name)
82 struct zone_hack *scan;
84 /* from osfmk/kern/zalloc.c */
85 extern zone_t first_zone;
87 LASSERT(name != NULL);
89 simple_lock(&all_zones_lock);
90 for (scan = (struct zone_hack *)first_zone;
91 scan != NULL; scan = scan->next_zone) {
92 if (!strcmp(scan->zone_name, name))
95 simple_unlock(&all_zones_lock);
100 * our wrapper around kern/zalloc.c:zinit()
102 * Creates copy of name and calls zinit() to do real work. Needed because zone
103 * survives kext unloading, so that @name cannot be just static string
104 * embedded into kext image.
106 zone_t cfs_zinit(vm_size_t size, vm_size_t max, int alloc, const char *name)
110 cname = _MALLOC(strlen(name) + 1, M_TEMP, M_WAITOK);
111 LASSERT(cname != NULL);
112 return zinit(size, max, alloc, strcpy(cname, name));
116 cfs_mem_cache_create (const char *name, size_t objsize, size_t off, unsigned long arg1,
117 void (*arg2)(void *, cfs_mem_cache_t *, unsigned long),
118 void (*arg3)(void *, cfs_mem_cache_t *, unsigned long))
120 cfs_mem_cache_t *new = NULL;
122 MALLOC(new, cfs_mem_cache_t *, objsize, M_TEMP, M_WAITOK|M_ZERO);
124 CERROR("cfs_mem_cache created fail!\n");
128 CFS_INIT_LIST_HEAD(&new->link);
129 strncpy(new->name, name, 1 + strlen(name));
130 new->zone = cfs_find_zone(name);
131 if (new->zone == NULL) {
132 new->zone = cfs_zinit (objsize, KMEM_MAX_ZONE * objsize, 0, name);
133 if (new->zone == NULL) {
134 CERROR("zone create fault!\n");
143 cfs_mem_cache_destroy (cfs_mem_cache_t *cachep)
145 FREE (cachep, M_TEMP);
150 cfs_mem_cache_alloc (cfs_mem_cache_t *cachep, int flags)
152 return (void *)zalloc(cachep->zone);
156 cfs_mem_cache_free (cfs_mem_cache_t *cachep, void *objp)
158 zfree (cachep->zone, (vm_address_t)objp);
161 /* ---------------------------------------------------------------------------
164 * --------------------------------------------------------------------------- */
170 extern vm_map_t zone_map;
171 static inline vm_map_t page_map(struct xnu_raw_page *pg)
175 return pg->order == 0 ? zone_map : kernel_map;
178 static int raw_page_init(struct xnu_raw_page *pg)
180 vm_size_t size = (1UL << pg->order) * PAGE_SIZE;
181 int upl_flags = UPL_SET_INTERNAL |
182 UPL_SET_LITE | UPL_SET_IO_WIRE | UPL_COPYOUT_FROM;
185 /* XXX is it necessary? */
186 kr = vm_map_get_upl(page_map(pg),
187 pg->virtual, &size, &pg->upl, 0, 0, &upl_flags, 0);
191 static void raw_page_done(struct xnu_raw_page *pg)
193 ubc_upl_abort(pg->upl, UPL_ABORT_FREE_ON_EMPTY);
197 static struct xnu_page_ops raw_page_ops;
198 static struct xnu_page_ops *page_ops[XNU_PAGE_NTYPES] = {
199 [XNU_PAGE_RAW] = &raw_page_ops
202 static int page_type_is_valid(cfs_page_t *page)
204 LASSERT(page != NULL);
205 return 0 <= page->type && page->type < XNU_PAGE_NTYPES;
208 static int page_is_raw(cfs_page_t *page)
210 return page->type == XNU_PAGE_RAW;
213 static struct xnu_raw_page *as_raw(cfs_page_t *page)
215 LASSERT(page_is_raw(page));
216 return list_entry(page, struct xnu_raw_page, header);
219 static void *raw_page_address(cfs_page_t *pg)
221 return (void *)as_raw(pg)->virtual;
224 static void *raw_page_map(cfs_page_t *pg)
226 return (void *)as_raw(pg)->virtual;
229 static void raw_page_unmap(cfs_page_t *pg)
233 static struct xnu_page_ops raw_page_ops = {
234 .page_map = raw_page_map,
235 .page_unmap = raw_page_unmap,
236 .page_address = raw_page_address
240 extern vm_size_t kalloc_max;
241 extern vm_size_t kalloc_max_prerounded;
242 extern int first_k_zone;
243 extern struct zone *k_zone[16];
244 extern vm_offset_t zalloc_canblock( register zone_t, boolean_t );
245 extern vm_map_t zone_map;
247 static inline vm_address_t
248 page_zone_alloc(int flags, int order)
251 register vm_size_t allocsize;
252 vm_size_t size = (1UL << order) * PAGE_SIZE;
257 if (size > PAGE_SIZE){
259 * zalloc_canblock() call kernel_memory_allocate to allocate
260 * pages, kernel_memory_allocate cannot guarantee contig pages!
261 * So any request bigger then PAGE_SIZE should not call zalloc()
263 * NB. kmem_alloc_contig could be very slow!!!! Anyway, I dont
264 * know what will happen if order >= 1 :-(
266 CDEBUG(D_MALLOC, "Allocate contig pages!\n");
267 kr = kmem_alloc_contig(kernel_map, &addr, size, 0, 0);
272 allocsize = KALLOC_MINSIZE;
273 zindex = first_k_zone;
274 while (allocsize < size) {
278 assert(allocsize < kalloc_max);
279 if (flags & M_NOWAIT != 0)
280 addr = zalloc_canblock(k_zone[zindex], FALSE);
282 addr = zalloc_canblock(k_zone[zindex], TRUE);
286 /* Allocate a "page", actually upl of darwin */
287 struct xnu_raw_page *alloc_raw_pages(u_int32_t flags, u_int32_t order)
290 vm_size_t size = (1UL << order) * PAGE_SIZE;
291 u_int32_t mflags = 0;
292 struct xnu_raw_page *pg;
294 if (flags & CFS_ALLOC_ATOMIC != 0)
298 if (flags & CFS_ALLOC_ZERO != 0)
301 MALLOC (pg, struct xnu_raw_page *, sizeof *pg, M_TEMP, mflags);
304 pg->header.type = XNU_PAGE_RAW;
306 cfs_set_page_count(&pg->header, 1);
307 pg->virtual = page_zone_alloc(flags, order);
310 * XXX nikita: Liang, shouldn't pg be freed here?
314 kr = raw_page_init(pg);
316 size = (1UL << order) * PAGE_SIZE;
317 kmem_free(page_map(pg), pg->virtual, size);
324 void free_raw_pages(struct xnu_raw_page *pg, u_int32_t order)
326 vm_size_t size = (1UL << order) * PAGE_SIZE;
328 if (!atomic_dec_and_test(&pg->count))
331 kmem_free(page_map(pg), pg->virtual, size);
335 cfs_page_t *cfs_alloc_pages(u_int32_t flags, u_int32_t order)
337 return &alloc_raw_pages(flags, order)->header;
340 cfs_page_t *cfs_alloc_page(u_int32_t flags)
342 return cfs_alloc_pages(flags, 0);
345 void cfs_free_pages(cfs_page_t *pages, int order)
347 free_raw_pages(as_raw(pages), order);
350 void cfs_free_page(cfs_page_t *page)
352 cfs_free_pages(page, 0);
355 void cfs_get_page(cfs_page_t *p)
357 atomic_inc(&as_raw(p)->count);
360 int cfs_put_page_testzero(cfs_page_t *p)
362 return atomic_dec_and_test(&as_raw(p)->count);
365 int cfs_page_count(cfs_page_t *p)
367 return atomic_read(&as_raw(p)->count);
370 void cfs_set_page_count(cfs_page_t *p, int v)
372 atomic_set(&as_raw(p)->count, v);
376 * Generic page operations
379 void *cfs_page_address(cfs_page_t *pg)
381 LASSERT(page_type_is_valid(pg));
382 return page_ops[pg->type]->page_address(pg);
385 void *cfs_kmap(cfs_page_t *pg)
387 LASSERT(page_type_is_valid(pg));
388 return page_ops[pg->type]->page_map(pg);
391 void cfs_kunmap(cfs_page_t *pg)
393 LASSERT(page_type_is_valid(pg));
394 return page_ops[pg->type]->page_unmap(pg);
397 void xnu_page_ops_register(int type, struct xnu_page_ops *ops)
399 LASSERT(0 <= type && type < XNU_PAGE_NTYPES);
400 LASSERT(ops != NULL);
401 LASSERT(page_ops[type] == NULL);
403 page_ops[type] = ops;
406 void xnu_page_ops_unregister(int type)
408 LASSERT(0 <= type && type < XNU_PAGE_NTYPES);
409 LASSERT(page_ops[type] != NULL);
411 page_ops[type] = NULL;
415 * Portable memory allocator API
417 #ifdef HAVE_GET_PREEMPTION_LEVEL
418 extern int get_preemption_level(void);
420 #define get_preemption_level() (0)
423 void *cfs_alloc(size_t nr_bytes, u_int32_t flags)
428 if (flags & CFS_ALLOC_ATOMIC != 0) {
429 mflags |= 0 /* M_NOWAIT */;
431 LASSERT(get_preemption_level() == 0);
435 if (flags & CFS_ALLOC_ZERO != 0)
438 return _MALLOC(nr_bytes, M_TEMP, mflags);
441 void cfs_free(void *addr)
443 return _FREE(addr, M_TEMP);
446 void *cfs_alloc_large(size_t nr_bytes)
448 LASSERT(get_preemption_level() == 0);
449 return _MALLOC(nr_bytes, M_TEMP, M_WAITOK);
452 void cfs_free_large(void *addr)
454 return _FREE(addr, M_TEMP);