| /* |
| * Copyright © 2009 CNRS |
| * Copyright © 2009-2011 INRIA. All rights reserved. |
| * Copyright © 2009-2011 Université Bordeaux 1 |
| * Copyright © 2011 Cisco Systems, Inc. All rights reserved. |
| * See COPYING in top-level directory. |
| */ |
| |
| #include <private/autogen/config.h> |
| |
| #include <sys/types.h> |
| #include <dirent.h> |
| #include <unistd.h> |
| #include <string.h> |
| #include <errno.h> |
| #include <stdio.h> |
| #include <sys/stat.h> |
| #include <fcntl.h> |
| #include <pthread.h> |
| |
| #include <hwloc.h> |
| #include <private/private.h> |
| #include <private/debug.h> |
| |
| #include <numa.h> |
| #include <radset.h> |
| #include <cpuset.h> |
| #include <sys/mman.h> |
| |
| /* |
| * TODO |
| * |
| * nsg_init(), nsg_attach_pid(), RAD_MIGRATE/RAD_WAIT |
| * assign_pid_to_pset() |
| * |
| * pthread_use_only_cpu too? |
| */ |
| |
| static int |
| prepare_radset(hwloc_topology_t topology, radset_t *radset, hwloc_const_bitmap_t hwloc_set) |
| { |
| unsigned cpu; |
| cpuset_t target_cpuset; |
| cpuset_t cpuset, xor_cpuset; |
| radid_t radid; |
| int ret = 0; |
| int ret_errno = 0; |
| |
| cpusetcreate(&target_cpuset); |
| cpuemptyset(target_cpuset); |
| hwloc_bitmap_foreach_begin(cpu, hwloc_set) |
| cpuaddset(target_cpuset, cpu); |
| hwloc_bitmap_foreach_end(); |
| |
| cpusetcreate(&cpuset); |
| cpusetcreate(&xor_cpuset); |
| for (radid = 0; radid < topology->backend_params.osf.nbnodes; radid++) { |
| cpuemptyset(cpuset); |
| if (rad_get_cpus(radid, cpuset)==-1) { |
| fprintf(stderr,"rad_get_cpus(%d) failed: %s\n",radid,strerror(errno)); |
| continue; |
| } |
| cpuxorset(target_cpuset, cpuset, xor_cpuset); |
| if (cpucountset(xor_cpuset) == 0) { |
| /* Found it */ |
| radsetcreate(radset); |
| rademptyset(*radset); |
| radaddset(*radset, radid); |
| ret = 1; |
| goto out; |
| } |
| } |
| /* radset containing exactly this set of CPUs not found */ |
| ret_errno = EXDEV; |
| |
| out: |
| cpusetdestroy(&target_cpuset); |
| cpusetdestroy(&cpuset); |
| cpusetdestroy(&xor_cpuset); |
| errno = ret_errno; |
| return ret; |
| } |
| |
| /* Note: get_cpubind not available on OSF */ |
| |
| static int |
| hwloc_osf_set_thread_cpubind(hwloc_topology_t topology, hwloc_thread_t thread, hwloc_const_bitmap_t hwloc_set, int flags) |
| { |
| radset_t radset; |
| |
| if (hwloc_bitmap_isequal(hwloc_set, hwloc_topology_get_complete_cpuset(topology))) { |
| if ((errno = pthread_rad_detach(thread))) |
| return -1; |
| return 0; |
| } |
| |
| /* Apparently OSF migrates pages */ |
| if (flags & HWLOC_CPUBIND_NOMEMBIND) { |
| errno = ENOSYS; |
| return -1; |
| } |
| |
| if (!prepare_radset(topology, &radset, hwloc_set)) |
| return -1; |
| |
| if (flags & HWLOC_CPUBIND_STRICT) { |
| if ((errno = pthread_rad_bind(thread, radset, RAD_INSIST | RAD_WAIT))) |
| return -1; |
| } else { |
| if ((errno = pthread_rad_attach(thread, radset, RAD_WAIT))) |
| return -1; |
| } |
| radsetdestroy(&radset); |
| |
| return 0; |
| } |
| |
| static int |
| hwloc_osf_set_proc_cpubind(hwloc_topology_t topology, hwloc_pid_t pid, hwloc_const_bitmap_t hwloc_set, int flags) |
| { |
| radset_t radset; |
| |
| if (hwloc_bitmap_isequal(hwloc_set, hwloc_topology_get_complete_cpuset(topology))) { |
| if (rad_detach_pid(pid)) |
| return -1; |
| return 0; |
| } |
| |
| /* Apparently OSF migrates pages */ |
| if (flags & HWLOC_CPUBIND_NOMEMBIND) { |
| errno = ENOSYS; |
| return -1; |
| } |
| |
| if (!prepare_radset(topology, &radset, hwloc_set)) |
| return -1; |
| |
| if (flags & HWLOC_CPUBIND_STRICT) { |
| if (rad_bind_pid(pid, radset, RAD_INSIST | RAD_WAIT)) |
| return -1; |
| } else { |
| if (rad_attach_pid(pid, radset, RAD_WAIT)) |
| return -1; |
| } |
| radsetdestroy(&radset); |
| |
| return 0; |
| } |
| |
| static int |
| hwloc_osf_set_thisthread_cpubind(hwloc_topology_t topology, hwloc_const_bitmap_t hwloc_set, int flags) |
| { |
| return hwloc_osf_set_thread_cpubind(topology, pthread_self(), hwloc_set, flags); |
| } |
| |
| static int |
| hwloc_osf_set_thisproc_cpubind(hwloc_topology_t topology, hwloc_const_bitmap_t hwloc_set, int flags) |
| { |
| return hwloc_osf_set_proc_cpubind(topology, getpid(), hwloc_set, flags); |
| } |
| |
| static int |
| hwloc_osf_prepare_mattr(hwloc_topology_t topology __hwloc_attribute_unused, memalloc_attr_t *mattr, hwloc_const_nodeset_t nodeset, hwloc_membind_policy_t policy, int flags __hwloc_attribute_unused) |
| { |
| unsigned long osf_policy; |
| int node; |
| |
| switch (policy) { |
| case HWLOC_MEMBIND_FIRSTTOUCH: |
| osf_policy = MPOL_THREAD; |
| break; |
| case HWLOC_MEMBIND_DEFAULT: |
| case HWLOC_MEMBIND_BIND: |
| osf_policy = MPOL_DIRECTED; |
| break; |
| case HWLOC_MEMBIND_INTERLEAVE: |
| osf_policy = MPOL_STRIPPED; |
| break; |
| case HWLOC_MEMBIND_REPLICATE: |
| osf_policy = MPOL_REPLICATED; |
| break; |
| default: |
| errno = ENOSYS; |
| return -1; |
| } |
| |
| memset(mattr, 0, sizeof(*mattr)); |
| mattr->mattr_policy = osf_policy; |
| mattr->mattr_rad = RAD_NONE; |
| radsetcreate(&mattr->mattr_radset); |
| rademptyset(mattr->mattr_radset); |
| |
| hwloc_bitmap_foreach_begin(node, nodeset) |
| radaddset(mattr->mattr_radset, node); |
| hwloc_bitmap_foreach_end(); |
| return 0; |
| } |
| |
| static int |
| hwloc_osf_set_area_membind(hwloc_topology_t topology, const void *addr, size_t len, hwloc_const_nodeset_t nodeset, hwloc_membind_policy_t policy, int flags) |
| { |
| memalloc_attr_t mattr; |
| int behavior = 0; |
| int ret; |
| |
| if (flags & HWLOC_MEMBIND_MIGRATE) |
| behavior |= MADV_CURRENT; |
| if (flags & HWLOC_MEMBIND_STRICT) |
| behavior |= MADV_INSIST; |
| |
| if (hwloc_osf_prepare_mattr(topology, &mattr, nodeset, policy, flags)) |
| return -1; |
| |
| ret = nmadvise(addr, len, MADV_CURRENT, &mattr); |
| radsetdestroy(&mattr.mattr_radset); |
| return ret; |
| } |
| |
| static void * |
| hwloc_osf_alloc_membind(hwloc_topology_t topology, size_t len, hwloc_const_nodeset_t nodeset, hwloc_membind_policy_t policy, int flags) |
| { |
| memalloc_attr_t mattr; |
| void *ptr; |
| |
| if (hwloc_osf_prepare_mattr(topology, &mattr, nodeset, policy, flags)) |
| return hwloc_alloc_or_fail(topology, len, flags); |
| |
| /* TODO: rather use acreate/amalloc ? */ |
| ptr = nmmap(NULL, len, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, |
| 0, &mattr); |
| radsetdestroy(&mattr.mattr_radset); |
| return ptr; |
| } |
| |
| void |
| hwloc_look_osf(struct hwloc_topology *topology) |
| { |
| cpu_cursor_t cursor; |
| unsigned nbnodes; |
| radid_t radid, radid2; |
| radset_t radset, radset2; |
| cpuid_t cpuid; |
| cpuset_t cpuset; |
| struct hwloc_obj *obj; |
| unsigned distance; |
| |
| topology->backend_params.osf.nbnodes = nbnodes = rad_get_num(); |
| |
| cpusetcreate(&cpuset); |
| radsetcreate(&radset); |
| radsetcreate(&radset2); |
| { |
| hwloc_obj_t *nodes = calloc(nbnodes, sizeof(hwloc_obj_t)); |
| unsigned *indexes = calloc(nbnodes, sizeof(unsigned)); |
| float *distances = calloc(nbnodes*nbnodes, sizeof(float)); |
| unsigned nfound; |
| numa_attr_t attr; |
| |
| attr.nattr_type = R_RAD; |
| attr.nattr_descr.rd_radset = radset; |
| attr.nattr_flags = 0; |
| |
| for (radid = 0; radid < (radid_t) nbnodes; radid++) { |
| rademptyset(radset); |
| radaddset(radset, radid); |
| cpuemptyset(cpuset); |
| if (rad_get_cpus(radid, cpuset)==-1) { |
| fprintf(stderr,"rad_get_cpus(%d) failed: %s\n",radid,strerror(errno)); |
| continue; |
| } |
| |
| indexes[radid] = radid; |
| nodes[radid] = obj = hwloc_alloc_setup_object(HWLOC_OBJ_NODE, radid); |
| obj->cpuset = hwloc_bitmap_alloc(); |
| obj->memory.local_memory = rad_get_physmem(radid) * getpagesize(); |
| obj->memory.page_types_len = 2; |
| obj->memory.page_types = malloc(2*sizeof(*obj->memory.page_types)); |
| memset(obj->memory.page_types, 0, 2*sizeof(*obj->memory.page_types)); |
| obj->memory.page_types[0].size = getpagesize(); |
| #ifdef HAVE__SC_LARGE_PAGESIZE |
| obj->memory.page_types[1].size = sysconf(_SC_LARGE_PAGESIZE); |
| #endif |
| |
| cursor = SET_CURSOR_INIT; |
| while((cpuid = cpu_foreach(cpuset, 0, &cursor)) != CPU_NONE) |
| hwloc_bitmap_set(obj->cpuset, cpuid); |
| |
| hwloc_debug_1arg_bitmap("node %d has cpuset %s\n", |
| radid, obj->cpuset); |
| |
| hwloc_insert_object_by_cpuset(topology, obj); |
| |
| nfound = 0; |
| for (radid2 = 0; radid2 < (radid_t) nbnodes; radid2++) |
| distances[radid*nbnodes+radid2] = RAD_DIST_REMOTE; |
| for (distance = RAD_DIST_LOCAL; distance < RAD_DIST_REMOTE; distance++) { |
| attr.nattr_distance = distance; |
| /* get set of NUMA nodes at distance <= DISTANCE */ |
| if (nloc(&attr, radset2)) { |
| fprintf(stderr,"nloc failed: %s\n", strerror(errno)); |
| continue; |
| } |
| cursor = SET_CURSOR_INIT; |
| while ((radid2 = rad_foreach(radset2, 0, &cursor)) != RAD_NONE) { |
| if (distances[radid*nbnodes+radid2] == RAD_DIST_REMOTE) { |
| distances[radid*nbnodes+radid2] = (float) distance; |
| nfound++; |
| } |
| } |
| if (nfound == nbnodes) |
| /* Finished finding distances, no need to go up to RAD_DIST_REMOTE */ |
| break; |
| } |
| } |
| |
| hwloc_topology__set_distance_matrix(topology, HWLOC_OBJ_NODE, nbnodes, indexes, nodes, distances); |
| } |
| radsetdestroy(&radset2); |
| radsetdestroy(&radset); |
| cpusetdestroy(&cpuset); |
| |
| /* add PU objects */ |
| hwloc_setup_pu_level(topology, hwloc_fallback_nbprocessors(topology)); |
| |
| hwloc_add_object_info(topology->levels[0][0], "Backend", "OSF"); |
| } |
| |
| void |
| hwloc_set_osf_hooks(struct hwloc_topology *topology) |
| { |
| topology->set_thread_cpubind = hwloc_osf_set_thread_cpubind; |
| topology->set_thisthread_cpubind = hwloc_osf_set_thisthread_cpubind; |
| topology->set_proc_cpubind = hwloc_osf_set_proc_cpubind; |
| topology->set_thisproc_cpubind = hwloc_osf_set_thisproc_cpubind; |
| topology->set_area_membind = hwloc_osf_set_area_membind; |
| topology->alloc_membind = hwloc_osf_alloc_membind; |
| topology->alloc = hwloc_alloc_mmap; |
| topology->free_membind = hwloc_free_mmap; |
| topology->support.membind->firsttouch_membind = 1; |
| topology->support.membind->bind_membind = 1; |
| topology->support.membind->interleave_membind = 1; |
| topology->support.membind->replicate_membind = 1; |
| } |