00001
00002
00003
00004
00005
00006
00007
00008
00009
00010
00011
00012
00013
00014
00015
00016
00017
00018
00019
00020
00021
00022
00023
00024
00025
00026
00027
00028
00029
00030
00031
00032
00033 #include "config.h"
00034 #if defined HAVE_STDINT_H
00035 #include <stdint.h>
00036 #elif defined HAVE_INTTYPES_H
00037 #include <inttypes.h>
00038 #else
00039 #include <sys/types.h>
00040 #endif
00041 #include <unistd.h>
00042 #include <fcntl.h>
00043 #include <sys/mman.h>
00044 #include "system-alloc.h"
00045 #include "internal_spinlock.h"
00046 #include "internal_logging.h"
00047 #include "base/commandlineflags.h"
00048
00049
00050 union MemoryAligner {
00051 void* p;
00052 double d;
00053 size_t s;
00054 };
00055
00056 static SpinLock spinlock = SPINLOCK_INITIALIZER;
00057
00058
00059 static size_t pagesize = 0;
00060
00061
00062
00063
00064
00065
00066
00067 static bool use_devmem = true;
00068 static bool use_sbrk = true;
00069 static bool use_mmap = true;
00070
00071
00072 static bool devmem_failure = false;
00073 static bool sbrk_failure = false;
00074 static bool mmap_failure = false;
00075
00076 DEFINE_int32(malloc_devmem_start, 0,
00077 "Physical memory starting location in MB for /dev/mem allocation."
00078 " Setting this to 0 disables /dev/mem allocation");
00079 DEFINE_int32(malloc_devmem_limit, 0,
00080 "Physical memory limit location in MB for /dev/mem allocation."
00081 " Setting this to 0 means no limit.");
00082
00083 #ifdef HAVE_SBRK
00084
00085 static void* TrySbrk(size_t size, size_t alignment) {
00086
00087
00088 if (static_cast<ptrdiff_t>(size + alignment) < 0) return NULL;
00089
00090 size = ((size + alignment - 1) / alignment) * alignment;
00091 void* result = sbrk(size);
00092 if (result == reinterpret_cast<void*>(-1)) {
00093 sbrk_failure = true;
00094 return NULL;
00095 }
00096
00097
00098 uintptr_t ptr = reinterpret_cast<uintptr_t>(result);
00099 if ((ptr & (alignment-1)) == 0) return result;
00100
00101
00102 size_t extra = alignment - (ptr & (alignment-1));
00103 void* r2 = sbrk(extra);
00104 if (reinterpret_cast<uintptr_t>(r2) == (ptr + size)) {
00105
00106 return reinterpret_cast<void*>(ptr + extra);
00107 }
00108
00109
00110
00111 result = sbrk(size + alignment - 1);
00112 if (result == reinterpret_cast<void*>(-1)) {
00113 sbrk_failure = true;
00114 return NULL;
00115 }
00116 ptr = reinterpret_cast<uintptr_t>(result);
00117 if ((ptr & (alignment-1)) != 0) {
00118 ptr += alignment - (ptr & (alignment-1));
00119 }
00120 return reinterpret_cast<void*>(ptr);
00121 }
00122
00123 #endif
00124
00125 #ifdef HAVE_MMAP
00126
00127 static void* TryMmap(size_t size, size_t alignment) {
00128
00129 if (pagesize == 0) pagesize = getpagesize();
00130 if (alignment < pagesize) alignment = pagesize;
00131 size = ((size + alignment - 1) / alignment) * alignment;
00132
00133
00134 size_t extra = 0;
00135 if (alignment > pagesize) {
00136 extra = alignment - pagesize;
00137 }
00138
00139
00140
00141
00142
00143 void* result = mmap(NULL, size + extra,
00144 PROT_READ|PROT_WRITE,
00145 MAP_PRIVATE|MAP_ANON,
00146 -1, 0);
00147 if (result == reinterpret_cast<void*>(MAP_FAILED)) {
00148 mmap_failure = true;
00149 return NULL;
00150 }
00151
00152
00153 uintptr_t ptr = reinterpret_cast<uintptr_t>(result);
00154 size_t adjust = 0;
00155 if ((ptr & (alignment - 1)) != 0) {
00156 adjust = alignment - (ptr & (alignment - 1));
00157 }
00158
00159
00160 if (adjust > 0) {
00161 munmap(reinterpret_cast<void*>(ptr), adjust);
00162 }
00163 if (adjust < extra) {
00164 munmap(reinterpret_cast<void*>(ptr + adjust + size), extra - adjust);
00165 }
00166
00167 ptr += adjust;
00168 return reinterpret_cast<void*>(ptr);
00169 }
00170
00171 #endif
00172
00173 static void* TryDevMem(size_t size, size_t alignment) {
00174 static bool initialized = false;
00175 static off_t physmem_base;
00176 static off_t physmem_limit;
00177 static int physmem_fd;
00178
00179
00180
00181
00182
00183 if (FLAGS_malloc_devmem_start == 0) {
00184
00185
00186 return NULL;
00187 }
00188
00189 if (!initialized) {
00190 physmem_fd = open("/dev/mem", O_RDWR);
00191 if (physmem_fd < 0) {
00192 devmem_failure = true;
00193 return NULL;
00194 }
00195 physmem_base = FLAGS_malloc_devmem_start*1024LL*1024LL;
00196 physmem_limit = FLAGS_malloc_devmem_limit*1024LL*1024LL;
00197 initialized = true;
00198 }
00199
00200
00201 if (pagesize == 0) pagesize = getpagesize();
00202 if (alignment < pagesize) alignment = pagesize;
00203 size = ((size + alignment - 1) / alignment) * alignment;
00204
00205
00206 size_t extra = 0;
00207 if (alignment > pagesize) {
00208 extra = alignment - pagesize;
00209 }
00210
00211
00212 if (physmem_limit != 0 &&
00213 ((size + extra) > (physmem_limit - physmem_base))) {
00214 devmem_failure = true;
00215 return NULL;
00216 }
00217
00218
00219
00220
00221
00222 void *result = mmap(0, size + extra, PROT_WRITE|PROT_READ,
00223 MAP_SHARED, physmem_fd, physmem_base);
00224 if (result == reinterpret_cast<void*>(MAP_FAILED)) {
00225 devmem_failure = true;
00226 return NULL;
00227 }
00228 uintptr_t ptr = reinterpret_cast<uintptr_t>(result);
00229
00230
00231 size_t adjust = 0;
00232 if ((ptr & (alignment - 1)) != 0) {
00233 adjust = alignment - (ptr & (alignment - 1));
00234 }
00235
00236
00237 if (adjust > 0) {
00238 munmap(reinterpret_cast<void*>(ptr), adjust);
00239 }
00240 if (adjust < extra) {
00241 munmap(reinterpret_cast<void*>(ptr + adjust + size), extra - adjust);
00242 }
00243
00244 ptr += adjust;
00245 physmem_base += adjust + size;
00246
00247 return reinterpret_cast<void*>(ptr);
00248 }
00249
00250 void* TCMalloc_SystemAlloc(size_t size, size_t alignment) {
00251
00252 if (size + alignment < size) return NULL;
00253
00254 if (TCMallocDebug::level >= TCMallocDebug::kVerbose) {
00255 MESSAGE("TCMalloc_SystemAlloc(%" PRIuS ", %" PRIuS")\n",
00256 size, alignment);
00257 }
00258 SpinLockHolder lock_holder(&spinlock);
00259
00260
00261 if (alignment < sizeof(MemoryAligner)) alignment = sizeof(MemoryAligner);
00262
00263
00264
00265 for (int i = 0; i < 2; i++) {
00266 if (use_devmem && !devmem_failure) {
00267 void* result = TryDevMem(size, alignment);
00268 if (result != NULL) return result;
00269 }
00270
00271 #ifdef HAVE_SBRK
00272 if (use_sbrk && !sbrk_failure) {
00273 void* result = TrySbrk(size, alignment);
00274 if (result != NULL) return result;
00275 }
00276 #endif
00277
00278 #ifdef HAVE_MMAP
00279 if (use_mmap && !mmap_failure) {
00280 void* result = TryMmap(size, alignment);
00281 if (result != NULL) return result;
00282 }
00283 #endif
00284
00285
00286 devmem_failure = false;
00287 sbrk_failure = false;
00288 mmap_failure = false;
00289 }
00290 return NULL;
00291 }