00001
00002
00003
00004
00005
00006
00007
00008
00009
00010
00011
00012
00013
00014
00015
00016
00017
00018
00019
00020
00021
00022
00023
00024
00025
00026
00027
00028
00029
00030
00031
00032
00033 #include "config.h"
00034 #if defined HAVE_STDINT_H
00035 #include <stdint.h>
00036 #elif defined HAVE_INTTYPES_H
00037 #include <inttypes.h>
00038 #else
00039 #include <sys/types.h>
00040 #endif
00041 #ifdef HAVE_UNISTD_H
00042 #include <unistd.h>
00043 #endif
00044 #include <fcntl.h>
00045 #ifdef HAVE_MMAP
00046 #include <sys/mman.h>
00047 #endif
00048 #include <sys/resource.h>
00049 #include <errno.h>
00050 #include "system-alloc.h"
00051 #include "internal_logging.h"
00052 #include "base/logging.h"
00053 #include "base/commandlineflags.h"
00054 #include "base/spinlock.h"
00055
00056
00057
00058 #ifndef MAP_ANONYMOUS
00059 # define MAP_ANONYMOUS MAP_ANON
00060 #endif
00061
00062
00063
00064 #if defined(__sun) && defined(__SVR4)
00065 # include <sys/types.h>
00066 extern "C" { extern int madvise(caddr_t, size_t, int); }
00067 #endif
00068
00069
00070 union MemoryAligner {
00071 void* p;
00072 double d;
00073 size_t s;
00074 };
00075
00076 static SpinLock spinlock(SpinLock::LINKER_INITIALIZED);
00077
00078
00079 static size_t pagesize = 0;
00080
00081
00082
00083
00084
00085
00086
00087
00088 DEFINE_int32(malloc_devmem_start, 0,
00089 "Physical memory starting location in MB for /dev/mem allocation."
00090 " Setting this to 0 disables /dev/mem allocation");
00091 DEFINE_int32(malloc_devmem_limit, 0,
00092 "Physical memory limit location in MB for /dev/mem allocation."
00093 " Setting this to 0 means no limit.");
00094
00095
00096 class SbrkSysAllocator : public SysAllocator {
00097 public:
00098 SbrkSysAllocator() : SysAllocator() {
00099 }
00100 void* Alloc(size_t size, size_t *actual_size, size_t alignment);
00101 };
00102 static char sbrk_space[sizeof(SbrkSysAllocator)];
00103
00104 class MmapSysAllocator : public SysAllocator {
00105 public:
00106 MmapSysAllocator() : SysAllocator() {
00107 }
00108 void* Alloc(size_t size, size_t *actual_size, size_t alignment);
00109 };
00110 static char mmap_space[sizeof(MmapSysAllocator)];
00111
00112 class DevMemSysAllocator : public SysAllocator {
00113 public:
00114 DevMemSysAllocator() : SysAllocator() {
00115 }
00116 void* Alloc(size_t size, size_t *actual_size, size_t alignment);
00117 };
00118 static char devmem_space[sizeof(DevMemSysAllocator)];
00119
00120 static const int kStaticAllocators = 3;
00121
00122 static const int kMaxAllocators = 5;
00123 SysAllocator *allocators[kMaxAllocators];
00124
00125 bool RegisterSystemAllocator(SysAllocator *a, int priority) {
00126 SpinLockHolder lock_holder(&spinlock);
00127
00128
00129
00130 CHECK(allocators[priority] == NULL);
00131 allocators[priority] = a;
00132 return true;
00133 }
00134
00135
00136 void* SbrkSysAllocator::Alloc(size_t size, size_t *actual_size,
00137 size_t alignment) {
00138
00139
00140 if (static_cast<ptrdiff_t>(size + alignment) < 0) return NULL;
00141
00142
00143
00144 if (actual_size) {
00145 *actual_size = size;
00146 }
00147
00148
00149
00150 size = ((size + alignment - 1) / alignment) * alignment;
00151 void* result = sbrk(size);
00152 if (result == reinterpret_cast<void*>(-1)) {
00153 failed_ = true;
00154 return NULL;
00155 }
00156
00157
00158 uintptr_t ptr = reinterpret_cast<uintptr_t>(result);
00159 if ((ptr & (alignment-1)) == 0) return result;
00160
00161
00162 size_t extra = alignment - (ptr & (alignment-1));
00163 void* r2 = sbrk(extra);
00164 if (reinterpret_cast<uintptr_t>(r2) == (ptr + size)) {
00165
00166 return reinterpret_cast<void*>(ptr + extra);
00167 }
00168
00169
00170
00171 result = sbrk(size + alignment - 1);
00172 if (result == reinterpret_cast<void*>(-1)) {
00173 failed_ = true;
00174 return NULL;
00175 }
00176 ptr = reinterpret_cast<uintptr_t>(result);
00177 if ((ptr & (alignment-1)) != 0) {
00178 ptr += alignment - (ptr & (alignment-1));
00179 }
00180 return reinterpret_cast<void*>(ptr);
00181 }
00182
00183 void* MmapSysAllocator::Alloc(size_t size, size_t *actual_size,
00184 size_t alignment) {
00185
00186
00187 if (actual_size) {
00188 *actual_size = size;
00189 }
00190
00191
00192 if (pagesize == 0) pagesize = getpagesize();
00193 if (alignment < pagesize) alignment = pagesize;
00194 size_t aligned_size = ((size + alignment - 1) / alignment) * alignment;
00195 if (aligned_size < size) {
00196 return NULL;
00197 }
00198 size = aligned_size;
00199
00200
00201 size_t extra = 0;
00202 if (alignment > pagesize) {
00203 extra = alignment - pagesize;
00204 }
00205
00206
00207
00208
00209
00210 void* result = mmap(NULL, size + extra,
00211 PROT_READ|PROT_WRITE,
00212 MAP_PRIVATE|MAP_ANONYMOUS,
00213 -1, 0);
00214 if (result == reinterpret_cast<void*>(MAP_FAILED)) {
00215 failed_ = true;
00216 return NULL;
00217 }
00218
00219
00220 uintptr_t ptr = reinterpret_cast<uintptr_t>(result);
00221 size_t adjust = 0;
00222 if ((ptr & (alignment - 1)) != 0) {
00223 adjust = alignment - (ptr & (alignment - 1));
00224 }
00225
00226
00227 if (adjust > 0) {
00228 munmap(reinterpret_cast<void*>(ptr), adjust);
00229 }
00230 if (adjust < extra) {
00231 munmap(reinterpret_cast<void*>(ptr + adjust + size), extra - adjust);
00232 }
00233
00234 ptr += adjust;
00235 return reinterpret_cast<void*>(ptr);
00236 }
00237
00238 void* DevMemSysAllocator::Alloc(size_t size, size_t *actual_size,
00239 size_t alignment) {
00240 static bool initialized = false;
00241 static off_t physmem_base;
00242 static off_t physmem_limit;
00243 static int physmem_fd;
00244
00245
00246
00247
00248
00249 if (FLAGS_malloc_devmem_start == 0) {
00250
00251
00252 return NULL;
00253 }
00254
00255 if (!initialized) {
00256 physmem_fd = open("/dev/mem", O_RDWR);
00257 if (physmem_fd < 0) {
00258 failed_ = true;
00259 return NULL;
00260 }
00261 physmem_base = FLAGS_malloc_devmem_start*1024LL*1024LL;
00262 physmem_limit = FLAGS_malloc_devmem_limit*1024LL*1024LL;
00263 initialized = true;
00264 }
00265
00266
00267
00268 if (actual_size) {
00269 *actual_size = size;
00270 }
00271
00272
00273 if (pagesize == 0) pagesize = getpagesize();
00274 if (alignment < pagesize) alignment = pagesize;
00275 size_t aligned_size = ((size + alignment - 1) / alignment) * alignment;
00276 if (aligned_size < size) {
00277 return NULL;
00278 }
00279 size = aligned_size;
00280
00281
00282 size_t extra = 0;
00283 if (alignment > pagesize) {
00284 extra = alignment - pagesize;
00285 }
00286
00287
00288 if (physmem_limit != 0 &&
00289 ((size + extra) > (physmem_limit - physmem_base))) {
00290 failed_ = true;
00291 return NULL;
00292 }
00293
00294
00295
00296
00297
00298 void *result = mmap(0, size + extra, PROT_WRITE|PROT_READ,
00299 MAP_SHARED, physmem_fd, physmem_base);
00300 if (result == reinterpret_cast<void*>(MAP_FAILED)) {
00301 failed_ = true;
00302 return NULL;
00303 }
00304 uintptr_t ptr = reinterpret_cast<uintptr_t>(result);
00305
00306
00307 size_t adjust = 0;
00308 if ((ptr & (alignment - 1)) != 0) {
00309 adjust = alignment - (ptr & (alignment - 1));
00310 }
00311
00312
00313 if (adjust > 0) {
00314 munmap(reinterpret_cast<void*>(ptr), adjust);
00315 }
00316 if (adjust < extra) {
00317 munmap(reinterpret_cast<void*>(ptr + adjust + size), extra - adjust);
00318 }
00319
00320 ptr += adjust;
00321 physmem_base += adjust + size;
00322
00323 return reinterpret_cast<void*>(ptr);
00324 }
00325
00326 static bool system_alloc_inited = false;
00327 void InitSystemAllocators(void) {
00328
00329 int i = kMaxDynamicAllocators;
00330 allocators[i++] = new (devmem_space) DevMemSysAllocator();
00331 allocators[i++] = new (sbrk_space) SbrkSysAllocator();
00332 allocators[i++] = new (mmap_space) MmapSysAllocator();
00333 }
00334
00335 #if defined(__APPLE__)
00336 static uint64_t allocated_bytes = 0;
00337 static uint64_t allowedBytes()
00338 {
00339 rlimit rlp;
00340 getrlimit(RLIMIT_AS, &rlp);
00341 return rlp.rlim_cur;
00342 }
00343 #endif
00344 void* TCMalloc_SystemAlloc(size_t size, size_t *actual_size,
00345 size_t alignment) {
00346 #if defined(__APPLE__)
00347 static const uint64_t allowed_bytes = allowedBytes();
00348 if (__builtin_expect(allocated_bytes+size >= allowed_bytes, 0)) {
00349 fprintf(stderr, "allocation failed %lld (max %lld)\n", (uint64_t)size, allowed_bytes);
00350 return NULL;
00351 }
00352 allocated_bytes += size;
00353 #endif
00354
00355 if (size + alignment < size) return NULL;
00356
00357 SpinLockHolder lock_holder(&spinlock);
00358
00359 if (!system_alloc_inited) {
00360 InitSystemAllocators();
00361 system_alloc_inited = true;
00362 }
00363
00364
00365 if (alignment < sizeof(MemoryAligner)) alignment = sizeof(MemoryAligner);
00366
00367
00368
00369 for (int i = 0; i < 2; i++) {
00370 for (int j = 0; j < kMaxAllocators; j++) {
00371 SysAllocator *a = allocators[j];
00372 if (a == NULL) continue;
00373 if (a->usable_ && !a->failed_) {
00374 void* result = a->Alloc(size, actual_size, alignment);
00375 if (result != NULL) return result;
00376 }
00377 }
00378
00379
00380 for (int j = 0; j < kMaxAllocators; j++) {
00381 SysAllocator *a = allocators[j];
00382 if (a == NULL) continue;
00383 a->failed_ = false;
00384 }
00385 }
00386 return NULL;
00387 }
00388
00389 void TCMalloc_SystemRelease(void* start, size_t length) {
00390 #ifdef MADV_DONTNEED
00391 if (FLAGS_malloc_devmem_start) {
00392
00393
00394 return;
00395 }
00396 if (pagesize == 0) pagesize = getpagesize();
00397 const size_t pagemask = pagesize - 1;
00398
00399 size_t new_start = reinterpret_cast<size_t>(start);
00400 size_t end = new_start + length;
00401 size_t new_end = end;
00402
00403
00404
00405 new_start = (new_start + pagesize - 1) & ~pagemask;
00406 new_end = new_end & ~pagemask;
00407
00408 ASSERT((new_start & pagemask) == 0);
00409 ASSERT((new_end & pagemask) == 0);
00410 ASSERT(new_start >= reinterpret_cast<size_t>(start));
00411 ASSERT(new_end <= end);
00412
00413 if (new_end > new_start) {
00414
00415
00416 while (madvise(reinterpret_cast<char*>(new_start), new_end - new_start,
00417 MADV_DONTNEED) == -1 &&
00418 errno == EAGAIN) {
00419
00420 }
00421 }
00422 #endif
00423 }