arch/mips/mach-octeon/cvmx-bootmem.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (C) 2018-2020 Marvell International Ltd.
   4  */
   5
   6 /*
   7  * Simple allocate only memory allocator. Used to allocate memory at
   8  * application start time.
   9  */
  10
  11 #include <asm/global_data.h>
  12
  13 #include <linux/compat.h>
  14 #include <linux/io.h>
  15 #include <linux/types.h>
  16
  17 #include <mach/octeon-model.h>
  18 #include <mach/cvmx-bootmem.h>
  19 #include <mach/cvmx-coremask.h>
  20 #include <mach/cvmx-regs.h>
  21
  22 DECLARE_GLOBAL_DATA_PTR;
  23
  24 #define CVMX_MIPS32_SPACE_KSEG0         1L
  25 #define CVMX_MIPS_SPACE_XKPHYS          2LL
  26
  27 #define CVMX_ADD_SEG(seg, add)          ((((u64)(seg)) << 62) | (add))
  28 #define CVMX_ADD_SEG32(seg, add)        (((u32)(seg) << 31) | (u32)(add))
  29
  30 /**
  31  * This is the physical location of a struct cvmx_bootmem_desc
  32  * structure in Octeon's memory. Note that dues to addressing
  33  * limits or runtime environment it might not be possible to
  34  * create a C pointer to this structure.
  35  */
  36 static u64 cvmx_bootmem_desc_addr;
  37
  38 /**
  39  * This macro returns the size of a member of a structure.
  40  * Logically it is the same as "sizeof(s::field)" in C++, but
  41  * C lacks the "::" operator.
  42  */
  43 #define SIZEOF_FIELD(s, field) sizeof(((s *)NULL)->field)
  44
  45 /**
  46  * This macro returns a member of the struct cvmx_bootmem_desc
  47  * structure. These members can't be directly addressed as
  48  * they might be in memory not directly reachable. In the case
  49  * where bootmem is compiled with LINUX_HOST, the structure
  50  * itself might be located on a remote Octeon. The argument
  51  * "field" is the member name of the struct cvmx_bootmem_desc to read.
  52  * Regardless of the type of the field, the return type is always
  53  * a u64.
  54  */
  55 #define CVMX_BOOTMEM_DESC_GET_FIELD(field)                              \
  56         __cvmx_bootmem_desc_get(cvmx_bootmem_desc_addr,                 \
  57                                 offsetof(struct cvmx_bootmem_desc, field), \
  58                                 SIZEOF_FIELD(struct cvmx_bootmem_desc, field))
  59
  60 /**
  61  * This macro writes a member of the struct cvmx_bootmem_desc
  62  * structure. These members can't be directly addressed as
  63  * they might be in memory not directly reachable. In the case
  64  * where bootmem is compiled with LINUX_HOST, the structure
  65  * itself might be located on a remote Octeon. The argument
  66  * "field" is the member name of the struct cvmx_bootmem_desc to write.
  67  */
  68 #define CVMX_BOOTMEM_DESC_SET_FIELD(field, value)                       \
  69         __cvmx_bootmem_desc_set(cvmx_bootmem_desc_addr,                 \
  70                                 offsetof(struct cvmx_bootmem_desc, field), \
  71                                 SIZEOF_FIELD(struct cvmx_bootmem_desc, field), \
  72                                 value)
  73
  74 /**
  75  * This macro returns a member of the
  76  * struct cvmx_bootmem_named_block_desc structure. These members can't
  77  * be directly addressed as they might be in memory not directly
  78  * reachable. In the case where bootmem is compiled with
  79  * LINUX_HOST, the structure itself might be located on a remote
  80  * Octeon. The argument "field" is the member name of the
  81  * struct cvmx_bootmem_named_block_desc to read. Regardless of the type
  82  * of the field, the return type is always a u64. The "addr"
  83  * parameter is the physical address of the structure.
  84  */
  85 #define CVMX_BOOTMEM_NAMED_GET_FIELD(addr, field)                       \
  86         __cvmx_bootmem_desc_get(addr,                                   \
  87                 offsetof(struct cvmx_bootmem_named_block_desc,  field), \
  88                 SIZEOF_FIELD(struct cvmx_bootmem_named_block_desc, field))
  89
  90 /**
  91  * This macro writes a member of the struct cvmx_bootmem_named_block_desc
  92  * structure. These members can't be directly addressed as
  93  * they might be in memory not directly reachable. In the case
  94  * where bootmem is compiled with LINUX_HOST, the structure
  95  * itself might be located on a remote Octeon. The argument
  96  * "field" is the member name of the
  97  * struct cvmx_bootmem_named_block_desc to write. The "addr" parameter
  98  * is the physical address of the structure.
  99  */
 100 #define CVMX_BOOTMEM_NAMED_SET_FIELD(addr, field, value)                \
 101         __cvmx_bootmem_desc_set(addr,                                   \
 102                 offsetof(struct cvmx_bootmem_named_block_desc, field),  \
 103                 SIZEOF_FIELD(struct cvmx_bootmem_named_block_desc, field), \
 104                                 value)
 105
 106 /**
 107  * This function is the implementation of the get macros defined
 108  * for individual structure members. The argument are generated
 109  * by the macros inorder to read only the needed memory.
 110  *
 111  * @param base   64bit physical address of the complete structure
 112  * @param offset Offset from the beginning of the structure to the member being
 113  *               accessed.
 114  * @param size   Size of the structure member.
 115  *
 116  * @return Value of the structure member promoted into a u64.
 117  */
 118 static inline u64 __cvmx_bootmem_desc_get(u64 base, int offset,
 119                                           int size)
 120 {
 121         base = (1ull << 63) | (base + offset);
 122         switch (size) {
 123         case 4:
 124                 return cvmx_read64_uint32(base);
 125         case 8:
 126                 return cvmx_read64_uint64(base);
 127         default:
 128                 return 0;
 129         }
 130 }
 131
 132 /**
 133  * This function is the implementation of the set macros defined
 134  * for individual structure members. The argument are generated
 135  * by the macros in order to write only the needed memory.
 136  *
 137  * @param base   64bit physical address of the complete structure
 138  * @param offset Offset from the beginning of the structure to the member being
 139  *               accessed.
 140  * @param size   Size of the structure member.
 141  * @param value  Value to write into the structure
 142  */
 143 static inline void __cvmx_bootmem_desc_set(u64 base, int offset, int size,
 144                                            u64 value)
 145 {
 146         base = (1ull << 63) | (base + offset);
 147         switch (size) {
 148         case 4:
 149                 cvmx_write64_uint32(base, value);
 150                 break;
 151         case 8:
 152                 cvmx_write64_uint64(base, value);
 153                 break;
 154         default:
 155                 break;
 156         }
 157 }
 158
 159 /**
 160  * This function returns the address of the bootmem descriptor lock.
 161  *
 162  * @return 64-bit address in KSEG0 of the bootmem descriptor block
 163  */
 164 static inline u64 __cvmx_bootmem_get_lock_addr(void)
 165 {
 166         return (1ull << 63) |
 167                 (cvmx_bootmem_desc_addr + offsetof(struct cvmx_bootmem_desc, lock));
 168 }
 169
 170 /**
 171  * This function retrieves the string name of a named block. It is
 172  * more complicated than a simple memcpy() since the named block
 173  * descriptor may not be directly accessible.
 174  *
 175  * @param addr   Physical address of the named block descriptor
 176  * @param str    String to receive the named block string name
 177  * @param len    Length of the string buffer, which must match the length
 178  *               stored in the bootmem descriptor.
 179  */
 180 static void CVMX_BOOTMEM_NAMED_GET_NAME(u64 addr, char *str, int len)
 181 {
 182         int l = len;
 183         char *ptr = str;
 184
 185         addr |= (1ull << 63);
 186         addr += offsetof(struct cvmx_bootmem_named_block_desc, name);
 187         while (l) {
 188                 /*
 189                  * With big-endian in memory byte order, this gives uniform
 190                  * results for the CPU in either big or Little endian mode.
 191                  */
 192                 u64 blob = cvmx_read64_uint64(addr);
 193                 int sa = 56;
 194
 195                 addr += sizeof(u64);
 196                 while (l && sa >= 0) {
 197                         *ptr++ = (char)(blob >> sa);
 198                         l--;
 199                         sa -= 8;
 200                 }
 201         }
 202         str[len] = 0;
 203 }
 204
 205 /**
 206  * This function stores the string name of a named block. It is
 207  * more complicated than a simple memcpy() since the named block
 208  * descriptor may not be directly accessible.
 209  *
 210  * @param addr   Physical address of the named block descriptor
 211  * @param str    String to store into the named block string name
 212  * @param len    Length of the string buffer, which must match the length
 213  *               stored in the bootmem descriptor.
 214  */
 215 void CVMX_BOOTMEM_NAMED_SET_NAME(u64 addr, const char *str, int len)
 216 {
 217         int l = len;
 218
 219         addr |= (1ull << 63);
 220         addr += offsetof(struct cvmx_bootmem_named_block_desc, name);
 221
 222         while (l) {
 223                 /*
 224                  * With big-endian in memory byte order, this gives uniform
 225                  * results for the CPU in either big or Little endian mode.
 226                  */
 227                 u64 blob = 0;
 228                 int sa = 56;
 229
 230                 while (l && sa >= 0) {
 231                         u64 c = (u8)(*str++);
 232
 233                         l--;
 234                         if (l == 0)
 235                                 c = 0;
 236                         blob |= c << sa;
 237                         sa -= 8;
 238                 }
 239                 cvmx_write64_uint64(addr, blob);
 240                 addr += sizeof(u64);
 241         }
 242 }
 243
 244 /* See header file for descriptions of functions */
 245
 246 /*
 247  * Wrapper functions are provided for reading/writing the size and next block
 248  * values as these may not be directly addressible (in 32 bit applications, for
 249  * instance.)
 250  *
 251  * Offsets of data elements in bootmem list, must match
 252  * struct cvmx_bootmem_block_header
 253  */
 254 #define NEXT_OFFSET 0
 255 #define SIZE_OFFSET 8
 256
 257 static void cvmx_bootmem_phy_set_size(u64 addr, u64 size)
 258 {
 259         cvmx_write64_uint64((addr + SIZE_OFFSET) | (1ull << 63), size);
 260 }
 261
 262 static void cvmx_bootmem_phy_set_next(u64 addr, u64 next)
 263 {
 264         cvmx_write64_uint64((addr + NEXT_OFFSET) | (1ull << 63), next);
 265 }
 266
 267 static u64 cvmx_bootmem_phy_get_size(u64 addr)
 268 {
 269         return cvmx_read64_uint64((addr + SIZE_OFFSET) | (1ull << 63));
 270 }
 271
 272 static u64 cvmx_bootmem_phy_get_next(u64 addr)
 273 {
 274         return cvmx_read64_uint64((addr + NEXT_OFFSET) | (1ull << 63));
 275 }
 276
 277 /**
 278  * Check the version information on the bootmem descriptor
 279  *
 280  * @param exact_match
 281  *               Exact major version to check against. A zero means
 282  *               check that the version supports named blocks.
 283  *
 284  * @return Zero if the version is correct. Negative if the version is
 285  *         incorrect. Failures also cause a message to be displayed.
 286  */
 287 static int __cvmx_bootmem_check_version(int exact_match)
 288 {
 289         int major_version;
 290
 291         major_version = CVMX_BOOTMEM_DESC_GET_FIELD(major_version);
 292         if (major_version > 3 ||
 293             (exact_match && major_version) != exact_match) {
 294                 debug("ERROR: Incompatible bootmem descriptor version: %d.%d at addr: 0x%llx\n",
 295                       major_version,
 296                       (int)CVMX_BOOTMEM_DESC_GET_FIELD(minor_version),
 297                       CAST_ULL(cvmx_bootmem_desc_addr));
 298                 return -1;
 299         } else {
 300                 return 0;
 301         }
 302 }
 303
 304 /**
 305  * Get the low level bootmem descriptor lock. If no locking
 306  * is specified in the flags, then nothing is done.
 307  *
 308  * @param flags  CVMX_BOOTMEM_FLAG_NO_LOCKING means this functions should do
 309  *               nothing. This is used to support nested bootmem calls.
 310  */
 311 static inline void __cvmx_bootmem_lock(u32 flags)
 312 {
 313         if (!(flags & CVMX_BOOTMEM_FLAG_NO_LOCKING)) {
 314                 /*
 315                  * Unfortunately we can't use the normal cvmx-spinlock code as
 316                  * the memory for the bootmem descriptor may be not accessible
 317                  * by a C pointer. We use a 64bit XKPHYS address to access the
 318                  * memory directly
 319                  */
 320                 u64 lock_addr = (1ull << 63) |
 321                         (cvmx_bootmem_desc_addr + offsetof(struct cvmx_bootmem_desc,
 322                                                            lock));
 323                 unsigned int tmp;
 324
 325                 __asm__ __volatile__(".set noreorder\n"
 326                                      "1: ll   %[tmp], 0(%[addr])\n"
 327                                      "   bnez %[tmp], 1b\n"
 328                                      "   li   %[tmp], 1\n"
 329                                      "   sc   %[tmp], 0(%[addr])\n"
 330                                      "   beqz %[tmp], 1b\n"
 331                                      "   nop\n"
 332                                      ".set reorder\n"
 333                                      : [tmp] "=&r"(tmp)
 334                                      : [addr] "r"(lock_addr)
 335                                      : "memory");
 336         }
 337 }
 338
 339 /**
 340  * Release the low level bootmem descriptor lock. If no locking
 341  * is specified in the flags, then nothing is done.
 342  *
 343  * @param flags  CVMX_BOOTMEM_FLAG_NO_LOCKING means this functions should do
 344  *               nothing. This is used to support nested bootmem calls.
 345  */
 346 static inline void __cvmx_bootmem_unlock(u32 flags)
 347 {
 348         if (!(flags & CVMX_BOOTMEM_FLAG_NO_LOCKING)) {
 349                 /*
 350                  * Unfortunately we can't use the normal cvmx-spinlock code as
 351                  * the memory for the bootmem descriptor may be not accessible
 352                  * by a C pointer. We use a 64bit XKPHYS address to access the
 353                  * memory directly
 354                  */
 355                 u64 lock_addr = __cvmx_bootmem_get_lock_addr();
 356
 357                 CVMX_SYNCW;
 358                 __asm__ __volatile__("sw $0, 0(%[addr])\n"
 359                                      : : [addr] "r"(lock_addr)
 360                                      : "memory");
 361                 CVMX_SYNCW;
 362         }
 363 }
 364
 365 /*
 366  * Some of the cvmx-bootmem functions dealing with C pointers are not
 367  * supported when we are compiling for CVMX_BUILD_FOR_LINUX_HOST. This
 368  * ifndef removes these functions when they aren't needed.
 369  *
 370  * This functions takes an address range and adjusts it as necessary
 371  * to match the ABI that is currently being used.  This is required to
 372  * ensure that bootmem_alloc* functions only return valid pointers for
 373  * 32 bit ABIs
 374  */
 375 static int __cvmx_validate_mem_range(u64 *min_addr_ptr,
 376                                      u64 *max_addr_ptr)
 377 {
 378         u64 max_phys = (1ull << 29) - 0x10;     /* KSEG0 */
 379
 380         *min_addr_ptr = min_t(u64, max_t(u64, *min_addr_ptr, 0x0), max_phys);
 381         if (!*max_addr_ptr) {
 382                 *max_addr_ptr = max_phys;
 383         } else {
 384                 *max_addr_ptr = max_t(u64, min_t(u64, *max_addr_ptr,
 385                                                  max_phys), 0x0);
 386         }
 387
 388         return 0;
 389 }
 390
 391 u64 cvmx_bootmem_phy_alloc_range(u64 size, u64 alignment,
 392                                  u64 min_addr, u64 max_addr)
 393 {
 394         s64 address;
 395
 396         __cvmx_validate_mem_range(&min_addr, &max_addr);
 397         address = cvmx_bootmem_phy_alloc(size, min_addr, max_addr,
 398                                          alignment, 0);
 399         if (address > 0)
 400                 return address;
 401         else
 402                 return 0;
 403 }
 404
 405 void *cvmx_bootmem_alloc_range(u64 size, u64 alignment,
 406                                u64 min_addr, u64 max_addr)
 407 {
 408         s64 address;
 409
 410         __cvmx_validate_mem_range(&min_addr, &max_addr);
 411         address = cvmx_bootmem_phy_alloc(size, min_addr, max_addr,
 412                                          alignment, 0);
 413
 414         if (address > 0)
 415                 return cvmx_phys_to_ptr(address);
 416         else
 417                 return NULL;
 418 }
 419
 420 void *cvmx_bootmem_alloc_address(u64 size, u64 address,
 421                                  u64 alignment)
 422 {
 423         return cvmx_bootmem_alloc_range(size, alignment, address,
 424                                         address + size);
 425 }
 426
 427 void *cvmx_bootmem_alloc_node(u64 node, u64 size, u64 alignment)
 428 {
 429         return cvmx_bootmem_alloc_range(size, alignment,
 430                                         node << CVMX_NODE_MEM_SHIFT,
 431                                         ((node + 1) << CVMX_NODE_MEM_SHIFT) - 1);
 432 }
 433
 434 void *cvmx_bootmem_alloc(u64 size, u64 alignment)
 435 {
 436         return cvmx_bootmem_alloc_range(size, alignment, 0, 0);
 437 }
 438
 439 void *cvmx_bootmem_alloc_named_range_once(u64 size, u64 min_addr,
 440                                           u64 max_addr, u64 align,
 441                                           const char *name,
 442                                           void (*init)(void *))
 443 {
 444         u64 named_block_desc_addr;
 445         void *ptr;
 446         s64 addr;
 447
 448         __cvmx_bootmem_lock(0);
 449
 450         __cvmx_validate_mem_range(&min_addr, &max_addr);
 451         named_block_desc_addr =
 452                 cvmx_bootmem_phy_named_block_find(name,
 453                                                   CVMX_BOOTMEM_FLAG_NO_LOCKING);
 454
 455         if (named_block_desc_addr) {
 456                 addr = CVMX_BOOTMEM_NAMED_GET_FIELD(named_block_desc_addr,
 457                                                     base_addr);
 458                 __cvmx_bootmem_unlock(0);
 459                 return cvmx_phys_to_ptr(addr);
 460         }
 461
 462         addr = cvmx_bootmem_phy_named_block_alloc(size, min_addr, max_addr,
 463                                                   align, name,
 464                                                   CVMX_BOOTMEM_FLAG_NO_LOCKING);
 465
 466         if (addr < 0) {
 467                 __cvmx_bootmem_unlock(0);
 468                 return NULL;
 469         }
 470         ptr = cvmx_phys_to_ptr(addr);
 471
 472         if (init)
 473                 init(ptr);
 474         else
 475                 memset(ptr, 0, size);
 476
 477         __cvmx_bootmem_unlock(0);
 478         return ptr;
 479 }
 480
 481 void *cvmx_bootmem_alloc_named_range_flags(u64 size, u64 min_addr,
 482                                            u64 max_addr, u64 align,
 483                                            const char *name, u32 flags)
 484 {
 485         s64 addr;
 486
 487         __cvmx_validate_mem_range(&min_addr, &max_addr);
 488         addr = cvmx_bootmem_phy_named_block_alloc(size, min_addr, max_addr,
 489                                                   align, name, flags);
 490         if (addr >= 0)
 491                 return cvmx_phys_to_ptr(addr);
 492         else
 493                 return NULL;
 494 }
 495
 496 void *cvmx_bootmem_alloc_named_range(u64 size, u64 min_addr,
 497                                      u64 max_addr, u64 align,
 498                                      const char *name)
 499 {
 500         return cvmx_bootmem_alloc_named_range_flags(size, min_addr, max_addr,
 501                                                     align, name, 0);
 502 }
 503
 504 void *cvmx_bootmem_alloc_named_address(u64 size, u64 address,
 505                                        const char *name)
 506 {
 507         return cvmx_bootmem_alloc_named_range(size, address, address + size,
 508                                               0, name);
 509 }
 510
 511 void *cvmx_bootmem_alloc_named(u64 size, u64 alignment,
 512                                const char *name)
 513 {
 514         return cvmx_bootmem_alloc_named_range(size, 0, 0, alignment, name);
 515 }
 516
 517 void *cvmx_bootmem_alloc_named_flags(u64 size, u64 alignment,
 518                                      const char *name, u32 flags)
 519 {
 520         return cvmx_bootmem_alloc_named_range_flags(size, 0, 0, alignment,
 521                                                     name, flags);
 522 }
 523
 524 int cvmx_bootmem_free_named(const char *name)
 525 {
 526         return cvmx_bootmem_phy_named_block_free(name, 0);
 527 }
 528
 529 /**
 530  * Find a named block with flags
 531  *
 532  * @param name is the block name
 533  * @param flags indicates the need to use locking during search
 534  * @return pointer to named block descriptor
 535  *
 536  * Note: this function returns a pointer to a static structure,
 537  * and is therefore not re-entrant.
 538  * Making this function re-entrant will break backward compatibility.
 539  */
 540 const struct cvmx_bootmem_named_block_desc *
 541 __cvmx_bootmem_find_named_block_flags(const char *name, u32 flags)
 542 {
 543         static struct cvmx_bootmem_named_block_desc desc;
 544         u64 named_addr = cvmx_bootmem_phy_named_block_find(name, flags);
 545
 546         if (named_addr) {
 547                 desc.base_addr = CVMX_BOOTMEM_NAMED_GET_FIELD(named_addr,
 548                                                               base_addr);
 549                 desc.size = CVMX_BOOTMEM_NAMED_GET_FIELD(named_addr, size);
 550                 strncpy(desc.name, name, sizeof(desc.name));
 551                 desc.name[sizeof(desc.name) - 1] = 0;
 552                 return &desc;
 553         } else {
 554                 return NULL;
 555         }
 556 }
 557
 558 const struct cvmx_bootmem_named_block_desc *
 559 cvmx_bootmem_find_named_block(const char *name)
 560 {
 561         return __cvmx_bootmem_find_named_block_flags(name, 0);
 562 }
 563
 564 void cvmx_bootmem_print_named(void)
 565 {
 566         cvmx_bootmem_phy_named_block_print();
 567 }
 568
 569 int cvmx_bootmem_init(u64 mem_desc_addr)
 570 {
 571         if (!cvmx_bootmem_desc_addr)
 572                 cvmx_bootmem_desc_addr = mem_desc_addr;
 573
 574         return 0;
 575 }
 576
 577 u64 cvmx_bootmem_available_mem(u64 min_block_size)
 578 {
 579         return cvmx_bootmem_phy_available_mem(min_block_size);
 580 }
 581
 582 /*
 583  * The cvmx_bootmem_phy* functions below return 64 bit physical
 584  * addresses, and expose more features that the cvmx_bootmem_functions
 585  * above.  These are required for full memory space access in 32 bit
 586  * applications, as well as for using some advance features.  Most
 587  * applications should not need to use these.
 588  */
 589
 590 s64 cvmx_bootmem_phy_alloc(u64 req_size, u64 address_min,
 591                            u64 address_max, u64 alignment,
 592                            u32 flags)
 593 {
 594         u64 head_addr, ent_addr, ent_size;
 595         u64 target_ent_addr = 0, target_prev_addr = 0;
 596         u64 target_size = ~0ull;
 597         u64 free_start, free_end;
 598         u64 next_addr, prev_addr = 0;
 599         u64 new_ent_addr = 0, new_ent_size;
 600         u64 desired_min_addr, usable_max;
 601         u64 align, align_mask;
 602
 603         debug("%s: req_size: 0x%llx, min_addr: 0x%llx, max_addr: 0x%llx, align: 0x%llx\n",
 604               __func__, CAST_ULL(req_size), CAST_ULL(address_min),
 605               CAST_ULL(address_max), CAST_ULL(alignment));
 606
 607         if (__cvmx_bootmem_check_version(0))
 608                 return -1;
 609
 610         /*
 611          * Do a variety of checks to validate the arguments.  The
 612          * allocator code will later assume that these checks have
 613          * been made.  We validate that the requested constraints are
 614          * not self-contradictory before we look through the list of
 615          * available memory
 616          */
 617
 618         /* 0 is not a valid req_size for this allocator */
 619         if (!req_size)
 620                 return -1;
 621
 622         /* Round req_size up to multiple of minimum alignment bytes */
 623         req_size = (req_size + (CVMX_BOOTMEM_ALIGNMENT_SIZE - 1)) &
 624                 ~(CVMX_BOOTMEM_ALIGNMENT_SIZE - 1);
 625
 626         /* Make sure alignment is power of 2, and at least the minimum */
 627         for (align = CVMX_BOOTMEM_ALIGNMENT_SIZE;
 628              align < (1ull << 48);
 629              align <<= 1) {
 630                 if (align >= alignment)
 631                         break;
 632         }
 633
 634         align_mask = ~(align - 1);
 635
 636         /*
 637          * Adjust address minimum based on requested alignment (round
 638          * up to meet alignment).  Do this here so we can reject
 639          * impossible requests up front. (NOP for address_min == 0)
 640          */
 641         address_min = (address_min + (align - 1)) & align_mask;
 642
 643         /*
 644          * Convert !0 address_min and 0 address_max to special case of
 645          * range that specifies an exact memory block to allocate.  Do
 646          * this before other checks and adjustments so that this
 647          * tranformation will be validated
 648          */
 649         if (address_min && !address_max)
 650                 address_max = address_min + req_size;
 651         else if (!address_min && !address_max)
 652                 address_max = ~0ull;    /* If no limits given, use max */
 653
 654         /*
 655          * Reject inconsistent args.  We have adjusted these, so this
 656          * may fail due to our internal changes even if this check
 657          * would pass for the values the user supplied.
 658          */
 659         if (req_size > address_max - address_min)
 660                 return -1;
 661
 662         __cvmx_bootmem_lock(flags);
 663
 664         /* Walk through the list entries to find the right fit */
 665         head_addr = CVMX_BOOTMEM_DESC_GET_FIELD(head_addr);
 666
 667         for (ent_addr = head_addr;
 668              ent_addr != 0ULL && ent_addr < address_max;
 669              prev_addr = ent_addr,
 670                      ent_addr = cvmx_bootmem_phy_get_next(ent_addr)) {
 671                 /* Raw free block size */
 672                 ent_size = cvmx_bootmem_phy_get_size(ent_addr);
 673                 next_addr = cvmx_bootmem_phy_get_next(ent_addr);
 674
 675                 /* Validate the free list ascending order */
 676                 if (ent_size < CVMX_BOOTMEM_ALIGNMENT_SIZE ||
 677                     (next_addr && ent_addr > next_addr)) {
 678                         debug("ERROR: %s: bad free list ent: %#llx, next: %#llx\n",
 679                               __func__, CAST_ULL(ent_addr),
 680                               CAST_ULL(next_addr));
 681                         goto error_out;
 682                 }
 683
 684                 /* adjust free block edges for alignment */
 685                 free_start = (ent_addr + align - 1) & align_mask;
 686                 free_end = (ent_addr + ent_size) &  align_mask;
 687
 688                 /* check that free block is large enough */
 689                 if ((free_start + req_size) > free_end)
 690                         continue;
 691
 692                 /* check that desired start is within the free block */
 693                 if (free_end < address_min || free_start > address_max)
 694                         continue;
 695                 if ((free_end - address_min) < req_size)
 696                         continue;
 697                 if ((address_max - free_start) < req_size)
 698                         continue;
 699
 700                 /* Found usebale free block */
 701                 target_ent_addr = ent_addr;
 702                 target_prev_addr = prev_addr;
 703                 target_size = ent_size;
 704
 705                 /* Continue looking for highest/best block that fits */
 706         }
 707
 708         /* Bail if the search has resulted in no eligible free blocks */
 709         if (target_ent_addr == 0) {
 710                 debug("%s: eligible free block not found\n", __func__);
 711                 goto error_out;
 712         }
 713
 714         /* Found the free block to allocate from */
 715         ent_addr = target_ent_addr;
 716         prev_addr = target_prev_addr;
 717         ent_size = target_size;
 718
 719         debug("%s: using free block at %#010llx size %#llx\n",
 720               __func__, CAST_ULL(ent_addr), CAST_ULL(ent_size));
 721
 722         /* Always allocate from the end of a free block */
 723         usable_max = min_t(u64, address_max, ent_addr + ent_size);
 724         desired_min_addr = usable_max - req_size;
 725         desired_min_addr &= align_mask;
 726
 727         /* Split current free block into up to 3 free blocks */
 728
 729         /* Check for head room */
 730         if (desired_min_addr > ent_addr) {
 731                 /* Create a new free block at the allocation address */
 732                 new_ent_addr = desired_min_addr;
 733                 new_ent_size = ent_size - (desired_min_addr - ent_addr);
 734
 735                 cvmx_bootmem_phy_set_next(new_ent_addr,
 736                                           cvmx_bootmem_phy_get_next(ent_addr));
 737                 cvmx_bootmem_phy_set_size(new_ent_addr, new_ent_size);
 738
 739                 /* Split out head room into a new free block */
 740                 ent_size -= new_ent_size;
 741                 cvmx_bootmem_phy_set_next(ent_addr, new_ent_addr);
 742                 cvmx_bootmem_phy_set_size(ent_addr, ent_size);
 743
 744                 debug("%s: splitting head, addr %#llx size %#llx\n",
 745                       __func__, CAST_ULL(ent_addr), CAST_ULL(ent_size));
 746
 747                 /* Make the allocation target the current free block */
 748                 prev_addr = ent_addr;
 749                 ent_addr = new_ent_addr;
 750                 ent_size = new_ent_size;
 751         }
 752
 753         /* Check for tail room */
 754         if ((desired_min_addr + req_size) < (ent_addr + ent_size)) {
 755                 new_ent_addr = ent_addr + req_size;
 756                 new_ent_size = ent_size - req_size;
 757
 758                 /* Create a new free block from tail room */
 759                 cvmx_bootmem_phy_set_next(new_ent_addr,
 760                                           cvmx_bootmem_phy_get_next(ent_addr));
 761                 cvmx_bootmem_phy_set_size(new_ent_addr, new_ent_size);
 762
 763                 debug("%s: splitting tail, addr %#llx size %#llx\n",
 764                       __func__, CAST_ULL(new_ent_addr), CAST_ULL(new_ent_size));
 765
 766                 /* Adjust the current block to exclude tail room */
 767                 ent_size = ent_size - new_ent_size;
 768                 cvmx_bootmem_phy_set_next(ent_addr, new_ent_addr);
 769                 cvmx_bootmem_phy_set_size(ent_addr, ent_size);
 770         }
 771
 772         /* The current free block IS the allocation target */
 773         if (desired_min_addr != ent_addr || ent_size != req_size)
 774                 debug("ERROR: %s: internal error - addr %#llx %#llx size %#llx %#llx\n",
 775                       __func__, CAST_ULL(desired_min_addr), CAST_ULL(ent_addr),
 776                       CAST_ULL(ent_size), CAST_ULL(req_size));
 777
 778         /* Remove the current free block from list */
 779         if (prev_addr) {
 780                 cvmx_bootmem_phy_set_next(prev_addr,
 781                                           cvmx_bootmem_phy_get_next(ent_addr));
 782         } else {
 783                 /* head of list being returned, so update head ptr */
 784                 CVMX_BOOTMEM_DESC_SET_FIELD(head_addr,
 785                                             cvmx_bootmem_phy_get_next(ent_addr));
 786         }
 787
 788         __cvmx_bootmem_unlock(flags);
 789         debug("%s: allocated size: %#llx, at addr: %#010llx\n",
 790               __func__,
 791               CAST_ULL(req_size),
 792               CAST_ULL(desired_min_addr));
 793
 794         return desired_min_addr;
 795
 796 error_out:
 797         /* Requested memory not found or argument error */
 798         __cvmx_bootmem_unlock(flags);
 799         return -1;
 800 }
 801
 802 int __cvmx_bootmem_phy_free(u64 phy_addr, u64 size, u32 flags)
 803 {
 804         u64 cur_addr;
 805         u64 prev_addr = 0;      /* zero is invalid */
 806         int retval = 0;
 807
 808         debug("%s addr: %#llx, size: %#llx\n", __func__,
 809               CAST_ULL(phy_addr), CAST_ULL(size));
 810
 811         if (__cvmx_bootmem_check_version(0))
 812                 return 0;
 813
 814         /* 0 is not a valid size for this allocator */
 815         if (!size || !phy_addr)
 816                 return 0;
 817
 818         /* Round size up to mult of minimum alignment bytes */
 819         size = (size + (CVMX_BOOTMEM_ALIGNMENT_SIZE - 1)) &
 820                 ~(CVMX_BOOTMEM_ALIGNMENT_SIZE - 1);
 821
 822         __cvmx_bootmem_lock(flags);
 823         cur_addr = CVMX_BOOTMEM_DESC_GET_FIELD(head_addr);
 824         if (cur_addr == 0 || phy_addr < cur_addr) {
 825                 /* add at front of list - special case with changing head ptr */
 826                 if (cur_addr && phy_addr + size > cur_addr)
 827                         goto bootmem_free_done; /* error, overlapping section */
 828                 else if (phy_addr + size == cur_addr) {
 829                         /* Add to front of existing first block */
 830                         cvmx_bootmem_phy_set_next(phy_addr,
 831                                                   cvmx_bootmem_phy_get_next(cur_addr));
 832                         cvmx_bootmem_phy_set_size(phy_addr,
 833                                                   cvmx_bootmem_phy_get_size(cur_addr) + size);
 834                         CVMX_BOOTMEM_DESC_SET_FIELD(head_addr, phy_addr);
 835
 836                 } else {
 837                         /* New block before first block */
 838                         /* OK if cur_addr is 0 */
 839                         cvmx_bootmem_phy_set_next(phy_addr, cur_addr);
 840                         cvmx_bootmem_phy_set_size(phy_addr, size);
 841                         CVMX_BOOTMEM_DESC_SET_FIELD(head_addr, phy_addr);
 842                 }
 843                 retval = 1;
 844                 goto bootmem_free_done;
 845         }
 846
 847         /* Find place in list to add block */
 848         while (cur_addr && phy_addr > cur_addr) {
 849                 prev_addr = cur_addr;
 850                 cur_addr = cvmx_bootmem_phy_get_next(cur_addr);
 851         }
 852
 853         if (!cur_addr) {
 854                 /*
 855                  * We have reached the end of the list, add on to end, checking
 856                  * to see if we need to combine with last block
 857                  */
 858                 if (prev_addr + cvmx_bootmem_phy_get_size(prev_addr) == phy_addr) {
 859                         cvmx_bootmem_phy_set_size(prev_addr,
 860                                                   cvmx_bootmem_phy_get_size(prev_addr) + size);
 861                 } else {
 862                         cvmx_bootmem_phy_set_next(prev_addr, phy_addr);
 863                         cvmx_bootmem_phy_set_size(phy_addr, size);
 864                         cvmx_bootmem_phy_set_next(phy_addr, 0);
 865                 }
 866                 retval = 1;
 867                 goto bootmem_free_done;
 868         } else {
 869                 /*
 870                  * insert between prev and cur nodes, checking for merge with
 871                  * either/both
 872                  */
 873                 if (prev_addr + cvmx_bootmem_phy_get_size(prev_addr) == phy_addr) {
 874                         /* Merge with previous */
 875                         cvmx_bootmem_phy_set_size(prev_addr,
 876                                                   cvmx_bootmem_phy_get_size(prev_addr) + size);
 877                         if (phy_addr + size == cur_addr) {
 878                                 /* Also merge with current */
 879                                 cvmx_bootmem_phy_set_size(prev_addr,
 880                                                           cvmx_bootmem_phy_get_size(cur_addr) +
 881                                                           cvmx_bootmem_phy_get_size(prev_addr));
 882                                 cvmx_bootmem_phy_set_next(prev_addr,
 883                                                           cvmx_bootmem_phy_get_next(cur_addr));
 884                         }
 885                         retval = 1;
 886                         goto bootmem_free_done;
 887                 } else if (phy_addr + size == cur_addr) {
 888                         /* Merge with current */
 889                         cvmx_bootmem_phy_set_size(phy_addr,
 890                                                   cvmx_bootmem_phy_get_size(cur_addr) + size);
 891                         cvmx_bootmem_phy_set_next(phy_addr,
 892                                                   cvmx_bootmem_phy_get_next(cur_addr));
 893                         cvmx_bootmem_phy_set_next(prev_addr, phy_addr);
 894                         retval = 1;
 895                         goto bootmem_free_done;
 896                 }
 897
 898                 /* It is a standalone block, add in between prev and cur */
 899                 cvmx_bootmem_phy_set_size(phy_addr, size);
 900                 cvmx_bootmem_phy_set_next(phy_addr, cur_addr);
 901                 cvmx_bootmem_phy_set_next(prev_addr, phy_addr);
 902         }
 903         retval = 1;
 904
 905 bootmem_free_done:
 906         __cvmx_bootmem_unlock(flags);
 907         return retval;
 908 }
 909
 910 void cvmx_bootmem_phy_list_print(void)
 911 {
 912         u64 addr;
 913
 914         addr = CVMX_BOOTMEM_DESC_GET_FIELD(head_addr);
 915         printf("\n\n\nPrinting bootmem block list, descriptor: 0x%llx, head is 0x%llx\n",
 916                CAST_ULL(cvmx_bootmem_desc_addr), CAST_ULL(addr));
 917         printf("Descriptor version: %d.%d\n",
 918                (int)CVMX_BOOTMEM_DESC_GET_FIELD(major_version),
 919                (int)CVMX_BOOTMEM_DESC_GET_FIELD(minor_version));
 920         if (CVMX_BOOTMEM_DESC_GET_FIELD(major_version) > 3)
 921                 debug("Warning: Bootmem descriptor version is newer than expected\n");
 922
 923         if (!addr)
 924                 printf("mem list is empty!\n");
 925
 926         while (addr) {
 927                 printf("Block address: 0x%08llx, size: 0x%08llx, next: 0x%08llx\n", CAST_ULL(addr),
 928                        CAST_ULL(cvmx_bootmem_phy_get_size(addr)),
 929                        CAST_ULL(cvmx_bootmem_phy_get_next(addr)));
 930                 addr = cvmx_bootmem_phy_get_next(addr);
 931         }
 932         printf("\n\n");
 933 }
 934
 935 u64 cvmx_bootmem_phy_available_mem(u64 min_block_size)
 936 {
 937         u64 addr;
 938
 939         u64 available_mem = 0;
 940
 941         __cvmx_bootmem_lock(0);
 942         addr = CVMX_BOOTMEM_DESC_GET_FIELD(head_addr);
 943         while (addr) {
 944                 if (cvmx_bootmem_phy_get_size(addr) >= min_block_size)
 945                         available_mem += cvmx_bootmem_phy_get_size(addr);
 946                 addr = cvmx_bootmem_phy_get_next(addr);
 947         }
 948         __cvmx_bootmem_unlock(0);
 949         return available_mem;
 950 }
 951
 952 u64 cvmx_bootmem_phy_named_block_find(const char *name, u32 flags)
 953 {
 954         u64 result = 0;
 955
 956         debug("%s: %s\n", __func__, name);
 957
 958         __cvmx_bootmem_lock(flags);
 959         if (!__cvmx_bootmem_check_version(3)) {
 960                 int i;
 961                 u64 named_block_array_addr =
 962                         CVMX_BOOTMEM_DESC_GET_FIELD(named_block_array_addr);
 963                 int num_blocks =
 964                         CVMX_BOOTMEM_DESC_GET_FIELD(named_block_num_blocks);
 965                 int name_length =
 966                         CVMX_BOOTMEM_DESC_GET_FIELD(named_block_name_len);
 967                 u64 named_addr = named_block_array_addr;
 968
 969                 for (i = 0; i < num_blocks; i++) {
 970                         u64 named_size =
 971                                 CVMX_BOOTMEM_NAMED_GET_FIELD(named_addr, size);
 972                         if (name && named_size) {
 973                                 char name_tmp[name_length + 1];
 974
 975                                 CVMX_BOOTMEM_NAMED_GET_NAME(named_addr,
 976                                                             name_tmp,
 977                                                             name_length);
 978                                 if (!strncmp(name, name_tmp, name_length)) {
 979                                         result = named_addr;
 980                                         break;
 981                                 }
 982                         } else if (!name && !named_size) {
 983                                 result = named_addr;
 984                                 break;
 985                         }
 986
 987                         named_addr +=
 988                                 sizeof(struct cvmx_bootmem_named_block_desc);
 989                 }
 990         }
 991         __cvmx_bootmem_unlock(flags);
 992         return result;
 993 }
 994
 995 int cvmx_bootmem_phy_named_block_free(const char *name, u32 flags)
 996 {
 997         u64 named_block_addr;
 998
 999         if (__cvmx_bootmem_check_version(3))
1000                 return 0;
1001
1002         debug("%s: %s\n", __func__, name);
1003
1004         /*
1005          * Take lock here, as name lookup/block free/name free need to be
1006          * atomic
1007          */
1008         __cvmx_bootmem_lock(flags);
1009
1010         named_block_addr = cvmx_bootmem_phy_named_block_find(name,
1011                                                              CVMX_BOOTMEM_FLAG_NO_LOCKING);
1012         if (named_block_addr) {
1013                 u64 named_addr =
1014                         CVMX_BOOTMEM_NAMED_GET_FIELD(named_block_addr,
1015                                                      base_addr);
1016                 u64 named_size =
1017                         CVMX_BOOTMEM_NAMED_GET_FIELD(named_block_addr, size);
1018
1019                 debug("%s: %s, base: 0x%llx, size: 0x%llx\n",
1020                       __func__, name, CAST_ULL(named_addr),
1021                       CAST_ULL(named_size));
1022
1023                 __cvmx_bootmem_phy_free(named_addr, named_size,
1024                                         CVMX_BOOTMEM_FLAG_NO_LOCKING);
1025
1026                 /* Set size to zero to indicate block not used. */
1027                 CVMX_BOOTMEM_NAMED_SET_FIELD(named_block_addr, size, 0);
1028         }
1029
1030         __cvmx_bootmem_unlock(flags);
1031         return !!named_block_addr;      /* 0 on failure, 1 on success */
1032 }
1033
1034 s64 cvmx_bootmem_phy_named_block_alloc(u64 size, u64 min_addr,
1035                                        u64 max_addr,
1036                                        u64 alignment, const char *name,
1037                                        u32 flags)
1038 {
1039         s64 addr_allocated;
1040         u64 named_block_desc_addr;
1041
1042         debug("%s: size: 0x%llx, min: 0x%llx, max: 0x%llx, align: 0x%llx, name: %s\n",
1043               __func__, CAST_ULL(size), CAST_ULL(min_addr), CAST_ULL(max_addr),
1044               CAST_ULL(alignment), name);
1045
1046         if (__cvmx_bootmem_check_version(3))
1047                 return -1;
1048
1049         /*
1050          * Take lock here, as name lookup/block alloc/name add need to be
1051          * atomic
1052          */
1053         __cvmx_bootmem_lock(flags);
1054
1055         named_block_desc_addr =
1056                 cvmx_bootmem_phy_named_block_find(name, flags |
1057                                                   CVMX_BOOTMEM_FLAG_NO_LOCKING);
1058         if (named_block_desc_addr) {
1059                 __cvmx_bootmem_unlock(flags);
1060                 return -1;
1061         }
1062
1063         /* Get pointer to first available named block descriptor */
1064         named_block_desc_addr =
1065                 cvmx_bootmem_phy_named_block_find(NULL, flags |
1066                                                   CVMX_BOOTMEM_FLAG_NO_LOCKING);
1067         if (!named_block_desc_addr) {
1068                 __cvmx_bootmem_unlock(flags);
1069                 return -1;
1070         }
1071
1072         /*
1073          * Round size up to mult of minimum alignment bytes
1074          * We need the actual size allocated to allow for blocks to be
1075          * coallesced when they are freed.  The alloc routine does the
1076          * same rounding up on all allocations.
1077          */
1078         size = (size + (CVMX_BOOTMEM_ALIGNMENT_SIZE - 1)) &
1079                 ~(CVMX_BOOTMEM_ALIGNMENT_SIZE - 1);
1080
1081         addr_allocated = cvmx_bootmem_phy_alloc(size, min_addr, max_addr,
1082                                                 alignment,
1083                                                 flags | CVMX_BOOTMEM_FLAG_NO_LOCKING);
1084         if (addr_allocated >= 0) {
1085                 CVMX_BOOTMEM_NAMED_SET_FIELD(named_block_desc_addr, base_addr,
1086                                              addr_allocated);
1087                 CVMX_BOOTMEM_NAMED_SET_FIELD(named_block_desc_addr, size, size);
1088                 CVMX_BOOTMEM_NAMED_SET_NAME(named_block_desc_addr, name,
1089                                             CVMX_BOOTMEM_DESC_GET_FIELD(named_block_name_len));
1090         }
1091
1092         __cvmx_bootmem_unlock(flags);
1093         return addr_allocated;
1094 }
1095
1096 void cvmx_bootmem_phy_named_block_print(void)
1097 {
1098         int i;
1099         int printed = 0;
1100
1101         u64 named_block_array_addr =
1102                 CVMX_BOOTMEM_DESC_GET_FIELD(named_block_array_addr);
1103         int num_blocks = CVMX_BOOTMEM_DESC_GET_FIELD(named_block_num_blocks);
1104         int name_length = CVMX_BOOTMEM_DESC_GET_FIELD(named_block_name_len);
1105         u64 named_block_addr = named_block_array_addr;
1106
1107         debug("%s: desc addr: 0x%llx\n",
1108               __func__, CAST_ULL(cvmx_bootmem_desc_addr));
1109
1110         if (__cvmx_bootmem_check_version(3))
1111                 return;
1112
1113         printf("List of currently allocated named bootmem blocks:\n");
1114         for (i = 0; i < num_blocks; i++) {
1115                 u64 named_size =
1116                         CVMX_BOOTMEM_NAMED_GET_FIELD(named_block_addr, size);
1117                 if (named_size) {
1118                         char name_tmp[name_length + 1];
1119                         u64 named_addr =
1120                                 CVMX_BOOTMEM_NAMED_GET_FIELD(named_block_addr,
1121                                                              base_addr);
1122                         CVMX_BOOTMEM_NAMED_GET_NAME(named_block_addr, name_tmp,
1123                                                     name_length);
1124                         printed++;
1125                         printf("Name: %s, address: 0x%08llx, size: 0x%08llx, index: %d\n", name_tmp,
1126                                CAST_ULL(named_addr),
1127                                CAST_ULL(named_size), i);
1128                 }
1129                 named_block_addr +=
1130                         sizeof(struct cvmx_bootmem_named_block_desc);
1131         }
1132
1133         if (!printed)
1134                 printf("No named bootmem blocks exist.\n");
1135 }
1136
1137 s64 cvmx_bootmem_phy_mem_list_init(u64 mem_size,
1138                                    u32 low_reserved_bytes,
1139                                    struct cvmx_bootmem_desc *desc_buffer)
1140 {
1141         u64 cur_block_addr;
1142         s64 addr;
1143         int i;
1144
1145         debug("%s (arg desc ptr: %p, cvmx_bootmem_desc: 0x%llx)\n",
1146               __func__, desc_buffer, CAST_ULL(cvmx_bootmem_desc_addr));
1147
1148         /*
1149          * Descriptor buffer needs to be in 32 bit addressable space to be
1150          * compatible with 32 bit applications
1151          */
1152         if (!desc_buffer) {
1153                 debug("ERROR: no memory for cvmx_bootmem descriptor provided\n");
1154                 return 0;
1155         }
1156
1157         if (mem_size > OCTEON_MAX_PHY_MEM_SIZE) {
1158                 mem_size = OCTEON_MAX_PHY_MEM_SIZE;
1159                 debug("ERROR: requested memory size too large, truncating to maximum size\n");
1160         }
1161
1162         if (cvmx_bootmem_desc_addr)
1163                 return 1;
1164
1165         /* Initialize cvmx pointer to descriptor */
1166         cvmx_bootmem_init(cvmx_ptr_to_phys(desc_buffer));
1167
1168         /* Fill the bootmem descriptor */
1169         CVMX_BOOTMEM_DESC_SET_FIELD(lock, 0);
1170         CVMX_BOOTMEM_DESC_SET_FIELD(flags, 0);
1171         CVMX_BOOTMEM_DESC_SET_FIELD(head_addr, 0);
1172         CVMX_BOOTMEM_DESC_SET_FIELD(major_version, CVMX_BOOTMEM_DESC_MAJ_VER);
1173         CVMX_BOOTMEM_DESC_SET_FIELD(minor_version, CVMX_BOOTMEM_DESC_MIN_VER);
1174         CVMX_BOOTMEM_DESC_SET_FIELD(app_data_addr, 0);
1175         CVMX_BOOTMEM_DESC_SET_FIELD(app_data_size, 0);
1176
1177         /*
1178          * Set up global pointer to start of list, exclude low 64k for exception
1179          * vectors, space for global descriptor
1180          */
1181         cur_block_addr = (OCTEON_DDR0_BASE + low_reserved_bytes);
1182
1183         if (mem_size <= OCTEON_DDR0_SIZE) {
1184                 __cvmx_bootmem_phy_free(cur_block_addr,
1185                                         mem_size - low_reserved_bytes, 0);
1186                 goto frees_done;
1187         }
1188
1189         __cvmx_bootmem_phy_free(cur_block_addr,
1190                                 OCTEON_DDR0_SIZE - low_reserved_bytes, 0);
1191
1192         mem_size -= OCTEON_DDR0_SIZE;
1193
1194         /* Add DDR2 block next if present */
1195         if (mem_size > OCTEON_DDR1_SIZE) {
1196                 __cvmx_bootmem_phy_free(OCTEON_DDR1_BASE, OCTEON_DDR1_SIZE, 0);
1197                 __cvmx_bootmem_phy_free(OCTEON_DDR2_BASE,
1198                                         mem_size - OCTEON_DDR1_SIZE, 0);
1199         } else {
1200                 __cvmx_bootmem_phy_free(OCTEON_DDR1_BASE, mem_size, 0);
1201         }
1202 frees_done:
1203
1204         /* Initialize the named block structure */
1205         CVMX_BOOTMEM_DESC_SET_FIELD(named_block_name_len, CVMX_BOOTMEM_NAME_LEN);
1206         CVMX_BOOTMEM_DESC_SET_FIELD(named_block_num_blocks,
1207                                     CVMX_BOOTMEM_NUM_NAMED_BLOCKS);
1208         CVMX_BOOTMEM_DESC_SET_FIELD(named_block_array_addr, 0);
1209
1210         /* Allocate this near the top of the low 256 MBytes of memory */
1211         addr = cvmx_bootmem_phy_alloc(CVMX_BOOTMEM_NUM_NAMED_BLOCKS *
1212                                       sizeof(struct cvmx_bootmem_named_block_desc),
1213                                       0, 0x10000000, 0,
1214                                       CVMX_BOOTMEM_FLAG_END_ALLOC);
1215         if (addr >= 0)
1216                 CVMX_BOOTMEM_DESC_SET_FIELD(named_block_array_addr, addr);
1217
1218         debug("%s: named_block_array_addr: 0x%llx)\n",
1219               __func__, CAST_ULL(addr));
1220
1221         if (addr < 0) {
1222                 debug("FATAL ERROR: unable to allocate memory for bootmem descriptor!\n");
1223                 return 0;
1224         }
1225
1226         for (i = 0; i < CVMX_BOOTMEM_NUM_NAMED_BLOCKS; i++) {
1227                 CVMX_BOOTMEM_NAMED_SET_FIELD(addr, base_addr, 0);
1228                 CVMX_BOOTMEM_NAMED_SET_FIELD(addr, size, 0);
1229                 addr += sizeof(struct cvmx_bootmem_named_block_desc);
1230         }
1231
1232         return 1;
1233 }
1234
1235 s64 cvmx_bootmem_phy_mem_list_init_multi(u8 node_mask,
1236                                          u32 mem_sizes[],
1237                                          u32 low_reserved_bytes,
1238                                          struct cvmx_bootmem_desc *desc_buffer)
1239 {
1240         u64 cur_block_addr;
1241         u64 mem_size;
1242         s64 addr;
1243         int i;
1244         int node;
1245         u64 node_base;  /* Make u64 to reduce type casting */
1246
1247         mem_sizes[0] = gd->ram_size / (1024 * 1024);
1248
1249         debug("cvmx_bootmem_phy_mem_list_init (arg desc ptr: %p, cvmx_bootmem_desc: 0x%llx)\n",
1250               desc_buffer, CAST_ULL(cvmx_bootmem_desc_addr));
1251
1252         /*
1253          * Descriptor buffer needs to be in 32 bit addressable space to be
1254          * compatible with 32 bit applications
1255          */
1256         if (!desc_buffer) {
1257                 debug("ERROR: no memory for cvmx_bootmem descriptor provided\n");
1258                 return 0;
1259         }
1260
1261         cvmx_coremask_for_each_node(node, node_mask) {
1262                 if ((mem_sizes[node] * 1024 * 1024) > OCTEON_MAX_PHY_MEM_SIZE) {
1263                         mem_sizes[node] = OCTEON_MAX_PHY_MEM_SIZE /
1264                                 (1024 * 1024);
1265                         debug("ERROR node#%lld: requested memory size too large, truncating to maximum size\n",
1266                               CAST_ULL(node));
1267                 }
1268         }
1269
1270         if (cvmx_bootmem_desc_addr)
1271                 return 1;
1272
1273         /* Initialize cvmx pointer to descriptor */
1274         cvmx_bootmem_init(cvmx_ptr_to_phys(desc_buffer));
1275
1276         /* Fill the bootmem descriptor */
1277         CVMX_BOOTMEM_DESC_SET_FIELD(lock, 0);
1278         CVMX_BOOTMEM_DESC_SET_FIELD(flags, 0);
1279         CVMX_BOOTMEM_DESC_SET_FIELD(head_addr, 0);
1280         CVMX_BOOTMEM_DESC_SET_FIELD(major_version, CVMX_BOOTMEM_DESC_MAJ_VER);
1281         CVMX_BOOTMEM_DESC_SET_FIELD(minor_version, CVMX_BOOTMEM_DESC_MIN_VER);
1282         CVMX_BOOTMEM_DESC_SET_FIELD(app_data_addr, 0);
1283         CVMX_BOOTMEM_DESC_SET_FIELD(app_data_size, 0);
1284
1285         cvmx_coremask_for_each_node(node, node_mask) {
1286                 if (node != 0)  /* do not reserve memory on remote nodes */
1287                         low_reserved_bytes = 0;
1288
1289                 mem_size = (u64)mem_sizes[node] * (1024 * 1024); /* MBytes */
1290
1291                 /*
1292                  * Set up global pointer to start of list, exclude low 64k
1293                  * for exception vectors, space for global descriptor
1294                  */
1295
1296                 node_base = (u64)node << CVMX_NODE_MEM_SHIFT;
1297                 cur_block_addr = (OCTEON_DDR0_BASE + low_reserved_bytes) |
1298                         node_base;
1299
1300                 if (mem_size <= OCTEON_DDR0_SIZE) {
1301                         __cvmx_bootmem_phy_free(cur_block_addr,
1302                                                 mem_size - low_reserved_bytes,
1303                                                 0);
1304                         continue;
1305                 }
1306
1307                 __cvmx_bootmem_phy_free(cur_block_addr,
1308                                         OCTEON_DDR0_SIZE - low_reserved_bytes,
1309                                         0);
1310
1311                 mem_size -= OCTEON_DDR0_SIZE;
1312
1313                 /* Add DDR2 block next if present */
1314                 if (mem_size > OCTEON_DDR1_SIZE) {
1315                         __cvmx_bootmem_phy_free(OCTEON_DDR1_BASE |
1316                                                 node_base,
1317                                                 OCTEON_DDR1_SIZE, 0);
1318                         __cvmx_bootmem_phy_free(OCTEON_DDR2_BASE |
1319                                                 node_base,
1320                                                 mem_size - OCTEON_DDR1_SIZE, 0);
1321                 } else {
1322                         __cvmx_bootmem_phy_free(OCTEON_DDR1_BASE |
1323                                                 node_base,
1324                                                 mem_size, 0);
1325                 }
1326         }
1327
1328         debug("%s: Initialize the named block\n", __func__);
1329
1330         /* Initialize the named block structure */
1331         CVMX_BOOTMEM_DESC_SET_FIELD(named_block_name_len, CVMX_BOOTMEM_NAME_LEN);
1332         CVMX_BOOTMEM_DESC_SET_FIELD(named_block_num_blocks,
1333                                     CVMX_BOOTMEM_NUM_NAMED_BLOCKS);
1334         CVMX_BOOTMEM_DESC_SET_FIELD(named_block_array_addr, 0);
1335
1336         /* Allocate this near the top of the low 256 MBytes of memory */
1337         addr = cvmx_bootmem_phy_alloc(CVMX_BOOTMEM_NUM_NAMED_BLOCKS *
1338                                       sizeof(struct cvmx_bootmem_named_block_desc),
1339                                       0, 0x10000000, 0,
1340                                       CVMX_BOOTMEM_FLAG_END_ALLOC);
1341         if (addr >= 0)
1342                 CVMX_BOOTMEM_DESC_SET_FIELD(named_block_array_addr, addr);
1343
1344         debug("cvmx_bootmem_phy_mem_list_init: named_block_array_addr: 0x%llx)\n",
1345               CAST_ULL(addr));
1346
1347         if (addr < 0) {
1348                 debug("FATAL ERROR: unable to allocate memory for bootmem descriptor!\n");
1349                 return 0;
1350         }
1351
1352         for (i = 0; i < CVMX_BOOTMEM_NUM_NAMED_BLOCKS; i++) {
1353                 CVMX_BOOTMEM_NAMED_SET_FIELD(addr, base_addr, 0);
1354                 CVMX_BOOTMEM_NAMED_SET_FIELD(addr, size, 0);
1355                 addr += sizeof(struct cvmx_bootmem_named_block_desc);
1356         }
1357
1358         // test-only: DEBUG ifdef???
1359         cvmx_bootmem_phy_list_print();
1360
1361         return 1;
1362 }
1363
1364 int cvmx_bootmem_reserve_memory(u64 start_addr, u64 size,
1365                                 const char *name, u32 flags)
1366 {
1367         u64 addr;
1368         int rc = 1;
1369         static unsigned int block_num;
1370         char block_name[CVMX_BOOTMEM_NAME_LEN];
1371
1372         debug("%s: start %#llx, size: %#llx, name: %s, flags:%#x)\n",
1373               __func__, CAST_ULL(start_addr), CAST_ULL(size), name, flags);
1374
1375         if (__cvmx_bootmem_check_version(3))
1376                 return 0;
1377
1378         addr = CVMX_BOOTMEM_DESC_GET_FIELD(head_addr);
1379         if (!addr)
1380                 return 0;
1381
1382         if (!name)
1383                 name = "__cvmx_bootmem_reserved";
1384
1385         while (addr && rc) {
1386                 u64 block_size = cvmx_bootmem_phy_get_size(addr);
1387                 u64 reserve_size = 0;
1388
1389                 if (addr >= start_addr && addr < start_addr + size) {
1390                         reserve_size = size - (addr - start_addr);
1391                         if (block_size < reserve_size)
1392                                 reserve_size = block_size;
1393                 } else if (start_addr > addr &&
1394                            start_addr < (addr + block_size)) {
1395                         reserve_size = block_size - (start_addr - addr);
1396                 }
1397
1398                 if (reserve_size) {
1399                         snprintf(block_name, sizeof(block_name),
1400                                  "%.32s_%012llx_%u",
1401                                  name, (unsigned long long)start_addr,
1402                                  (unsigned int)block_num);
1403
1404                         debug("%s: Reserving 0x%llx bytes at address 0x%llx with name %s\n",
1405                               __func__, CAST_ULL(reserve_size),
1406                               CAST_ULL(addr), block_name);
1407
1408                         if (cvmx_bootmem_phy_named_block_alloc(reserve_size,
1409                                                                addr, 0, 0,
1410                                                                block_name,
1411                                                                flags) == -1) {
1412                                 debug("%s: Failed to reserve 0x%llx bytes at address 0x%llx\n",
1413                                       __func__, CAST_ULL(reserve_size),
1414                                       (unsigned long long)addr);
1415                                 rc = 0;
1416                                 break;
1417                         }
1418
1419                         debug("%s: Reserved 0x%llx bytes at address 0x%llx with name %s\n",
1420                               __func__, CAST_ULL(reserve_size),
1421                               CAST_ULL(addr), block_name);
1422                 }
1423
1424                 addr = cvmx_bootmem_phy_get_next(addr);
1425                 block_num++;
1426         }
1427
1428         return rc;
1429 }
1430
1431 void cvmx_bootmem_lock(void)
1432 {
1433         __cvmx_bootmem_lock(0);
1434 }
1435
1436 void cvmx_bootmem_unlock(void)
1437 {
1438         __cvmx_bootmem_unlock(0);
1439 }
1440
1441 void *__cvmx_phys_addr_to_ptr(u64 phys, int size)
1442 {
1443         void *tmp;
1444
1445         if (sizeof(void *) == 8) {
1446                 tmp = CASTPTR(void, CVMX_ADD_SEG(CVMX_MIPS_SPACE_XKPHYS, phys));
1447         } else {
1448                 u32 phy32 = (u32)(phys & 0x7fffffffULL);
1449
1450                 tmp = CASTPTR(void, CVMX_ADD_SEG32(CVMX_MIPS32_SPACE_KSEG0,
1451                                                    phy32));
1452         }
1453
1454         return tmp;
1455 }
1456
1457 void *__cvmx_bootmem_internal_get_desc_ptr(void)
1458 {
1459         return cvmx_phys_to_ptr(cvmx_bootmem_desc_addr);
1460 }