1 // SPDX-License-Identifier: GPL-2.0
3 * Copyright (C) 2018-2020 Marvell International Ltd.
7 * Simple allocate only memory allocator. Used to allocate memory at
8 * application start time.
11 #include <asm/global_data.h>
13 #include <linux/compat.h>
15 #include <linux/types.h>
17 #include <mach/octeon-model.h>
18 #include <mach/cvmx-bootmem.h>
19 #include <mach/cvmx-coremask.h>
20 #include <mach/cvmx-regs.h>
22 DECLARE_GLOBAL_DATA_PTR;
24 #define CVMX_MIPS32_SPACE_KSEG0 1L
25 #define CVMX_MIPS_SPACE_XKPHYS 2LL
27 #define CVMX_ADD_SEG(seg, add) ((((u64)(seg)) << 62) | (add))
28 #define CVMX_ADD_SEG32(seg, add) (((u32)(seg) << 31) | (u32)(add))
31 * This is the physical location of a struct cvmx_bootmem_desc
32 * structure in Octeon's memory. Note that dues to addressing
33 * limits or runtime environment it might not be possible to
34 * create a C pointer to this structure.
36 static u64 cvmx_bootmem_desc_addr;
39 * This macro returns the size of a member of a structure.
40 * Logically it is the same as "sizeof(s::field)" in C++, but
41 * C lacks the "::" operator.
43 #define SIZEOF_FIELD(s, field) sizeof(((s *)NULL)->field)
46 * This macro returns a member of the struct cvmx_bootmem_desc
47 * structure. These members can't be directly addressed as
48 * they might be in memory not directly reachable. In the case
49 * where bootmem is compiled with LINUX_HOST, the structure
50 * itself might be located on a remote Octeon. The argument
51 * "field" is the member name of the struct cvmx_bootmem_desc to read.
52 * Regardless of the type of the field, the return type is always
55 #define CVMX_BOOTMEM_DESC_GET_FIELD(field) \
56 __cvmx_bootmem_desc_get(cvmx_bootmem_desc_addr, \
57 offsetof(struct cvmx_bootmem_desc, field), \
58 SIZEOF_FIELD(struct cvmx_bootmem_desc, field))
61 * This macro writes a member of the struct cvmx_bootmem_desc
62 * structure. These members can't be directly addressed as
63 * they might be in memory not directly reachable. In the case
64 * where bootmem is compiled with LINUX_HOST, the structure
65 * itself might be located on a remote Octeon. The argument
66 * "field" is the member name of the struct cvmx_bootmem_desc to write.
68 #define CVMX_BOOTMEM_DESC_SET_FIELD(field, value) \
69 __cvmx_bootmem_desc_set(cvmx_bootmem_desc_addr, \
70 offsetof(struct cvmx_bootmem_desc, field), \
71 SIZEOF_FIELD(struct cvmx_bootmem_desc, field), \
75 * This macro returns a member of the
76 * struct cvmx_bootmem_named_block_desc structure. These members can't
77 * be directly addressed as they might be in memory not directly
78 * reachable. In the case where bootmem is compiled with
79 * LINUX_HOST, the structure itself might be located on a remote
80 * Octeon. The argument "field" is the member name of the
81 * struct cvmx_bootmem_named_block_desc to read. Regardless of the type
82 * of the field, the return type is always a u64. The "addr"
83 * parameter is the physical address of the structure.
85 #define CVMX_BOOTMEM_NAMED_GET_FIELD(addr, field) \
86 __cvmx_bootmem_desc_get(addr, \
87 offsetof(struct cvmx_bootmem_named_block_desc, field), \
88 SIZEOF_FIELD(struct cvmx_bootmem_named_block_desc, field))
91 * This macro writes a member of the struct cvmx_bootmem_named_block_desc
92 * structure. These members can't be directly addressed as
93 * they might be in memory not directly reachable. In the case
94 * where bootmem is compiled with LINUX_HOST, the structure
95 * itself might be located on a remote Octeon. The argument
96 * "field" is the member name of the
97 * struct cvmx_bootmem_named_block_desc to write. The "addr" parameter
98 * is the physical address of the structure.
100 #define CVMX_BOOTMEM_NAMED_SET_FIELD(addr, field, value) \
101 __cvmx_bootmem_desc_set(addr, \
102 offsetof(struct cvmx_bootmem_named_block_desc, field), \
103 SIZEOF_FIELD(struct cvmx_bootmem_named_block_desc, field), \
107 * This function is the implementation of the get macros defined
108 * for individual structure members. The argument are generated
109 * by the macros inorder to read only the needed memory.
111 * @param base 64bit physical address of the complete structure
112 * @param offset Offset from the beginning of the structure to the member being
114 * @param size Size of the structure member.
116 * @return Value of the structure member promoted into a u64.
118 static inline u64 __cvmx_bootmem_desc_get(u64 base, int offset,
121 base = (1ull << 63) | (base + offset);
124 return cvmx_read64_uint32(base);
126 return cvmx_read64_uint64(base);
133 * This function is the implementation of the set macros defined
134 * for individual structure members. The argument are generated
135 * by the macros in order to write only the needed memory.
137 * @param base 64bit physical address of the complete structure
138 * @param offset Offset from the beginning of the structure to the member being
140 * @param size Size of the structure member.
141 * @param value Value to write into the structure
143 static inline void __cvmx_bootmem_desc_set(u64 base, int offset, int size,
146 base = (1ull << 63) | (base + offset);
149 cvmx_write64_uint32(base, value);
152 cvmx_write64_uint64(base, value);
160 * This function returns the address of the bootmem descriptor lock.
162 * @return 64-bit address in KSEG0 of the bootmem descriptor block
164 static inline u64 __cvmx_bootmem_get_lock_addr(void)
166 return (1ull << 63) |
167 (cvmx_bootmem_desc_addr + offsetof(struct cvmx_bootmem_desc, lock));
171 * This function retrieves the string name of a named block. It is
172 * more complicated than a simple memcpy() since the named block
173 * descriptor may not be directly accessible.
175 * @param addr Physical address of the named block descriptor
176 * @param str String to receive the named block string name
177 * @param len Length of the string buffer, which must match the length
178 * stored in the bootmem descriptor.
180 static void CVMX_BOOTMEM_NAMED_GET_NAME(u64 addr, char *str, int len)
185 addr |= (1ull << 63);
186 addr += offsetof(struct cvmx_bootmem_named_block_desc, name);
189 * With big-endian in memory byte order, this gives uniform
190 * results for the CPU in either big or Little endian mode.
192 u64 blob = cvmx_read64_uint64(addr);
196 while (l && sa >= 0) {
197 *ptr++ = (char)(blob >> sa);
206 * This function stores the string name of a named block. It is
207 * more complicated than a simple memcpy() since the named block
208 * descriptor may not be directly accessible.
210 * @param addr Physical address of the named block descriptor
211 * @param str String to store into the named block string name
212 * @param len Length of the string buffer, which must match the length
213 * stored in the bootmem descriptor.
215 void CVMX_BOOTMEM_NAMED_SET_NAME(u64 addr, const char *str, int len)
219 addr |= (1ull << 63);
220 addr += offsetof(struct cvmx_bootmem_named_block_desc, name);
224 * With big-endian in memory byte order, this gives uniform
225 * results for the CPU in either big or Little endian mode.
230 while (l && sa >= 0) {
231 u64 c = (u8)(*str++);
239 cvmx_write64_uint64(addr, blob);
244 /* See header file for descriptions of functions */
247 * Wrapper functions are provided for reading/writing the size and next block
248 * values as these may not be directly addressible (in 32 bit applications, for
251 * Offsets of data elements in bootmem list, must match
252 * struct cvmx_bootmem_block_header
254 #define NEXT_OFFSET 0
255 #define SIZE_OFFSET 8
257 static void cvmx_bootmem_phy_set_size(u64 addr, u64 size)
259 cvmx_write64_uint64((addr + SIZE_OFFSET) | (1ull << 63), size);
262 static void cvmx_bootmem_phy_set_next(u64 addr, u64 next)
264 cvmx_write64_uint64((addr + NEXT_OFFSET) | (1ull << 63), next);
267 static u64 cvmx_bootmem_phy_get_size(u64 addr)
269 return cvmx_read64_uint64((addr + SIZE_OFFSET) | (1ull << 63));
272 static u64 cvmx_bootmem_phy_get_next(u64 addr)
274 return cvmx_read64_uint64((addr + NEXT_OFFSET) | (1ull << 63));
278 * Check the version information on the bootmem descriptor
281 * Exact major version to check against. A zero means
282 * check that the version supports named blocks.
284 * @return Zero if the version is correct. Negative if the version is
285 * incorrect. Failures also cause a message to be displayed.
287 static int __cvmx_bootmem_check_version(int exact_match)
291 major_version = CVMX_BOOTMEM_DESC_GET_FIELD(major_version);
292 if (major_version > 3 ||
293 (exact_match && major_version) != exact_match) {
294 debug("ERROR: Incompatible bootmem descriptor version: %d.%d at addr: 0x%llx\n",
296 (int)CVMX_BOOTMEM_DESC_GET_FIELD(minor_version),
297 CAST_ULL(cvmx_bootmem_desc_addr));
305 * Get the low level bootmem descriptor lock. If no locking
306 * is specified in the flags, then nothing is done.
308 * @param flags CVMX_BOOTMEM_FLAG_NO_LOCKING means this functions should do
309 * nothing. This is used to support nested bootmem calls.
311 static inline void __cvmx_bootmem_lock(u32 flags)
313 if (!(flags & CVMX_BOOTMEM_FLAG_NO_LOCKING)) {
315 * Unfortunately we can't use the normal cvmx-spinlock code as
316 * the memory for the bootmem descriptor may be not accessible
317 * by a C pointer. We use a 64bit XKPHYS address to access the
320 u64 lock_addr = (1ull << 63) |
321 (cvmx_bootmem_desc_addr + offsetof(struct cvmx_bootmem_desc,
325 __asm__ __volatile__(".set noreorder\n"
326 "1: ll %[tmp], 0(%[addr])\n"
329 " sc %[tmp], 0(%[addr])\n"
334 : [addr] "r"(lock_addr)
340 * Release the low level bootmem descriptor lock. If no locking
341 * is specified in the flags, then nothing is done.
343 * @param flags CVMX_BOOTMEM_FLAG_NO_LOCKING means this functions should do
344 * nothing. This is used to support nested bootmem calls.
346 static inline void __cvmx_bootmem_unlock(u32 flags)
348 if (!(flags & CVMX_BOOTMEM_FLAG_NO_LOCKING)) {
350 * Unfortunately we can't use the normal cvmx-spinlock code as
351 * the memory for the bootmem descriptor may be not accessible
352 * by a C pointer. We use a 64bit XKPHYS address to access the
355 u64 lock_addr = __cvmx_bootmem_get_lock_addr();
358 __asm__ __volatile__("sw $0, 0(%[addr])\n"
359 : : [addr] "r"(lock_addr)
366 * Some of the cvmx-bootmem functions dealing with C pointers are not
367 * supported when we are compiling for CVMX_BUILD_FOR_LINUX_HOST. This
368 * ifndef removes these functions when they aren't needed.
370 * This functions takes an address range and adjusts it as necessary
371 * to match the ABI that is currently being used. This is required to
372 * ensure that bootmem_alloc* functions only return valid pointers for
375 static int __cvmx_validate_mem_range(u64 *min_addr_ptr,
378 u64 max_phys = (1ull << 29) - 0x10; /* KSEG0 */
380 *min_addr_ptr = min_t(u64, max_t(u64, *min_addr_ptr, 0x0), max_phys);
381 if (!*max_addr_ptr) {
382 *max_addr_ptr = max_phys;
384 *max_addr_ptr = max_t(u64, min_t(u64, *max_addr_ptr,
391 u64 cvmx_bootmem_phy_alloc_range(u64 size, u64 alignment,
392 u64 min_addr, u64 max_addr)
396 __cvmx_validate_mem_range(&min_addr, &max_addr);
397 address = cvmx_bootmem_phy_alloc(size, min_addr, max_addr,
405 void *cvmx_bootmem_alloc_range(u64 size, u64 alignment,
406 u64 min_addr, u64 max_addr)
410 __cvmx_validate_mem_range(&min_addr, &max_addr);
411 address = cvmx_bootmem_phy_alloc(size, min_addr, max_addr,
415 return cvmx_phys_to_ptr(address);
420 void *cvmx_bootmem_alloc_address(u64 size, u64 address,
423 return cvmx_bootmem_alloc_range(size, alignment, address,
427 void *cvmx_bootmem_alloc_node(u64 node, u64 size, u64 alignment)
429 return cvmx_bootmem_alloc_range(size, alignment,
430 node << CVMX_NODE_MEM_SHIFT,
431 ((node + 1) << CVMX_NODE_MEM_SHIFT) - 1);
434 void *cvmx_bootmem_alloc(u64 size, u64 alignment)
436 return cvmx_bootmem_alloc_range(size, alignment, 0, 0);
439 void *cvmx_bootmem_alloc_named_range_once(u64 size, u64 min_addr,
440 u64 max_addr, u64 align,
442 void (*init)(void *))
444 u64 named_block_desc_addr;
448 __cvmx_bootmem_lock(0);
450 __cvmx_validate_mem_range(&min_addr, &max_addr);
451 named_block_desc_addr =
452 cvmx_bootmem_phy_named_block_find(name,
453 CVMX_BOOTMEM_FLAG_NO_LOCKING);
455 if (named_block_desc_addr) {
456 addr = CVMX_BOOTMEM_NAMED_GET_FIELD(named_block_desc_addr,
458 __cvmx_bootmem_unlock(0);
459 return cvmx_phys_to_ptr(addr);
462 addr = cvmx_bootmem_phy_named_block_alloc(size, min_addr, max_addr,
464 CVMX_BOOTMEM_FLAG_NO_LOCKING);
467 __cvmx_bootmem_unlock(0);
470 ptr = cvmx_phys_to_ptr(addr);
475 memset(ptr, 0, size);
477 __cvmx_bootmem_unlock(0);
481 void *cvmx_bootmem_alloc_named_range_flags(u64 size, u64 min_addr,
482 u64 max_addr, u64 align,
483 const char *name, u32 flags)
487 __cvmx_validate_mem_range(&min_addr, &max_addr);
488 addr = cvmx_bootmem_phy_named_block_alloc(size, min_addr, max_addr,
491 return cvmx_phys_to_ptr(addr);
496 void *cvmx_bootmem_alloc_named_range(u64 size, u64 min_addr,
497 u64 max_addr, u64 align,
500 return cvmx_bootmem_alloc_named_range_flags(size, min_addr, max_addr,
504 void *cvmx_bootmem_alloc_named_address(u64 size, u64 address,
507 return cvmx_bootmem_alloc_named_range(size, address, address + size,
511 void *cvmx_bootmem_alloc_named(u64 size, u64 alignment,
514 return cvmx_bootmem_alloc_named_range(size, 0, 0, alignment, name);
517 void *cvmx_bootmem_alloc_named_flags(u64 size, u64 alignment,
518 const char *name, u32 flags)
520 return cvmx_bootmem_alloc_named_range_flags(size, 0, 0, alignment,
524 int cvmx_bootmem_free_named(const char *name)
526 return cvmx_bootmem_phy_named_block_free(name, 0);
530 * Find a named block with flags
532 * @param name is the block name
533 * @param flags indicates the need to use locking during search
534 * @return pointer to named block descriptor
536 * Note: this function returns a pointer to a static structure,
537 * and is therefore not re-entrant.
538 * Making this function re-entrant will break backward compatibility.
540 const struct cvmx_bootmem_named_block_desc *
541 __cvmx_bootmem_find_named_block_flags(const char *name, u32 flags)
543 static struct cvmx_bootmem_named_block_desc desc;
544 u64 named_addr = cvmx_bootmem_phy_named_block_find(name, flags);
547 desc.base_addr = CVMX_BOOTMEM_NAMED_GET_FIELD(named_addr,
549 desc.size = CVMX_BOOTMEM_NAMED_GET_FIELD(named_addr, size);
550 strncpy(desc.name, name, sizeof(desc.name));
551 desc.name[sizeof(desc.name) - 1] = 0;
558 const struct cvmx_bootmem_named_block_desc *
559 cvmx_bootmem_find_named_block(const char *name)
561 return __cvmx_bootmem_find_named_block_flags(name, 0);
564 void cvmx_bootmem_print_named(void)
566 cvmx_bootmem_phy_named_block_print();
569 int cvmx_bootmem_init(u64 mem_desc_addr)
571 if (!cvmx_bootmem_desc_addr)
572 cvmx_bootmem_desc_addr = mem_desc_addr;
577 u64 cvmx_bootmem_available_mem(u64 min_block_size)
579 return cvmx_bootmem_phy_available_mem(min_block_size);
583 * The cvmx_bootmem_phy* functions below return 64 bit physical
584 * addresses, and expose more features that the cvmx_bootmem_functions
585 * above. These are required for full memory space access in 32 bit
586 * applications, as well as for using some advance features. Most
587 * applications should not need to use these.
590 s64 cvmx_bootmem_phy_alloc(u64 req_size, u64 address_min,
591 u64 address_max, u64 alignment,
594 u64 head_addr, ent_addr, ent_size;
595 u64 target_ent_addr = 0, target_prev_addr = 0;
596 u64 target_size = ~0ull;
597 u64 free_start, free_end;
598 u64 next_addr, prev_addr = 0;
599 u64 new_ent_addr = 0, new_ent_size;
600 u64 desired_min_addr, usable_max;
601 u64 align, align_mask;
603 debug("%s: req_size: 0x%llx, min_addr: 0x%llx, max_addr: 0x%llx, align: 0x%llx\n",
604 __func__, CAST_ULL(req_size), CAST_ULL(address_min),
605 CAST_ULL(address_max), CAST_ULL(alignment));
607 if (__cvmx_bootmem_check_version(0))
611 * Do a variety of checks to validate the arguments. The
612 * allocator code will later assume that these checks have
613 * been made. We validate that the requested constraints are
614 * not self-contradictory before we look through the list of
618 /* 0 is not a valid req_size for this allocator */
622 /* Round req_size up to multiple of minimum alignment bytes */
623 req_size = (req_size + (CVMX_BOOTMEM_ALIGNMENT_SIZE - 1)) &
624 ~(CVMX_BOOTMEM_ALIGNMENT_SIZE - 1);
626 /* Make sure alignment is power of 2, and at least the minimum */
627 for (align = CVMX_BOOTMEM_ALIGNMENT_SIZE;
628 align < (1ull << 48);
630 if (align >= alignment)
634 align_mask = ~(align - 1);
637 * Adjust address minimum based on requested alignment (round
638 * up to meet alignment). Do this here so we can reject
639 * impossible requests up front. (NOP for address_min == 0)
641 address_min = (address_min + (align - 1)) & align_mask;
644 * Convert !0 address_min and 0 address_max to special case of
645 * range that specifies an exact memory block to allocate. Do
646 * this before other checks and adjustments so that this
647 * tranformation will be validated
649 if (address_min && !address_max)
650 address_max = address_min + req_size;
651 else if (!address_min && !address_max)
652 address_max = ~0ull; /* If no limits given, use max */
655 * Reject inconsistent args. We have adjusted these, so this
656 * may fail due to our internal changes even if this check
657 * would pass for the values the user supplied.
659 if (req_size > address_max - address_min)
662 __cvmx_bootmem_lock(flags);
664 /* Walk through the list entries to find the right fit */
665 head_addr = CVMX_BOOTMEM_DESC_GET_FIELD(head_addr);
667 for (ent_addr = head_addr;
668 ent_addr != 0ULL && ent_addr < address_max;
669 prev_addr = ent_addr,
670 ent_addr = cvmx_bootmem_phy_get_next(ent_addr)) {
671 /* Raw free block size */
672 ent_size = cvmx_bootmem_phy_get_size(ent_addr);
673 next_addr = cvmx_bootmem_phy_get_next(ent_addr);
675 /* Validate the free list ascending order */
676 if (ent_size < CVMX_BOOTMEM_ALIGNMENT_SIZE ||
677 (next_addr && ent_addr > next_addr)) {
678 debug("ERROR: %s: bad free list ent: %#llx, next: %#llx\n",
679 __func__, CAST_ULL(ent_addr),
680 CAST_ULL(next_addr));
684 /* adjust free block edges for alignment */
685 free_start = (ent_addr + align - 1) & align_mask;
686 free_end = (ent_addr + ent_size) & align_mask;
688 /* check that free block is large enough */
689 if ((free_start + req_size) > free_end)
692 /* check that desired start is within the free block */
693 if (free_end < address_min || free_start > address_max)
695 if ((free_end - address_min) < req_size)
697 if ((address_max - free_start) < req_size)
700 /* Found usebale free block */
701 target_ent_addr = ent_addr;
702 target_prev_addr = prev_addr;
703 target_size = ent_size;
705 /* Continue looking for highest/best block that fits */
708 /* Bail if the search has resulted in no eligible free blocks */
709 if (target_ent_addr == 0) {
710 debug("%s: eligible free block not found\n", __func__);
714 /* Found the free block to allocate from */
715 ent_addr = target_ent_addr;
716 prev_addr = target_prev_addr;
717 ent_size = target_size;
719 debug("%s: using free block at %#010llx size %#llx\n",
720 __func__, CAST_ULL(ent_addr), CAST_ULL(ent_size));
722 /* Always allocate from the end of a free block */
723 usable_max = min_t(u64, address_max, ent_addr + ent_size);
724 desired_min_addr = usable_max - req_size;
725 desired_min_addr &= align_mask;
727 /* Split current free block into up to 3 free blocks */
729 /* Check for head room */
730 if (desired_min_addr > ent_addr) {
731 /* Create a new free block at the allocation address */
732 new_ent_addr = desired_min_addr;
733 new_ent_size = ent_size - (desired_min_addr - ent_addr);
735 cvmx_bootmem_phy_set_next(new_ent_addr,
736 cvmx_bootmem_phy_get_next(ent_addr));
737 cvmx_bootmem_phy_set_size(new_ent_addr, new_ent_size);
739 /* Split out head room into a new free block */
740 ent_size -= new_ent_size;
741 cvmx_bootmem_phy_set_next(ent_addr, new_ent_addr);
742 cvmx_bootmem_phy_set_size(ent_addr, ent_size);
744 debug("%s: splitting head, addr %#llx size %#llx\n",
745 __func__, CAST_ULL(ent_addr), CAST_ULL(ent_size));
747 /* Make the allocation target the current free block */
748 prev_addr = ent_addr;
749 ent_addr = new_ent_addr;
750 ent_size = new_ent_size;
753 /* Check for tail room */
754 if ((desired_min_addr + req_size) < (ent_addr + ent_size)) {
755 new_ent_addr = ent_addr + req_size;
756 new_ent_size = ent_size - req_size;
758 /* Create a new free block from tail room */
759 cvmx_bootmem_phy_set_next(new_ent_addr,
760 cvmx_bootmem_phy_get_next(ent_addr));
761 cvmx_bootmem_phy_set_size(new_ent_addr, new_ent_size);
763 debug("%s: splitting tail, addr %#llx size %#llx\n",
764 __func__, CAST_ULL(new_ent_addr), CAST_ULL(new_ent_size));
766 /* Adjust the current block to exclude tail room */
767 ent_size = ent_size - new_ent_size;
768 cvmx_bootmem_phy_set_next(ent_addr, new_ent_addr);
769 cvmx_bootmem_phy_set_size(ent_addr, ent_size);
772 /* The current free block IS the allocation target */
773 if (desired_min_addr != ent_addr || ent_size != req_size)
774 debug("ERROR: %s: internal error - addr %#llx %#llx size %#llx %#llx\n",
775 __func__, CAST_ULL(desired_min_addr), CAST_ULL(ent_addr),
776 CAST_ULL(ent_size), CAST_ULL(req_size));
778 /* Remove the current free block from list */
780 cvmx_bootmem_phy_set_next(prev_addr,
781 cvmx_bootmem_phy_get_next(ent_addr));
783 /* head of list being returned, so update head ptr */
784 CVMX_BOOTMEM_DESC_SET_FIELD(head_addr,
785 cvmx_bootmem_phy_get_next(ent_addr));
788 __cvmx_bootmem_unlock(flags);
789 debug("%s: allocated size: %#llx, at addr: %#010llx\n",
792 CAST_ULL(desired_min_addr));
794 return desired_min_addr;
797 /* Requested memory not found or argument error */
798 __cvmx_bootmem_unlock(flags);
802 int __cvmx_bootmem_phy_free(u64 phy_addr, u64 size, u32 flags)
805 u64 prev_addr = 0; /* zero is invalid */
808 debug("%s addr: %#llx, size: %#llx\n", __func__,
809 CAST_ULL(phy_addr), CAST_ULL(size));
811 if (__cvmx_bootmem_check_version(0))
814 /* 0 is not a valid size for this allocator */
815 if (!size || !phy_addr)
818 /* Round size up to mult of minimum alignment bytes */
819 size = (size + (CVMX_BOOTMEM_ALIGNMENT_SIZE - 1)) &
820 ~(CVMX_BOOTMEM_ALIGNMENT_SIZE - 1);
822 __cvmx_bootmem_lock(flags);
823 cur_addr = CVMX_BOOTMEM_DESC_GET_FIELD(head_addr);
824 if (cur_addr == 0 || phy_addr < cur_addr) {
825 /* add at front of list - special case with changing head ptr */
826 if (cur_addr && phy_addr + size > cur_addr)
827 goto bootmem_free_done; /* error, overlapping section */
828 else if (phy_addr + size == cur_addr) {
829 /* Add to front of existing first block */
830 cvmx_bootmem_phy_set_next(phy_addr,
831 cvmx_bootmem_phy_get_next(cur_addr));
832 cvmx_bootmem_phy_set_size(phy_addr,
833 cvmx_bootmem_phy_get_size(cur_addr) + size);
834 CVMX_BOOTMEM_DESC_SET_FIELD(head_addr, phy_addr);
837 /* New block before first block */
838 /* OK if cur_addr is 0 */
839 cvmx_bootmem_phy_set_next(phy_addr, cur_addr);
840 cvmx_bootmem_phy_set_size(phy_addr, size);
841 CVMX_BOOTMEM_DESC_SET_FIELD(head_addr, phy_addr);
844 goto bootmem_free_done;
847 /* Find place in list to add block */
848 while (cur_addr && phy_addr > cur_addr) {
849 prev_addr = cur_addr;
850 cur_addr = cvmx_bootmem_phy_get_next(cur_addr);
855 * We have reached the end of the list, add on to end, checking
856 * to see if we need to combine with last block
858 if (prev_addr + cvmx_bootmem_phy_get_size(prev_addr) == phy_addr) {
859 cvmx_bootmem_phy_set_size(prev_addr,
860 cvmx_bootmem_phy_get_size(prev_addr) + size);
862 cvmx_bootmem_phy_set_next(prev_addr, phy_addr);
863 cvmx_bootmem_phy_set_size(phy_addr, size);
864 cvmx_bootmem_phy_set_next(phy_addr, 0);
867 goto bootmem_free_done;
870 * insert between prev and cur nodes, checking for merge with
873 if (prev_addr + cvmx_bootmem_phy_get_size(prev_addr) == phy_addr) {
874 /* Merge with previous */
875 cvmx_bootmem_phy_set_size(prev_addr,
876 cvmx_bootmem_phy_get_size(prev_addr) + size);
877 if (phy_addr + size == cur_addr) {
878 /* Also merge with current */
879 cvmx_bootmem_phy_set_size(prev_addr,
880 cvmx_bootmem_phy_get_size(cur_addr) +
881 cvmx_bootmem_phy_get_size(prev_addr));
882 cvmx_bootmem_phy_set_next(prev_addr,
883 cvmx_bootmem_phy_get_next(cur_addr));
886 goto bootmem_free_done;
887 } else if (phy_addr + size == cur_addr) {
888 /* Merge with current */
889 cvmx_bootmem_phy_set_size(phy_addr,
890 cvmx_bootmem_phy_get_size(cur_addr) + size);
891 cvmx_bootmem_phy_set_next(phy_addr,
892 cvmx_bootmem_phy_get_next(cur_addr));
893 cvmx_bootmem_phy_set_next(prev_addr, phy_addr);
895 goto bootmem_free_done;
898 /* It is a standalone block, add in between prev and cur */
899 cvmx_bootmem_phy_set_size(phy_addr, size);
900 cvmx_bootmem_phy_set_next(phy_addr, cur_addr);
901 cvmx_bootmem_phy_set_next(prev_addr, phy_addr);
906 __cvmx_bootmem_unlock(flags);
910 void cvmx_bootmem_phy_list_print(void)
914 addr = CVMX_BOOTMEM_DESC_GET_FIELD(head_addr);
915 printf("\n\n\nPrinting bootmem block list, descriptor: 0x%llx, head is 0x%llx\n",
916 CAST_ULL(cvmx_bootmem_desc_addr), CAST_ULL(addr));
917 printf("Descriptor version: %d.%d\n",
918 (int)CVMX_BOOTMEM_DESC_GET_FIELD(major_version),
919 (int)CVMX_BOOTMEM_DESC_GET_FIELD(minor_version));
920 if (CVMX_BOOTMEM_DESC_GET_FIELD(major_version) > 3)
921 debug("Warning: Bootmem descriptor version is newer than expected\n");
924 printf("mem list is empty!\n");
927 printf("Block address: 0x%08llx, size: 0x%08llx, next: 0x%08llx\n", CAST_ULL(addr),
928 CAST_ULL(cvmx_bootmem_phy_get_size(addr)),
929 CAST_ULL(cvmx_bootmem_phy_get_next(addr)));
930 addr = cvmx_bootmem_phy_get_next(addr);
935 u64 cvmx_bootmem_phy_available_mem(u64 min_block_size)
939 u64 available_mem = 0;
941 __cvmx_bootmem_lock(0);
942 addr = CVMX_BOOTMEM_DESC_GET_FIELD(head_addr);
944 if (cvmx_bootmem_phy_get_size(addr) >= min_block_size)
945 available_mem += cvmx_bootmem_phy_get_size(addr);
946 addr = cvmx_bootmem_phy_get_next(addr);
948 __cvmx_bootmem_unlock(0);
949 return available_mem;
952 u64 cvmx_bootmem_phy_named_block_find(const char *name, u32 flags)
956 debug("%s: %s\n", __func__, name);
958 __cvmx_bootmem_lock(flags);
959 if (!__cvmx_bootmem_check_version(3)) {
961 u64 named_block_array_addr =
962 CVMX_BOOTMEM_DESC_GET_FIELD(named_block_array_addr);
964 CVMX_BOOTMEM_DESC_GET_FIELD(named_block_num_blocks);
966 CVMX_BOOTMEM_DESC_GET_FIELD(named_block_name_len);
967 u64 named_addr = named_block_array_addr;
969 for (i = 0; i < num_blocks; i++) {
971 CVMX_BOOTMEM_NAMED_GET_FIELD(named_addr, size);
972 if (name && named_size) {
973 char name_tmp[name_length + 1];
975 CVMX_BOOTMEM_NAMED_GET_NAME(named_addr,
978 if (!strncmp(name, name_tmp, name_length)) {
982 } else if (!name && !named_size) {
988 sizeof(struct cvmx_bootmem_named_block_desc);
991 __cvmx_bootmem_unlock(flags);
995 int cvmx_bootmem_phy_named_block_free(const char *name, u32 flags)
997 u64 named_block_addr;
999 if (__cvmx_bootmem_check_version(3))
1002 debug("%s: %s\n", __func__, name);
1005 * Take lock here, as name lookup/block free/name free need to be
1008 __cvmx_bootmem_lock(flags);
1010 named_block_addr = cvmx_bootmem_phy_named_block_find(name,
1011 CVMX_BOOTMEM_FLAG_NO_LOCKING);
1012 if (named_block_addr) {
1014 CVMX_BOOTMEM_NAMED_GET_FIELD(named_block_addr,
1017 CVMX_BOOTMEM_NAMED_GET_FIELD(named_block_addr, size);
1019 debug("%s: %s, base: 0x%llx, size: 0x%llx\n",
1020 __func__, name, CAST_ULL(named_addr),
1021 CAST_ULL(named_size));
1023 __cvmx_bootmem_phy_free(named_addr, named_size,
1024 CVMX_BOOTMEM_FLAG_NO_LOCKING);
1026 /* Set size to zero to indicate block not used. */
1027 CVMX_BOOTMEM_NAMED_SET_FIELD(named_block_addr, size, 0);
1030 __cvmx_bootmem_unlock(flags);
1031 return !!named_block_addr; /* 0 on failure, 1 on success */
1034 s64 cvmx_bootmem_phy_named_block_alloc(u64 size, u64 min_addr,
1036 u64 alignment, const char *name,
1040 u64 named_block_desc_addr;
1042 debug("%s: size: 0x%llx, min: 0x%llx, max: 0x%llx, align: 0x%llx, name: %s\n",
1043 __func__, CAST_ULL(size), CAST_ULL(min_addr), CAST_ULL(max_addr),
1044 CAST_ULL(alignment), name);
1046 if (__cvmx_bootmem_check_version(3))
1050 * Take lock here, as name lookup/block alloc/name add need to be
1053 __cvmx_bootmem_lock(flags);
1055 named_block_desc_addr =
1056 cvmx_bootmem_phy_named_block_find(name, flags |
1057 CVMX_BOOTMEM_FLAG_NO_LOCKING);
1058 if (named_block_desc_addr) {
1059 __cvmx_bootmem_unlock(flags);
1063 /* Get pointer to first available named block descriptor */
1064 named_block_desc_addr =
1065 cvmx_bootmem_phy_named_block_find(NULL, flags |
1066 CVMX_BOOTMEM_FLAG_NO_LOCKING);
1067 if (!named_block_desc_addr) {
1068 __cvmx_bootmem_unlock(flags);
1073 * Round size up to mult of minimum alignment bytes
1074 * We need the actual size allocated to allow for blocks to be
1075 * coallesced when they are freed. The alloc routine does the
1076 * same rounding up on all allocations.
1078 size = (size + (CVMX_BOOTMEM_ALIGNMENT_SIZE - 1)) &
1079 ~(CVMX_BOOTMEM_ALIGNMENT_SIZE - 1);
1081 addr_allocated = cvmx_bootmem_phy_alloc(size, min_addr, max_addr,
1083 flags | CVMX_BOOTMEM_FLAG_NO_LOCKING);
1084 if (addr_allocated >= 0) {
1085 CVMX_BOOTMEM_NAMED_SET_FIELD(named_block_desc_addr, base_addr,
1087 CVMX_BOOTMEM_NAMED_SET_FIELD(named_block_desc_addr, size, size);
1088 CVMX_BOOTMEM_NAMED_SET_NAME(named_block_desc_addr, name,
1089 CVMX_BOOTMEM_DESC_GET_FIELD(named_block_name_len));
1092 __cvmx_bootmem_unlock(flags);
1093 return addr_allocated;
1096 void cvmx_bootmem_phy_named_block_print(void)
1101 u64 named_block_array_addr =
1102 CVMX_BOOTMEM_DESC_GET_FIELD(named_block_array_addr);
1103 int num_blocks = CVMX_BOOTMEM_DESC_GET_FIELD(named_block_num_blocks);
1104 int name_length = CVMX_BOOTMEM_DESC_GET_FIELD(named_block_name_len);
1105 u64 named_block_addr = named_block_array_addr;
1107 debug("%s: desc addr: 0x%llx\n",
1108 __func__, CAST_ULL(cvmx_bootmem_desc_addr));
1110 if (__cvmx_bootmem_check_version(3))
1113 printf("List of currently allocated named bootmem blocks:\n");
1114 for (i = 0; i < num_blocks; i++) {
1116 CVMX_BOOTMEM_NAMED_GET_FIELD(named_block_addr, size);
1118 char name_tmp[name_length + 1];
1120 CVMX_BOOTMEM_NAMED_GET_FIELD(named_block_addr,
1122 CVMX_BOOTMEM_NAMED_GET_NAME(named_block_addr, name_tmp,
1125 printf("Name: %s, address: 0x%08llx, size: 0x%08llx, index: %d\n", name_tmp,
1126 CAST_ULL(named_addr),
1127 CAST_ULL(named_size), i);
1130 sizeof(struct cvmx_bootmem_named_block_desc);
1134 printf("No named bootmem blocks exist.\n");
1137 s64 cvmx_bootmem_phy_mem_list_init(u64 mem_size,
1138 u32 low_reserved_bytes,
1139 struct cvmx_bootmem_desc *desc_buffer)
1145 debug("%s (arg desc ptr: %p, cvmx_bootmem_desc: 0x%llx)\n",
1146 __func__, desc_buffer, CAST_ULL(cvmx_bootmem_desc_addr));
1149 * Descriptor buffer needs to be in 32 bit addressable space to be
1150 * compatible with 32 bit applications
1153 debug("ERROR: no memory for cvmx_bootmem descriptor provided\n");
1157 if (mem_size > OCTEON_MAX_PHY_MEM_SIZE) {
1158 mem_size = OCTEON_MAX_PHY_MEM_SIZE;
1159 debug("ERROR: requested memory size too large, truncating to maximum size\n");
1162 if (cvmx_bootmem_desc_addr)
1165 /* Initialize cvmx pointer to descriptor */
1166 cvmx_bootmem_init(cvmx_ptr_to_phys(desc_buffer));
1168 /* Fill the bootmem descriptor */
1169 CVMX_BOOTMEM_DESC_SET_FIELD(lock, 0);
1170 CVMX_BOOTMEM_DESC_SET_FIELD(flags, 0);
1171 CVMX_BOOTMEM_DESC_SET_FIELD(head_addr, 0);
1172 CVMX_BOOTMEM_DESC_SET_FIELD(major_version, CVMX_BOOTMEM_DESC_MAJ_VER);
1173 CVMX_BOOTMEM_DESC_SET_FIELD(minor_version, CVMX_BOOTMEM_DESC_MIN_VER);
1174 CVMX_BOOTMEM_DESC_SET_FIELD(app_data_addr, 0);
1175 CVMX_BOOTMEM_DESC_SET_FIELD(app_data_size, 0);
1178 * Set up global pointer to start of list, exclude low 64k for exception
1179 * vectors, space for global descriptor
1181 cur_block_addr = (OCTEON_DDR0_BASE + low_reserved_bytes);
1183 if (mem_size <= OCTEON_DDR0_SIZE) {
1184 __cvmx_bootmem_phy_free(cur_block_addr,
1185 mem_size - low_reserved_bytes, 0);
1189 __cvmx_bootmem_phy_free(cur_block_addr,
1190 OCTEON_DDR0_SIZE - low_reserved_bytes, 0);
1192 mem_size -= OCTEON_DDR0_SIZE;
1194 /* Add DDR2 block next if present */
1195 if (mem_size > OCTEON_DDR1_SIZE) {
1196 __cvmx_bootmem_phy_free(OCTEON_DDR1_BASE, OCTEON_DDR1_SIZE, 0);
1197 __cvmx_bootmem_phy_free(OCTEON_DDR2_BASE,
1198 mem_size - OCTEON_DDR1_SIZE, 0);
1200 __cvmx_bootmem_phy_free(OCTEON_DDR1_BASE, mem_size, 0);
1204 /* Initialize the named block structure */
1205 CVMX_BOOTMEM_DESC_SET_FIELD(named_block_name_len, CVMX_BOOTMEM_NAME_LEN);
1206 CVMX_BOOTMEM_DESC_SET_FIELD(named_block_num_blocks,
1207 CVMX_BOOTMEM_NUM_NAMED_BLOCKS);
1208 CVMX_BOOTMEM_DESC_SET_FIELD(named_block_array_addr, 0);
1210 /* Allocate this near the top of the low 256 MBytes of memory */
1211 addr = cvmx_bootmem_phy_alloc(CVMX_BOOTMEM_NUM_NAMED_BLOCKS *
1212 sizeof(struct cvmx_bootmem_named_block_desc),
1214 CVMX_BOOTMEM_FLAG_END_ALLOC);
1216 CVMX_BOOTMEM_DESC_SET_FIELD(named_block_array_addr, addr);
1218 debug("%s: named_block_array_addr: 0x%llx)\n",
1219 __func__, CAST_ULL(addr));
1222 debug("FATAL ERROR: unable to allocate memory for bootmem descriptor!\n");
1226 for (i = 0; i < CVMX_BOOTMEM_NUM_NAMED_BLOCKS; i++) {
1227 CVMX_BOOTMEM_NAMED_SET_FIELD(addr, base_addr, 0);
1228 CVMX_BOOTMEM_NAMED_SET_FIELD(addr, size, 0);
1229 addr += sizeof(struct cvmx_bootmem_named_block_desc);
1235 s64 cvmx_bootmem_phy_mem_list_init_multi(u8 node_mask,
1237 u32 low_reserved_bytes,
1238 struct cvmx_bootmem_desc *desc_buffer)
1245 u64 node_base; /* Make u64 to reduce type casting */
1247 mem_sizes[0] = gd->ram_size / (1024 * 1024);
1249 debug("cvmx_bootmem_phy_mem_list_init (arg desc ptr: %p, cvmx_bootmem_desc: 0x%llx)\n",
1250 desc_buffer, CAST_ULL(cvmx_bootmem_desc_addr));
1253 * Descriptor buffer needs to be in 32 bit addressable space to be
1254 * compatible with 32 bit applications
1257 debug("ERROR: no memory for cvmx_bootmem descriptor provided\n");
1261 cvmx_coremask_for_each_node(node, node_mask) {
1262 if ((mem_sizes[node] * 1024 * 1024) > OCTEON_MAX_PHY_MEM_SIZE) {
1263 mem_sizes[node] = OCTEON_MAX_PHY_MEM_SIZE /
1265 debug("ERROR node#%lld: requested memory size too large, truncating to maximum size\n",
1270 if (cvmx_bootmem_desc_addr)
1273 /* Initialize cvmx pointer to descriptor */
1274 cvmx_bootmem_init(cvmx_ptr_to_phys(desc_buffer));
1276 /* Fill the bootmem descriptor */
1277 CVMX_BOOTMEM_DESC_SET_FIELD(lock, 0);
1278 CVMX_BOOTMEM_DESC_SET_FIELD(flags, 0);
1279 CVMX_BOOTMEM_DESC_SET_FIELD(head_addr, 0);
1280 CVMX_BOOTMEM_DESC_SET_FIELD(major_version, CVMX_BOOTMEM_DESC_MAJ_VER);
1281 CVMX_BOOTMEM_DESC_SET_FIELD(minor_version, CVMX_BOOTMEM_DESC_MIN_VER);
1282 CVMX_BOOTMEM_DESC_SET_FIELD(app_data_addr, 0);
1283 CVMX_BOOTMEM_DESC_SET_FIELD(app_data_size, 0);
1285 cvmx_coremask_for_each_node(node, node_mask) {
1286 if (node != 0) /* do not reserve memory on remote nodes */
1287 low_reserved_bytes = 0;
1289 mem_size = (u64)mem_sizes[node] * (1024 * 1024); /* MBytes */
1292 * Set up global pointer to start of list, exclude low 64k
1293 * for exception vectors, space for global descriptor
1296 node_base = (u64)node << CVMX_NODE_MEM_SHIFT;
1297 cur_block_addr = (OCTEON_DDR0_BASE + low_reserved_bytes) |
1300 if (mem_size <= OCTEON_DDR0_SIZE) {
1301 __cvmx_bootmem_phy_free(cur_block_addr,
1302 mem_size - low_reserved_bytes,
1307 __cvmx_bootmem_phy_free(cur_block_addr,
1308 OCTEON_DDR0_SIZE - low_reserved_bytes,
1311 mem_size -= OCTEON_DDR0_SIZE;
1313 /* Add DDR2 block next if present */
1314 if (mem_size > OCTEON_DDR1_SIZE) {
1315 __cvmx_bootmem_phy_free(OCTEON_DDR1_BASE |
1317 OCTEON_DDR1_SIZE, 0);
1318 __cvmx_bootmem_phy_free(OCTEON_DDR2_BASE |
1320 mem_size - OCTEON_DDR1_SIZE, 0);
1322 __cvmx_bootmem_phy_free(OCTEON_DDR1_BASE |
1328 debug("%s: Initialize the named block\n", __func__);
1330 /* Initialize the named block structure */
1331 CVMX_BOOTMEM_DESC_SET_FIELD(named_block_name_len, CVMX_BOOTMEM_NAME_LEN);
1332 CVMX_BOOTMEM_DESC_SET_FIELD(named_block_num_blocks,
1333 CVMX_BOOTMEM_NUM_NAMED_BLOCKS);
1334 CVMX_BOOTMEM_DESC_SET_FIELD(named_block_array_addr, 0);
1336 /* Allocate this near the top of the low 256 MBytes of memory */
1337 addr = cvmx_bootmem_phy_alloc(CVMX_BOOTMEM_NUM_NAMED_BLOCKS *
1338 sizeof(struct cvmx_bootmem_named_block_desc),
1340 CVMX_BOOTMEM_FLAG_END_ALLOC);
1342 CVMX_BOOTMEM_DESC_SET_FIELD(named_block_array_addr, addr);
1344 debug("cvmx_bootmem_phy_mem_list_init: named_block_array_addr: 0x%llx)\n",
1348 debug("FATAL ERROR: unable to allocate memory for bootmem descriptor!\n");
1352 for (i = 0; i < CVMX_BOOTMEM_NUM_NAMED_BLOCKS; i++) {
1353 CVMX_BOOTMEM_NAMED_SET_FIELD(addr, base_addr, 0);
1354 CVMX_BOOTMEM_NAMED_SET_FIELD(addr, size, 0);
1355 addr += sizeof(struct cvmx_bootmem_named_block_desc);
1358 // test-only: DEBUG ifdef???
1359 cvmx_bootmem_phy_list_print();
1364 int cvmx_bootmem_reserve_memory(u64 start_addr, u64 size,
1365 const char *name, u32 flags)
1369 static unsigned int block_num;
1370 char block_name[CVMX_BOOTMEM_NAME_LEN];
1372 debug("%s: start %#llx, size: %#llx, name: %s, flags:%#x)\n",
1373 __func__, CAST_ULL(start_addr), CAST_ULL(size), name, flags);
1375 if (__cvmx_bootmem_check_version(3))
1378 addr = CVMX_BOOTMEM_DESC_GET_FIELD(head_addr);
1383 name = "__cvmx_bootmem_reserved";
1385 while (addr && rc) {
1386 u64 block_size = cvmx_bootmem_phy_get_size(addr);
1387 u64 reserve_size = 0;
1389 if (addr >= start_addr && addr < start_addr + size) {
1390 reserve_size = size - (addr - start_addr);
1391 if (block_size < reserve_size)
1392 reserve_size = block_size;
1393 } else if (start_addr > addr &&
1394 start_addr < (addr + block_size)) {
1395 reserve_size = block_size - (start_addr - addr);
1399 snprintf(block_name, sizeof(block_name),
1401 name, (unsigned long long)start_addr,
1402 (unsigned int)block_num);
1404 debug("%s: Reserving 0x%llx bytes at address 0x%llx with name %s\n",
1405 __func__, CAST_ULL(reserve_size),
1406 CAST_ULL(addr), block_name);
1408 if (cvmx_bootmem_phy_named_block_alloc(reserve_size,
1412 debug("%s: Failed to reserve 0x%llx bytes at address 0x%llx\n",
1413 __func__, CAST_ULL(reserve_size),
1414 (unsigned long long)addr);
1419 debug("%s: Reserved 0x%llx bytes at address 0x%llx with name %s\n",
1420 __func__, CAST_ULL(reserve_size),
1421 CAST_ULL(addr), block_name);
1424 addr = cvmx_bootmem_phy_get_next(addr);
1431 void cvmx_bootmem_lock(void)
1433 __cvmx_bootmem_lock(0);
1436 void cvmx_bootmem_unlock(void)
1438 __cvmx_bootmem_unlock(0);
1441 void *__cvmx_phys_addr_to_ptr(u64 phys, int size)
1445 if (sizeof(void *) == 8) {
1446 tmp = CASTPTR(void, CVMX_ADD_SEG(CVMX_MIPS_SPACE_XKPHYS, phys));
1448 u32 phy32 = (u32)(phys & 0x7fffffffULL);
1450 tmp = CASTPTR(void, CVMX_ADD_SEG32(CVMX_MIPS32_SPACE_KSEG0,
1457 void *__cvmx_bootmem_internal_get_desc_ptr(void)
1459 return cvmx_phys_to_ptr(cvmx_bootmem_desc_addr);