[prev in list] [next in list] [prev in thread] [next in thread]
List: sbcl-devel
Subject: Re: [Sbcl-devel] [Sbcl-commits] master: Move corefile PTE loading into coreparse
From: Stas Boukarev <stassats () gmail ! com>
Date: 2023-07-31 17:27:41
Message-ID: CAF63=12cJHw1FPzk3gndUVY_j23ROynb3YkR4e8ouA_s8eWSQA () mail ! gmail ! com
[Download RAW message or body]
[Attachment #2 (multipart/alternative)]
cc -g -Wall -Wundef -Wsign-compare -Wpointer-arith -O3
-fno-omit-frame-pointer -I. -isystem/usr/local/include -c -o coreparse.o
coreparse.c
coreparse.c:1182:36: error: subscripted value is not an array, pointer, or
vector
immobile_space_coreparse(spaces[IMMOBILE_FIXEDOBJ_CORE_SPACE_ID].len,
On Mon, Jul 31, 2023 at 8:04 PM snuglas via Sbcl-commits <
sbcl-commits@lists.sourceforge.net> wrote:
> The branch "master" has been updated in SBCL:
> via 2d4525a334c09a68fa1ba2202f280fdae685bff1 (commit)
> from 3490d6eb3124fb8079a42f904f86cc51f1f57c0d (commit)
>
> - Log -----------------------------------------------------------------
> commit 2d4525a334c09a68fa1ba2202f280fdae685bff1
> Author: Douglas Katzman <dougk@google.com>
> Date: Mon Jul 31 12:17:38 2023 -0400
>
> Move corefile PTE loading into coreparse
>
> Move hopscotch_create out of gc_allocate_ptes, and other changes to
> facilitate sharing this code with mark-region collector.
> ---
> src/runtime/coreparse.c | 374
> +++++++++++++++++++++++++++++++++++++++++++-----
> src/runtime/gencgc.c | 296 +-------------------------------------
> 2 files changed, 340 insertions(+), 330 deletions(-)
>
> diff --git a/src/runtime/coreparse.c b/src/runtime/coreparse.c
> index 9e4ecf83b..6143433d4 100644
> --- a/src/runtime/coreparse.c
> +++ b/src/runtime/coreparse.c
> @@ -40,6 +40,7 @@
> #include "validate.h"
> #include "gc-internal.h"
> #include "gc-private.h"
> +#include "gencgc-private.h"
> #include "code.h"
> #include "graphvisit.h"
>
> @@ -705,6 +706,14 @@ static os_vm_address_t reserve_space(int space_id,
> int attr,
> return addr;
> }
>
> +struct coreparse_space {
> + size_t desired_size; // size wanted, ORed with 1 if addr must be <2GB
> + // Values from the core file:
> + uword_t len; // length in bytes, as an integral multiple of
> os_vm_page_size
> + uword_t base;
> + lispobj** pfree_pointer; // pointer to x_free_pointer
> +} spaces;
> +
> /* TODO: If static + readonly were mapped as desired without disabling
> ASLR
> * but one of the large spaces couldn't be mapped as desired, start over
> from
> * the top, disabling ASLR. This should help to avoid relocating the heap
> @@ -716,35 +725,9 @@ static void
> process_directory(int count, struct ndir_entry *entry,
> int fd, os_vm_offset_t file_offset,
> int __attribute__((unused)) merge_core_pages,
> - struct heap_adjust __attribute__((unused)) *adj)
> + struct coreparse_space *spaces,
> + struct heap_adjust *adj)
> {
> - extern void immobile_space_coreparse(uword_t,uword_t);
> -
> - struct {
> - size_t desired_size; // size wanted, ORed with 1 if addr must be
> <2GB
> - // Values from the core file:
> - uword_t len; // length in bytes, as an integral multiple of
> os_vm_page_size
> - uword_t base;
> - lispobj** pfree_pointer; // pointer to x_free_pointer
> - } spaces[MAX_CORE_SPACE_ID+1] = {
> - {0, 0, 0, 0}, // blank for space ID 0
> - {dynamic_space_size, 0, DYNAMIC_SPACE_START, 0},
> - // This order is determined by constants in
> compiler/generic/genesis
> - {0, 0, STATIC_SPACE_START, &static_space_free_pointer},
> -
> - {0, 0, READ_ONLY_SPACE_START, &read_only_space_free_pointer},
> -
> -#ifdef LISP_FEATURE_DARWIN_JIT
> - {0, 0, STATIC_CODE_SPACE_START, &static_code_space_free_pointer},
> -#endif
> -
> -#ifdef LISP_FEATURE_IMMOBILE_SPACE
> - {FIXEDOBJ_SPACE_SIZE | 1, 0,
> - FIXEDOBJ_SPACE_START, &fixedobj_free_pointer},
> - {1, 0, TEXT_SPACE_START, &text_space_highwatermark}
> -#endif
> - };
> -
> #if ELFCORE
> if (&lisp_code_start) {
> TEXT_SPACE_START = (uword_t)&lisp_code_start;
> @@ -966,6 +949,228 @@ process_directory(int count, struct ndir_entry
> *entry,
> spaces[IMMOBILE_TEXT_CORE_SPACE_ID].base, //
> expected
> spaces[IMMOBILE_TEXT_CORE_SPACE_ID].len);
> #endif
> +}
> +
> +static void sanity_check_loaded_core(lispobj);
> +
> +/** routines for loading a core using the heap organization of gencgc
> + ** or other GC that is compatible with it **/
> +
> +bool gc_allocate_ptes()
> +{
> + /* Compute the number of pages needed for the dynamic space.
> + * Dynamic space size should be aligned on page size. */
> + page_table_pages = dynamic_space_size/GENCGC_PAGE_BYTES;
> + gc_assert(dynamic_space_size == npage_bytes(page_table_pages));
> +
> + /* Assert that a cons whose car has MOST-POSITIVE-WORD
> + * can not be considered a valid cons, which is to say, even though
> + * MOST-POSITIVE-WORD seems to satisfy is_lisp_pointer(),
> + * it's OK to use as a filler marker. */
> + if (find_page_index((void*)(uword_t)-1) >= 0)
> + lose("dynamic space too large");
> +
> + /* Default nursery size to 5% of the total dynamic space size,
> + * min 1Mb. */
> + bytes_consed_between_gcs = dynamic_space_size/(os_vm_size_t)20;
> + if (bytes_consed_between_gcs < (1024*1024))
> + bytes_consed_between_gcs = 1024*1024;
> +
> + /* The page_table is allocated using "calloc" to zero-initialize it.
> + * The C library typically implements this efficiently with mmap() if
> the
> + * size is large enough. To further avoid touching each page
> structure
> + * until first use, FREE_PAGE_FLAG must be 0, statically asserted
> here:
> + */
> +#if FREE_PAGE_FLAG != 0
> +#error "FREE_PAGE_FLAG is not 0"
> +#endif
> +
> + /* An extra 'struct page' exists at each end of the page table acting
> as
> + * a sentinel.
> + *
> + * For for leading sentinel:
> + * - all fields are zero except that 'gen' has an illegal value
> + * which makes from_space_p() and new_space_p() both return false
> + *
> + * For the trailing sentinel:
> + * - all fields are zero which makes page_ends_contiguous_block_p()
> + * return true for the last in-range page index (so the "illegal"
> + * index at 1+ appears to start a contiguous block even though
> + * it corresponds to no page)
> + */
> + page_table = calloc(page_table_pages+2, sizeof(struct page));
> + gc_assert(page_table);
> + page_table[0].gen = 9; // an arbitrary never-used value
> + ++page_table;
> + gc_page_pins = calloc(page_table_pages, 1);
> + gc_assert(gc_page_pins);
> +
> + // The card table size is a power of 2 at *least* as large
> + // as the number of cards. These are the default values.
> + int nbits = 13;
> + long num_gc_cards = 1L << nbits;
> +
> + // Sure there's a fancier way to round up to a power-of-2
> + // but this is executed exactly once, so KISS.
> + while (num_gc_cards < page_table_pages*CARDS_PER_PAGE) { ++nbits;
> num_gc_cards <<= 1; }
> + // 2 Gigacards should suffice for now. That would span 2TiB of memory
> + // using 1Kb card size, or more if larger card size.
> + gc_assert(nbits < 32);
> + // If the space size is less than or equal to the number of cards
> + // that 'gc_card_table_nbits' cover, we're fine. Otherwise, problem.
> + // 'nbits' is what we need, 'gc_card_table_nbits' is what the core
> was compiled for.
> + int patch_card_index_mask_fixups = 0;
> + if (nbits > gc_card_table_nbits) {
> + gc_card_table_nbits = nbits;
> + // The value needed based on dynamic space size exceeds the value
> that the
> + // core was compiled for, so we need to patch all code blobs.
> + patch_card_index_mask_fixups = 1;
> + }
> + // Regardless of the mask implied by space size, it has to be
> gc_card_table_nbits wide
> + // even if that is excessive - when the core is restarted using a
> _smaller_ dynamic space
> + // size than saved at - otherwise lisp could overrun the mark table.
> + num_gc_cards = 1L << gc_card_table_nbits;
> +
> + gc_card_table_mask = num_gc_cards - 1;
> + gc_card_mark = successful_malloc(num_gc_cards);
> + /* The mark array used to work "by accident" if the numeric value of
> CARD_MARKED
> + * is 0 - or equivalently the "WP'ed" state - which is the value that
> calloc()
> + * fills with. If using malloc() we have to fill with CARD_MARKED,
> + * as I discovered when I changed that to a nonzero value */
> + memset(gc_card_mark, CARD_MARKED, num_gc_cards);
> +
> + gc_common_init();
> +
> + /* Initialize the generations. */
> + int i;
> + for (i = 0; i < NUM_GENERATIONS; i++) {
> + struct generation* gen = &generations[i];
> + gen->bytes_allocated = 0;
> + gen->gc_trigger = 2000000;
> + gen->num_gc = 0;
> + gen->cum_sum_bytes_allocated = 0;
> + /* the tune-able parameters */
> + gen->bytes_consed_between_gc
> + =
> bytes_consed_between_gcs/(os_vm_size_t)HIGHEST_NORMAL_GENERATION;
> + gen->number_of_gcs_before_promotion = 1;
> + gen->minimum_age_before_gc = 0.75;
> + }
> +
> + /* Initialize gc_alloc. */
> + gc_init_region(mixed_region);
> + gc_init_region(small_mixed_region);
> + gc_init_region(boxed_region);
> + gc_init_region(unboxed_region);
> + gc_init_region(code_region);
> + gc_init_region(cons_region);
> + return patch_card_index_mask_fixups;
> +}
> +
> +extern void gcbarrier_patch_code(void*, int);
> +#if !(defined LISP_FEATURE_MIPS || defined LISP_FEATURE_PPC64 \
> + || defined LISP_FEATURE_X86 || defined LISP_FEATURE_X86_64)
> +#define gcbarrier_patch_code(dummy1,dummy2)
> +#endif
> +
> +static void gengcbarrier_patch_code_range(uword_t start, lispobj* limit)
> +{
> + struct varint_unpacker unpacker;
> + lispobj *where = (lispobj*)start;
> + for ( ; where < limit ; where += object_size(where) ) {
> + struct code* code = (void*)where;
> + if (widetag_of(where) != CODE_HEADER_WIDETAG || !code->fixups)
> continue;
> + varint_unpacker_init(&unpacker, code->fixups);
> + // There are two other data streams preceding the one we want
> + skip_data_stream(&unpacker);
> + skip_data_stream(&unpacker);
> + char* instructions = code_text_start(code);
> + int prev_loc = 0, loc;
> + while (varint_unpack(&unpacker, &loc) && loc != 0) {
> + loc += prev_loc;
> + prev_loc = loc;
> + gcbarrier_patch_code(instructions + loc, gc_card_table_nbits);
> + }
> + }
> +}
> +
> +#ifdef LISP_FEATURE_DARWIN_JIT
> +/* Inexplicably, an executable page can generate spurious faults if
> + * it's not written to after changing its protection flags.
> + * Touch every page... */
> +void darwin_jit_code_pages_kludge () {
> + THREAD_JIT(0);
> + page_index_t page;
> + for (page = 0; page < next_free_page; page++) {
> + if(is_code(page_table[page].type)) {
> + char* addr = page_address(page);
> + for (unsigned i = 0; i < GENCGC_PAGE_BYTES; i+=4096) {
> + volatile char* page_start = addr + i;
> + page_start[0] = page_start[0];
> + }
> + }
> + }
> + THREAD_JIT(1);
> +}
> +#endif
> +
> +/* Read corefile ptes from 'fd' which has already been positioned
> + * and store into the page table */
> +void gc_load_corefile_ptes(int card_table_nbits,
> + core_entry_elt_t n_ptes, core_entry_elt_t
> total_bytes,
> + os_vm_offset_t offset, int fd,
> + struct heap_adjust *adj)
> +{
> + gc_assert(ALIGN_UP(n_ptes * sizeof (struct corefile_pte),
> N_WORD_BYTES)
> + == (size_t)total_bytes);
> + if (next_free_page != n_ptes)
> + lose("n_PTEs=%"PAGE_INDEX_FMT" but expected %"PAGE_INDEX_FMT,
> + n_ptes, next_free_page);
> +
> + gc_card_table_nbits = card_table_nbits;
> + bool patchp = gc_allocate_ptes();
> +
> + if (LSEEK(fd, offset, SEEK_SET) != offset) lose("failed seek");
> +
> + char data[8192];
> + // Process an integral number of ptes on each read.
> + // Parentheses around sizeof (type) are necessary to suppress a
> + // clang warning (-Wsizeof-array-div) that we're dividing the array
> size
> + // by a divisor that is not the size of one element in that array.
> + page_index_t max_pages_per_read = sizeof data / (sizeof (struct
> corefile_pte));
> + page_index_t page = 0;
> + generation_index_t gen = CORE_PAGE_GENERATION;
> + while (page < n_ptes) {
> + page_index_t pages_remaining = n_ptes - page;
> + page_index_t npages =
> + pages_remaining < max_pages_per_read ? pages_remaining :
> max_pages_per_read;
> + ssize_t bytes = npages * sizeof (struct corefile_pte);
> + if (read(fd, data, bytes) != bytes) lose("failed read");
> + int i;
> + for ( i = 0 ; i < npages ; ++i, ++page ) {
> + struct corefile_pte pte;
> + memcpy(&pte, data+i*sizeof (struct corefile_pte), sizeof pte);
> + // Low 3 bits of the scan_start hold the 'type' flags.
> + // Low bit of words_used indicates a large (a/k/a single)
> object.
> + char type = ((pte.words_used & 1) ? SINGLE_OBJECT_FLAG : 0)
> + | (pte.sso & 0x07);
> + page_table[page].type = type;
> + pte.words_used &= ~1;
> + /* It is possible, though rare, for the saved page table
> + * to contain free pages below alloc_ptr. */
> + if (type != FREE_PAGE_FLAG) {
> + gc_assert(pte.words_used);
> + page_table[page].words_used_ = pte.words_used;
> + set_page_scan_start_offset(page, pte.sso & ~0x07);
> + page_table[page].gen = gen;
> + }
> + bytes_allocated += pte.words_used << WORD_SHIFT;
> + }
> + }
> + generations[gen].bytes_allocated = bytes_allocated;
> + gc_assert((ssize_t)bytes_allocated <= (ssize_t)(n_ptes *
> GENCGC_PAGE_BYTES));
> +
> + // Adjust for discrepancies between actually-allocated space addresses
> + // and desired addresses.
> if (adj->n_ranges) relocate_heap(adj);
>
> #ifdef LISP_FEATURE_IMMOBILE_SPACE
> @@ -973,24 +1178,83 @@ process_directory(int count, struct ndir_entry
> *entry,
> * (tbh it would be better to output the immobile-space page tables
> to the core file).
> * This used to depend critically on space relocation already having
> been performed.
> * It doesn't any more, but this is an OK time to do it */
> + extern void immobile_space_coreparse(uword_t,uword_t);
> immobile_space_coreparse(spaces[IMMOBILE_FIXEDOBJ_CORE_SPACE_ID].len,
> spaces[IMMOBILE_TEXT_CORE_SPACE_ID].len);
> calc_immobile_space_bounds();
> #endif
> #ifdef LISP_FEATURE_X86_64
> - tune_asm_routines_for_microarch(); // before WPing immobile space
> + tune_asm_routines_for_microarch();
> #endif
> #ifdef LISP_FEATURE_DARWIN_JIT
> if (!static_code_space_free_pointer)
> static_code_space_free_pointer = (lispobj
> *)STATIC_CODE_SPACE_START;
> #endif
> +
> + if (patchp) {
> + gengcbarrier_patch_code_range(READ_ONLY_SPACE_START,
> read_only_space_free_pointer);
> + gengcbarrier_patch_code_range(STATIC_SPACE_START,
> static_space_free_pointer);
> + gengcbarrier_patch_code_range(DYNAMIC_SPACE_START,
> (lispobj*)dynamic_space_highwatermark());
> +#ifdef LISP_FEATURE_IMMOBILE_SPACE
> + gengcbarrier_patch_code_range(TEXT_SPACE_START,
> text_space_highwatermark);
> +#endif
> + }
> +
> + // Apply physical page protection as needed.
> + // The non-soft-card-mark code is disgusting and I do not understand
> it.
> + if (gen != 0 && ENABLE_PAGE_PROTECTION) {
> +#ifdef LISP_FEATURE_SOFT_CARD_MARKS
> + page_index_t p;
> + for (p = 0; p < next_free_page; ++p)
> + if (page_words_used(p)) assign_page_card_marks(p,
> CARD_UNMARKED);
> +#else
> + // coreparse can avoid hundreds to thousands of mprotect() calls
> by
> + // treating the whole range from the corefile as protectable,
> except
> + // that soft-marked code pages must NOT be subject to mprotect.
> + // So just watch out for empty pages and code. Unboxed object
> pages
> + // will get unprotected on demand.
> +#define non_protectable_page_p(x) !page_words_used(x) ||
> is_code(page_table[x].type)
> + page_index_t start = 0, end;
> + // cf. write_protect_generation_pages()
> + while (start < next_free_page) {
> +#ifdef LISP_FEATURE_DARWIN_JIT
> + if(is_code(page_table[start].type)) {
> + SET_PAGE_PROTECTED(start,1);
> + for (end = start + 1; end < next_free_page; end++) {
> + if (!page_words_used(end) ||
> !is_code(page_table[end].type))
> + break;
> + SET_PAGE_PROTECTED(end,1);
> + }
> + os_protect(page_address(start), npage_bytes(end - start),
> OS_VM_PROT_ALL);
> + start = end+1;
> + continue;
> + }
> +#endif
> + if (non_protectable_page_p(start)) {
> + ++start;
> + continue;
> + }
> + SET_PAGE_PROTECTED(start,1);
> + for (end = start + 1; end < next_free_page; end++) {
> + if (non_protectable_page_p(end))
> + break;
> + SET_PAGE_PROTECTED(end,1);
> + }
> + os_protect(page_address(start), npage_bytes(end - start),
> OS_VM_PROT_JIT_READ);
> + start = end;
> + }
> +#endif
> + }
> +
> +#ifdef LISP_FEATURE_DARWIN_JIT
> + darwin_jit_code_pages_kludge();
> + /* For some reason doing an early pthread_jit_write_protect_np
> sometimes fails.
> + Which is weird, because it's done many times in
> arch_write_linkage_table_entry later.
> + Adding the executable bit here avoids calling
> pthread_jit_write_protect_np */
> + os_protect((os_vm_address_t)STATIC_CODE_SPACE_START,
> STATIC_CODE_SPACE_SIZE, OS_VM_PROT_ALL);
> +#endif
> }
>
> -extern void gc_load_corefile_ptes(int, core_entry_elt_t, core_entry_elt_t,
> - os_vm_offset_t offset, int fd);
> -
> -static void sanity_check_loaded_core(lispobj);
> -
> /* 'merge_core_pages': Tri-state flag to determine whether we attempt to
> mark
> * pages as targets for virtual memory deduplication via MADV_MERGEABLE.
> * 1: Yes
> @@ -1030,6 +1294,25 @@ load_core_file(char *file, os_vm_offset_t
> file_offset, int merge_core_pages)
> lose("invalid magic number in core: %"OBJ_FMTX" should have been
> %x",
> (lispobj)val, CORE_MAGIC);
>
> + struct coreparse_space spaces[MAX_CORE_SPACE_ID+1] = {
> + {0, 0, 0, 0}, // blank for space ID 0
> + {dynamic_space_size, 0, DYNAMIC_SPACE_START, 0},
> + // This order is determined by constants in
> compiler/generic/genesis
> + {0, 0, STATIC_SPACE_START, &static_space_free_pointer},
> +
> + {0, 0, READ_ONLY_SPACE_START, &read_only_space_free_pointer},
> +
> +#ifdef LISP_FEATURE_DARWIN_JIT
> + {0, 0, STATIC_CODE_SPACE_START, &static_code_space_free_pointer},
> +#endif
> +
> +#ifdef LISP_FEATURE_IMMOBILE_SPACE
> + {FIXEDOBJ_SPACE_SIZE | 1, 0,
> + FIXEDOBJ_SPACE_START, &fixedobj_free_pointer},
> + {1, 0, TEXT_SPACE_START, &text_space_highwatermark}
> +#endif
> + };
> +
> for ( ; ; ptr += remaining_len) {
> val = *ptr++;
> len = *ptr++;
> @@ -1046,12 +1329,13 @@ load_core_file(char *file, os_vm_offset_t
> file_offset, int merge_core_pages)
> case DIRECTORY_CORE_ENTRY_TYPE_CODE:
> process_directory(remaining_len / NDIR_ENTRY_LENGTH,
> (struct ndir_entry*)ptr, fd, file_offset,
> - merge_core_pages, &adj);
> + merge_core_pages, spaces, &adj);
> break;
> case PAGE_TABLE_CORE_ENTRY_TYPE_CODE:
> // elements = gencgc-card-table-index-nbits, n-ptes, nbytes,
> data-page
> gc_load_corefile_ptes(ptr[0], ptr[1], ptr[2],
> - file_offset + (ptr[3] + 1) *
> os_vm_page_size, fd);
> + file_offset + (ptr[3] + 1) *
> os_vm_page_size, fd,
> + &adj);
> break;
> case INITIAL_FUN_CORE_ENTRY_TYPE_CODE:
> initial_function = adjust_word(&adj, (lispobj)*ptr);
> @@ -1370,3 +1654,21 @@ static void sanity_check_loaded_core(lispobj
> initial_function)
> #else
> static void sanity_check_loaded_core(lispobj __attribute__((unused))
> initial_function) {}
> #endif
> +
> +/* Prepare the array of corefile_ptes for save */
> +void gc_store_corefile_ptes(struct corefile_pte *ptes)
> +{
> + page_index_t i;
> + for (i = 0; i < next_free_page; i++) {
> + /* Thanks to alignment requirements, the two three bits
> + * are always zero, so we can use them to store the
> + * allocation type -- region is always closed, so only
> + * the three low bits of allocation flags matter. */
> + uword_t word = page_scan_start_offset(i);
> + gc_assert((word & 0x07) == 0);
> + ptes[i].sso = word | (0x07 & page_table[i].type);
> + int used = page_table[i].words_used_;
> + gc_assert(!(used & 1));
> + ptes[i].words_used = used | page_single_obj_p(i);
> + }
> +}
> diff --git a/src/runtime/gencgc.c b/src/runtime/gencgc.c
> index 00a23aabb..d04a90ab1 100644
> --- a/src/runtime/gencgc.c
> +++ b/src/runtime/gencgc.c
> @@ -4278,6 +4278,8 @@ collect_garbage(generation_index_t last_gen)
> void
> gc_init(void)
> {
> + hopscotch_create(&pinned_objects, HOPSCOTCH_HASH_FUN_DEFAULT, 0 /*
> hashset */,
> + 32 /* logical bin count */, 0 /* default range */);
> #ifdef LISP_FEATURE_WIN32
> InitializeCriticalSection(&free_pages_lock);
> #endif
> @@ -4289,153 +4291,6 @@ gc_init(void)
>
> int gc_card_table_nbits;
> long gc_card_table_mask;
> -
> -static void __attribute__((unused)) gcbarrier_patch_code_range(uword_t
> start, void* limit)
> -{
> - extern void gcbarrier_patch_code(void*, int);
> - struct varint_unpacker unpacker;
> - struct code* code;
> - lispobj *where = (lispobj*)start;
> - while (where < (lispobj*)limit) {
> - if (widetag_of(where) == CODE_HEADER_WIDETAG && ((struct
> code*)where)->fixups) {
> - code = (struct code*)where;
> - varint_unpacker_init(&unpacker, code->fixups);
> - // There are two other data streams preceding the one we want
> - skip_data_stream(&unpacker);
> - skip_data_stream(&unpacker);
> - char* instructions = code_text_start(code);
> - int prev_loc = 0, loc;
> - while (varint_unpack(&unpacker, &loc) && loc != 0) {
> - loc += prev_loc;
> - prev_loc = loc;
> - void* patch_where = instructions + loc;
> - gcbarrier_patch_code(patch_where, gc_card_table_nbits);
> - }
> - }
> - where += object_size(where);
> - }
> -}
> -void gc_allocate_ptes()
> -{
> - page_index_t i;
> -
> - /* Compute the number of pages needed for the dynamic space.
> - * Dynamic space size should be aligned on page size. */
> - page_table_pages = dynamic_space_size/GENCGC_PAGE_BYTES;
> - gc_assert(dynamic_space_size == npage_bytes(page_table_pages));
> -
> - /* Assert that a cons whose car has MOST-POSITIVE-WORD
> - * can not be considered a valid cons, which is to say, even though
> - * MOST-POSITIVE-WORD seems to satisfy is_lisp_pointer(),
> - * it's OK to use as a filler marker. */
> - if (find_page_index((void*)(uword_t)-1) >= 0)
> - lose("dynamic space too large");
> -
> - /* Default nursery size to 5% of the total dynamic space size,
> - * min 1Mb. */
> - bytes_consed_between_gcs = dynamic_space_size/(os_vm_size_t)20;
> - if (bytes_consed_between_gcs < (1024*1024))
> - bytes_consed_between_gcs = 1024*1024;
> -
> - /* The page_table is allocated using "calloc" to zero-initialize it.
> - * The C library typically implements this efficiently with mmap() if
> the
> - * size is large enough. To further avoid touching each page
> structure
> - * until first use, FREE_PAGE_FLAG must be 0, statically asserted
> here:
> - */
> -#if FREE_PAGE_FLAG != 0
> -#error "FREE_PAGE_FLAG is not 0"
> -#endif
> -
> - /* An extra 'struct page' exists at each end of the page table acting
> as
> - * a sentinel.
> - *
> - * For for leading sentinel:
> - * - all fields are zero except that 'gen' has an illegal value
> - * which makes from_space_p() and new_space_p() both return false
> - *
> - * For the trailing sentinel:
> - * - all fields are zero which makes page_ends_contiguous_block_p()
> - * return true for the last in-range page index (so the "illegal"
> - * index at 1+ appears to start a contiguous block even though
> - * it corresponds to no page)
> - */
> - page_table = calloc(page_table_pages+2, sizeof(struct page));
> - gc_assert(page_table);
> - page_table[0].gen = 9; // an arbitrary never-used value
> - ++page_table;
> - gc_page_pins = calloc(page_table_pages, 1);
> - gc_assert(gc_page_pins);
> -
> - // The card table size is a power of 2 at *least* as large
> - // as the number of cards. These are the default values.
> - int nbits = 13;
> - long num_gc_cards = 1L << nbits;
> -
> - // Sure there's a fancier way to round up to a power-of-2
> - // but this is executed exactly once, so KISS.
> - while (num_gc_cards < page_table_pages*CARDS_PER_PAGE) { ++nbits;
> num_gc_cards <<= 1; }
> - // 2 Gigacards should suffice for now. That would span 2TiB of memory
> - // using 1Kb card size, or more if larger card size.
> - gc_assert(nbits < 32);
> - // If the space size is less than or equal to the number of cards
> - // that 'gc_card_table_nbits' cover, we're fine. Otherwise, problem.
> - // 'nbits' is what we need, 'gc_card_table_nbits' is what the core
> was compiled for.
> - if (nbits > gc_card_table_nbits) {
> - gc_card_table_nbits = nbits;
> -#if defined LISP_FEATURE_MIPS || defined LISP_FEATURE_PPC64 \
> - || defined LISP_FEATURE_X86 || defined LISP_FEATURE_X86_64
> - // The value needed based on dynamic space size exceeds the value
> that the
> - // core was compiled for, so we need to patch all code blobs.
> - gcbarrier_patch_code_range(READ_ONLY_SPACE_START,
> read_only_space_free_pointer);
> - gcbarrier_patch_code_range(STATIC_SPACE_START,
> static_space_free_pointer);
> - gcbarrier_patch_code_range(DYNAMIC_SPACE_START,
> (lispobj*)dynamic_space_highwatermark());
> -#ifdef LISP_FEATURE_IMMOBILE_SPACE
> - gcbarrier_patch_code_range(TEXT_SPACE_START,
> text_space_highwatermark);
> -#endif
> -#endif
> - }
> - // Regardless of the mask implied by space size, it has to be
> gc_card_table_nbits wide
> - // even if that is excessive - when the core is restarted using a
> _smaller_ dynamic space
> - // size than saved at - otherwise lisp could overrun the mark table.
> - num_gc_cards = 1L << gc_card_table_nbits;
> -
> - gc_card_table_mask = num_gc_cards - 1;
> - gc_card_mark = successful_malloc(num_gc_cards);
> - /* The mark array used to work "by accident" if the numeric value of
> CARD_MARKED
> - * is 0 - or equivalently the "WP'ed" state - which is the value that
> calloc()
> - * fills with. If using malloc() we have to fill with CARD_MARKED,
> - * as I discovered when I changed that to a nonzero value */
> - memset(gc_card_mark, CARD_MARKED, num_gc_cards);
> -
> - gc_common_init();
> - hopscotch_create(&pinned_objects, HOPSCOTCH_HASH_FUN_DEFAULT, 0 /*
> hashset */,
> - 32 /* logical bin count */, 0 /* default range */);
> -
> - bytes_allocated = 0;
> -
> - /* Initialize the generations. */
> - for (i = 0; i < NUM_GENERATIONS; i++) {
> - struct generation* gen = &generations[i];
> - gen->bytes_allocated = 0;
> - gen->gc_trigger = 2000000;
> - gen->num_gc = 0;
> - gen->cum_sum_bytes_allocated = 0;
> - /* the tune-able parameters */
> - gen->bytes_consed_between_gc
> - =
> bytes_consed_between_gcs/(os_vm_size_t)HIGHEST_NORMAL_GENERATION;
> - gen->number_of_gcs_before_promotion = 1;
> - gen->minimum_age_before_gc = 0.75;
> - }
> -
> - /* Initialize gc_alloc. */
> - gc_alloc_generation = 0;
> - gc_init_region(mixed_region);
> - gc_init_region(small_mixed_region);
> - gc_init_region(boxed_region);
> - gc_init_region(unboxed_region);
> - gc_init_region(code_region);
> - gc_init_region(cons_region);
> -}
>
>
> /* alloc() and alloc_list() are external interfaces for memory allocation.
> @@ -4924,153 +4779,6 @@ gc_and_save(char *filename, bool prepend_runtime,
> bool purify,
> lose("Attempt to save core after non-conservative GC failed.");
> }
>
> -#ifdef LISP_FEATURE_DARWIN_JIT
> -/* Inexplicably, an executable page can generate spurious faults if
> - * it's not written to after changing its protection flags.
> - * Touch every page... */
> -void darwin_jit_code_pages_kludge () {
> - THREAD_JIT(0);
> - page_index_t page;
> - for (page = 0; page < next_free_page; page++) {
> - if(is_code(page_table[page].type)) {
> - char* addr = page_address(page);
> - for (unsigned i = 0; i < GENCGC_PAGE_BYTES; i+=4096) {
> - volatile char* page_start = addr + i;
> - page_start[0] = page_start[0];
> - }
> - }
> - }
> - THREAD_JIT(1);
> -}
> -#endif
> -
> -/* Read corefile ptes from 'fd' which has already been positioned
> - * and store into the page table */
> -void gc_load_corefile_ptes(int card_table_nbits,
> - core_entry_elt_t n_ptes, core_entry_elt_t
> total_bytes,
> - os_vm_offset_t offset, int fd)
> -{
> - gc_assert(ALIGN_UP(n_ptes * sizeof (struct corefile_pte),
> N_WORD_BYTES)
> - == (size_t)total_bytes);
> - if (next_free_page != n_ptes)
> - lose("n_PTEs=%"PAGE_INDEX_FMT" but expected %"PAGE_INDEX_FMT,
> - n_ptes, next_free_page);
> -
> - // Allocation of PTEs is delayed 'til now so that calloc() doesn't
> - // consume addresses that would have been taken by a mapped space.
> - gc_card_table_nbits = card_table_nbits;
> - gc_allocate_ptes();
> -
> - if (LSEEK(fd, offset, SEEK_SET) != offset) lose("failed seek");
> -
> - char data[8192];
> - // Process an integral number of ptes on each read.
> - // Parentheses around sizeof (type) are necessary to suppress a
> - // clang warning (-Wsizeof-array-div) that we're dividing the array
> size
> - // by a divisor that is not the size of one element in that array.
> - page_index_t max_pages_per_read = sizeof data / (sizeof (struct
> corefile_pte));
> - page_index_t page = 0;
> - generation_index_t gen = CORE_PAGE_GENERATION;
> - while (page < n_ptes) {
> - page_index_t pages_remaining = n_ptes - page;
> - page_index_t npages =
> - pages_remaining < max_pages_per_read ? pages_remaining :
> max_pages_per_read;
> - ssize_t bytes = npages * sizeof (struct corefile_pte);
> - if (read(fd, data, bytes) != bytes) lose("failed read");
> - int i;
> - for ( i = 0 ; i < npages ; ++i, ++page ) {
> - struct corefile_pte pte;
> - memcpy(&pte, data+i*sizeof (struct corefile_pte), sizeof pte);
> - // Low 3 bits of the scan_start hold the 'type' flags.
> - // Low bit of words_used indicates a large (a/k/a single)
> object.
> - char type = ((pte.words_used & 1) ? SINGLE_OBJECT_FLAG : 0)
> - | (pte.sso & 0x07);
> - page_table[page].type = type;
> - pte.words_used &= ~1;
> - /* It is possible, though rare, for the saved page table
> - * to contain free pages below alloc_ptr. */
> - if (type != FREE_PAGE_FLAG) {
> - gc_assert(pte.words_used);
> - page_table[page].words_used_ = pte.words_used;
> - set_page_scan_start_offset(page, pte.sso & ~0x07);
> - page_table[page].gen = gen;
> - }
> - bytes_allocated += pte.words_used << WORD_SHIFT;
> - }
> - }
> - generations[gen].bytes_allocated = bytes_allocated;
> - gc_assert((ssize_t)bytes_allocated <= (ssize_t)(n_ptes *
> GENCGC_PAGE_BYTES));
> - if (gen != 0 && ENABLE_PAGE_PROTECTION) {
> -#ifdef LISP_FEATURE_SOFT_CARD_MARKS
> - page_index_t p;
> - for (p = 0; p < next_free_page; ++p)
> - if (page_words_used(p)) assign_page_card_marks(p,
> CARD_UNMARKED);
> -#else
> - // coreparse can avoid hundreds to thousands of mprotect() calls
> by
> - // treating the whole range from the corefile as protectable,
> except
> - // that soft-marked code pages must NOT be subject to mprotect.
> - // So just watch out for empty pages and code. Unboxed object
> pages
> - // will get unprotected on demand.
> -#define non_protectable_page_p(x) !page_words_used(x) ||
> is_code(page_table[x].type)
> - page_index_t start = 0, end;
> - // cf. write_protect_generation_pages()
> - while (start < next_free_page) {
> -#ifdef LISP_FEATURE_DARWIN_JIT
> - if(is_code(page_table[start].type)) {
> - SET_PAGE_PROTECTED(start,1);
> - for (end = start + 1; end < next_free_page; end++) {
> - if (!page_words_used(end) ||
> !is_code(page_table[end].type))
> - break;
> - SET_PAGE_PROTECTED(end,1);
> - }
> - os_protect(page_address(start), npage_bytes(end - start),
> OS_VM_PROT_ALL);
> - start = end+1;
> - continue;
> - }
> -#endif
> - if (non_protectable_page_p(start)) {
> - ++start;
> - continue;
> - }
> - SET_PAGE_PROTECTED(start,1);
> - for (end = start + 1; end < next_free_page; end++) {
> - if (non_protectable_page_p(end))
> - break;
> - SET_PAGE_PROTECTED(end,1);
> - }
> - os_protect(page_address(start), npage_bytes(end - start),
> OS_VM_PROT_JIT_READ);
> - start = end;
> - }
> -#endif
> - }
> -
> -#ifdef LISP_FEATURE_DARWIN_JIT
> - darwin_jit_code_pages_kludge();
> - /* For some reason doing an early pthread_jit_write_protect_np
> sometimes fails.
> - Which is weird, because it's done many times in
> arch_write_linkage_table_entry later.
> - Adding the executable bit here avoids calling
> pthread_jit_write_protect_np */
> - os_protect((os_vm_address_t)STATIC_CODE_SPACE_START,
> STATIC_CODE_SPACE_SIZE, OS_VM_PROT_ALL);
> -#endif
> -}
> -
> -/* Prepare the array of corefile_ptes for save */
> -void gc_store_corefile_ptes(struct corefile_pte *ptes)
> -{
> - page_index_t i;
> - for (i = 0; i < next_free_page; i++) {
> - /* Thanks to alignment requirements, the two three bits
> - * are always zero, so we can use them to store the
> - * allocation type -- region is always closed, so only
> - * the three low bits of allocation flags matter. */
> - uword_t word = page_scan_start_offset(i);
> - gc_assert((word & 0x07) == 0);
> - ptes[i].sso = word | (0x07 & page_table[i].type);
> - int used = page_table[i].words_used_;
> - gc_assert(!(used & 1));
> - ptes[i].words_used = used | page_single_obj_p(i);
> - }
> -}
> -
> #define ARTIFICIALLY_HIGH_GEN 8
> generation_index_t gc_gen_of(lispobj obj, int defaultval) {
> int page = find_page_index((void*)obj);
>
> -----------------------------------------------------------------------
>
>
> hooks/post-receive
> --
> SBCL
>
>
> _______________________________________________
> Sbcl-commits mailing list
> Sbcl-commits@lists.sourceforge.net
> https://lists.sourceforge.net/lists/listinfo/sbcl-commits
>
[Attachment #5 (text/html)]
<div dir="ltr"><div><div>cc -g -Wall -Wundef -Wsign-compare -Wpointer-arith -O3 \
-fno-omit-frame-pointer -I. -isystem/usr/local/include -c -o coreparse.o \
coreparse.c<br>coreparse.c:1182:36: error: subscripted value is not an array, \
pointer, or vector<br> \
immobile_space_coreparse(spaces[IMMOBILE_FIXEDOBJ_CORE_SPACE_ID].len,<br></div></div></div><br><div \
class="gmail_quote"><div dir="ltr" class="gmail_attr">On Mon, Jul 31, 2023 at \
8:04 PM snuglas via Sbcl-commits <<a \
href="mailto:sbcl-commits@lists.sourceforge.net">sbcl-commits@lists.sourceforge.net</a>> \
wrote:<br></div><blockquote class="gmail_quote" style="margin:0px 0px 0px \
0.8ex;border-left:1px solid rgb(204,204,204);padding-left:1ex">The branch \
"master" has been updated in SBCL:<br> via \
2d4525a334c09a68fa1ba2202f280fdae685bff1 (commit)<br> from \
3490d6eb3124fb8079a42f904f86cc51f1f57c0d (commit)<br> <br>
- Log -----------------------------------------------------------------<br>
commit 2d4525a334c09a68fa1ba2202f280fdae685bff1<br>
Author: Douglas Katzman <<a href="mailto:dougk@google.com" \
target="_blank">dougk@google.com</a>><br>
Date: Mon Jul 31 12:17:38 2023 -0400<br>
<br>
Move corefile PTE loading into coreparse<br>
<br>
Move hopscotch_create out of gc_allocate_ptes, and other changes to<br>
facilitate sharing this code with mark-region collector.<br>
---<br>
src/runtime/coreparse.c | 374 +++++++++++++++++++++++++++++++++++++++++++-----<br>
src/runtime/gencgc.c | 296 +-------------------------------------<br>
2 files changed, 340 insertions(+), 330 deletions(-)<br>
<br>
diff --git a/src/runtime/coreparse.c b/src/runtime/coreparse.c<br>
index 9e4ecf83b..6143433d4 100644<br>
--- a/src/runtime/coreparse.c<br>
+++ b/src/runtime/coreparse.c<br>
@@ -40,6 +40,7 @@<br>
#include "validate.h"<br>
#include "gc-internal.h"<br>
#include "gc-private.h"<br>
+#include "gencgc-private.h"<br>
#include "code.h"<br>
#include "graphvisit.h"<br>
<br>
@@ -705,6 +706,14 @@ static os_vm_address_t reserve_space(int space_id, int attr,<br>
return addr;<br>
}<br>
<br>
+struct coreparse_space {<br>
+ size_t desired_size; // size wanted, ORed with 1 if addr must be <2GB<br>
+ // Values from the core file:<br>
+ uword_t len; // length in bytes, as an integral multiple of \
os_vm_page_size<br> + uword_t base;<br>
+ lispobj** pfree_pointer; // pointer to x_free_pointer<br>
+} spaces;<br>
+<br>
/* TODO: If static + readonly were mapped as desired without disabling ASLR<br>
* but one of the large spaces couldn't be mapped as desired, start over \
from<br>
* the top, disabling ASLR. This should help to avoid relocating the heap<br>
@@ -716,35 +725,9 @@ static void<br>
process_directory(int count, struct ndir_entry *entry,<br>
int fd, os_vm_offset_t file_offset,<br>
int __attribute__((unused)) merge_core_pages,<br>
- struct heap_adjust __attribute__((unused)) *adj)<br>
+ struct coreparse_space *spaces,<br>
+ struct heap_adjust *adj)<br>
{<br>
- extern void immobile_space_coreparse(uword_t,uword_t);<br>
-<br>
- struct {<br>
- size_t desired_size; // size wanted, ORed with 1 if addr must be \
<2GB<br>
- // Values from the core file:<br>
- uword_t len; // length in bytes, as an integral multiple of \
os_vm_page_size<br>
- uword_t base;<br>
- lispobj** pfree_pointer; // pointer to x_free_pointer<br>
- } spaces[MAX_CORE_SPACE_ID+1] = {<br>
- {0, 0, 0, 0}, // blank for space ID 0<br>
- {dynamic_space_size, 0, DYNAMIC_SPACE_START, 0},<br>
- // This order is determined by constants in compiler/generic/genesis<br>
- {0, 0, STATIC_SPACE_START, &static_space_free_pointer},<br>
-<br>
- {0, 0, READ_ONLY_SPACE_START, &read_only_space_free_pointer},<br>
-<br>
-#ifdef LISP_FEATURE_DARWIN_JIT<br>
- {0, 0, STATIC_CODE_SPACE_START, \
&static_code_space_free_pointer},<br>
-#endif<br>
-<br>
-#ifdef LISP_FEATURE_IMMOBILE_SPACE<br>
- {FIXEDOBJ_SPACE_SIZE | 1, 0,<br>
- FIXEDOBJ_SPACE_START, &fixedobj_free_pointer},<br>
- {1, 0, TEXT_SPACE_START, &text_space_highwatermark}<br>
-#endif<br>
- };<br>
-<br>
#if ELFCORE<br>
if (&lisp_code_start) {<br>
TEXT_SPACE_START = (uword_t)&lisp_code_start;<br>
@@ -966,6 +949,228 @@ process_directory(int count, struct ndir_entry *entry,<br>
spaces[IMMOBILE_TEXT_CORE_SPACE_ID].base, // \
expected<br>
spaces[IMMOBILE_TEXT_CORE_SPACE_ID].len);<br>
#endif<br>
+}<br>
+<br>
+static void sanity_check_loaded_core(lispobj);<br>
+<br>
+/** routines for loading a core using the heap organization of gencgc<br>
+ ** or other GC that is compatible with it **/<br>
+<br>
+bool gc_allocate_ptes()<br>
+{<br>
+ /* Compute the number of pages needed for the dynamic space.<br>
+ * Dynamic space size should be aligned on page size. */<br>
+ page_table_pages = dynamic_space_size/GENCGC_PAGE_BYTES;<br>
+ gc_assert(dynamic_space_size == npage_bytes(page_table_pages));<br>
+<br>
+ /* Assert that a cons whose car has MOST-POSITIVE-WORD<br>
+ * can not be considered a valid cons, which is to say, even though<br>
+ * MOST-POSITIVE-WORD seems to satisfy is_lisp_pointer(),<br>
+ * it's OK to use as a filler marker. */<br>
+ if (find_page_index((void*)(uword_t)-1) >= 0)<br>
+ lose("dynamic space too large");<br>
+<br>
+ /* Default nursery size to 5% of the total dynamic space size,<br>
+ * min 1Mb. */<br>
+ bytes_consed_between_gcs = dynamic_space_size/(os_vm_size_t)20;<br>
+ if (bytes_consed_between_gcs < (1024*1024))<br>
+ bytes_consed_between_gcs = 1024*1024;<br>
+<br>
+ /* The page_table is allocated using "calloc" to zero-initialize \
it.<br> + * The C library typically implements this efficiently with mmap() if \
the<br> + * size is large enough. To further avoid touching each page \
structure<br> + * until first use, FREE_PAGE_FLAG must be 0, statically \
asserted here:<br> + */<br>
+#if FREE_PAGE_FLAG != 0<br>
+#error "FREE_PAGE_FLAG is not 0"<br>
+#endif<br>
+<br>
+ /* An extra 'struct page' exists at each end of the page table acting \
as<br> + * a sentinel.<br>
+ *<br>
+ * For for leading sentinel:<br>
+ * - all fields are zero except that 'gen' has an illegal value<br>
+ * which makes from_space_p() and new_space_p() both return false<br>
+ *<br>
+ * For the trailing sentinel:<br>
+ * - all fields are zero which makes page_ends_contiguous_block_p()<br>
+ * return true for the last in-range page index (so the \
"illegal"<br> + * index at 1+ appears to start a contiguous \
block even though<br> + * it corresponds to no page)<br>
+ */<br>
+ page_table = calloc(page_table_pages+2, sizeof(struct page));<br>
+ gc_assert(page_table);<br>
+ page_table[0].gen = 9; // an arbitrary never-used value<br>
+ ++page_table;<br>
+ gc_page_pins = calloc(page_table_pages, 1);<br>
+ gc_assert(gc_page_pins);<br>
+<br>
+ // The card table size is a power of 2 at *least* as large<br>
+ // as the number of cards. These are the default values.<br>
+ int nbits = 13;<br>
+ long num_gc_cards = 1L << nbits;<br>
+<br>
+ // Sure there's a fancier way to round up to a power-of-2<br>
+ // but this is executed exactly once, so KISS.<br>
+ while (num_gc_cards < page_table_pages*CARDS_PER_PAGE) { ++nbits; \
num_gc_cards <<= 1; }<br> + // 2 Gigacards should suffice for now. That \
would span 2TiB of memory<br> + // using 1Kb card size, or more if larger card \
size.<br> + gc_assert(nbits < 32);<br>
+ // If the space size is less than or equal to the number of cards<br>
+ // that 'gc_card_table_nbits' cover, we're fine. Otherwise, \
problem.<br> + // 'nbits' is what we need, 'gc_card_table_nbits' \
is what the core was compiled for.<br> + int patch_card_index_mask_fixups = \
0;<br> + if (nbits > gc_card_table_nbits) {<br>
+ gc_card_table_nbits = nbits;<br>
+ // The value needed based on dynamic space size exceeds the value that \
the<br> + // core was compiled for, so we need to patch all code \
blobs.<br> + patch_card_index_mask_fixups = 1;<br>
+ }<br>
+ // Regardless of the mask implied by space size, it has to be \
gc_card_table_nbits wide<br> + // even if that is excessive - when the core is \
restarted using a _smaller_ dynamic space<br> + // size than saved at - \
otherwise lisp could overrun the mark table.<br> + num_gc_cards = 1L << \
gc_card_table_nbits;<br> +<br>
+ gc_card_table_mask = num_gc_cards - 1;<br>
+ gc_card_mark = successful_malloc(num_gc_cards);<br>
+ /* The mark array used to work "by accident" if the numeric value of \
CARD_MARKED<br> + * is 0 - or equivalently the "WP'ed" state - \
which is the value that calloc()<br> + * fills with. If using malloc() we have \
to fill with CARD_MARKED,<br> + * as I discovered when I changed that to a \
nonzero value */<br> + memset(gc_card_mark, CARD_MARKED, num_gc_cards);<br>
+<br>
+ gc_common_init();<br>
+<br>
+ /* Initialize the generations. */<br>
+ int i;<br>
+ for (i = 0; i < NUM_GENERATIONS; i++) {<br>
+ struct generation* gen = &generations[i];<br>
+ gen->bytes_allocated = 0;<br>
+ gen->gc_trigger = 2000000;<br>
+ gen->num_gc = 0;<br>
+ gen->cum_sum_bytes_allocated = 0;<br>
+ /* the tune-able parameters */<br>
+ gen->bytes_consed_between_gc<br>
+ = \
bytes_consed_between_gcs/(os_vm_size_t)HIGHEST_NORMAL_GENERATION;<br> + \
gen->number_of_gcs_before_promotion = 1;<br> + \
gen->minimum_age_before_gc = 0.75;<br> + }<br>
+<br>
+ /* Initialize gc_alloc. */<br>
+ gc_init_region(mixed_region);<br>
+ gc_init_region(small_mixed_region);<br>
+ gc_init_region(boxed_region);<br>
+ gc_init_region(unboxed_region);<br>
+ gc_init_region(code_region);<br>
+ gc_init_region(cons_region);<br>
+ return patch_card_index_mask_fixups;<br>
+}<br>
+<br>
+extern void gcbarrier_patch_code(void*, int);<br>
+#if !(defined LISP_FEATURE_MIPS || defined LISP_FEATURE_PPC64 \<br>
+ || defined LISP_FEATURE_X86 || defined LISP_FEATURE_X86_64)<br>
+#define gcbarrier_patch_code(dummy1,dummy2)<br>
+#endif<br>
+<br>
+static void gengcbarrier_patch_code_range(uword_t start, lispobj* limit)<br>
+{<br>
+ struct varint_unpacker unpacker;<br>
+ lispobj *where = (lispobj*)start;<br>
+ for ( ; where < limit ; where += object_size(where) ) {<br>
+ struct code* code = (void*)where;<br>
+ if (widetag_of(where) != CODE_HEADER_WIDETAG || !code->fixups) \
continue;<br> + varint_unpacker_init(&unpacker, code->fixups);<br>
+ // There are two other data streams preceding the one we want<br>
+ skip_data_stream(&unpacker);<br>
+ skip_data_stream(&unpacker);<br>
+ char* instructions = code_text_start(code);<br>
+ int prev_loc = 0, loc;<br>
+ while (varint_unpack(&unpacker, &loc) && loc != 0) {<br>
+ loc += prev_loc;<br>
+ prev_loc = loc;<br>
+ gcbarrier_patch_code(instructions + loc, gc_card_table_nbits);<br>
+ }<br>
+ }<br>
+}<br>
+<br>
+#ifdef LISP_FEATURE_DARWIN_JIT<br>
+/* Inexplicably, an executable page can generate spurious faults if<br>
+ * it's not written to after changing its protection flags.<br>
+ * Touch every page... */<br>
+void darwin_jit_code_pages_kludge () {<br>
+ THREAD_JIT(0);<br>
+ page_index_t page;<br>
+ for (page = 0; page < next_free_page; page++) {<br>
+ if(is_code(page_table[page].type)) {<br>
+ char* addr = page_address(page);<br>
+ for (unsigned i = 0; i < GENCGC_PAGE_BYTES; i+=4096) {<br>
+ volatile char* page_start = addr + i;<br>
+ page_start[0] = page_start[0];<br>
+ }<br>
+ }<br>
+ }<br>
+ THREAD_JIT(1);<br>
+}<br>
+#endif<br>
+<br>
+/* Read corefile ptes from 'fd' which has already been positioned<br>
+ * and store into the page table */<br>
+void gc_load_corefile_ptes(int card_table_nbits,<br>
+ core_entry_elt_t n_ptes, core_entry_elt_t \
total_bytes,<br> + os_vm_offset_t offset, int \
fd,<br> + struct heap_adjust *adj)<br>
+{<br>
+ gc_assert(ALIGN_UP(n_ptes * sizeof (struct corefile_pte), N_WORD_BYTES)<br>
+ == (size_t)total_bytes);<br>
+ if (next_free_page != n_ptes)<br>
+ lose("n_PTEs=%"PAGE_INDEX_FMT" but expected \
%"PAGE_INDEX_FMT,<br> + n_ptes, next_free_page);<br>
+<br>
+ gc_card_table_nbits = card_table_nbits;<br>
+ bool patchp = gc_allocate_ptes();<br>
+<br>
+ if (LSEEK(fd, offset, SEEK_SET) != offset) lose("failed seek");<br>
+<br>
+ char data[8192];<br>
+ // Process an integral number of ptes on each read.<br>
+ // Parentheses around sizeof (type) are necessary to suppress a<br>
+ // clang warning (-Wsizeof-array-div) that we're dividing the array \
size<br> + // by a divisor that is not the size of one element in that \
array.<br> + page_index_t max_pages_per_read = sizeof data / (sizeof (struct \
corefile_pte));<br> + page_index_t page = 0;<br>
+ generation_index_t gen = CORE_PAGE_GENERATION;<br>
+ while (page < n_ptes) {<br>
+ page_index_t pages_remaining = n_ptes - page;<br>
+ page_index_t npages =<br>
+ pages_remaining < max_pages_per_read ? pages_remaining : \
max_pages_per_read;<br> + ssize_t bytes = npages * sizeof (struct \
corefile_pte);<br> + if (read(fd, data, bytes) != bytes) lose("failed \
read");<br> + int i;<br>
+ for ( i = 0 ; i < npages ; ++i, ++page ) {<br>
+ struct corefile_pte pte;<br>
+ memcpy(&pte, data+i*sizeof (struct corefile_pte), sizeof \
pte);<br> + // Low 3 bits of the scan_start hold the 'type' \
flags.<br> + // Low bit of words_used indicates a large (a/k/a \
single) object.<br> + char type = ((pte.words_used & 1) ? \
SINGLE_OBJECT_FLAG : 0)<br> + | (pte.sso & \
0x07);<br> + page_table[page].type = type;<br>
+ pte.words_used &= ~1;<br>
+ /* It is possible, though rare, for the saved page table<br>
+ * to contain free pages below alloc_ptr. */<br>
+ if (type != FREE_PAGE_FLAG) {<br>
+ gc_assert(pte.words_used);<br>
+ page_table[page].words_used_ = pte.words_used;<br>
+ set_page_scan_start_offset(page, pte.sso & ~0x07);<br>
+ page_table[page].gen = gen;<br>
+ }<br>
+ bytes_allocated += pte.words_used << WORD_SHIFT;<br>
+ }<br>
+ }<br>
+ generations[gen].bytes_allocated = bytes_allocated;<br>
+ gc_assert((ssize_t)bytes_allocated <= (ssize_t)(n_ptes * \
GENCGC_PAGE_BYTES));<br> +<br>
+ // Adjust for discrepancies between actually-allocated space addresses<br>
+ // and desired addresses.<br>
if (adj->n_ranges) relocate_heap(adj);<br>
<br>
#ifdef LISP_FEATURE_IMMOBILE_SPACE<br>
@@ -973,24 +1178,83 @@ process_directory(int count, struct ndir_entry *entry,<br>
* (tbh it would be better to output the immobile-space page tables to the \
core file).<br>
* This used to depend critically on space relocation already having been \
performed.<br>
* It doesn't any more, but this is an OK time to do it */<br>
+ extern void immobile_space_coreparse(uword_t,uword_t);<br>
immobile_space_coreparse(spaces[IMMOBILE_FIXEDOBJ_CORE_SPACE_ID].len,<br>
\
spaces[IMMOBILE_TEXT_CORE_SPACE_ID].len);<br> calc_immobile_space_bounds();<br>
#endif<br>
#ifdef LISP_FEATURE_X86_64<br>
- tune_asm_routines_for_microarch(); // before WPing immobile space<br>
+ tune_asm_routines_for_microarch();<br>
#endif<br>
#ifdef LISP_FEATURE_DARWIN_JIT<br>
if (!static_code_space_free_pointer)<br>
static_code_space_free_pointer = (lispobj \
*)STATIC_CODE_SPACE_START;<br> #endif<br>
+<br>
+ if (patchp) {<br>
+ gengcbarrier_patch_code_range(READ_ONLY_SPACE_START, \
read_only_space_free_pointer);<br> + \
gengcbarrier_patch_code_range(STATIC_SPACE_START, static_space_free_pointer);<br> + \
gengcbarrier_patch_code_range(DYNAMIC_SPACE_START, \
(lispobj*)dynamic_space_highwatermark());<br> +#ifdef LISP_FEATURE_IMMOBILE_SPACE<br>
+ gengcbarrier_patch_code_range(TEXT_SPACE_START, \
text_space_highwatermark);<br> +#endif<br>
+ }<br>
+<br>
+ // Apply physical page protection as needed.<br>
+ // The non-soft-card-mark code is disgusting and I do not understand it.<br>
+ if (gen != 0 && ENABLE_PAGE_PROTECTION) {<br>
+#ifdef LISP_FEATURE_SOFT_CARD_MARKS<br>
+ page_index_t p;<br>
+ for (p = 0; p < next_free_page; ++p)<br>
+ if (page_words_used(p)) assign_page_card_marks(p, \
CARD_UNMARKED);<br> +#else<br>
+ // coreparse can avoid hundreds to thousands of mprotect() calls by<br>
+ // treating the whole range from the corefile as protectable, except<br>
+ // that soft-marked code pages must NOT be subject to mprotect.<br>
+ // So just watch out for empty pages and code. Unboxed object \
pages<br> + // will get unprotected on demand.<br>
+#define non_protectable_page_p(x) !page_words_used(x) || \
is_code(page_table[x].type)<br> + page_index_t start = 0, end;<br>
+ // cf. write_protect_generation_pages()<br>
+ while (start < next_free_page) {<br>
+#ifdef LISP_FEATURE_DARWIN_JIT<br>
+ if(is_code(page_table[start].type)) {<br>
+ SET_PAGE_PROTECTED(start,1);<br>
+ for (end = start + 1; end < next_free_page; end++) {<br>
+ if (!page_words_used(end) || \
!is_code(page_table[end].type))<br> + break;<br>
+ SET_PAGE_PROTECTED(end,1);<br>
+ }<br>
+ os_protect(page_address(start), npage_bytes(end - start), \
OS_VM_PROT_ALL);<br> + start = end+1;<br>
+ continue;<br>
+ }<br>
+#endif<br>
+ if (non_protectable_page_p(start)) {<br>
+ ++start;<br>
+ continue;<br>
+ }<br>
+ SET_PAGE_PROTECTED(start,1);<br>
+ for (end = start + 1; end < next_free_page; end++) {<br>
+ if (non_protectable_page_p(end))<br>
+ break;<br>
+ SET_PAGE_PROTECTED(end,1);<br>
+ }<br>
+ os_protect(page_address(start), npage_bytes(end - start), \
OS_VM_PROT_JIT_READ);<br> + start = end;<br>
+ }<br>
+#endif<br>
+ }<br>
+<br>
+#ifdef LISP_FEATURE_DARWIN_JIT<br>
+ darwin_jit_code_pages_kludge();<br>
+ /* For some reason doing an early pthread_jit_write_protect_np sometimes \
fails.<br> + Which is weird, because it's done many times in \
arch_write_linkage_table_entry later.<br> + Adding the executable bit here \
avoids calling pthread_jit_write_protect_np */<br> + \
os_protect((os_vm_address_t)STATIC_CODE_SPACE_START, STATIC_CODE_SPACE_SIZE, \
OS_VM_PROT_ALL);<br> +#endif<br>
}<br>
<br>
-extern void gc_load_corefile_ptes(int, core_entry_elt_t, core_entry_elt_t,<br>
- os_vm_offset_t offset, int \
fd);<br>
-<br>
-static void sanity_check_loaded_core(lispobj);<br>
-<br>
/* 'merge_core_pages': Tri-state flag to determine whether we attempt to \
mark<br>
* pages as targets for virtual memory deduplication via MADV_MERGEABLE.<br>
* 1: Yes<br>
@@ -1030,6 +1294,25 @@ load_core_file(char *file, os_vm_offset_t file_offset, int \
merge_core_pages)<br>
lose("invalid magic number in core: %"OBJ_FMTX" should \
have been %x",<br> (lispobj)val, CORE_MAGIC);<br>
<br>
+ struct coreparse_space spaces[MAX_CORE_SPACE_ID+1] = {<br>
+ {0, 0, 0, 0}, // blank for space ID 0<br>
+ {dynamic_space_size, 0, DYNAMIC_SPACE_START, 0},<br>
+ // This order is determined by constants in compiler/generic/genesis<br>
+ {0, 0, STATIC_SPACE_START, &static_space_free_pointer},<br>
+<br>
+ {0, 0, READ_ONLY_SPACE_START, &read_only_space_free_pointer},<br>
+<br>
+#ifdef LISP_FEATURE_DARWIN_JIT<br>
+ {0, 0, STATIC_CODE_SPACE_START, \
&static_code_space_free_pointer},<br> +#endif<br>
+<br>
+#ifdef LISP_FEATURE_IMMOBILE_SPACE<br>
+ {FIXEDOBJ_SPACE_SIZE | 1, 0,<br>
+ FIXEDOBJ_SPACE_START, &fixedobj_free_pointer},<br>
+ {1, 0, TEXT_SPACE_START, &text_space_highwatermark}<br>
+#endif<br>
+ };<br>
+<br>
for ( ; ; ptr += remaining_len) {<br>
val = *ptr++;<br>
len = *ptr++;<br>
@@ -1046,12 +1329,13 @@ load_core_file(char *file, os_vm_offset_t file_offset, int \
merge_core_pages)<br> case DIRECTORY_CORE_ENTRY_TYPE_CODE:<br>
process_directory(remaining_len / NDIR_ENTRY_LENGTH,<br>
(struct ndir_entry*)ptr, fd, \
file_offset,<br>
- merge_core_pages, &adj);<br>
+ merge_core_pages, spaces, \
&adj);<br> break;<br>
case PAGE_TABLE_CORE_ENTRY_TYPE_CODE:<br>
// elements = gencgc-card-table-index-nbits, n-ptes, nbytes, \
data-page<br> gc_load_corefile_ptes(ptr[0], ptr[1], ptr[2],<br>
- file_offset + (ptr[3] + 1) * \
os_vm_page_size, fd);<br> + \
file_offset + (ptr[3] + 1) * os_vm_page_size, fd,<br> + \
&adj);<br> break;<br>
case INITIAL_FUN_CORE_ENTRY_TYPE_CODE:<br>
initial_function = adjust_word(&adj, (lispobj)*ptr);<br>
@@ -1370,3 +1654,21 @@ static void sanity_check_loaded_core(lispobj \
initial_function)<br> #else<br>
static void sanity_check_loaded_core(lispobj __attribute__((unused)) \
initial_function) {}<br> #endif<br>
+<br>
+/* Prepare the array of corefile_ptes for save */<br>
+void gc_store_corefile_ptes(struct corefile_pte *ptes)<br>
+{<br>
+ page_index_t i;<br>
+ for (i = 0; i < next_free_page; i++) {<br>
+ /* Thanks to alignment requirements, the two three bits<br>
+ * are always zero, so we can use them to store the<br>
+ * allocation type -- region is always closed, so only<br>
+ * the three low bits of allocation flags matter. */<br>
+ uword_t word = page_scan_start_offset(i);<br>
+ gc_assert((word & 0x07) == 0);<br>
+ ptes[i].sso = word | (0x07 & page_table[i].type);<br>
+ int used = page_table[i].words_used_;<br>
+ gc_assert(!(used & 1));<br>
+ ptes[i].words_used = used | page_single_obj_p(i);<br>
+ }<br>
+}<br>
diff --git a/src/runtime/gencgc.c b/src/runtime/gencgc.c<br>
index 00a23aabb..d04a90ab1 100644<br>
--- a/src/runtime/gencgc.c<br>
+++ b/src/runtime/gencgc.c<br>
@@ -4278,6 +4278,8 @@ collect_garbage(generation_index_t last_gen)<br>
void<br>
gc_init(void)<br>
{<br>
+ hopscotch_create(&pinned_objects, HOPSCOTCH_HASH_FUN_DEFAULT, 0 /* hashset \
*/,<br> + 32 /* logical bin count */, 0 /* default \
range */);<br> #ifdef LISP_FEATURE_WIN32<br>
InitializeCriticalSection(&free_pages_lock);<br>
#endif<br>
@@ -4289,153 +4291,6 @@ gc_init(void)<br>
<br>
int gc_card_table_nbits;<br>
long gc_card_table_mask;<br>
-<br>
-static void __attribute__((unused)) gcbarrier_patch_code_range(uword_t start, void* \
limit)<br>
-{<br>
- extern void gcbarrier_patch_code(void*, int);<br>
- struct varint_unpacker unpacker;<br>
- struct code* code;<br>
- lispobj *where = (lispobj*)start;<br>
- while (where < (lispobj*)limit) {<br>
- if (widetag_of(where) == CODE_HEADER_WIDETAG && ((struct \
code*)where)->fixups) {<br>
- code = (struct code*)where;<br>
- varint_unpacker_init(&unpacker, code->fixups);<br>
- // There are two other data streams preceding the one we want<br>
- skip_data_stream(&unpacker);<br>
- skip_data_stream(&unpacker);<br>
- char* instructions = code_text_start(code);<br>
- int prev_loc = 0, loc;<br>
- while (varint_unpack(&unpacker, &loc) && loc != 0) \
{<br>
- loc += prev_loc;<br>
- prev_loc = loc;<br>
- void* patch_where = instructions + loc;<br>
- gcbarrier_patch_code(patch_where, gc_card_table_nbits);<br>
- }<br>
- }<br>
- where += object_size(where);<br>
- }<br>
-}<br>
-void gc_allocate_ptes()<br>
-{<br>
- page_index_t i;<br>
-<br>
- /* Compute the number of pages needed for the dynamic space.<br>
- * Dynamic space size should be aligned on page size. */<br>
- page_table_pages = dynamic_space_size/GENCGC_PAGE_BYTES;<br>
- gc_assert(dynamic_space_size == npage_bytes(page_table_pages));<br>
-<br>
- /* Assert that a cons whose car has MOST-POSITIVE-WORD<br>
- * can not be considered a valid cons, which is to say, even though<br>
- * MOST-POSITIVE-WORD seems to satisfy is_lisp_pointer(),<br>
- * it's OK to use as a filler marker. */<br>
- if (find_page_index((void*)(uword_t)-1) >= 0)<br>
- lose("dynamic space too large");<br>
-<br>
- /* Default nursery size to 5% of the total dynamic space size,<br>
- * min 1Mb. */<br>
- bytes_consed_between_gcs = dynamic_space_size/(os_vm_size_t)20;<br>
- if (bytes_consed_between_gcs < (1024*1024))<br>
- bytes_consed_between_gcs = 1024*1024;<br>
-<br>
- /* The page_table is allocated using "calloc" to zero-initialize \
it.<br>
- * The C library typically implements this efficiently with mmap() if the<br>
- * size is large enough. To further avoid touching each page structure<br>
- * until first use, FREE_PAGE_FLAG must be 0, statically asserted here:<br>
- */<br>
-#if FREE_PAGE_FLAG != 0<br>
-#error "FREE_PAGE_FLAG is not 0"<br>
-#endif<br>
-<br>
- /* An extra 'struct page' exists at each end of the page table acting \
as<br>
- * a sentinel.<br>
- *<br>
- * For for leading sentinel:<br>
- * - all fields are zero except that 'gen' has an illegal value<br>
- * which makes from_space_p() and new_space_p() both return false<br>
- *<br>
- * For the trailing sentinel:<br>
- * - all fields are zero which makes page_ends_contiguous_block_p()<br>
- * return true for the last in-range page index (so the \
"illegal"<br>
- * index at 1+ appears to start a contiguous block even though<br>
- * it corresponds to no page)<br>
- */<br>
- page_table = calloc(page_table_pages+2, sizeof(struct page));<br>
- gc_assert(page_table);<br>
- page_table[0].gen = 9; // an arbitrary never-used value<br>
- ++page_table;<br>
- gc_page_pins = calloc(page_table_pages, 1);<br>
- gc_assert(gc_page_pins);<br>
-<br>
- // The card table size is a power of 2 at *least* as large<br>
- // as the number of cards. These are the default values.<br>
- int nbits = 13;<br>
- long num_gc_cards = 1L << nbits;<br>
-<br>
- // Sure there's a fancier way to round up to a power-of-2<br>
- // but this is executed exactly once, so KISS.<br>
- while (num_gc_cards < page_table_pages*CARDS_PER_PAGE) { ++nbits; \
num_gc_cards <<= 1; }<br>
- // 2 Gigacards should suffice for now. That would span 2TiB of memory<br>
- // using 1Kb card size, or more if larger card size.<br>
- gc_assert(nbits < 32);<br>
- // If the space size is less than or equal to the number of cards<br>
- // that 'gc_card_table_nbits' cover, we're fine. Otherwise, \
problem.<br>
- // 'nbits' is what we need, 'gc_card_table_nbits' is what the \
core was compiled for.<br>
- if (nbits > gc_card_table_nbits) {<br>
- gc_card_table_nbits = nbits;<br>
-#if defined LISP_FEATURE_MIPS || defined LISP_FEATURE_PPC64 \<br>
- || defined LISP_FEATURE_X86 || defined LISP_FEATURE_X86_64<br>
- // The value needed based on dynamic space size exceeds the value that \
the<br>
- // core was compiled for, so we need to patch all code blobs.<br>
- gcbarrier_patch_code_range(READ_ONLY_SPACE_START, \
read_only_space_free_pointer);<br>
- gcbarrier_patch_code_range(STATIC_SPACE_START, \
static_space_free_pointer);<br>
- gcbarrier_patch_code_range(DYNAMIC_SPACE_START, \
(lispobj*)dynamic_space_highwatermark());<br>
-#ifdef LISP_FEATURE_IMMOBILE_SPACE<br>
- gcbarrier_patch_code_range(TEXT_SPACE_START, \
text_space_highwatermark);<br>
-#endif<br>
-#endif<br>
- }<br>
- // Regardless of the mask implied by space size, it has to be \
gc_card_table_nbits wide<br>
- // even if that is excessive - when the core is restarted using a _smaller_ \
dynamic space<br>
- // size than saved at - otherwise lisp could overrun the mark table.<br>
- num_gc_cards = 1L << gc_card_table_nbits;<br>
-<br>
- gc_card_table_mask = num_gc_cards - 1;<br>
- gc_card_mark = successful_malloc(num_gc_cards);<br>
- /* The mark array used to work "by accident" if the numeric value of \
CARD_MARKED<br>
- * is 0 - or equivalently the "WP'ed" state - which is the \
value that calloc()<br>
- * fills with. If using malloc() we have to fill with CARD_MARKED,<br>
- * as I discovered when I changed that to a nonzero value */<br>
- memset(gc_card_mark, CARD_MARKED, num_gc_cards);<br>
-<br>
- gc_common_init();<br>
- hopscotch_create(&pinned_objects, HOPSCOTCH_HASH_FUN_DEFAULT, 0 /* hashset \
*/,<br>
- 32 /* logical bin count */, 0 /* default range \
*/);<br>
-<br>
- bytes_allocated = 0;<br>
-<br>
- /* Initialize the generations. */<br>
- for (i = 0; i < NUM_GENERATIONS; i++) {<br>
- struct generation* gen = &generations[i];<br>
- gen->bytes_allocated = 0;<br>
- gen->gc_trigger = 2000000;<br>
- gen->num_gc = 0;<br>
- gen->cum_sum_bytes_allocated = 0;<br>
- /* the tune-able parameters */<br>
- gen->bytes_consed_between_gc<br>
- = \
bytes_consed_between_gcs/(os_vm_size_t)HIGHEST_NORMAL_GENERATION;<br>
- gen->number_of_gcs_before_promotion = 1;<br>
- gen->minimum_age_before_gc = 0.75;<br>
- }<br>
-<br>
- /* Initialize gc_alloc. */<br>
- gc_alloc_generation = 0;<br>
- gc_init_region(mixed_region);<br>
- gc_init_region(small_mixed_region);<br>
- gc_init_region(boxed_region);<br>
- gc_init_region(unboxed_region);<br>
- gc_init_region(code_region);<br>
- gc_init_region(cons_region);<br>
-}<br>
<br>
<br>
/* alloc() and alloc_list() are external interfaces for memory allocation.<br>
@@ -4924,153 +4779,6 @@ gc_and_save(char *filename, bool prepend_runtime, bool \
purify,<br>
lose("Attempt to save core after non-conservative GC failed.");<br>
}<br>
<br>
-#ifdef LISP_FEATURE_DARWIN_JIT<br>
-/* Inexplicably, an executable page can generate spurious faults if<br>
- * it's not written to after changing its protection flags.<br>
- * Touch every page... */<br>
-void darwin_jit_code_pages_kludge () {<br>
- THREAD_JIT(0);<br>
- page_index_t page;<br>
- for (page = 0; page < next_free_page; page++) {<br>
- if(is_code(page_table[page].type)) {<br>
- char* addr = page_address(page);<br>
- for (unsigned i = 0; i < GENCGC_PAGE_BYTES; i+=4096) {<br>
- volatile char* page_start = addr + i;<br>
- page_start[0] = page_start[0];<br>
- }<br>
- }<br>
- }<br>
- THREAD_JIT(1);<br>
-}<br>
-#endif<br>
-<br>
-/* Read corefile ptes from 'fd' which has already been positioned<br>
- * and store into the page table */<br>
-void gc_load_corefile_ptes(int card_table_nbits,<br>
- core_entry_elt_t n_ptes, core_entry_elt_t \
total_bytes,<br>
- os_vm_offset_t offset, int fd)<br>
-{<br>
- gc_assert(ALIGN_UP(n_ptes * sizeof (struct corefile_pte), N_WORD_BYTES)<br>
- == (size_t)total_bytes);<br>
- if (next_free_page != n_ptes)<br>
- lose("n_PTEs=%"PAGE_INDEX_FMT" but expected \
%"PAGE_INDEX_FMT,<br>
- n_ptes, next_free_page);<br>
-<br>
- // Allocation of PTEs is delayed 'til now so that calloc() doesn't<br>
- // consume addresses that would have been taken by a mapped space.<br>
- gc_card_table_nbits = card_table_nbits;<br>
- gc_allocate_ptes();<br>
-<br>
- if (LSEEK(fd, offset, SEEK_SET) != offset) lose("failed seek");<br>
-<br>
- char data[8192];<br>
- // Process an integral number of ptes on each read.<br>
- // Parentheses around sizeof (type) are necessary to suppress a<br>
- // clang warning (-Wsizeof-array-div) that we're dividing the array \
size<br>
- // by a divisor that is not the size of one element in that array.<br>
- page_index_t max_pages_per_read = sizeof data / (sizeof (struct \
corefile_pte));<br>
- page_index_t page = 0;<br>
- generation_index_t gen = CORE_PAGE_GENERATION;<br>
- while (page < n_ptes) {<br>
- page_index_t pages_remaining = n_ptes - page;<br>
- page_index_t npages =<br>
- pages_remaining < max_pages_per_read ? pages_remaining : \
max_pages_per_read;<br>
- ssize_t bytes = npages * sizeof (struct corefile_pte);<br>
- if (read(fd, data, bytes) != bytes) lose("failed read");<br>
- int i;<br>
- for ( i = 0 ; i < npages ; ++i, ++page ) {<br>
- struct corefile_pte pte;<br>
- memcpy(&pte, data+i*sizeof (struct corefile_pte), sizeof \
pte);<br>
- // Low 3 bits of the scan_start hold the 'type' flags.<br>
- // Low bit of words_used indicates a large (a/k/a single) \
object.<br>
- char type = ((pte.words_used & 1) ? SINGLE_OBJECT_FLAG : \
0)<br>
- | (pte.sso & 0x07);<br>
- page_table[page].type = type;<br>
- pte.words_used &= ~1;<br>
- /* It is possible, though rare, for the saved page table<br>
- * to contain free pages below alloc_ptr. */<br>
- if (type != FREE_PAGE_FLAG) {<br>
- gc_assert(pte.words_used);<br>
- page_table[page].words_used_ = pte.words_used;<br>
- set_page_scan_start_offset(page, pte.sso & ~0x07);<br>
- page_table[page].gen = gen;<br>
- }<br>
- bytes_allocated += pte.words_used << WORD_SHIFT;<br>
- }<br>
- }<br>
- generations[gen].bytes_allocated = bytes_allocated;<br>
- gc_assert((ssize_t)bytes_allocated <= (ssize_t)(n_ptes * \
GENCGC_PAGE_BYTES));<br>
- if (gen != 0 && ENABLE_PAGE_PROTECTION) {<br>
-#ifdef LISP_FEATURE_SOFT_CARD_MARKS<br>
- page_index_t p;<br>
- for (p = 0; p < next_free_page; ++p)<br>
- if (page_words_used(p)) assign_page_card_marks(p, \
CARD_UNMARKED);<br>
-#else<br>
- // coreparse can avoid hundreds to thousands of mprotect() calls by<br>
- // treating the whole range from the corefile as protectable, except<br>
- // that soft-marked code pages must NOT be subject to mprotect.<br>
- // So just watch out for empty pages and code. Unboxed object \
pages<br>
- // will get unprotected on demand.<br>
-#define non_protectable_page_p(x) !page_words_used(x) || \
is_code(page_table[x].type)<br>
- page_index_t start = 0, end;<br>
- // cf. write_protect_generation_pages()<br>
- while (start < next_free_page) {<br>
-#ifdef LISP_FEATURE_DARWIN_JIT<br>
- if(is_code(page_table[start].type)) {<br>
- SET_PAGE_PROTECTED(start,1);<br>
- for (end = start + 1; end < next_free_page; end++) {<br>
- if (!page_words_used(end) || \
!is_code(page_table[end].type))<br>
- break;<br>
- SET_PAGE_PROTECTED(end,1);<br>
- }<br>
- os_protect(page_address(start), npage_bytes(end - start), \
OS_VM_PROT_ALL);<br>
- start = end+1;<br>
- continue;<br>
- }<br>
-#endif<br>
- if (non_protectable_page_p(start)) {<br>
- ++start;<br>
- continue;<br>
- }<br>
- SET_PAGE_PROTECTED(start,1);<br>
- for (end = start + 1; end < next_free_page; end++) {<br>
- if (non_protectable_page_p(end))<br>
- break;<br>
- SET_PAGE_PROTECTED(end,1);<br>
- }<br>
- os_protect(page_address(start), npage_bytes(end - start), \
OS_VM_PROT_JIT_READ);<br>
- start = end;<br>
- }<br>
-#endif<br>
- }<br>
-<br>
-#ifdef LISP_FEATURE_DARWIN_JIT<br>
- darwin_jit_code_pages_kludge();<br>
- /* For some reason doing an early pthread_jit_write_protect_np sometimes \
fails.<br>
- Which is weird, because it's done many times in \
arch_write_linkage_table_entry later.<br>
- Adding the executable bit here avoids calling \
pthread_jit_write_protect_np */<br>
- os_protect((os_vm_address_t)STATIC_CODE_SPACE_START, STATIC_CODE_SPACE_SIZE, \
OS_VM_PROT_ALL);<br>
-#endif<br>
-}<br>
-<br>
-/* Prepare the array of corefile_ptes for save */<br>
-void gc_store_corefile_ptes(struct corefile_pte *ptes)<br>
-{<br>
- page_index_t i;<br>
- for (i = 0; i < next_free_page; i++) {<br>
- /* Thanks to alignment requirements, the two three bits<br>
- * are always zero, so we can use them to store the<br>
- * allocation type -- region is always closed, so only<br>
- * the three low bits of allocation flags matter. */<br>
- uword_t word = page_scan_start_offset(i);<br>
- gc_assert((word & 0x07) == 0);<br>
- ptes[i].sso = word | (0x07 & page_table[i].type);<br>
- int used = page_table[i].words_used_;<br>
- gc_assert(!(used & 1));<br>
- ptes[i].words_used = used | page_single_obj_p(i);<br>
- }<br>
-}<br>
-<br>
#define ARTIFICIALLY_HIGH_GEN 8<br>
generation_index_t gc_gen_of(lispobj obj, int defaultval) {<br>
int page = find_page_index((void*)obj);<br>
<br>
-----------------------------------------------------------------------<br>
<br>
<br>
hooks/post-receive<br>
-- <br>
SBCL<br>
<br>
<br>
_______________________________________________<br>
Sbcl-commits mailing list<br>
<a href="mailto:Sbcl-commits@lists.sourceforge.net" \
target="_blank">Sbcl-commits@lists.sourceforge.net</a><br> <a \
href="https://lists.sourceforge.net/lists/listinfo/sbcl-commits" rel="noreferrer" \
target="_blank">https://lists.sourceforge.net/lists/listinfo/sbcl-commits</a><br> \
</blockquote></div>
_______________________________________________
Sbcl-devel mailing list
Sbcl-devel@lists.sourceforge.net
https://lists.sourceforge.net/lists/listinfo/sbcl-devel
[prev in list] [next in list] [prev in thread] [next in thread]
Configure |
About |
News |
Add a list |
Sponsored by KoreLogic