/* StarForth — Steady-State Virtual Machine Runtime Copyright (c) 2023–2025 Robert A. James All rights reserved. This file is part of the StarForth project. Licensed under the StarForth License, Version 1.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at: https://github.com/star.4th@proton.me/StarForth/LICENSE.txt This software is provided "AS IS", WITHOUT WARRANTY OF ANY KIND, express or implied, including but not limited to the warranties of merchantability, fitness for a particular purpose, and noninfringement. See the License for the specific language governing permissions and limitations under the License. StarForth — Steady-State Virtual Machine Runtime Copyright (c) 2023–2025 Robert A. James All rights reserved. This file is part of the StarForth project. Licensed under the StarForth License, Version 1.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at: https://github.com/star.4th@proton.me/StarForth/LICENSE.txt This software is provided "AS IS", WITHOUT WARRANTY OF ANY KIND, express or implied, including but not limited to the warranties of merchantability, fitness for a particular purpose, and noninfringement. See the License for the specific language governing permissions and limitations under the License. */ /* * StarForth dictionary_management.c — FORTH-79 + ANSI C99 * Optimized: incremental FC index + capacity reuse + newest-first search */ #include "../include/vm.h" #include "../include/io.h" #include "../include/log.h" #include "../include/physics_metadata.h" #include "../include/physics_hotwords_cache.h" #include "../include/physics_pipelining_metrics.h" #include "../include/dictionary_heat_optimization.h" #include "../include/platform_alloc.h" #include #include #include #ifdef __STARKERNEL__ #include "starkernel/vm/stadium_words.h" /* item 4.1: FORGET coherence hook */ #include "starkernel/vm_uuid.h" /* vm_uuid_hera() */ #endif /* LIKELY/UNLIKELY macros are defined in vm.h */ #ifndef SF_FC_BUCKETS #define SF_FC_BUCKETS 256 #endif /* Buckets of entry pointers, sizes, and capacities (we reuse memory). */ /* Note: sf_fc_list, sf_fc_count, and sf_fc_cap are not static to allow access from * dictionary_heat_optimization.c for heat-aware bucket reorganization with proper * capacity checks (ASan fix: 2025-12-09). */ DictEntry **sf_fc_list[SF_FC_BUCKETS]; size_t sf_fc_count[SF_FC_BUCKETS]; size_t sf_fc_cap[SF_FC_BUCKETS]; static DictEntry *sf_cached_latest = NULL; /* last head we indexed */ static uint32_t vm_dictionary_acquire_word_id(VM *vm) { if (!vm) { return WORD_ID_INVALID; } if (vm->recycled_word_id_count > 0) { return vm->recycled_word_ids[--vm->recycled_word_id_count]; } if (vm->next_word_id < DICTIONARY_SIZE) { return vm->next_word_id++; } return WORD_ID_INVALID; } void vm_dictionary_track_entry(VM *vm, DictEntry *entry) { if (!vm || !entry) { return; } uint32_t word_id = vm_dictionary_acquire_word_id(vm); entry->word_id = word_id; if (word_id == WORD_ID_INVALID || word_id >= DICTIONARY_SIZE) { log_message(LOG_WARN, "vm_dictionary_track_entry: out of stable IDs, disabling metrics for '%.*s'", (int)entry->name_len, entry->name); return; } vm->word_id_map[word_id] = entry; } void vm_dictionary_untrack_entry(VM *vm, DictEntry *entry) { if (!vm || !entry) { return; } /* Cache coherence: the entry is about to be removed (FORGET frees it) — * a stale pointer left in the hot-words ring would be a use-after-free, * and an older same-named word may become visible again. * §17.3 (item 4.1): retired under __STARKERNEL__ -- the kernel word * layer stays inert to this mechanism entirely, so there is nothing to * keep coherent here on the kernel side. Hosted is unaffected. */ #ifndef __STARKERNEL__ hotwords_cache_evict_entry(vm->hotwords_cache, entry); #endif uint32_t word_id = entry->word_id; if (word_id == WORD_ID_INVALID || word_id >= DICTIONARY_SIZE) { return; } #ifdef __STARKERNEL__ /* item 4.1 coherence hook: if this word_id is resident on the Stadium, * reclaim its cell (crediting heat back to the reservoir) BEFORE the id * is recycled below -- otherwise the next word assigned this same id * would alias onto the forgotten word's stale cell (same failure class * as the 2026-08-02 block_words.c aliasing bug). */ stadium_word_forget(vm->stadium_vm_id, word_id); #endif if (vm->word_id_map[word_id] == entry) { vm->word_id_map[word_id] = NULL; } if (vm->recycled_word_id_count < DICTIONARY_SIZE) { vm->recycled_word_ids[vm->recycled_word_id_count++] = word_id; } entry->word_id = WORD_ID_INVALID; } DictEntry *vm_dictionary_lookup_by_word_id(VM *vm, uint32_t word_id) { if (!vm || word_id >= DICTIONARY_SIZE) { return NULL; } return vm->word_id_map[word_id]; } /* --- helpers ------------------------------------------------------------- */ static inline void *sf_xrealloc(void *p, size_t nbytes) { void *r = sf_realloc(p, nbytes); if (!r) { sf_free(p); return NULL; } return r; } static void sf_fc_clear_all(void) { for (size_t i = 0; i < SF_FC_BUCKETS; ++i) { sf_free(sf_fc_list[i]); sf_fc_list[i] = NULL; sf_fc_count[i] = 0; sf_fc_cap[i] = 0; } } /* Ensure bucket i has at least `need` capacity; grow geometrically. */ static int sf_fc_reserve(size_t i, size_t need) { if (sf_fc_cap[i] >= need) return 1; size_t newcap = sf_fc_cap[i] ? sf_fc_cap[i] : 8; while (newcap < need) newcap <<= 1; DictEntry **old = sf_fc_list[i]; DictEntry **newv = (DictEntry **) sf_xrealloc(old, newcap * sizeof(*newv)); if (!newv) return 0; /* Kernel sf_realloc is a bump allocator that does not copy; patch that up. * On hosted Linux, realloc already copied before freeing old — do NOT memcpy * from freed memory (UB; glibc overwrites freed block with heap metadata). */ #ifdef __STARKERNEL__ { size_t old_count = sf_fc_count[i]; if (newv != old && old && old_count > 0) memcpy(newv, old, old_count * sizeof(*newv)); } #endif sf_fc_list[i] = newv; sf_fc_cap[i] = newcap; return 1; } /* Full rebuild: count → reserve → fill oldest→newest (so backward scan = newest-first). */ static void sf_rebuild_fc_index(VM *vm) { /* Reset counts, keep capacity to avoid churn. */ for (size_t i = 0; i < SF_FC_BUCKETS; ++i) sf_fc_count[i] = 0; /* Count first chars. */ for (DictEntry *e = vm->latest; e; e = e->link) { unsigned fc = (unsigned char) e->name[0]; sf_fc_count[fc]++; } /* Reserve memory once per bucket. */ for (size_t i = 0; i < SF_FC_BUCKETS; ++i) { if (sf_fc_count[i] && !sf_fc_reserve(i, sf_fc_count[i])) { /* OOM for this bucket → treat as empty; we’ll just miss the fast path. */ sf_fc_count[i] = 0; } } /* Back-fill so [0]=oldest, [count-1]=newest. */ size_t fill_at[SF_FC_BUCKETS]; for (size_t i = 0; i < SF_FC_BUCKETS; ++i) fill_at[i] = sf_fc_count[i]; for (DictEntry *e = vm->latest; e; e = e->link) { unsigned fc = (unsigned char) e->name[0]; if (sf_fc_count[fc] == 0 || !sf_fc_list[fc]) continue; sf_fc_list[fc][--fill_at[fc]] = e; } sf_cached_latest = vm->latest; } /* Fast path: if exactly one new word was pushed, append it without rebuilding. */ static void sf_try_fast_append(VM *vm) { if (!vm || vm->latest == sf_cached_latest) return; DictEntry *newest = vm->latest; /* single push if the new head links to the previously cached head */ if (newest && newest->link == sf_cached_latest) { unsigned fc = (unsigned char) newest->name[0]; size_t n = sf_fc_count[fc]; if (sf_fc_reserve(fc, n + 1)) { /* Keep oldest→…→newest ordering: append at the end. */ sf_fc_list[fc][n] = newest; sf_fc_count[fc] = n + 1; sf_cached_latest = newest; return; } } /* Otherwise fall back to a full rebuild (multiple changes, deletes, etc.). */ sf_rebuild_fc_index(vm); } /* --- API ----------------------------------------------------------------- */ /** * @brief Return whichever of two entries is the more recent definition. * * Walks the definition chain from @c vm->latest; the first of (@p a, @p b) * encountered is the newer one. The link chain is the only reliable age * order — word_id values are recycled and bucket positions are shuffled by * @c dict_reorganize_buckets_by_heat(), so neither can arbitrate age. * Only invoked when a bucket holds two visible entries with the same name * (redefinition/shadowing), so the walk cost is off the common path. */ static DictEntry *dict_newer_of(VM *vm, DictEntry *a, DictEntry *b) { for (DictEntry *e = vm->latest; e; e = e->link) { if (e == a) return a; if (e == b) return b; } return a; } DictEntry *vm_dict_resolve_in_bucket(VM *vm, DictEntry **bucket, size_t n, const char *name, size_t len) { if (!vm || !bucket || n == 0 || !name || len == 0) return NULL; const unsigned char last = (len > 1) ? (unsigned char) name[len - 1] : 0; DictEntry *best = NULL; /* Full scan: bucket order is heat-shuffled, so no prefix of the bucket * is authoritative — every same-named candidate must be considered and * the newest visible definition wins (FORTH-79 shadowing). */ for (size_t i = n; i-- > 0;) { DictEntry *e = bucket[i]; if (!e) continue; if ((size_t) e->name_len != len) continue; #ifdef WORD_HIDDEN if (UNLIKELY(e->flags & WORD_HIDDEN)) continue; #endif #ifdef WORD_SMUDGED if (UNLIKELY(e->flags & WORD_SMUDGED)) continue; #endif if (len > 1 && (unsigned char) e->name[len - 1] != last) continue; if (memcmp(e->name, name, len) != 0) continue; best = best ? dict_newer_of(vm, best, e) : e; } return best; } /* Find by name: newest-first within the first-character bucket. */ /** * @brief Finds a word in the dictionary by name * * Searches for a word in the dictionary using an optimized first-character index. * The search is performed newest-first within the matching first-character bucket. * * @param vm The virtual machine context * @param name The name to search for * @param len Length of the name string * @return DictEntry* Pointer to found dictionary entry or NULL if not found */ DictEntry *vm_find_word(VM *vm, const char *name, size_t len) { if (UNLIKELY(!vm || !name || len == 0)) return NULL; /* DoE counter: dictionary lookups */ vm->heartbeat.dictionary_lookups++; /* Thread safety: Acquire dict_lock for read access to dictionary */ sf_mutex_lock(&vm->dict_lock); /* Keep the index in sync with dictionary head, cheaply if possible. */ if (UNLIKELY(sf_cached_latest != vm->latest)) sf_try_fast_append(vm); const unsigned char first = (unsigned char) name[0]; DictEntry **bucket = sf_fc_list[first]; size_t n = sf_fc_count[first]; if (UNLIKELY(!bucket || n == 0)) { sf_mutex_unlock(&vm->dict_lock); return NULL; } /* Use hot-words cache for physics-driven frequency-based acceleration. * The cache's bucket fallback resolves via vm_dict_resolve_in_bucket(), * so a non-NULL result already honors newest-first shadowing. * §17.3 (item 4.1): bypassed under __STARKERNEL__ -- the kernel side * feeds the Stadium instead (vm_core.c's dispatch sites), and this * lookup fast path is retired in favor of the ordinary bucket scan * below. Hosted is unaffected. */ #ifndef __STARKERNEL__ DictEntry *found = hotwords_cache_lookup(vm, vm->hotwords_cache, bucket, n, name, len); if (found) { sf_mutex_unlock(&vm->dict_lock); return found; } #endif /* Phase 2: Choose lookup strategy based on pattern diversity */ if (vm->lookup_strategy == 1) { /* Heat-aware lookup: search by heat percentiles (hot first) */ DictEntry *result = dict_find_word_heat_aware(vm, name, len); sf_mutex_unlock(&vm->dict_lock); return result; } /* Fallback/Default: arbitrated scan — buckets may be heat-reordered, so * first-match-wins is unsound; the resolver picks the newest visible * definition among all same-named candidates. */ DictEntry *e = vm_dict_resolve_in_bucket(vm, bucket, n, name, len); sf_mutex_unlock(&vm->dict_lock); return e; } /* Create a new word (unchanged layout); index append happens lazily on next lookup. */ /** * @brief Creates a new word in the dictionary * * Allocates and initializes a new dictionary entry with the given name and function. * The word is added to the dictionary but index update is deferred until next lookup. * * @param vm The virtual machine context * @param name Name for the new word * @param len Length of the name string * @param func Function pointer for the word's behavior * @return DictEntry* Pointer to new dictionary entry or NULL on failure */ DictEntry *vm_create_word(VM *vm, const char *name, size_t len, word_func_t func) { if (!vm || !name || len == 0 || len > WORD_NAME_MAX) { if (vm) { vm->error = 1; log_message(LOG_ERROR, "vm_create_word: invalid parameters (vm=%p, name=%p, len=%zu, max=%d)", (void *) vm, (const void *) name, len, WORD_NAME_MAX); } return NULL; } DictEntry *entry = NULL; int lock_held = 0; sf_mutex_lock(&vm->dict_lock); lock_held = 1; /* Pin is permanent: no word may shadow a pinned entry — ever, by anyone */ for (DictEntry *scan = vm->latest; scan; scan = scan->link) { if (scan->acl_pinned && scan->name_len == (uint8_t)len && memcmp(scan->name, name, len) == 0) { vm->error = 1; log_message(LOG_WARN, "vm_create_word: cannot shadow pinned word '%.*s'", (int)len, name); entry = NULL; goto create_cleanup; } } size_t base = offsetof(DictEntry, name); size_t name_bytes = len + 1; size_t align = sizeof(cell_t); size_t df_off = (base + name_bytes + (align - 1)) & ~(align - 1); size_t total = df_off + sizeof(cell_t); entry = (DictEntry *) sf_malloc(total); if (!entry) { vm->error = 1; log_message(LOG_ERROR, "vm_create_word: malloc failed for '%.*s' (%zu bytes)", (int) len, name, total); entry = NULL; goto create_cleanup; } entry->link = vm->latest; entry->func = func; entry->flags = 0; entry->name_len = (uint8_t) len; entry->execution_heat = 0; /* Initialize execution heat counter */ entry->acl_default = ACL_USER_DEFAULT; entry->acl_ttl = 0; /* first execution always rechecks */ entry->acl_allow = 1; /* permissive until ACL.4th loads */ entry->acl_mode = ACL_MODE_TTL; entry->acl_pinned = 0; entry->word_id = WORD_ID_INVALID; uint32_t header_bytes = (total > UINT32_MAX) ? UINT32_MAX : (uint32_t) total; physics_metadata_init(entry, header_bytes); physics_metadata_apply_seed(entry); /* Initialize word transition metrics for pipelining (Phase 1) */ entry->transition_metrics = (WordTransitionMetrics *)sf_malloc(sizeof(WordTransitionMetrics)); if (entry->transition_metrics) { transition_metrics_init(entry->transition_metrics); } else { log_message(LOG_WARN, "vm_create_word: transition metrics malloc failed for '%.*s'", (int)len, name); } memcpy(entry->name, name, len); entry->name[len] = '\0'; if (df_off > base + name_bytes) { memset(((uint8_t *) entry) + base + name_bytes, 0, df_off - (base + name_bytes)); } *(cell_t *) (((uint8_t *) entry) + df_off) = 0; vm->latest = entry; vm_dictionary_track_entry(vm, entry); /* Cache coherence: this definition may shadow an older word of the same * name that the hot-words cache is still serving. Evict the name so the * next lookup re-resolves through the arbitrated bucket scan. * §17.3 (item 4.1): retired under __STARKERNEL__, same as the other * hotwords_cache_* call sites in this file. */ #ifndef __STARKERNEL__ hotwords_cache_evict_name(vm->hotwords_cache, name, len); #endif log_message(LOG_DEBUG, "vm_create_word: '%.*s' len=%zu total=%zu @%p", (int) len, name, len, total, (void *) entry); create_cleanup: if (lock_held) sf_mutex_unlock(&vm->dict_lock); return entry; } /* Linear scan by func (left as-is; usually cold path). */ /** * @brief Finds a word by its function pointer * * Performs a linear scan of the dictionary to find an entry with matching function. * * @param vm The virtual machine context * @param func Function pointer to search for * @return DictEntry* Pointer to found dictionary entry or NULL if not found */ DictEntry *vm_dictionary_find_by_func(VM *vm, word_func_t func) { for (DictEntry *e = vm ? vm->latest : NULL; e; e = e->link) { if (e->func == func) return e; } return NULL; } DictEntry *vm_dictionary_find_latest_by_func(VM *vm, word_func_t func) { return vm_dictionary_find_by_func(vm, func); } /** * @brief Gets pointer to a word's data field * * Calculates and returns pointer to the aligned data field following the name field. * * @param entry Dictionary entry to get data field from * @return cell_t* Pointer to data field or NULL if entry is NULL */ cell_t *vm_dictionary_get_data_field(DictEntry *entry) { if (!entry) return NULL; size_t base = offsetof(DictEntry, name); size_t name_bytes = (size_t) entry->name_len + 1; size_t align = sizeof(cell_t); size_t df_off = (base + name_bytes + (align - 1)) & ~(align - 1); return (cell_t *) (((uint8_t *) entry) + df_off); } /** * @brief Marks the latest word as hidden * * Sets the WORD_HIDDEN flag on the most recently defined word. * * @param vm The virtual machine context */ void vm_hide_word(VM *vm) { if (vm && vm->latest) { vm->latest->flags |= WORD_HIDDEN; physics_metadata_refresh_state(vm->latest); /* Visibility changed: this name may now resolve to an older entry. * §17.3 (item 4.1): retired under __STARKERNEL__. */ #ifndef __STARKERNEL__ hotwords_cache_evict_name(vm->hotwords_cache, vm->latest->name, vm->latest->name_len); #endif } } void vm_smudge_word(VM *vm) { if (!vm || !vm->latest) { if (vm) { vm->error = 1; log_message(LOG_ERROR, "vm_smudge_word: invalid vm or no latest word"); } return; } vm->latest->flags ^= WORD_SMUDGED; physics_metadata_refresh_state(vm->latest); /* Visibility changed in either direction: force re-resolution. * §17.3 (item 4.1): retired under __STARKERNEL__. */ #ifndef __STARKERNEL__ hotwords_cache_evict_name(vm->hotwords_cache, vm->latest->name, vm->latest->name_len); #endif } void vm_pin_execution_heat(VM *vm) { if (!vm || !vm->latest) { if (vm) { vm->error = 1; log_message(LOG_ERROR, "vm_pin_execution_heat: invalid vm or no latest word"); } return; } vm->latest->flags |= WORD_PINNED; physics_metadata_refresh_state(vm->latest); } void vm_unpin_execution_heat(VM *vm) { if (!vm || !vm->latest) { if (vm) { vm->error = 1; log_message(LOG_ERROR, "vm_unpin_execution_heat: invalid vm or no latest word"); } return; } vm->latest->flags &= ~WORD_PINNED; physics_metadata_refresh_state(vm->latest); } /* If you ever need to hard-reset (e.g., switching vocabularies wholesale) */ /** * @brief Resets the dictionary index * * Clears all index data and cached state. Used when switching vocabularies. */ void vm_dictionary_index_reset(void) { sf_cached_latest = NULL; sf_fc_clear_all(); }